From 02fed8fdf9880c543c6210734e859e0ef6f3e2bd Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 12:56:02 -0600 Subject: [PATCH 01/47] docs(agent): design Windows lifecycle durability --- ...gent-helper-lifecycle-durability-design.md | 499 ++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md diff --git a/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md b/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md new file mode 100644 index 0000000000..fbfe0c90e8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md @@ -0,0 +1,499 @@ +# Windows agent and helper lifecycle durability + +**Date:** 2026-07-14 +**Status:** Design approved, awaiting spec review +**Scope:** Windows session helpers, RDS support, main-agent exclusivity, watchdog recovery, and diagnostics +**Branch:** `ToddHebebrand/helper-sessions-in-remote-desktop` + +## Problem + +Windows terminal servers can accumulate far more `breeze-user-helper.exe` +processes than there are eligible sessions. The reported example showed dozens +of SYSTEM-owned helpers on a server with no more than 20 users. Separate Windows +11 reports describe a Breeze service that stops while a Breeze process remains, +multiple Breeze processes running at once, and the device remaining offline. + +These reports expose several related lifecycle gaps. + +### RDS helper accumulation + +The lifecycle manager intentionally targets one SYSTEM helper per active or +connected Windows session and one user helper per active session. However, +Windows IPC admission is keyed only by SID and permits five connections per +identity. Every SYSTEM helper has SID `S-1-5-18`, so a terminal server exhausts +that shared bucket after five SYSTEM helpers authenticate. + +Additional helpers receive a transient pre-auth rejection and remain alive in +their reconnect loop. Because they never authenticated, reconciliation cannot +see them and launches replacements every 30 seconds, up to ten attempts per +session. Pre-auth-rejected helpers are also absent from registered-session PID +cleanup. The result is a growing process population without a corresponding +increase in usable helper sessions. + +### RDP user helpers are rejected + +The broker currently binds the user helper role to the physical active console +session. A correctly tokenized user helper in an RDP session therefore fails +authorization even when the OS-derived peer session matches the claimed +session. This prevents PAM, run-as-user, notifications, and related per-user +operations from working reliably in RDP sessions. + +### Multiple `breeze-agent.exe` processes are ambiguous + +When `breeze-user-helper.exe` is missing, quarantined, or absent during a +partial upgrade, the broker deliberately falls back to launching +`breeze-agent.exe user-helper --role ...`. Task Manager can consequently show +several `breeze-agent.exe` processes even though only one is the SCM-managed +main service. + +There is also no process-wide guard on the full `breeze-agent.exe run` path. An +elevated manual or scheduled invocation can initialize a second full agent next +to the service. The SCM normally serializes its own service instance, but it +does not prevent a console-mode invocation of the same command. + +### Watchdog recovery can race normal shutdown + +The watchdog's first recovery attempt requests a service stop, waits up to 15 +seconds, and then calls `Start` even if the service never reached `Stopped`. +Normal bounded shutdown can legitimately consume approximately 21 seconds +across subsystem cancellation, command drain, WebSocket stop, and heartbeat +stop stages. The watchdog can therefore call `Start` while the service is still +`StopPending`. Windows should reject that start instead of creating a second +SCM instance, but the failed recovery extends the offline window and can look +like a stuck service. + +The watchdog's forced attempt also uses the PID most recently read from the +agent state file. A duplicate console agent or stale state file can make that +PID different from the process currently owned by the BreezeAgent service. + +## Goals + +- Support the intended number of SYSTEM and user helpers on multi-session RDS + hosts without weakening helper authentication. +- Enforce exactly one live helper for each `(Windows session, helper role)`. +- Make helper ownership explicit and terminate helpers on logoff, service + shutdown, agent crash, and replacement. +- Authorize RDP user helpers using OS-derived identity and session evidence. +- Prevent more than one full Windows agent from initializing. +- Make watchdog recovery wait for verified service and process transitions. +- Make every Breeze process's role evident in local diagnostics and support + collection. +- Preserve compatibility with partially upgraded installations that must use + the main binary as a temporary helper fallback. + +## Non-goals + +- Replacing the Windows service/watchdog architecture with a new supervisor. +- Removing the companion helper fallback in this change. +- Changing Linux or macOS helper admission semantics. +- Using executable names alone to kill processes. +- Treating every Breeze process shown in Task Manager as a full-agent duplicate. +- Adding customer-facing configuration or requiring guide changes. +- Diagnosing a specific reported stuck process without its command line, SCM + state, logs, and process dump. This design prevents known lifecycle gaps but + does not invent an unsupported root cause for a particular endpoint. + +## Design principles + +1. The operating system is the authority for SID, token session, service PID, + and process lifetime. Helper-supplied metadata cannot override it. +2. Lifecycle ownership and IPC authentication are separate controls. Admission + must remain safe even when reconciliation is wrong, and reconciliation must + remain bounded even when admission rejects a process. +3. Every replacement must follow `observe -> stop -> verify -> start -> verify`. +4. Destructive cleanup must prove process ownership and role. It must never kill + an arbitrary process based only on a PID or image name. +5. Existing fallback paths remain observable and bounded during version skew. + +## Architecture + +The change is additive and stays within the current service, session broker, +helper client, and watchdog components. + +### 1. Session-aware IPC admission + +On Windows, the pre-auth identity key becomes a composite of the OS-derived +peer SID and peer Windows session ID: + +```text +windows::session: +``` + +This key is used for pre-auth connection counts and rate limiting. It prevents +all SYSTEM helpers across an RDS host from competing for the same five-slot +bucket while retaining a bounded quota within each Windows session. + +After the helper hello is authenticated, the broker applies a stricter logical +key: + +```text + +``` + +Only one registered helper is allowed for a given Windows session and role. +When a replacement authenticates, the broker either rejects it as a duplicate +or replaces a provably stale registered session; it never keeps two active +sessions for the same key. + +Unix identity behavior remains unchanged. + +### 2. RDP-safe user-role authorization + +The physical-console-only gate is replaced on Windows with the following user +helper requirements: + +- The peer token is not LocalSystem. +- The OS-derived peer Windows session ID is nonzero and equals the session ID + claimed in the authenticated helper hello. +- The broker's session detector recognizes that session as eligible for the + requested role. +- Existing binary identity, token, protocol, and helper-role checks still pass. + +The helper's username or claimed session is not trusted independently. The +kernel token session is the binding evidence. This admits a legitimate user +helper in an RDP session without allowing a helper in one session to impersonate +another. + +SYSTEM-role authorization continues to require LocalSystem and matching +OS-derived/claimed session IDs. + +### 3. Lifecycle-owned helper registry + +The Windows lifecycle manager maintains a registry keyed by: + +```go +type HelperKey struct { + WindowsSessionID uint32 + Role string +} +``` + +Each entry records the PID, process handle, launch time, executable path, +command mode, and current state (`starting`, `connected`, `stopping`, or +`exited`). The spawner returns a process reference instead of only reporting +whether launch succeeded. + +Reconciliation follows these rules: + +- If the desired key has a connected broker session, do nothing. +- If it has a tracked process that is still alive or still within the startup + grace period, do not spawn another. +- If the tracked process exited, record the exit classification and apply the + existing retry/cooldown policy before replacing it. +- If the session is no longer eligible, terminate and reap the tracked process, + close any broker session, and remove retry state. +- A broker connect/disconnect callback updates the corresponding tracked entry + rather than relying only on polling. + +This closes the current gap where pre-auth-rejected helpers are alive but +invisible to reconciliation. + +### 4. Windows Job Object ownership + +All proactively spawned Windows helpers are created suspended, assigned to an +agent-owned Job Object configured with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`, +and resumed only after assignment succeeds. This prevents a fast-starting +helper from escaping ownership between process creation and job assignment. + +Consequences: + +- Graceful agent shutdown explicitly terminates/reaps helpers and then closes + the job. +- An agent crash or forced termination closes its job handle and Windows kills + remaining child helpers. +- A restarted agent starts with no surviving children from the previous + owner under normal Job Object semantics. + +If assigning a newly created helper to the job fails, the spawner terminates +the still-suspended helper, closes its handles, and reports launch failure. It +must not leave an unowned process. + +The existing scheduled user-helper path is not necessarily a child of the +agent and therefore cannot rely on this job. Broker-side one-per-key admission +and role/session validation remain authoritative for it. + +### 5. Logoff, disconnect, and shutdown behavior + +- `logoff`: terminate both SYSTEM and user role processes for the session and + remove their retry state. +- `disconnect`: keep or stop the SYSTEM role according to the existing desired + state for connected sessions; stop the user role when the session is no + longer active. +- `reconnect`/`logon`: reconcile immediately, still protected by tracked-process + deduplication. +- agent service shutdown: stop accepting new helper connections, request helper + shutdown, wait for a short bounded grace period, terminate remaining tracked + processes, and close the Job Object. + +Startup cleanup is conservative. The agent may remove stale broker records and +tracked metadata, but it does not enumerate and kill processes merely because +their filename begins with `breeze`. Any orphan cleanup outside Job Object +ownership requires verified executable path, command mode, Windows session, +and a process creation time predating the current agent owner. + +### 6. Main-agent single-instance guard + +Only the full Windows `run` path acquires an exclusive, no-sharing file handle +before config, IPC, heartbeat, or collector initialization. The lock file lives +in Breeze's hardened machine-wide ProgramData directory. Helper, +service-management, enrollment, and diagnostic subcommands do not acquire it. + +The protected parent directory is the security boundary: an unprivileged local +process cannot pre-create or replace the file to block service startup. A plain +global named mutex is not authoritative because its name can be squatted before +the service creates and applies its intended ACL. The lock file contains +diagnostic metadata (PID, Windows session, launch mode, executable path, and +creation time), but the held file handle—not the metadata—is the lock. + +If exclusive open fails because another full agent owns the file: + +- Log and print a clear `main agent already running` diagnostic. +- Include the current process PID, session, and launch mode. +- Exit with a dedicated nonzero code before initializing agent components. + +The handle is held for the lifetime of the full agent and released by process +termination. Helper fallback invocations such as +`breeze-agent.exe user-helper --role user` are unaffected. + +### 7. Verified watchdog recovery + +The Windows service controller exposes explicit operations for querying service +state/PID, requesting stop, waiting for a state, validating process identity, +terminating the service process, and starting the service. + +Graceful attempt: + +1. Query the service. +2. If already stopped, proceed to start. +3. Otherwise request stop and wait for `Stopped` using a timeout greater than + the documented maximum normal shutdown budget, with margin. +4. If the timeout expires, return a typed stop-timeout error. Do not call + `Start`. +5. Start and verify that the service reaches `Running` with a nonzero PID. + +Forced attempt: + +1. Query the current PID from SCM immediately before termination; do not trust + the agent state-file PID for destructive action. +2. Verify that the PID is the SCM-owned BreezeAgent process and that its image + path matches the configured service binary. +3. Terminate it, wait for process exit, and wait for SCM state `Stopped`. +4. Start the service and verify `Running` with a new live PID. + +`RecoveryManager.Attempt` will accept a richer recovery result so journal +entries identify the phase, old/new PID, service states, elapsed time, and +failure class. Existing heartbeat-based post-restart verification remains the +definition of application-level recovery. + +### 8. Recovery authority coordination + +SCM recovery actions remain enabled for immediate process crashes. The watchdog +remains responsible for a live-but-unhealthy agent and for verified escalation. +They coordinate through SCM state: + +- The watchdog does nothing while the service is `StartPending`, `StopPending`, + `ContinuePending`, or `PausePending`, except bounded observation and logging. +- It does not issue a second start for `Running` or transitional states. +- A watchdog attempt that observes an SCM recovery already in progress waits + for the transition and then enters heartbeat verification. + +This avoids expanding the change into installer recovery-policy removal while +preventing competing side effects. + +### 9. Process-role diagnostics + +Every Breeze process writes a startup record containing: + +- binary name and resolved executable path; +- process and parent PID; +- Windows session ID; +- launch mode (`service-run`, `console-run`, `user-helper`, `system-helper`, or + other command); +- helper lifecycle key when applicable; +- whether the companion helper or main-binary fallback is in use; +- agent/helper version and creation timestamp. + +The main agent state remains reserved for the main agent. Helper processes must +not overwrite its PID. Watchdog journal entries include SCM state/PID separately +from the state-file PID so support can identify disagreement. + +The existing missing-helper warning is elevated into structured diagnostics and +rate-limited. Support collection can then distinguish: + +- one SCM service process in Session 0; +- fallback `breeze-agent.exe user-helper` processes in interactive sessions; +- true duplicate full `run` processes; +- companion `breeze-user-helper.exe` processes by role/session. + +## Failure handling + +- Admission quota rejection cannot cause process multiplication because the + lifecycle registry still owns and observes the rejected process. +- Duplicate logical helper registration causes the surplus helper to exit; it + does not reconnect forever. +- Failed Job Object assignment kills the just-created process. +- Failed helper termination is logged with PID, key, and error and retried only + within a bounded cleanup policy. +- Lock creation/security failure fails closed for the full agent and produces + a startup marker; it does not continue without exclusivity. +- Watchdog stop timeout leaves the service untouched in its observed state and + advances to the separately verified forced attempt after cooldown. +- PID validation failure blocks forced termination and enters failover rather + than risking termination of an unrelated process. + +## Compatibility and rollout + +The companion helper and main agent ship together, but rollout must tolerate +old/new mixtures: + +- New agent + old helper: protocol/auth version checks retain existing behavior; + lifecycle tracking prevents spawn multiplication. +- Old agent + new helper: the helper retains compatible hello/rejection parsing. +- Missing companion helper: main-binary fallback remains available and is + tracked using the same role/session key. +- Existing scheduled tasks: accepted through the same broker admission rules; + they are not assumed to be Job Object children. + +Rollout should use the normal agent fleet rollout controls, beginning with RDS +hosts and a Windows workstation cohort. Observe helper counts, rejection rates, +watchdog stop time, recovery success, and offline duration before broad release. + +No migration or customer configuration change is required. + +## Testing strategy + +Implementation follows test-driven development. + +### Session broker and lifecycle unit tests + +- Windows identity keys differ for the same SID in different OS sessions. +- Existing Unix identity keys remain unchanged. +- Five SYSTEM helpers in distinct RDS sessions do not share one quota bucket. +- Same-session connection/rate limits still apply. +- Only one authenticated helper is retained per session/role key. +- An alive pre-auth helper suppresses reconciliation respawn. +- An exited helper is replaced only according to cooldown rules. +- Logoff terminates and removes both role entries. +- Disconnect preserves/removes roles according to desired session state. +- Broker callbacks transition tracked process state safely under concurrency. +- Lifecycle stop is idempotent and reaps all tracked helpers. + +### Authorization tests + +- User helper with a non-SYSTEM token and matching RDP session is accepted. +- User helper claiming a different Windows session is rejected permanently. +- SYSTEM identity claiming user role is rejected. +- Non-SYSTEM identity claiming system role is rejected. +- SYSTEM role with matching OS-derived session is accepted. +- Physical console behavior remains supported. + +### Windows process ownership tests + +- Spawned helpers are assigned to the Job Object. +- Assignment failure terminates the process. +- Closing the job terminates remaining helper test processes. +- A tracked live process prevents duplicate launch under concurrent reconciles. +- Main-binary helper fallback does not acquire the main-agent lock. + +### Main-agent exclusivity tests + +- First full-agent instance acquires the guard. +- Second full-agent instance exits before component initialization. +- Guard is released after owner exit. +- Helper and administrative subcommands are not blocked. +- Lock ACL/open error paths fail closed and produce diagnostics. + +### Watchdog tests + +- Graceful recovery never starts before `Stopped`. +- A 15-second `StopPending` interval does not trigger premature start. +- Stop timeout returns a typed error without calling `Start`. +- Forced recovery uses the fresh SCM PID rather than the state-file PID. +- Mismatched executable/service ownership prevents termination. +- Forced recovery waits for process exit and `Stopped` before start. +- Start verification requires `Running` and a live nonzero PID. +- Existing heartbeat verification and flap accounting remain intact. +- Transitional SCM states do not cause competing recovery actions. + +### Integration and manual verification + +- Simulate more than five Windows sessions and confirm one SYSTEM helper per + eligible session without accumulation across several reconcile intervals. +- Exercise two active RDP users and verify user-role PAM/run-as-user routing is + session-correct. +- Kill the agent and verify Job Object cleanup followed by bounded repopulation. +- Launch a second elevated `breeze-agent.exe run` and verify immediate refusal. +- Hold a fake service in `StopPending` longer than 15 seconds and verify the + watchdog does not start it prematurely. +- Remove the companion helper in a test install and confirm fallback processes + are labeled, bounded, and self-healed on heartbeat. +- Run relevant Go packages with `go test -race` and run Windows-specific tests + in the Windows CI environment. + +## Observability and acceptance criteria + +The change is accepted when: + +- Helper process count converges to the desired role/session count plus only a + short, bounded replacement overlap. +- A 20-session RDS host can register its intended helpers without SID-wide + quota rejection. +- No session/role key has more than one registered helper. +- Pre-auth rejection does not increase process count on subsequent reconciles. +- RDP user helpers authenticate only into their own OS session. +- Agent crash or stop leaves no proactively spawned helper children behind. +- A second full Windows agent cannot initialize. +- Watchdog never calls `Start` until SCM reports `Stopped`. +- Forced termination only targets the freshly verified SCM service PID. +- Logs distinguish full agents from helper fallback processes without requiring + a process dump. +- Existing single-session Windows, Linux, and macOS behavior remains green. + +## Risks and mitigations + +- **Job Object compatibility:** a process may already belong to a non-breakaway + job. Create helpers with appropriate flags, test under common RDS policies, + and fail closed if ownership cannot be established. +- **Lock-file denial of service:** rely on the already hardened ProgramData + parent ACL, refuse reparse-point/path substitution, and acquire only in the + full-agent path. Do not treat editable file contents as ownership evidence. +- **Session-ID trust regression:** use the peer token's OS-derived session and + require equality with authenticated hello metadata. +- **PID reuse:** pair PID with current SCM service state, image path, creation + time where available, and a live process handle before termination. +- **Upgrade skew:** keep fallback compatibility and make new rejection reasons + intelligible to old clients. +- **Cleanup races:** serialize registry transitions by helper key and make stop, + disconnect, and exit callbacks idempotent. +- **Overlong recovery windows:** log phase timing and retain heartbeat-based + recovery verification so a successfully restarted but unhealthy agent still + escalates. + +## Rollback + +The feature requires no data migration. A binary rollback restores the previous +lifecycle behavior. Because Job Object ownership ends with process termination, +it leaves no persistent OS object. The empty lock file may remain after exit, +but it carries no ownership without an exclusive live handle and is safely +reused. Existing service configuration, scheduled tasks, and helper binaries +remain compatible. + +Operational rollback should restart BreezeAgent and verify that expected helper +processes reconnect. If helper admission is the reason for rollback, capture the +new structured process and rejection diagnostics first. + +## Implementation sequencing + +The implementation plan should split the work into independently reviewable, +test-first slices: + +1. Structured process-role diagnostics and test seams. +2. Session-aware admission and RDP-safe role authorization. +3. Lifecycle process registry, deduplication, and event cleanup. +4. Windows Job Object ownership. +5. Main-agent single-instance guard. +6. Verified watchdog service controller and recovery sequencing. +7. Cross-component integration tests, Windows validation, and rollout notes. + +This ordering establishes observability and authorization invariants before +introducing process termination or recovery changes. From 3ae6d27b6c0b62a36f6a0d866c4455b72a1f16fd Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 13:51:22 -0600 Subject: [PATCH 02/47] docs(agent): plan Windows lifecycle durability --- .../plans/2026-07-14-rds-helper-lifecycle.md | 829 ++++++++++++++++++ ...7-14-windows-agent-instance-diagnostics.md | 629 +++++++++++++ ...7-14-windows-watchdog-verified-recovery.md | 738 ++++++++++++++++ ...gent-helper-lifecycle-durability-design.md | 17 +- 4 files changed, 2207 insertions(+), 6 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-14-rds-helper-lifecycle.md create mode 100644 docs/superpowers/plans/2026-07-14-windows-agent-instance-diagnostics.md create mode 100644 docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md diff --git a/docs/superpowers/plans/2026-07-14-rds-helper-lifecycle.md b/docs/superpowers/plans/2026-07-14-rds-helper-lifecycle.md new file mode 100644 index 0000000000..a3ff6bae36 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-rds-helper-lifecycle.md @@ -0,0 +1,829 @@ +# RDS Helper Lifecycle Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make Windows helpers scale safely across RDS sessions while enforcing one owned process per Windows-session/role pair and preserving cross-user isolation. + +**Architecture:** Introduce a shared `HelperKey`, make pre-auth admission session-aware, atomically enforce one authenticated helper per key, and replace retry-only tracking with an injected process registry. Proactively spawned helpers are created suspended, assigned to a kill-on-close Job Object, and only then resumed. + +**Tech Stack:** Go, `golang.org/x/sys/windows`, `github.com/Microsoft/go-winio`, Windows SCM/WTS APIs, Go race detector. + +## Global Constraints + +- Operating-system-derived SID, PID, Windows session, process handle, and executable path are authoritative; helper claims are never authoritative by themselves. +- Generic Windows `runAs=user` remains physical-console-bound to preserve the #1009 cross-user interception defense. +- Explicit-user and explicit-Windows-session operations may use matching RDP user helpers. +- SYSTEM helper desired state is active or connected; user helper desired state is active only; Session 0 and service sessions are never eligible. +- Unix identity keys and helper authorization behavior must remain unchanged. +- Never kill by filename or an unverified PID. +- Every implementation task follows red-green-refactor and ends in a focused commit. +- Design source: `docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md`. +- This document is the RDS/helper phase of that design. Companion plans cover the main-agent guard/diagnostics and verified watchdog recovery. + +## File Structure + +- Create `agent/internal/sessionbroker/helper_key.go`: shared key, parsing, and desired-state predicate. +- Create `agent/internal/sessionbroker/helper_key_test.go`: host-neutral key and eligibility tests. +- Create `agent/internal/sessionbroker/broker_admission.go`: pure admission/logical-key helpers and atomic registration support. +- Create `agent/internal/sessionbroker/broker_admission_test.go`: session-aware quota and concurrent logical dedup tests. +- Modify `agent/internal/sessionbroker/broker.go`: early Windows session derivation, composite identity, logical-key map, role validation, and map cleanup. +- Modify `agent/internal/sessionbroker/console_session_gate_test.go`: RDP user-role authorization cases while retaining assist/console tests. +- Create `agent/internal/sessionbroker/lifecycle_registry.go`: host-neutral tracked-process state machine and bounded cleanup. +- Create `agent/internal/sessionbroker/lifecycle_registry_test.go`: fake process/spawner lifecycle tests. +- Create `agent/internal/sessionbroker/peer_process_windows.go`: kernel-bound handle for every authenticated Windows helper. +- Create `agent/internal/sessionbroker/peer_process_other.go`: no-op cross-platform peer ownership. +- Create `agent/internal/sessionbroker/peer_process_windows_test.go`: native handle lifetime/termination tests. +- Modify `agent/internal/sessionbroker/lifecycle.go`: dependency injection, `HelperKey` use, event cleanup, and process-aware reconciliation. +- Modify `agent/internal/heartbeat/heartbeat.go`: retain and synchronously stop the lifecycle manager after the broker stops accepting. +- Modify `agent/internal/heartbeat/heartbeat_test.go`: shutdown-order regression coverage. +- Modify `agent/internal/sessionbroker/spawner_windows.go`: process interface methods and suspended creation wiring. +- Create `agent/internal/sessionbroker/helper_job_windows.go`: Job Object wrapper. +- Create `agent/internal/sessionbroker/helper_job_windows_test.go`: native Windows ownership tests. +- Modify `agent/internal/sessionbroker/spawner_stub.go`: keep cross-platform compile parity. +- Modify `.github/workflows/ci.yml`: run Windows Go unit tests that cannot execute on Ubuntu. + +--- + +### Task 1: Shared helper key and eligibility contract + +**Files:** +- Create: `agent/internal/sessionbroker/helper_key.go` +- Create: `agent/internal/sessionbroker/helper_key_test.go` +- Modify: `agent/internal/sessionbroker/lifecycle.go:138-187` + +**Interfaces:** +- Produces: `HelperKey{WindowsSessionID uint32, Role string}`. +- Produces: `helperKeyFromDetected(DetectedSession, string) (HelperKey, bool)`. +- Produces: `helperRoleDesired(DetectedSession, string) bool`. +- Consumed by: broker registration, lifecycle registry, SCM cleanup, and targeted broker close operations. + +- [ ] **Step 1: Write failing table tests for key parsing and desired state** + +```go +package sessionbroker + +import "testing" + +func TestHelperRoleDesired(t *testing.T) { + tests := []struct { + name string + s DetectedSession + role string + want bool + }{ + {"system active", DetectedSession{Session: "7", State: "active", Type: "rdp"}, "system", true}, + {"system connected", DetectedSession{Session: "7", State: "connected", Type: "rdp"}, "system", true}, + {"user active", DetectedSession{Session: "7", State: "active", Type: "rdp"}, "user", true}, + {"user connected", DetectedSession{Session: "7", State: "connected", Type: "rdp"}, "user", false}, + {"session zero", DetectedSession{Session: "0", State: "active", Type: "rdp"}, "system", false}, + {"services", DetectedSession{Session: "8", State: "active", Type: "services"}, "system", false}, + {"disconnected", DetectedSession{Session: "8", State: "disconnected", Type: "rdp"}, "system", false}, + {"unknown role", DetectedSession{Session: "8", State: "active", Type: "rdp"}, "assist", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := helperRoleDesired(tt.s, tt.role); got != tt.want { + t.Fatalf("helperRoleDesired() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHelperKeyFromDetectedRejectsInvalidSession(t *testing.T) { + if _, ok := helperKeyFromDetected(DetectedSession{Session: "not-a-number", State: "active", Type: "rdp"}, "user"); ok { + t.Fatal("invalid Windows session unexpectedly produced a key") + } +} +``` + +- [ ] **Step 2: Run the tests and verify the missing-symbol failure** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestHelper(RoleDesired|KeyFromDetected)' -count=1` + +Expected: FAIL because `helperRoleDesired` and `helperKeyFromDetected` do not exist. + +- [ ] **Step 3: Add the shared key and predicate** + +```go +package sessionbroker + +import ( + "fmt" + "strconv" +) + +type HelperKey struct { + WindowsSessionID uint32 + Role string +} + +func (k HelperKey) String() string { + return fmt.Sprintf("%d-%s", k.WindowsSessionID, k.Role) +} + +func helperRoleDesired(s DetectedSession, role string) bool { + if s.Session == "0" || s.Type == "services" { + return false + } + switch role { + case "system": + return s.State == "active" || s.State == "connected" + case "user": + return s.State == "active" + default: + return false + } +} + +func helperKeyFromDetected(s DetectedSession, role string) (HelperKey, bool) { + if !helperRoleDesired(s, role) { + return HelperKey{}, false + } + id, err := strconv.ParseUint(s.Session, 10, 32) + if err != nil || id == 0 { + return HelperKey{}, false + } + return HelperKey{WindowsSessionID: uint32(id), Role: role}, true +} +``` + +Replace the duplicated desired-state checks in `reconcile` with calls to `helperKeyFromDetected` so admission and lifecycle cannot drift. + +- [ ] **Step 4: Run focused tests and the existing lifecycle tests** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestHelper|TestSpawnWithRetry|TestHandleSCMEvent' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit the shared contract** + +```bash +git add agent/internal/sessionbroker/helper_key.go agent/internal/sessionbroker/helper_key_test.go agent/internal/sessionbroker/lifecycle.go +git commit -m "refactor(agent): share Windows helper eligibility rules" +``` + +--- + +### Task 2: Session-aware admission and atomic one-per-key registration + +**Files:** +- Create: `agent/internal/sessionbroker/broker_admission.go` +- Create: `agent/internal/sessionbroker/broker_admission_test.go` +- Modify: `agent/internal/sessionbroker/broker.go:247-289,1264-1613,1662-1702` +- Modify: `agent/internal/sessionbroker/broker_windows.go:36-42` +- Modify: `agent/internal/sessionbroker/lifecycle.go:138-187` + +**Interfaces:** +- Consumes: `HelperKey` from Task 1. +- Produces: `admissionIdentityKey(base string, peerSession uint32, goos string) string`. +- Produces: `AuthenticatedHelperKey{PeerSID string; HelperKey}` plus one-per-role `Broker.helperByKey` enforcement. +- Produces: `Broker.UpdateDesiredHelperKeys(map[HelperKey]struct{})` for the detector-backed eligibility snapshot. +- Produces: `reserveWindowsHelper`, `commitWindowsHelper`, and `releaseWindowsHelper`; no session is published before the accepted response is sent and its HMAC key is installed. + +- [ ] **Step 1: Write failing pure and concurrent registration tests** + +```go +package sessionbroker + +import ( + "errors" + "sync" + "testing" +) + +func TestAdmissionIdentityKeyWindowsIncludesSession(t *testing.T) { + a := admissionIdentityKey("S-1-5-18", 7, "windows") + b := admissionIdentityKey("S-1-5-18", 8, "windows") + if a == b { + t.Fatalf("distinct RDS sessions shared key %q", a) + } + if got := admissionIdentityKey("1000", 7, "linux"); got != "1000" { + t.Fatalf("Unix key changed: %q", got) + } +} + +func TestReserveWindowsHelperAllowsOnlyOnePerHelperKey(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: "system"} + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + + var wg sync.WaitGroup + errs := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := b.reserveWindowsHelper("windows:S-1-5-18:session:7", "S-1-5-18", key) + errs <- err + }() + } + wg.Wait() + close(errs) + + duplicates := 0 + for err := range errs { + if errors.Is(err, errDuplicateHelperKey) { + duplicates++ + } + } + if duplicates != 1 || len(b.helperReservations) != 1 || len(b.sessions) != 0 { + t.Fatalf("duplicates=%d reservations=%d sessions=%d", duplicates, len(b.helperReservations), len(b.sessions)) + } +} + +func TestReserveWindowsHelperRejectsUndesiredWindowsKey(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + _, err := b.reserveWindowsHelper("windows:S-1-5-18:session:7", "S-1-5-18", HelperKey{WindowsSessionID: 7, Role: "system"}) + if !errors.Is(err, errHelperKeyNotDesired) { + t.Fatalf("err=%v, want errHelperKeyNotDesired", err) + } +} +``` + +Also add: + +- `TestReservationIsInvisibleUntilCommit`: `Sessions()` and `HasHelperForWinSessionRole` remain empty after reserve, then expose exactly one session after `SetSessionKey` and commit. +- `TestReleaseReservationAfterAcceptedWriteFailure`: release removes both logical and identity quota reservations without evicting an existing session. +- `TestTwentySystemSIDsAcrossWindowsSessionsHaveIndependentAdmission`: the same LocalSystem SID in sessions 1-20 receives twenty distinct identity buckets, while six reservations in one bucket hit the existing five-connection limit. +- `TestUnixAndNonLifecycleRolesBypassWindowsLogicalReservation`: Unix, assist, watchdog, and backup paths retain the current registration behavior. + +- [ ] **Step 2: Run the tests and verify they fail** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestAdmissionIdentityKey|TestReserveWindowsHelper|TestReservation|TestTwentySystem|TestUnixAndNonLifecycle' -count=1` + +Expected: FAIL because the reservation types, composite identity, and logical-key maps do not exist. + +- [ ] **Step 3: Add the session-aware identity and typed duplicate error** + +```go +package sessionbroker + +import ( + "errors" + "fmt" +) + +type AuthenticatedHelperKey struct { + PeerSID string + HelperKey +} + +type helperAuthReservation struct { + id uint64 + authKey AuthenticatedHelperKey + identityKey string + victim *Session +} + +var ( + errDuplicateHelperKey = errors.New("helper already registered for Windows session and role") + errHelperKeyNotDesired = errors.New("helper Windows session and role are not currently eligible") +) + +func admissionIdentityKey(base string, peerSession uint32, goos string) string { + if goos != "windows" { + return base + } + return fmt.Sprintf("windows:%s:session:%d", base, peerSession) +} + +func isWindowsLifecycleRole(goos, role string) bool { + return goos == "windows" && (role == ipc.HelperRoleSystem || role == ipc.HelperRoleUser) +} +``` + +Under `b.mu`, `reserveWindowsHelper` must (1) require the copied desired-key snapshot, (2) reject a live `helperByKey` owner even when its SID differs, (3) allow replacement only when `Session.IsClosed()` or its kernel-bound peer handle proves it exited, (4) reserve both the `AuthenticatedHelperKey` and identity quota without publishing a session, and (5) remember any idle quota victim without removing it. `commitWindowsHelper` validates the reservation, desired snapshot, and victim staleness/idle threshold again; if any changed, it releases and fails without eviction. Otherwise it removes the reserved victim, inserts into `sessions`, `byIdentity`, `helperByAuthKey`, and `helperByKey`, then publishes once. `releaseWindowsHelper` only removes pending reservation/quota state. Victim `Close` and observer callbacks run after unlocking. + +Initialize all maps and the monotonically increasing reservation ID in `New`. `UpdateDesiredHelperKeys` copies the caller's map and invalidates reservations whose role key is no longer desired. Delete logical map entries only when their stored session pointer equals the session being removed. + +In the existing Windows `reconcile`, compute the complete desired map first and call `UpdateDesiredHelperKeys` before checking broker connections or spawning. This makes the admission change deployable in the same commit; there is no intermediate build where every Windows helper is permanently rejected because eligibility was never published. + +- [ ] **Step 4: Move peer-session derivation before admission and make registration authoritative** + +In `handleConnection`, immediately after kernel credentials: + +```go +verifiedWinSessionID := peerWinSessionID(creds.PID) +identityKey := admissionIdentityKey(creds.IdentityKey(), verifiedWinSessionID, b.goos) +``` + +Only for Windows `system`/`user`, reserve after all credential, executable, protocol, role, and peer/claim checks. Then send the accepted auth response, install the derived HMAC key with `conn.SetSessionKey`, construct the `Session`, and commit it. Defer reservation release until commit succeeds. If commit fails because shutdown or eligibility changed after acceptance, close the authenticated connection without publishing it. Live duplicates and undesired keys receive permanent rejection so surplus or terminally ineligible scheduled helpers exit instead of reconnecting forever. Unix and every other role keep the existing accepted-response/key-install/registration path unchanged. + +- [ ] **Step 5: Run broker tests with the race detector** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestAdmission|TestReserveWindowsHelper|TestReservation|TestTwentySystem|TestUnixAndNonLifecycle|TestAdmit|TestIdentity' -count=1` + +Expected: PASS with no race report. + +- [ ] **Step 6: Commit admission and logical dedup** + +```bash +git add agent/internal/sessionbroker/broker.go agent/internal/sessionbroker/broker_windows.go agent/internal/sessionbroker/broker_admission.go agent/internal/sessionbroker/broker_admission_test.go agent/internal/sessionbroker/lifecycle.go +git commit -m "fix(agent): scope helper admission to Windows sessions" +``` + +--- + +### Task 3: RDP-safe user authorization without weakening generic routing + +**Files:** +- Modify: `agent/internal/sessionbroker/broker.go:1504-1583,1836-1879` +- Modify: `agent/internal/sessionbroker/console_session_gate_test.go:19-156` +- Modify: `agent/internal/sessionbroker/broker_test.go:74-99,166-221` + +**Interfaces:** +- Consumes: kernel `verifiedWinSessionID`, authenticated `AuthRequest.WinSessionID`, and `HelperKey`. +- Produces: `roleIdentityRejection(role, sid string, uid uint32, peer, claimed, console, goos string)`. +- Preserves: `PreferredRunAsUserSession()` console-only behavior. +- Preserves: `SessionForUser` and `LaunchProcessViaUserHelperForSession` targeted routing. + +- [ ] **Step 1: Replace the user-role table expectations with RDP-safe cases** + +```go +tests := []struct { + name, role, sid, peer, claimed, console, wantReason string + wantReject bool +}{ + {"RDP user matching kernel session", ipc.HelperRoleUser, nonSystemSID, "7", "7", "1", "", false}, + {"RDP user session mismatch", ipc.HelperRoleUser, nonSystemSID, "7", "8", "1", "user role session claim does not match peer token", true}, + {"RDP user unknown peer session", ipc.HelperRoleUser, nonSystemSID, "0", "7", "1", "user role requires an interactive peer session", true}, + {"SYSTEM cannot claim user", ipc.HelperRoleUser, systemSID, "7", "7", "1", "user role requires non-SYSTEM identity", true}, + {"non-SYSTEM cannot claim system", ipc.HelperRoleSystem, nonSystemSID, "7", "7", "1", "system role requires SYSTEM identity", true}, + {"SYSTEM matching RDP session", ipc.HelperRoleSystem, systemSID, "7", "7", "1", "", false}, + {"assist remains console bound", ipc.HelperRoleAssist, nonSystemSID, "7", "7", "1", "assist role requires the active console session", true}, +} +``` + +Call the revised gate with both peer and claimed session values. Retain the existing Unix test. + +- [ ] **Step 2: Run the table and verify the old gate fails** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestRoleIdentityRejection' -count=1` + +Expected: FAIL because user role is still console-bound and the function lacks `claimed`. + +- [ ] **Step 3: Implement the peer/claim equality gate** + +```go +func roleIdentityRejection(role, sid string, uid uint32, peer, claimed, console, goos string) (string, bool) { + if goos == "windows" { + switch { + case role == ipc.HelperRoleSystem && sid != systemSID: + return "system role requires SYSTEM identity", true + case role == ipc.HelperRoleUser && sid == systemSID: + return "user role requires non-SYSTEM identity", true + case role == ipc.HelperRoleAssist && sid == systemSID: + return "assist role requires non-SYSTEM identity", true + case role == ipc.HelperRoleWatchdog && sid != systemSID: + return "watchdog role requires SYSTEM identity", true + } + if role == ipc.HelperRoleUser || role == ipc.HelperRoleSystem { + if peer == "" || peer == "0" { + return role + " role requires an interactive peer session", true + } + if peer != claimed { + return role + " role session claim does not match peer token", true + } + } + if role == ipc.HelperRoleAssist && (console == "" || console == "0" || peer != console) { + return "assist role requires the active console session", true + } + return "", false + } + if (role == ipc.HelperRoleWatchdog || role == ipc.HelperRoleSystem) && uid != 0 { + return role + " role requires root identity", true + } + return "", false +} +``` + +Use the authenticated numeric claim converted to decimal. Reject mismatch rather than warning and silently substituting the kernel value. + +- [ ] **Step 4: Lock in the generic-versus-targeted routing boundary** + +Add `TestPreferredRunAsUserSessionIgnoresRDPHelper`, `TestSessionForUserSelectsRDPHelper`, and `TestLaunchProcessViaUserHelperForSessionTargetsMatchingRDPHelper`. The first expects only the physical-console session; the latter two expect the authenticated session-7 helper. + +- [ ] **Step 5: Run authorization and routing tests** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestRoleIdentityRejection|TestPreferredRunAsUserSession|TestSessionForUser|TestLaunchProcessViaUserHelperForSession' -count=1` + +Expected: PASS with generic console safety and targeted RDP routing both covered. + +- [ ] **Step 6: Commit RDP authorization** + +```bash +git add agent/internal/sessionbroker/broker.go agent/internal/sessionbroker/console_session_gate_test.go agent/internal/sessionbroker/broker_test.go +git commit -m "fix(agent): authenticate user helpers in matching RDP sessions" +``` + +--- + +### Task 4: Process-aware lifecycle registry and deterministic cleanup + +**Files:** +- Create: `agent/internal/sessionbroker/lifecycle_core.go` +- Create: `agent/internal/sessionbroker/lifecycle_registry.go` +- Create: `agent/internal/sessionbroker/lifecycle_registry_test.go` +- Create: `agent/internal/sessionbroker/peer_process_windows.go` +- Create: `agent/internal/sessionbroker/peer_process_other.go` +- Create: `agent/internal/sessionbroker/peer_process_windows_test.go` +- Modify: `agent/internal/sessionbroker/lifecycle.go:20-392` +- Modify: `agent/internal/sessionbroker/lifecycle_stub.go` +- Modify: `agent/internal/sessionbroker/broker.go:247-454,1264-1702` +- Modify: `agent/internal/sessionbroker/session.go:30-115` +- Modify: `agent/internal/heartbeat/heartbeat.go:820-870,1092-1118` +- Modify: `agent/internal/heartbeat/heartbeat_test.go` + +**Interfaces:** +- Consumes: `HelperKey` and broker logical-key registration. +- Produces: `helperProcess`, `helperSpawner`, `trackedHelper`, and `helperState`. +- Produces: idempotent `HelperLifecycleManager.Stop()` and `stopKey(HelperKey)`. +- Produces: lifecycle-specific broker observer registration without replacing heartbeat callbacks. +- Produces: a kernel-bound peer process handle for every authenticated Windows helper, including scheduled helpers outside the Job Object. +- Produces: synchronous shutdown order `broker stop accepting/pre-auth wait -> lifecycle terminate/reap -> broker session close -> remaining heartbeat shutdown`. + +- [ ] **Step 1: Write fake-process tests for the accumulation and race cases** + +```go +func TestReconcileDoesNotRespawnWhilePreAuthProcessLives(t *testing.T) { + proc := newFakeHelperProcess(4100) + spawner := &fakeHelperSpawner{processes: []helperProcess{proc}} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) + + m.reconcile() + m.reconcile() + + if got := spawner.SpawnCount(HelperKey{WindowsSessionID: 7, Role: "system"}); got != 1 { + t.Fatalf("system spawn count = %d, want 1", got) + } +} + +func TestStaleExitCannotClearReplacement(t *testing.T) { + oldProc := newFakeHelperProcess(4100) + newProc := newFakeHelperProcess(4200) + m := newLifecycleHarness(t, nil, &fakeHelperSpawner{}) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + oldGeneration := m.registry.attach(key, oldProc, "breeze-user-helper.exe", "system-helper") + // The old generation has already transitioned out of the one-process slot; + // its watcher is only late delivering the exit notification. + oldProc.markExited(0) + m.registry.detach(key, oldGeneration) + m.registry.attach(key, newProc, "breeze-user-helper.exe", "system-helper") + m.registry.noteExit(key, oldGeneration, 0) + if got := m.registry.processID(key); got != 4200 { + t.Fatalf("replacement PID = %d, want 4200", got) + } +} + +func TestStopSessionTerminatesBothRoles(t *testing.T) { + m := newLifecycleHarness(t, nil, &fakeHelperSpawner{}) + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "system"}, newFakeHelperProcess(1), "helper", "system-helper") + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "user"}, newFakeHelperProcess(2), "helper", "user-helper") + m.stopSession(7) + if got := m.registry.len(); got != 0 { + t.Fatalf("registry len = %d, want 0", got) + } +} +``` + +The fake process implements deterministic `Alive`, `Terminate`, `Wait`, and idempotent `Close`; the fake spawner counts by `HelperKey` and never calls Windows APIs. `detach` requires the matching generation to be stopping or observably non-live and leaves its watcher owning the process handle, so the stale-exit test proves exit before installing the replacement. Add `TestConcurrentReconcileStopAndExitOwnsHandleOnce` to assert one wait, one close, and no duplicate spawn under concurrent reconcile, stop, and exit delivery. + +- [ ] **Step 2: Run the registry tests and verify missing harness failures** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestReconcileDoesNotRespawn|TestStaleExit|TestStopSessionTerminates' -count=1` + +Expected: FAIL because the process registry and injected spawner do not exist. + +- [ ] **Step 3: Add lifecycle interfaces and generation-safe state** + +```go +type helperProcess interface { + ProcessID() uint32 + ExecutablePath() string + Alive() bool + Terminate() error + Wait() (int, error) + Close() error +} + +type helperSpawner interface { + Spawn(HelperKey) (helperProcess, error) + Close() error +} + +type ownedPeerProcess interface { + ProcessID() uint32 + Alive() (bool, error) + Terminate() error + Close() error +} + +type helperState string + +const ( + helperStarting helperState = "starting" + helperConnected helperState = "connected" + helperStopping helperState = "stopping" + helperExited helperState = "exited" +) + +type trackedHelper struct { + key HelperKey + process helperProcess + generation uint64 + state helperState + launchedAt time.Time + retryCount int + lastFailure time.Time + fatalExitUntil time.Time + executablePath string + commandMode string + done chan struct{} + exitCode int +} +``` + +Move the registry/reconcile engine into host-neutral `lifecycle_core.go`; keep WTS/SCM event translation in build-tagged `lifecycle.go`. Reserve a `starting` entry under the manager lock before spawning. Attach a returned process only if the same generation is still current. Exactly one watcher owns `Wait` and process-handle `Close`; it records the exit code, closes `done`, and updates the registry only when its generation is still current. Stop paths wait on `done`, call idempotent `Terminate` after the graceful deadline, and finally close the Job Object for any survivor—they never race the watcher by closing its process handle. After closing the job, `Stop` performs one final bounded wait for all watcher `done` channels and logs any unreaped handle instead of blocking service shutdown indefinitely. + +- [ ] **Step 4: Refactor reconciliation, SCM events, and shutdown** + +Make reconciliation operate on `map[HelperKey]bool` and publish an immutable copy through `Broker.UpdateDesiredHelperKeys` before spawning. If an entry has an alive process, suppress spawn even when no broker session authenticated. On logoff/terminate stop both keys and publish the reduced desired set before closing sessions. On disconnect stop user, retain system, publish, and reconcile. `Stop` first publishes an empty desired set, swaps the registry into stopping state, requests graceful close, waits a bounded grace, terminates survivors, and closes the spawner/job owner. + +After the existing credential/PID/session/executable verification succeeds during Windows authentication, open the peer PID once with query, synchronize, and terminate rights and store that handle in a session-owned `ownedPeerProcessRef`. This handle—not a later PID lookup—is the ownership proof for scheduled helpers. The ref has explicit `active -> termination-claimed -> consumed` states. `Broker.TerminateHelperKey`, while holding `b.mu` and before removing the matching logical owner, calls the ref's non-blocking `claimTermination`; it then removes ownership, releases `b.mu`, and the sole claim token performs `Terminate` followed by `Close`. No Windows call occurs under `b.mu`. Ordinary `Session.Close` and unexpected-disconnect cleanup route through the broker's same ownership-removal critical section before calling ref `close`; if termination is already claimed, `close` leaves the handle for the claim token, otherwise it consumes the active handle exactly once. Authentication rollback (which was never published as a logical owner) closes its ref directly. This ordering prevents an ordinary close from stealing a handle after terminal cleanup has selected the session, while still releasing every handle on ordinary disconnect and broker-wide close. Add `TestSessionCloseReleasesOwnedPeerProcessOnce`, `TestUnexpectedDisconnectReleasesOwnedPeerProcess`, and `TestConcurrentTerminateAndSessionCloseTerminatesAndConsumesPeerHandleOnce`; use a barrier after terminal claim and require exactly one `Terminate` and one `Close`, not merely one consumption. Lifecycle logoff/disconnect and shutdown call `TerminateHelperKey` as well as stopping any proactively tracked process. A helper rejected because its key is undesired receives a permanent rejection and exits before entering its reconnect loop. + +Add a lifecycle observer API to `Broker` rather than overwriting `SetSessionAuthenticatedHandler` or `SetSessionClosedHandler`, which heartbeat already owns: + +```go +type sessionLifecycleObserver struct { + authenticated func(*Session) + closed func(*Session) +} + +func (b *Broker) AddSessionLifecycleObserver(authenticated, closed func(*Session)) (remove func()) +``` + +Store observers by generated ID under `b.mu`; return an idempotent remover. Copy callbacks while locked and invoke them after unlocking. On authentication, mark the matching tracked generation `connected`; on close, mark it non-connected only when the broker session pointer is still the logical-key owner. Process liveness—not a close callback—remains authoritative for whether reconciliation may spawn a replacement. + +Retain the manager and its completion channel on `Heartbeat` instead of discarding the goroutine local. Add `Broker.StopAcceptingAndWait(context.Context) error`, backed by an accept/auth-handler wait group and a set of raw accepted connections registered before authentication. Close the listener and every still-pre-auth connection before waiting, but retain authenticated sessions and their peer handles for lifecycle termination; never hold `b.mu` while closing or waiting. `Heartbeat.Stop` supplies a bounded shutdown context, calls `StopAcceptingAndWait`, calls and awaits `HelperLifecycleManager.Stop` (which terminates logical helper sessions through their handles), then calls `Broker.Close` for remaining non-lifecycle sessions before completing existing teardown. Add `TestBrokerStopAcceptingAndWaitUnblocksStalledPreAuthConnection` and `TestHeartbeatStopOrdersBrokerBeforeLifecycleAndWaitsForReap` with injected ordered fakes. + +- [ ] **Step 5: Run registry tests under race detection** + +Run: `cd agent && go test -race ./internal/sessionbroker ./internal/heartbeat -run 'TestReconcile|TestStaleExit|TestStopSession|TestDisconnect|TestLifecycleStop|TestConcurrent|TestHeartbeatStopOrders|TestSessionClose|TestUnexpectedDisconnect' -count=1` + +Expected: PASS with exactly one spawn under concurrent reconcile/SCM events and no race report. + +- [ ] **Step 6: Commit lifecycle ownership** + +```bash +git add agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/lifecycle_registry_test.go agent/internal/sessionbroker/lifecycle.go agent/internal/sessionbroker/lifecycle_stub.go agent/internal/sessionbroker/peer_process_windows.go agent/internal/sessionbroker/peer_process_other.go agent/internal/sessionbroker/peer_process_windows_test.go agent/internal/sessionbroker/broker.go agent/internal/sessionbroker/session.go agent/internal/heartbeat/heartbeat.go agent/internal/heartbeat/heartbeat_test.go +git commit -m "fix(agent): own helper processes by session and role" +``` + +--- + +### Task 5: Kill-on-close Job Object and suspended helper creation + +**Files:** +- Create: `agent/internal/sessionbroker/helper_job_windows.go` +- Create: `agent/internal/sessionbroker/helper_job_windows_test.go` +- Modify: `agent/internal/sessionbroker/spawner_windows.go:21-228` +- Modify: `agent/internal/sessionbroker/spawner_stub.go:16-25` + +**Interfaces:** +- Consumes: `helperSpawner` and `helperProcess` from Task 4. +- Produces: `helperJob.Assign(windows.Handle) error` and `helperJob.Close() error`. +- Produces: `windowsHelperSpawner{job, closing}` as the single owner that serializes spawn/assign/resume against close. +- Produces: process methods on `SpawnedHelper`. + +- [ ] **Step 1: Write native Windows Job Object behavior tests** + +```go +//go:build windows + +func TestClosingHelperJobTerminatesAssignedProcess(t *testing.T) { + job, err := newHelperJob() + if err != nil { + t.Fatal(err) + } + proc := startSuspendedTestProcess(t) + defer proc.Close() + if err := job.Assign(proc.Handle); err != nil { + t.Fatal(err) + } + resumeTestProcess(t, proc.Thread) + if err := job.Close(); err != nil { + t.Fatal(err) + } + if event, err := windows.WaitForSingleObject(proc.Handle, 5_000); err != nil || event != windows.WAIT_OBJECT_0 { + t.Fatalf("process survived job close: event=%d err=%v", event, err) + } +} + +func TestSpawnedHelperTerminateIsIdempotent(t *testing.T) { + proc := startSuspendedTestProcess(t) + helper := &SpawnedHelper{PID: proc.PID, Handle: proc.Handle, BinaryPath: proc.Path} + if err := helper.Terminate(); err != nil { + t.Fatal(err) + } + if err := helper.Terminate(); err != nil { + t.Fatal(err) + } +} +``` + +- [ ] **Step 2: Run on Windows and verify missing-symbol failures** + +Run on Windows: `cd agent && go test -race ./internal/sessionbroker -run 'TestClosingHelperJob|TestSpawnedHelperTerminate' -count=1` + +Expected: FAIL because `newHelperJob`, `Assign`, and `Terminate` do not exist. + +- [ ] **Step 3: Implement the Job Object wrapper** + +```go +//go:build windows + +type helperJob struct { + mu sync.Mutex + handle windows.Handle +} + +func newHelperJob() (*helperJob, error) { + h, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("CreateJobObject: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject(h, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err != nil { + windows.CloseHandle(h) + return nil, fmt.Errorf("SetInformationJobObject: %w", err) + } + return &helperJob{handle: h}, nil +} + +func (j *helperJob) Assign(process windows.Handle) error { + j.mu.Lock() + defer j.mu.Unlock() + if j.handle == 0 { + return errors.New("helper job is closed") + } + return windows.AssignProcessToJobObject(j.handle, process) +} + +func (j *helperJob) Close() error { + j.mu.Lock() + defer j.mu.Unlock() + if j.handle == 0 { + return nil + } + err := windows.CloseHandle(j.handle) + j.handle = 0 + return err +} +``` + +- [ ] **Step 4: Create helpers suspended, assign, then resume** + +`windowsHelperSpawner.Spawn` holds its mutex from the closing check through process creation, job assignment, and thread resume; `Close` holds the same mutex while setting `closing` and closing the job. Both SYSTEM-token and user-token spawn paths use: + +```go +creationFlags := uint32(windows.CREATE_SUSPENDED | windows.CREATE_NO_WINDOW | windows.CREATE_UNICODE_ENVIRONMENT) +if err := windows.CreateProcessAsUser(token, nil, cmdLine, nil, nil, false, creationFlags, env, cwd, &si, &pi); err != nil { + return nil, err +} +cleanup := true +defer func() { + if cleanup { + _ = windows.TerminateProcess(pi.Process, 1) + _ = windows.CloseHandle(pi.Thread) + _ = windows.CloseHandle(pi.Process) + } +}() +if err := s.job.Assign(pi.Process); err != nil { + return nil, fmt.Errorf("assign helper to job: %w", err) +} +if _, err := windows.ResumeThread(pi.Thread); err != nil { + return nil, fmt.Errorf("resume helper: %w", err) +} +_ = windows.CloseHandle(pi.Thread) +cleanup = false +return &SpawnedHelper{PID: pi.ProcessId, Handle: pi.Process, BinaryPath: exePath}, nil +``` + +If job assignment fails, the helper is still suspended and is terminated before any user code runs. + +- [ ] **Step 5: Verify native behavior, cross-compilation, and races** + +Run on Windows: `cd agent && go test -race ./internal/sessionbroker -count=1` + +Run on non-Windows: `cd agent && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c -o /tmp/sessionbroker.test.exe ./internal/sessionbroker` + +Expected: native tests PASS; cross-compilation produces `/tmp/sessionbroker.test.exe`. + +- [ ] **Step 6: Commit Job Object ownership** + +```bash +git add agent/internal/sessionbroker/helper_job_windows.go agent/internal/sessionbroker/helper_job_windows_test.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/spawner_stub.go +git commit -m "fix(agent): contain Windows helpers in a kill-on-close job" +``` + +--- + +### Task 6: Multi-session integration coverage and Windows CI + +**Files:** +- Create: `agent/internal/sessionbroker/rds_lifecycle_integration_test.go` +- Modify: `.github/workflows/ci.yml` + +**Interfaces:** +- Consumes all prior RDS helper interfaces. +- Produces a required Windows test invocation for `internal/sessionbroker`. + +- [ ] **Step 1: Add a fake 20-session convergence test** + +```go +func TestRDSReconcileConvergesWithoutAccumulation(t *testing.T) { + sessions := make([]DetectedSession, 0, 20) + for id := 1; id <= 20; id++ { + sessions = append(sessions, DetectedSession{Session: strconv.Itoa(id), State: "active", Type: "rdp"}) + } + spawner := newCountingHelperSpawner() + m := newLifecycleHarness(t, sessions, spawner) + for i := 0; i < 10; i++ { + m.reconcile() + } + if got := spawner.TotalSpawnCount(); got != 40 { + t.Fatalf("spawned %d helpers, want 40", got) + } + for _, key := range spawner.Keys() { + if got := spawner.SpawnCount(key); got != 1 { + t.Fatalf("%s spawned %d times, want 1", key, got) + } + } +} +``` + +Add `TestRDSBrokerAdmissionAndLifecycleConvergeTwentySessions`: publish the 40 desired role keys, reserve/complete authenticated SYSTEM sessions for the same LocalSystem SID in Windows sessions 1-20, and assert all 20 commit through distinct composite identity buckets. Run reconciliation ten times; it must retain those 20 broker sessions, spawn exactly one user helper per active session, and spawn no replacement SYSTEM helpers. A paired same-session subtest reserves six pre-auth identity slots and requires the sixth to hit the existing five-connection bound. + +- [ ] **Step 2: Run the integration test with the race detector** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestRDS(Reconcile|BrokerAdmission)' -count=10` + +Expected: PASS on all ten repetitions: the process-only case owns exactly 40 processes, and the broker-integrated case retains 20 authenticated SYSTEM sessions plus exactly 20 spawned user processes. + +- [ ] **Step 3: Add Windows CI execution** + +Add this required job next to `test-agent`: + +```yaml + test-agent-windows: + name: Test Agent (Windows) + runs-on: windows-latest + needs: [lint] + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: agent/go.sum + + - name: Download Go dependencies + working-directory: agent + run: go mod download + + - name: Run Windows Go tests + working-directory: agent + run: go test -race ./internal/sessionbroker ./internal/heartbeat ./internal/agentapp ./internal/config ./internal/watchdog ./cmd/breeze-watchdog +``` + +Do not replace the existing Ubuntu `test-agent` job; Windows execution covers build-tagged runtime tests. Add `test-agent-windows` to `ci-success.needs`, expose it as `TEST_AGENT_WINDOWS_RESULT`, and require that value to equal `success` in the summary shell condition. + +- [ ] **Step 4: Run the complete agent suite** + +Run: `cd agent && go test -race ./...` + +Expected: PASS with no race report. + +- [ ] **Step 5: Commit integration coverage** + +```bash +git add agent/internal/sessionbroker/rds_lifecycle_integration_test.go .github/workflows/ci.yml +git commit -m "test(agent): cover RDS helper convergence on Windows" +``` + +## Plan Completion Gate + +- One composite admission bucket exists per SID and Windows session. +- One authenticated helper exists per `HelperKey`. +- A live pre-auth helper prevents another spawn. +- User helpers authenticate in their own RDP sessions; generic `runAs=user` stays console-bound. +- Logoff, disconnect, service stop, and agent crash have deterministic cleanup behavior. +- Every proactively spawned helper is assigned to the Job Object before it runs. +- `cd agent && go test -race ./...` passes. +- Native Windows build-tagged tests run in CI. diff --git a/docs/superpowers/plans/2026-07-14-windows-agent-instance-diagnostics.md b/docs/superpowers/plans/2026-07-14-windows-agent-instance-diagnostics.md new file mode 100644 index 0000000000..eb15769bdf --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-windows-agent-instance-diagnostics.md @@ -0,0 +1,629 @@ +# Windows Agent Instance Guard and Process Diagnostics Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prevent a second full Windows agent from initializing and make service, console, companion-helper, and main-binary-fallback processes unambiguous in diagnostics. + +**Architecture:** The full `runAgent` path acquires a no-sharing file handle inside the hardened Breeze ProgramData directory before any mutation or component initialization. A shared startup record classifies each process, while helper path resolution carries explicit fallback provenance instead of inferring it later from Task Manager names. + +**Tech Stack:** Go, Windows `CreateFile`, ProgramData DACLs, `golang.org/x/sys/windows`, structured Breeze logging. + +## Global Constraints + +- The instance guard is the first ownership/mutating action in `runAgent`; only side-effect-free Windows service detection and process-metadata collection may precede it. It is acquired before service-unit reconciliation, config loading, state writes, IPC, heartbeat, or collectors. +- Helper, enrollment, service-management, status, and diagnostic subcommands do not acquire the full-agent guard. +- The lock lives under `config.ConfigDir()`; current directories and temporary directories are forbidden. +- The ProgramData parent and final lock object must not be reparse points. +- Only sharing/lock violations mean another agent owns the lock; access, ACL, metadata, and path failures are security failures and fail closed. +- File contents are diagnostics only. The exclusive live handle is the ownership proof. +- No credential, token, certificate, or tenant secret may enter lock metadata or startup logs. +- Unix behavior remains a no-op guard. +- Design source: `docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md`. +- This document covers process diagnostics and main-agent exclusivity. The RDS lifecycle and watchdog phases are specified in the two sibling plans with the same date prefix. + +## File Structure + +- Create `agent/internal/agentapp/process_role.go`: startup record and pure launch-mode classification. +- Create `agent/internal/agentapp/process_role_test.go`: all process-role classifications. +- Create `agent/internal/agentapp/process_role_windows.go`: kernel-derived Windows session, parent PID, executable, and creation time. +- Create `agent/internal/agentapp/process_role_other.go`: neutral non-Windows metadata. +- Create `agent/internal/agentapp/main_instance.go`: guard interfaces, typed errors, and exit codes. +- Create `agent/internal/agentapp/main_instance_windows.go`: protected exclusive file-handle implementation. +- Create `agent/internal/agentapp/main_instance_other.go`: no-op guard. +- Create `agent/internal/agentapp/main_instance_windows_test.go`: native lock, stale-file, ACL, and reparse tests. +- Create `agent/internal/agentapp/instance_marker_windows.go`: bootstrap-safe Windows Event Log marker. +- Create `agent/internal/agentapp/instance_marker_other.go`: stderr-only non-Windows marker. +- Modify `agent/internal/eventlog/eventlog_windows.go`: expose error-returning lazy source registration/write for bootstrap markers. +- Modify `agent/internal/eventlog/eventlog.go`: non-Windows error-returning stub. +- Create `agent/internal/eventlog/eventlog_windows_test.go`: missing-source registration and open/write failure tests. +- Modify `agent/internal/config/permissions_windows.go`: prepare and verify the lock parent. +- Modify `agent/internal/config/permissions_windows_test.go`: lock-parent DACL assertions. +- Modify `agent/internal/agentapp/main.go`: early acquisition and structured startup logs. +- Modify `agent/internal/sessionbroker/userhelper_path.go`: typed companion/fallback result. +- Modify `agent/internal/sessionbroker/userhelper_path_windows.go`: carry fallback provenance. +- Modify `agent/internal/sessionbroker/userhelper_path_test.go`: path plus fallback-kind tests. +- Modify `agent/internal/sessionbroker/spawner_windows.go`: include command mode/fallback in `SpawnedHelper` and spawn logs. + +--- + +### Task 1: Pure startup classification and typed helper fallback + +**Files:** +- Create: `agent/internal/agentapp/process_role.go` +- Create: `agent/internal/agentapp/process_role_test.go` +- Modify: `agent/internal/sessionbroker/userhelper_path.go:42-56` +- Modify: `agent/internal/sessionbroker/userhelper_path_windows.go:20-26` +- Modify: `agent/internal/sessionbroker/userhelper_path_test.go:18-85` +- Modify: `agent/internal/sessionbroker/spawner_windows.go:21-228` +- Modify: `agent/internal/sessionbroker/spawner_stub.go:16-37` + +**Interfaces:** +- Produces: `ProcessStartup` and `classifyProcess`. +- Produces: `sessionbroker.ResolvedHelperExecutable{Path, MainBinaryFallback}`. +- Consumed by: instance metadata, main/helper startup logs, and `SpawnedHelper`. + +- [ ] **Step 1: Write the process-classification table** + +```go +package agentapp + +import "testing" + +func TestClassifyProcess(t *testing.T) { + tests := []struct { + name, command, role, exe string + service bool + wantMode string + wantFallback bool + }{ + {"SCM main", "run", "", "breeze-agent.exe", true, "service-run", false}, + {"console main", "run", "", "breeze-agent.exe", false, "console-run", false}, + {"companion user", "user-helper", "user", "breeze-user-helper.exe", false, "user-helper", false}, + {"fallback user", "user-helper", "user", "breeze-agent.exe", false, "user-helper", true}, + {"renamed fallback user", "user-helper", "user", "breeze-agent-0.70.exe", false, "user-helper", true}, + {"companion system", "user-helper", "system", "breeze-user-helper.exe", false, "system-helper", false}, + {"status", "status", "", "breeze-agent.exe", false, "other", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, fallback := classifyProcess(tt.command, tt.role, tt.exe, tt.service) + if mode != tt.wantMode || fallback != tt.wantFallback { + t.Fatalf("got (%q,%v), want (%q,%v)", mode, fallback, tt.wantMode, tt.wantFallback) + } + }) + } +} +``` + +Extend `userhelper_path_test.go` so the companion case asserts `MainBinaryFallback == false` and the missing-companion case asserts `true`. + +- [ ] **Step 2: Run tests and verify missing symbols** + +Run: `cd agent && go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestClassifyProcess|TestResolveUserHelperPath' -count=1` + +Expected: FAIL because the classifier and typed helper result do not exist. + +- [ ] **Step 3: Add the startup record and classifier** + +```go +package agentapp + +import ( + "path/filepath" + "strings" + "time" + + "github.com/breeze-rmm/agent/internal/sessionbroker" +) + +type ProcessStartup struct { + Binary string `json:"binary"` + ExecutablePath string `json:"executablePath"` + PID int `json:"pid"` + ParentPID int `json:"parentPid"` + WindowsSessionID uint32 `json:"windowsSessionId"` + LaunchMode string `json:"launchMode"` + HelperRole string `json:"helperRole,omitempty"` + LifecycleKey string `json:"lifecycleKey,omitempty"` + CompanionHelper bool `json:"companionHelper"` + MainBinaryFallback bool `json:"mainBinaryFallback"` + Version string `json:"version"` + CreatedAt time.Time `json:"createdAt"` +} + +func classifyProcess(command, role, executable string, service bool) (string, bool) { + base := strings.ToLower(filepath.Base(executable)) + fallback := command == "user-helper" && base != strings.ToLower(sessionbroker.UserHelperBinaryName) + switch { + case command == "run" && service: + return "service-run", false + case command == "run": + return "console-run", false + case command == "user-helper" && role == "system": + return "system-helper", fallback + case command == "user-helper" && role == "user": + return "user-helper", fallback + default: + return "other", false + } +} + +func processStartupFields(s ProcessStartup) map[string]any { + return map[string]any{ + "binary": s.Binary, "executablePath": s.ExecutablePath, + "pid": s.PID, "parentPid": s.ParentPID, + "windowsSessionId": s.WindowsSessionID, "launchMode": s.LaunchMode, + "helperRole": s.HelperRole, "lifecycleKey": s.LifecycleKey, + "companionHelper": s.CompanionHelper, + "mainBinaryFallback": s.MainBinaryFallback, + "version": s.Version, "createdAt": s.CreatedAt, + } +} +``` + +When constructing `ProcessStartup`, set `CompanionHelper` when `command == "user-helper" && !MainBinaryFallback`; this is derived from the resolved executable, not the caller's claim. + +- [ ] **Step 4: Return helper fallback provenance explicitly** + +```go +type ResolvedHelperExecutable struct { + Path string + MainBinaryFallback bool +} + +func resolveUserHelperPath(agentExe string) (ResolvedHelperExecutable, error) { + helper := filepath.Join(filepath.Dir(agentExe), UserHelperBinaryName) + _, err := os.Stat(helper) + if err == nil { + return ResolvedHelperExecutable{Path: helper}, nil + } + if errors.Is(err, fs.ErrNotExist) { + return ResolvedHelperExecutable{Path: agentExe, MainBinaryFallback: true}, nil + } + return ResolvedHelperExecutable{}, fmt.Errorf("stat %s: %w", helper, err) +} +``` + +Update both `SpawnHelperInSession` and `SpawnUserHelperInSession` to use `.Path`, and add `MainBinaryFallback bool` to both build-tagged `SpawnedHelper` definitions so the resolver result is retained immediately. Emit the existing missing-helper warning once per agent owner rather than once per reconcile. + +- [ ] **Step 5: Run focused tests and commit** + +Run: `cd agent && go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestClassifyProcess|TestResolveUserHelperPath|TestUserHelperExePath' -count=1` + +Expected: PASS. + +```bash +git add agent/internal/agentapp/process_role.go agent/internal/agentapp/process_role_test.go agent/internal/sessionbroker/userhelper_path.go agent/internal/sessionbroker/userhelper_path_windows.go agent/internal/sessionbroker/userhelper_path_test.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/spawner_stub.go +git commit -m "feat(agent): classify process roles and helper fallback" +``` + +--- + +### Task 2: Hardened ProgramData lock parent and exclusive Windows guard + +**Files:** +- Create: `agent/internal/agentapp/main_instance.go` +- Create: `agent/internal/agentapp/main_instance_windows.go` +- Create: `agent/internal/agentapp/main_instance_other.go` +- Create: `agent/internal/agentapp/main_instance_windows_test.go` +- Modify: `agent/internal/config/permissions_windows.go:22-119` +- Modify: `agent/internal/config/permissions_windows_test.go:20-109` + +**Interfaces:** +- Consumes: `ProcessStartup` from Task 1 and `config.ConfigDir()`. +- Produces: `mainAgentGuard`, `acquireMainAgentGuard(ProcessStartup)`, `ErrMainAgentAlreadyRunning`. +- Produces: `config.PrepareMainAgentLockDir() (string, error)`. + +- [ ] **Step 1: Write Windows lock ownership tests** + +```go +//go:build windows + +func TestMainAgentGuardExclusiveAndReacquirable(t *testing.T) { + dir := t.TempDir() + prepareMainAgentLockDirFn = func() (string, error) { return dir, nil } + t.Cleanup(func() { prepareMainAgentLockDirFn = config.PrepareMainAgentLockDir }) + meta := ProcessStartup{PID: os.Getpid(), LaunchMode: "console-run", CreatedAt: time.Now()} + + first, err := acquireMainAgentGuard(meta) + if err != nil { + t.Fatal(err) + } + if _, err := acquireMainAgentGuard(meta); !errors.Is(err, ErrMainAgentAlreadyRunning) { + t.Fatalf("second acquire err=%v, want ErrMainAgentAlreadyRunning", err) + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + third, err := acquireMainAgentGuard(meta) + if err != nil { + t.Fatalf("reacquire after close: %v", err) + } + _ = third.Close() +} + +func TestStaleLockFileDoesNotBlock(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, mainAgentLockFile), []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + prepareMainAgentLockDirFn = func() (string, error) { return dir, nil } + guard, err := acquireMainAgentGuard(ProcessStartup{PID: os.Getpid(), LaunchMode: "console-run"}) + if err != nil { + t.Fatal(err) + } + _ = guard.Close() +} +``` + +Add `TestPrepareMainAgentLockDirRejectsReparseParent`, `TestPrepareMainAgentLockDirRejectsUntrustedOwner`, `TestPrepareMainAgentLockDirSanitizesUntrustedExistingRunDir`, `TestPrepareMainAgentLockDirDetectsParentSwap`, `TestMainAgentGuardRejectsReparseFinalObject`, and `TestPrepareMainAgentLockDirDACLFailureIsSecurityError`. Introduce narrow Windows security/filesystem seams so these cases are deterministic on GitHub-hosted Windows runners; restore each seam with `t.Cleanup` and do not run those tests in parallel. Every case must return a wrapped security error that does not match `ErrMainAgentAlreadyRunning`. + +- [ ] **Step 2: Run on Windows and verify missing symbols** + +Run on Windows: `cd agent && go test -race ./internal/agentapp ./internal/config -run 'TestMainAgentGuard|TestStaleLock|TestPrepareMainAgentLockDir' -count=1` + +Expected: FAIL because the guard and directory preparation do not exist. + +- [ ] **Step 3: Add platform-neutral guard contracts** + +```go +package agentapp + +import ( + "errors" + "os" +) + +const ( + mainAgentLockFile = "agent.lock" + exitAlreadyRunning = 17 + exitInstanceGuardError = 18 +) + +var ErrMainAgentAlreadyRunning = errors.New("main agent already running") + +type mainAgentGuard interface { + Close() error +} + +var ( + acquireMainAgentGuardFn = acquireMainAgentGuard + mainAgentExitFn = os.Exit +) +``` + +The non-Windows implementation returns an idempotent no-op guard. + +- [ ] **Step 4: Prepare and verify the protected parent** + +Use a dedicated `filepath.Join(ConfigDir(), "run")` directory with private SDDL: + +```go +const ( + windowsConfigDirCreateSDDL = "O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;;FRFX;;;BU)" + windowsAgentRunDirSDDL = "O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)" +) +``` + +`PrepareMainAgentLockDir` performs this fixed sequence: + +1. Open `ConfigDir()` as a directory with `FILE_FLAG_OPEN_REPARSE_POINT|FILE_FLAG_BACKUP_SEMANTICS` and security-write rights; create it only through a Windows `CreateDirectory` call whose security attributes contain `windowsConfigDirCreateSDDL` and a BUILTIN\Administrators owner. +2. Through that handle, reject reparse attributes, set/repair the protected DACL and owner, then verify the owner is LocalSystem or BUILTIN\Administrators. An unprivileged or unknown owner that cannot be replaced is a security failure. +3. Reopen `ConfigDir()` by path and compare volume serial/file index to the original handle. A path swap is a security failure. +4. Atomically create `run` with `windowsAgentRunDirSDDL`. If it exists, open it no-follow, inspect owner/DACL, and repair through the handle. If its original owner was untrusted, treat its contents as hostile: remove the known `agent.lock` before use; failure to remove a held hostile file is a security failure, not a duplicate-agent result. +5. Reopen `run`, compare file identity, re-read owner/DACL/reparse attributes, and return only after proving it is the same trusted private directory. + +Use `windows.SetSecurityInfo` on handles with `OWNER_SECURITY_INFORMATION|GROUP_SECURITY_INFORMATION|DACL_SECURITY_INFORMATION|PROTECTED_DACL_SECURITY_INFORMATION`; path-based DACL application alone is insufficient. BUILTIN\Users retains read/traverse on `ConfigDir()` but has no access to the private `run` child. Fresh creation must be secure atomically—never `MkdirAll` followed by a DACL. + +- [ ] **Step 5: Implement exclusive file acquisition** + +```go +func acquireMainAgentGuard(meta ProcessStartup) (mainAgentGuard, error) { + dir, err := prepareMainAgentLockDirFn() + if err != nil { + return nil, fmt.Errorf("prepare instance-lock directory: %w", err) + } + path := filepath.Join(dir, mainAgentLockFile) + path16, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + sa, err := privateLockSecurityAttributes() // non-inheritable; trusted owner + SY/BA-only DACL + if err != nil { + return nil, err + } + h, err := windows.CreateFile(path16, windows.GENERIC_READ|windows.GENERIC_WRITE|windows.READ_CONTROL|windows.WRITE_DAC|windows.WRITE_OWNER, 0, sa, windows.OPEN_ALWAYS, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, 0) + if err != nil { + if errors.Is(err, windows.ERROR_SHARING_VIOLATION) || errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return nil, ErrMainAgentAlreadyRunning + } + return nil, fmt.Errorf("open main-agent lock: %w", err) + } + f := os.NewFile(uintptr(h), path) + if f == nil { + _ = windows.CloseHandle(h) + return nil, errors.New("wrap main-agent lock handle") + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(h, &info); err != nil { + _ = f.Close() + return nil, fmt.Errorf("inspect main-agent lock: %w", err) + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + _ = f.Close() + return nil, errors.New("refuse reparse-point main-agent lock") + } + if err := hardenAndVerifyLockHandle(h); err != nil { + _ = f.Close() + return nil, fmt.Errorf("verify main-agent lock security: %w", err) + } + if err := f.Truncate(0); err != nil { + _ = f.Close() + return nil, fmt.Errorf("truncate main-agent lock metadata: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + _ = f.Close() + return nil, err + } + if err := json.NewEncoder(f).Encode(meta); err != nil { + _ = f.Close() + return nil, fmt.Errorf("write main-agent lock metadata: %w", err) + } + if err := f.Sync(); err != nil { + _ = f.Close() + return nil, fmt.Errorf("flush main-agent lock metadata: %w", err) + } + return &fileMainAgentGuard{file: f}, nil +} +``` + +`hardenAndVerifyLockHandle` requires a LocalSystem/Administrators owner, the protected SY/BA-only DACL, and the same non-reparse handle after security application. `fileMainAgentGuard.Close` swaps its file pointer to nil under a mutex before closing, making repeated calls idempotent. `privateLockSecurityAttributes` sets `InheritHandle=0`. + +- [ ] **Step 6: Run native/cross-compile tests and commit** + +Run on Windows: `cd agent && go test -race ./internal/agentapp ./internal/config -run 'TestMainAgentGuard|TestStaleLock|TestPrepareMainAgentLockDir' -count=1` + +Run elsewhere: `cd agent && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c -o /tmp/agentapp.test.exe ./internal/agentapp` + +Expected: native tests PASS; cross-compilation succeeds. + +```bash +git add agent/internal/agentapp/main_instance.go agent/internal/agentapp/main_instance_windows.go agent/internal/agentapp/main_instance_other.go agent/internal/agentapp/main_instance_windows_test.go agent/internal/config/permissions_windows.go agent/internal/config/permissions_windows_test.go +git commit -m "fix(agent): enforce one full Windows agent instance" +``` + +--- + +### Task 3: Acquire and emit bootstrap-safe failure markers before initialization + +**Files:** +- Modify: `agent/internal/agentapp/main.go:910-985` +- Modify: `agent/internal/agentapp/main_test.go` +- Create: `agent/internal/agentapp/process_role_windows.go` +- Create: `agent/internal/agentapp/process_role_other.go` +- Create: `agent/internal/agentapp/instance_marker_windows.go` +- Create: `agent/internal/agentapp/instance_marker_other.go` +- Modify: `agent/internal/eventlog/eventlog_windows.go` +- Modify: `agent/internal/eventlog/eventlog.go` +- Create: `agent/internal/eventlog/eventlog_windows_test.go` + +**Interfaces:** +- Consumes: guard and `ProcessStartup` from Tasks 1-2. +- Produces: `currentProcessStartup(command, role string, service bool) ProcessStartup`. +- Guarantees: duplicate/security failure returns before `reconcileServiceUnitIfNeeded` and `startAgentFn`. +- Produces: `writeInstanceGuardMarker(ProcessStartup, error)` using Windows Event Log before normal logging exists. +- Extends: `agent/internal/eventlog` with `WriteError(source, message string) error`, preserving the existing best-effort `Error` API. + +- [ ] **Step 1: Add a test proving guard acquisition is first** + +```go +func TestRunAgentDuplicateStopsBeforeInitialization(t *testing.T) { + origAcquire := acquireMainAgentGuardFn + origExit := mainAgentExitFn + origMarker := writeInstanceGuardMarkerFn + origReconcile := reconcileServiceUnitIfNeededFn + origStart := startAgentFn + t.Cleanup(func() { + acquireMainAgentGuardFn = origAcquire + mainAgentExitFn = origExit + writeInstanceGuardMarkerFn = origMarker + reconcileServiceUnitIfNeededFn = origReconcile + startAgentFn = origStart + }) + + reconciled, started, markerWritten, exitCode := false, false, false, 0 + acquireMainAgentGuardFn = func(ProcessStartup) (mainAgentGuard, error) { return nil, ErrMainAgentAlreadyRunning } + writeInstanceGuardMarkerFn = func(ProcessStartup, error) { markerWritten = true } + mainAgentExitFn = func(code int) { exitCode = code } + reconcileServiceUnitIfNeededFn = func() { reconciled = true } + startAgentFn = func(*config.Config) (*agentComponents, error) { started = true; return nil, nil } + + runAgent() + if exitCode != exitAlreadyRunning || reconciled || started || !markerWritten { + t.Fatalf("exit=%d reconciled=%v started=%v marker=%v", exitCode, reconciled, started, markerWritten) + } +} +``` + +If reconciliation is not currently behind a seam, introduce `reconcileServiceUnitIfNeededFn = reconcileServiceUnitIfNeeded` and use it only in `runAgent`. + +- [ ] **Step 2: Run the test and verify it fails** + +Run: `cd agent && go test -race ./internal/agentapp -run TestRunAgentDuplicateStopsBeforeInitialization -count=1` + +Expected: FAIL because `runAgent` currently reconciles before acquiring a guard. + +- [ ] **Step 3: Wire the guard at the first line of `runAgent`** + +```go +func runAgent() { + serviceMode := isWindowsService() + startup := currentProcessStartup("run", "", serviceMode) + guard, err := acquireMainAgentGuardFn(startup) + if err != nil { + writeInstanceGuardMarkerFn(startup, err) + if errors.Is(err, ErrMainAgentAlreadyRunning) { + fmt.Fprintf(os.Stderr, "Breeze main agent is already running (pid=%d session=%d mode=%s)\n", startup.PID, startup.WindowsSessionID, startup.LaunchMode) + mainAgentExitFn(exitAlreadyRunning) + return + } + fmt.Fprintf(os.Stderr, "Breeze main-agent instance guard failed: %v\n", err) + mainAgentExitFn(exitInstanceGuardError) + return + } + defer guard.Close() + + reconcileServiceUnitIfNeededFn() + if serviceMode { + if err := runAsService(cfgFile); err != nil { + log.Error("service failed", "error", err.Error()) + mainAgentExitFn(1) + } + return + } + // Existing console-mode body remains after this point. +} +``` + +Cache `serviceMode`; do not call `isWindowsService` again for classification. + +The Windows marker calls the existing `agent/internal/eventlog` wrapper (imported with an unambiguous alias), not `eventlog.Open` directly. Extend that wrapper with `WriteError(source, message string) error`: on first use it attempts `InstallAsEventCreate` with info/warning/error severities, then opens the source and writes the event; cache both handle and initialization error behind the existing per-source `sync.Once`. Keep `Error` source-compatible by discarding `WriteError`'s result. This makes a fresh installation register `BreezeAgent` before opening it while still handling an already-registered source. `instance_marker_windows.go` writes a structured, secret-free message through `WriteError`; if registration/open/write fails it reports that failure to stderr without touching the untrusted config path. It must not call the existing config-directory startup marker because guard failure may mean that path is untrusted. + +Add `TestWriteErrorRegistersMissingSourceBeforeOpen`, `TestWriteErrorAlreadyRegisteredStillOpens`, and `TestWriteErrorReturnsRegistrationOpenFailure` in a Windows-tagged eventlog test using injected install/open/write seams; the missing-source test requires exact `install -> open -> write` order. Add separate agentapp duplicate and security-error tests asserting marker emission, dedicated exit codes, and zero initialization calls; inject `WriteError` there so tests do not depend on host registration. + +Implement `currentProcessStartup` once and cache its result for the lifetime of the process: + +```go +func currentProcessStartup(command, role string, service bool) ProcessStartup { + exe, _ := os.Executable() + meta := currentPlatformProcessMetadata() // kernel session ID, parent PID, and process creation time + mode, fallback := classifyProcess(command, role, exe, service) + lifecycleKey := "" + if meta.WindowsSessionID != 0 && (role == "user" || role == "system") { + lifecycleKey = fmt.Sprintf("%d-%s", meta.WindowsSessionID, role) + } + return ProcessStartup{ + Binary: filepath.Base(exe), ExecutablePath: exe, + PID: os.Getpid(), ParentPID: meta.ParentPID, + WindowsSessionID: meta.WindowsSessionID, + LaunchMode: mode, HelperRole: role, LifecycleKey: lifecycleKey, + CompanionHelper: command == "user-helper" && !fallback, + MainBinaryFallback: fallback, Version: version, + CreatedAt: meta.CreatedAt, + } +} +``` + +On Windows, `currentPlatformProcessMetadata` uses `ProcessIdToSessionId`, a Toolhelp parent-PID snapshot, and `GetProcessTimes`; each unavailable diagnostic field is left at its zero value without weakening guard acquisition. The non-Windows file returns parent PID from `os.Getppid`, session zero, and the current time. Metadata collection must never open another process with mutation rights. + +- [ ] **Step 4: Run main-agent tests** + +Run: `cd agent && go test -race ./internal/eventlog ./internal/agentapp -run 'TestWriteError|TestRunAgent|TestClassifyProcess' -count=1` + +Expected: PASS and duplicate/security error tests prove no initialization side effects. + +- [ ] **Step 5: Commit startup integration** + +```bash +git add agent/internal/agentapp/main.go agent/internal/agentapp/main_test.go agent/internal/agentapp/process_role_windows.go agent/internal/agentapp/process_role_other.go agent/internal/agentapp/instance_marker_windows.go agent/internal/agentapp/instance_marker_other.go agent/internal/eventlog/eventlog_windows.go agent/internal/eventlog/eventlog.go agent/internal/eventlog/eventlog_windows_test.go +git commit -m "fix(agent): acquire Windows instance guard before startup" +``` + +--- + +### Task 4: Structured process-role diagnostics + +**Files:** +- Modify: `agent/internal/agentapp/main.go:527-610,1426-1555` +- Modify: `agent/internal/sessionbroker/spawner_windows.go:21-228` +- Modify: `agent/internal/sessionbroker/spawner_cmdline_test.go:22-45` +- Modify: `agent/internal/agentapp/main_test.go` + +**Interfaces:** +- Consumes: `ProcessStartup`, typed fallback result, and `HelperKey` from the RDS plan. +- Produces one `process startup` event per process and enriched `SpawnedHelper` metadata. + +- [ ] **Step 1: Add pure field-map tests** + +```go +func TestProcessStartupFieldsContainRoleEvidenceOnly(t *testing.T) { + startup := ProcessStartup{ + Binary: "breeze-agent.exe", ExecutablePath: `C:\Program Files\Breeze\breeze-agent.exe`, + PID: 42, ParentPID: 4, WindowsSessionID: 7, LaunchMode: "user-helper", + HelperRole: "user", LifecycleKey: "7-user", MainBinaryFallback: true, + Version: "0.70.0", CreatedAt: time.Unix(100, 0), + } + fields := processStartupFields(startup) + for _, key := range []string{"pid", "parentPid", "windowsSessionId", "launchMode", "helperRole", "lifecycleKey", "mainBinaryFallback"} { + if _, ok := fields[key]; !ok { + t.Fatalf("missing field %q", key) + } + } + for _, forbidden := range []string{"authToken", "helperAuthToken", "mtlsKey"} { + if _, ok := fields[forbidden]; ok { + t.Fatalf("forbidden field %q", forbidden) + } + } +} +``` + +- [ ] **Step 2: Implement and wire one startup event per process** + +After bootstrap logging is available, log the cached `ProcessStartup` record. Replace the existing `starting agent` and `starting helper` field sets with `processStartupFields`; do not duplicate the event later in `startAgent`. + +Inside a helper process, build the startup record from the actual current executable plus the kernel-derived Windows session and authenticated role; `currentProcessStartup` computes `LifecycleKey`, companion mode, and fallback mode without trusting a claimed session. Separately, the parent agent's spawn event uses the typed `ResolvedHelperExecutable` retained in `SpawnedHelper`, so support sees the same fallback provenance even if the child exits before logging. + +Extend `SpawnedHelper` with: + +```go +type SpawnedHelper struct { + PID uint32 + Handle windows.Handle + BinaryPath string + CommandMode string + Role string + WindowsSessionID uint32 + MainBinaryFallback bool +} +``` + +Populate all fields from the actual resolved executable and explicit spawn role. + +- [ ] **Step 3: Run focused diagnostics/fallback tests** + +Run: `cd agent && go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestProcessStartup|TestClassifyProcess|TestResolveUserHelperPath|TestBuildUserHelperCmdLine' -count=1` + +Expected: PASS. + +- [ ] **Step 4: Perform Windows manual classification verification** + +On a Windows test endpoint: + +```powershell +Get-CimInstance Win32_Process | + Where-Object Name -in @('breeze-agent.exe','breeze-user-helper.exe') | + Select-Object ProcessId,ParentProcessId,SessionId,Name,CommandLine +``` + +Verify logs label the SCM PID as `service-run`, an elevated second `run` exits with code 17, companion helpers identify their session/role, and `breeze-agent.exe user-helper` is labeled `mainBinaryFallback=true` without being blocked by the guard. + +- [ ] **Step 5: Run the full agent suite and commit** + +Run: `cd agent && go test -race ./...` + +Expected: PASS with no race report. + +```bash +git add agent/internal/agentapp/main.go agent/internal/agentapp/main_test.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/spawner_cmdline_test.go +git commit -m "feat(agent): report Windows process roles and fallback mode" +``` + +## Plan Completion Gate + +- A second full `run` invocation exits before any mutation or component initialization. +- A stale lock file without a live exclusive handle does not block startup. +- Reparse/ACL/open failures fail closed and are not mislabeled as duplicates. +- Helper/admin commands never acquire the main-agent lock. +- Logs distinguish `service-run`, `console-run`, `user-helper`, `system-helper`, and fallback mode. +- Main helper processes never overwrite main-agent state. +- Native Windows tests and `cd agent && go test -race ./...` pass. diff --git a/docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md b/docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md new file mode 100644 index 0000000000..65686b8524 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md @@ -0,0 +1,738 @@ +# Windows Watchdog Verified Recovery Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make watchdog recovery verify every Windows service/process transition and never start or terminate the wrong agent process. + +**Architecture:** Replace coarse restart/start/kill calls with a structured recovery request/result and an OS-neutral, fakeable Windows recovery state machine. Concrete Windows adapters provide fresh SCM state/PID, configured binary identity, and process-handle operations; main-loop recovery and journals consume the structured result. + +**Tech Stack:** Go, Windows SCM, `golang.org/x/sys/windows/svc/mgr`, Windows process handles, watchdog state machine and health journal. + +## Global Constraints + +- Graceful stop timeout is 35 seconds, exceeding the current approximately 21-second maximum serial shutdown budget with margin. +- `Start` is called only from verified `Stopped` state. +- Initial `StartPending`, `StopPending`, `ContinuePending`, or `PausePending` is observation-only and does not consume an attempt/history slot. +- A failed side effect does consume the attempt/history slot. +- Forced recovery ignores the state-file PID for destructive action and re-queries the current SCM PID. +- Forced ordering is `validate -> terminate -> process exited -> (Stopped -> explicit start | SCM recovery already pending -> observe) -> Running with a new live PID`. +- PID, image, service ownership, or transition uncertainty fails closed. +- PID/image/ownership uncertainty returns a terminal failover disposition and cannot loop on attempt 2. +- Recovery observation and process-exit waits are cancellable so an SCM stop never waits for the 35-second recovery deadline. +- Existing heartbeat-based post-restart verification remains the application-health success criterion. +- Linux and macOS retain their existing behavior through structured adapter results. +- Design source: `docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md`. +- This document is the watchdog phase; the sibling RDS and main-agent plans cover the other approved-design slices. + +## File Structure + +- Modify `agent/internal/watchdog/recovery.go`: structured request/result/error and attempt accounting. +- Modify `agent/internal/watchdog/recovery_test.go`: dispatch, observation, counting, and typed-error tests. +- Create `agent/internal/watchdog/recovery_windows_logic.go`: OS-neutral Windows recovery orchestration. +- Create `agent/internal/watchdog/recovery_windows_logic_test.go`: deterministic transition/order tests. +- Replace `agent/internal/watchdog/recovery_windows.go`: SCM/process backend and concrete controller. +- Create `agent/internal/watchdog/recovery_windows_test.go`: native Windows path parsing and process adapter tests. +- Modify `agent/internal/watchdog/recovery_linux.go`: structured adapter compatibility. +- Modify `agent/internal/watchdog/recovery_darwin.go`: structured adapter compatibility. +- Modify `agent/internal/watchdog/integration_test.go`: structured recovery harness. +- Modify `agent/cmd/breeze-watchdog/main.go`: request/result flow, pending verification, and journal fields. +- Modify `agent/cmd/breeze-watchdog/main_test.go`: journal mapping tests. +- Modify `agent/cmd/breeze-watchdog/failover_dispatch_test.go`: structured failover recovery. + +--- + +### Task 1: Structured recovery contract and correct attempt accounting + +**Files:** +- Modify: `agent/internal/watchdog/recovery.go:23-139` +- Modify: `agent/internal/watchdog/recovery_test.go` +- Modify: `agent/internal/watchdog/recovery_linux.go` +- Modify: `agent/internal/watchdog/recovery_darwin.go` + +**Interfaces:** +- Produces: `RecoveryRequest`, `RecoveryResult`, `RecoveryError`, and `RecoveryFailureClass`. +- Replaces: three-method `serviceController` with `Recover(int, RecoveryRequest) (RecoveryResult, error)`. +- Changes: `RecoveryManager.Attempt(RecoveryRequest) (RecoveryResult, error)`. + +- [ ] **Step 1: Write failing accounting and propagation tests** + +```go +type fakeStructuredController struct { + result RecoveryResult + err error + calls []int +} + +func (f *fakeStructuredController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + f.calls = append(f.calls, attempt) + result := f.result + result.StateFilePID = req.StateFilePID + return result, f.err +} + +func TestObservationOnlyRecoveryDoesNotConsumeAttempt(t *testing.T) { + clk := &fakeClock{now: time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC)} + svc := &fakeStructuredController{result: RecoveryResult{Action: RecoveryActionObserve, ActionTaken: false}} + r := newRecoveryManagerWithDeps(3, 10*time.Minute, svc, clk) + result, err := r.Attempt(RecoveryRequest{StateFilePID: 44}) + if err != nil || result.ActionTaken || r.Attempts() != 0 || r.Count24h() != 0 { + t.Fatalf("result=%+v err=%v attempts=%d history=%d", result, err, r.Attempts(), r.Count24h()) + } +} + +func TestFailedSideEffectConsumesAttempt(t *testing.T) { + clk := &fakeClock{now: time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC)} + svc := &fakeStructuredController{ + result: RecoveryResult{Action: RecoveryActionGraceful, ActionTaken: true, Phase: "wait_stopped"}, + err: &RecoveryError{Class: RecoveryFailureStopTimeout, Phase: "wait_stopped"}, + } + r := newRecoveryManagerWithDeps(3, 10*time.Minute, svc, clk) + _, _ = r.Attempt(RecoveryRequest{StateFilePID: 44}) + if r.Attempts() != 1 || r.Count24h() != 1 { + t.Fatalf("attempts=%d history=%d, want 1 and 1", r.Attempts(), r.Count24h()) + } +} + +func TestStructuredResultPropagatesSCMPIDs(t *testing.T) { + svc := &fakeStructuredController{result: RecoveryResult{OldPID: 100, NewPID: 200, Disposition: RecoveryDispositionVerifyHeartbeat, ActionTaken: true}} + r := newRecoveryManagerWithDeps(3, 0, svc, &fakeClock{now: time.Now()}) + got, err := r.Attempt(RecoveryRequest{StateFilePID: 50}) + if err != nil || got.StateFilePID != 50 || got.OldPID != 100 || got.NewPID != 200 { + t.Fatalf("result=%+v err=%v", got, err) + } +} + +func TestTerminalIdentityFailureExhaustsRecoveryWithoutRestartHistory(t *testing.T) { + clk := &fakeClock{now: time.Now()} + svc := &fakeStructuredController{ + result: RecoveryResult{Action: RecoveryActionForced, Disposition: RecoveryDispositionFailover}, + err: &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_image"}, + } + r := newRecoveryManagerWithDeps(3, time.Minute, svc, clk) + _, _ = r.Attempt(RecoveryRequest{Intent: RecoveryIntentUnhealthy}) + clk.now = clk.now.Add(2 * time.Minute) // cooldown must not clear terminal exhaustion + if r.CanAttempt() || r.Count24h() != 0 { + t.Fatalf("canAttempt=%v history=%d, want false and 0", r.CanAttempt(), r.Count24h()) + } + r.Reset() + if !r.CanAttempt() { + t.Fatal("explicit reset did not clear terminal exhaustion") + } +} +``` + +- [ ] **Step 2: Run tests and verify interface failures** + +Run: `cd agent && go test -race ./internal/watchdog -run 'TestObservationOnly|TestFailedSideEffect|TestStructuredResult' -count=1` + +Expected: FAIL because the structured recovery types and controller method do not exist. + +- [ ] **Step 3: Add the structured contract** + +```go +type RecoveryAction string + +const ( + RecoveryActionObserve RecoveryAction = "observe" + RecoveryActionGraceful RecoveryAction = "graceful_restart" + RecoveryActionForced RecoveryAction = "forced_restart" + RecoveryActionStart RecoveryAction = "ensure_started" +) + +type RecoveryIntent string + +const ( + RecoveryIntentUnhealthy RecoveryIntent = "recover_unhealthy" + RecoveryIntentEnsureStart RecoveryIntent = "ensure_started" + RecoveryIntentRestart RecoveryIntent = "restart" +) + +type RecoveryDisposition string + +const ( + RecoveryDispositionNone RecoveryDisposition = "none" + RecoveryDispositionVerifyHeartbeat RecoveryDisposition = "verify_heartbeat" + RecoveryDispositionFailover RecoveryDisposition = "failover" +) + +type RecoveryFailureClass string + +const ( + RecoveryFailureQuery RecoveryFailureClass = "query_failed" + RecoveryFailureControl RecoveryFailureClass = "control_failed" + RecoveryFailureStopTimeout RecoveryFailureClass = "stop_timeout" + RecoveryFailureTransitionTimeout RecoveryFailureClass = "transition_timeout" + RecoveryFailureIdentityMismatch RecoveryFailureClass = "identity_mismatch" + RecoveryFailureProcessExitTimeout RecoveryFailureClass = "process_exit_timeout" + RecoveryFailureStartTimeout RecoveryFailureClass = "start_timeout" + RecoveryFailureCanceled RecoveryFailureClass = "canceled" +) + +type RecoveryRequest struct { + StateFilePID int + Intent RecoveryIntent + Context context.Context +} + +type RecoveryResult struct { + Intent RecoveryIntent + Action RecoveryAction + Phase string + InitialState string + FinalState string + StateFilePID int + OldPID int + NewPID int + Elapsed time.Duration + ActionTaken bool + Disposition RecoveryDisposition +} + +type RecoveryError struct { + Class RecoveryFailureClass + Phase string + State string + PID int + Err error +} + +func (e *RecoveryError) Error() string { + if e.Err == nil { + return fmt.Sprintf("recovery %s failed during %s", e.Class, e.Phase) + } + return fmt.Sprintf("recovery %s failed during %s: %v", e.Class, e.Phase, e.Err) +} + +func (e *RecoveryError) Unwrap() error { return e.Err } + +type serviceController interface { + Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) +} +``` + +Normalize an empty `Intent` to `RecoveryIntentUnhealthy` and a nil `Context` to `context.Background()` at the `RecoveryManager` boundary for source compatibility, but set both explicitly at every production call site. + +- [ ] **Step 4: Count only actual recovery actions** + +```go +func (r *RecoveryManager) Attempt(req RecoveryRequest) (RecoveryResult, error) { + if req.Intent == "" { + req.Intent = RecoveryIntentUnhealthy + } + if req.Context == nil { + req.Context = context.Background() + } + nextAttempt := r.attempts + 1 + result, err := r.svc.Recover(nextAttempt, req) + result.StateFilePID = req.StateFilePID + result.Intent = req.Intent + if result.Disposition == "" { + result.Disposition = RecoveryDispositionNone + } + if result.ActionTaken { + r.attempts++ + r.lastAttempt = r.clk.Now() + r.recordRestart(r.lastAttempt) + } + if result.Disposition == RecoveryDispositionFailover { + r.attempts = r.maxAttempts + r.terminalExhausted = true + } + return result, err +} +``` + +Add `terminalExhausted bool` to `RecoveryManager`. `CanAttempt` checks it before applying cooldown logic and always returns false while it is set; time passing must never clear it. Only explicit `Reset` clears the flag. Cancellation is not terminal failover: preserve `ActionTaken` for accounting if a control was already issued, return `RecoveryFailureCanceled`, and let watchdog shutdown proceed without another recovery transition. + +For `RecoveryIntentUnhealthy`, adapt Darwin/Linux controllers so attempt 1 performs their existing graceful restart, attempt 2 performs their existing state-PID kill plus start, and later attempts ensure start. `RecoveryIntentEnsureStart` always runs only their start/ensure operation; `RecoveryIntentRestart` always runs their graceful restart, independent of escalation count. Return minimal structured fields with `ActionTaken=true` whenever a side effect was issued. + +- [ ] **Step 5: Run watchdog unit tests and commit** + +Run: `cd agent && go test -race ./internal/watchdog -count=1` + +Expected: PASS after adapting existing history tests to pass `RecoveryRequest`. + +```bash +git add agent/internal/watchdog/recovery.go agent/internal/watchdog/recovery_test.go agent/internal/watchdog/recovery_linux.go agent/internal/watchdog/recovery_darwin.go +git commit -m "refactor(watchdog): return structured recovery outcomes" +``` + +--- + +### Task 2: Fakeable Windows transition state machine + +**Files:** +- Create: `agent/internal/watchdog/recovery_windows_logic.go` +- Create: `agent/internal/watchdog/recovery_windows_logic_test.go` + +**Interfaces:** +- Consumes: structured contract from Task 1. +- Produces: `windowsRecoveryBackend`, `serviceSnapshot`, `watchedProcess`, and `windowsRecoveryController`. +- Keeps Windows API imports out of logic tests. + +- [ ] **Step 1: Write ordered graceful and transitional tests** + +```go +func TestGracefulStopTimeoutNeverStarts(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + ) + controller := newTestWindowsRecoveryController(backend) + controller.stopTimeout = 35 * time.Second + result, err := controller.Recover(1, RecoveryRequest{StateFilePID: 99}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) { + t.Fatalf("err=%v, want RecoveryError", err) + } + if backend.startCalls != 0 || !result.ActionTaken { + t.Fatalf("startCalls=%d result=%+v", backend.startCalls, result) + } +} + +func TestInitialStopPendingObservesWithoutSideEffect(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + if err != nil || result.Action != RecoveryActionObserve || result.ActionTaken || backend.stopCalls != 0 || backend.startCalls != 0 { + t.Fatalf("result=%+v err=%v stop=%d start=%d", result, err, backend.stopCalls, backend.startCalls) + } +} + +func TestGracefulOrderIsStopStoppedStartRunning(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceStartPending, PID: 200}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + if err != nil || strings.Join(backend.operations, ",") != "query,stop,query,query,start,query,query" { + t.Fatalf("result=%+v err=%v operations=%v", result, err, backend.operations) + } +} +``` + +Add `TestGracefulAlreadyStoppedStartsWithoutStop`, `TestEnsureStartAlreadyRunningObserves`, `TestStartPendingToRunningReturnsHeartbeatDisposition`, `TestStartPendingToStoppedDefersStart`, `TestStartTimeout`, `TestRunningWithZeroPIDFails`, `TestRunningWithDeadPIDFails`, `TestStopControlRaceRequeriesAndObservesSCMRecovery`, and `TestStartControlRaceRequeriesWithoutSecondStart`. A `StartPending` observation that reaches `Running` returns `ActionTaken=false` and `Disposition=RecoveryDispositionVerifyHeartbeat`; an observation that ends in `Stopped` leaves disposition `none` so the next attempt can issue `Start`. + +Add `TestRecoveryCancellationInterruptsPendingObservation` and `TestRecoveryCancellationInterruptsProcessExitWait`. Each starts recovery with a cancelable request context, waits until the fake backend reaches its deterministic wait barrier, cancels, and requires a prompt `RecoveryFailureCanceled` return with no subsequent `Start`. The second preserves `ActionTaken=true` because termination already occurred. + +- [ ] **Step 2: Run and verify missing backend failures** + +Run: `cd agent && go test -race ./internal/watchdog -run 'TestGraceful|TestInitialStopPending|TestEnsureStart' -count=1` + +Expected: FAIL because the Windows logic layer does not exist. + +- [ ] **Step 3: Define OS-neutral backend contracts** + +```go +type serviceState string + +const ( + serviceStopped serviceState = "stopped" + serviceStartPending serviceState = "start_pending" + serviceStopPending serviceState = "stop_pending" + serviceRunning serviceState = "running" + serviceContinuePending serviceState = "continue_pending" + servicePausePending serviceState = "pause_pending" + servicePaused serviceState = "paused" +) + +type serviceSnapshot struct { + State serviceState + PID int +} + +type watchedProcess interface { + ImagePath() (string, error) + Alive() (bool, error) + Terminate() error + Wait(context.Context, time.Duration) error + Close() error +} + +type windowsRecoveryBackend interface { + Query() (serviceSnapshot, error) + ConfiguredBinaryPath() (string, error) + Stop() error + Start() error + OpenProcess(pid int) (watchedProcess, error) +} + +type recoveryClock interface { + Now() time.Time + Sleep(context.Context, time.Duration) error +} +``` + +- [ ] **Step 4: Implement graceful/observe/ensure state transitions** + +For `RecoveryIntentUnhealthy`, `Recover(1, req)` runs graceful recovery, `Recover(2, req)` runs forced recovery, and later attempts ensure start. `RecoveryIntentEnsureStart` always selects the ensure-start branch and `RecoveryIntentRestart` always selects graceful restart, regardless of the attempt argument. Pending states call only `Query`/cancellable sleep and return `ActionTaken=false`; they never call stop/start. Every transition loop checks `req.Context` before each query and uses `recoveryClock.Sleep(req.Context, interval)`; process-exit waits receive the same context. Cancellation returns `RecoveryFailureCanceled` promptly and never escalates to failover. If bounded observation reaches `Running`, return the heartbeat-verification disposition. Graceful recovery captures the original nonzero PID, sets `ActionTaken=true` immediately before issuing `Stop` or `Start`, waits up to 35 seconds for stopped, starts only from stopped, then verifies running with a nonzero, live PID different from the original. Thus even a failed control call is counted, while a query/observation failure is not. Each `Recover` call uses named returns plus a defer to populate `Elapsed` and the last reached `Phase` on success and failure. + +After any `Stop`/`Start` control error, immediately re-query SCM before classifying failure. If SCM is now pending, observe it without a competing control; if it is already in the requested terminal state, continue/verify; only an unchanged incompatible state returns `RecoveryFailureControl`. Add fake backend errors matching `ERROR_SERVICE_NOT_ACTIVE`, `ERROR_SERVICE_CANNOT_ACCEPT_CTRL`, and `ERROR_SERVICE_ALREADY_RUNNING` to prove these races do not escalate to forced termination. + +- [ ] **Step 5: Run deterministic logic tests and commit** + +Run: `cd agent && go test -race ./internal/watchdog -run 'TestGraceful|TestInitial|TestEnsure|TestStart|TestRecoveryCancellation' -count=1` + +Expected: PASS without wall-clock sleeps. + +```bash +git add agent/internal/watchdog/recovery_windows_logic.go agent/internal/watchdog/recovery_windows_logic_test.go +git commit -m "fix(watchdog): verify Windows service transitions" +``` + +--- + +### Task 3: Fresh SCM PID and fail-closed forced recovery + +**Files:** +- Modify: `agent/internal/watchdog/recovery_windows_logic.go` +- Modify: `agent/internal/watchdog/recovery_windows_logic_test.go` + +**Interfaces:** +- Consumes: `ConfiguredBinaryPath`, fresh `Query`, and process handles. +- Produces: `normalizeWindowsExecutablePath` and forced recovery order. + +- [ ] **Step 1: Write forced-recovery identity and order tests** + +```go +func TestForcedRecoveryUsesFreshSCMPIDNotStateFilePID(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 200}, + serviceSnapshot{State: serviceRunning, PID: 200}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 300}, + ) + backend.configuredPath = `C:\Program Files\Breeze\breeze-agent.exe` + backend.processes[200] = newFakeWatchedProcess(`C:\Program Files\Breeze\breeze-agent.exe`) + backend.processes[300] = newFakeWatchedProcess(`C:\Program Files\Breeze\breeze-agent.exe`) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{StateFilePID: 999}) + if err != nil || result.OldPID != 200 || result.NewPID != 300 || !reflect.DeepEqual(backend.openedPIDs, []int{200, 300}) { + t.Fatalf("result=%+v err=%v openedPIDs=%v", result, err, backend.openedPIDs) + } +} + +func TestForcedRecoveryImageMismatchDoesNotTerminateOrStart(t *testing.T) { + backend := newFakeWindowsBackend(serviceSnapshot{State: serviceRunning, PID: 200}) + backend.configuredPath = `C:\Program Files\Breeze\breeze-agent.exe` + proc := newFakeWatchedProcess(`C:\Windows\System32\notepad.exe`) + backend.processes[200] = proc + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch || result.Disposition != RecoveryDispositionFailover || proc.terminateCalls != 0 || backend.startCalls != 0 { + t.Fatalf("result=%+v err=%v terminate=%d start=%d", result, err, proc.terminateCalls, backend.startCalls) + } +} + +func TestForcedRecoveryOrdersExitBeforeStart(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + _, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + want := "query,config,open:100,image,alive,query,terminate,wait,query,start,query,open:200,image,alive" + if got := strings.Join(backend.operations, ","); got != want { + t.Fatalf("operations=%q, want %q", got, want) + } +} +``` + +Add `TestForcedRecoveryConfigQueryFailureEntersFailover`, `TestForcedRecoveryOpenProcessFailureEntersFailover`, `TestForcedRecoveryPIDChangesAfterOpenFailsClosed`, `TestForcedRecoverySamePIDPendingStateObservesWithoutTerminate`, `TestForcedRecoveryProcessExitTimeout`, `TestForcedRecoverySCMStoppedTimeout`, `TestForcedRecoveryRejectsSameNewPID`, `TestForcedRecoveryRejectsDeadNewProcess`, and `TestForcedRecoveryObservesAutomaticSCMRestartWithoutCallingStart`. Each asserts that no later unsafe operation occurs after the named failure. The same-PID pending-state case revalidates to `StopPending` and proves recovery boundedly observes without terminating. The automatic-restart case supplies `StartPending -> Running(new PID)` after termination, expects zero explicit `Start` calls, then requires the same new-process image/liveness verification and heartbeat disposition. + +- [ ] **Step 2: Run and verify forced cases fail** + +Run: `cd agent && go test -race ./internal/watchdog -run TestForcedRecovery -count=1` + +Expected: FAIL because attempt 2 still uses the coarse state-file PID path. + +- [ ] **Step 3: Implement fail-closed forced ordering** + +The forced function must: + +```go +snapshot, err := c.backend.Query() +if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("query_scm_pid", snapshot, err) +} +if snapshot.State == serviceStopped { + return c.ensureStarted(result) +} +if isPendingState(snapshot.State) { + return c.observeTransition(result, snapshot) +} +if snapshot.State != serviceRunning || snapshot.PID <= 0 { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_scm_owner", State: string(snapshot.State), PID: snapshot.PID} +} +result.OldPID = snapshot.PID +configured, err := c.backend.ConfiguredBinaryPath() +if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("service_config", snapshot, err) +} +proc, err := c.backend.OpenProcess(snapshot.PID) +if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("open_service_process", snapshot, err) +} +defer proc.Close() +actual, err := proc.ImagePath() +if err != nil || !sameWindowsExecutable(configured, actual) { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_image", PID: snapshot.PID, Err: err} +} +alive, err := proc.Alive() +if err != nil || !alive { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_process_live", PID: snapshot.PID, Err: err} +} +fresh, err := c.backend.Query() +if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "revalidate_scm_pid", PID: snapshot.PID, Err: err} +} +if isPendingState(fresh.State) { + // SCM recovery or another controller already owns the transition. + return c.observeTransition(result, fresh) +} +if fresh.State != serviceRunning || fresh.PID != snapshot.PID { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "revalidate_scm_owner", PID: snapshot.PID} +} +result.ActionTaken = true +if err := proc.Terminate(); err != nil { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureControl, Phase: "terminate", PID: snapshot.PID, Err: err} +} +if err := proc.Wait(req.Context, c.processExitTimeout); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return result, &RecoveryError{Class: RecoveryFailureCanceled, Phase: "wait_process_exit", PID: snapshot.PID, Err: err} + } + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureProcessExitTimeout, Phase: "wait_process_exit", PID: snapshot.PID, Err: err} +} +``` + +Configuration lookup, process open/image/liveness, PID revalidation, termination, process-exit, and post-termination ownership/transition uncertainty set `RecoveryDispositionFailover` before returning; `RecoveryManager` exhausts recovery, recording restart history only if a side effect occurred. After process exit, observe SCM. If it reaches `Stopped`, call `Start`; if SCM failure recovery has already entered `StartPending` or `Running`, do not call `Start` and observe it. In both branches require `Running`, a nonzero PID different from `OldPID`, open the new PID, and validate its image and liveness before returning the heartbeat-verification disposition. + +Use one pure comparison helper in the OS-neutral logic file: + +```go +func normalizeWindowsExecutablePath(path string) string { + path = strings.Trim(strings.TrimSpace(path), `"`) + path = strings.ReplaceAll(path, "/", `\`) + path = strings.TrimPrefix(path, `\\?\`) + return strings.ToLower(strings.TrimRight(path, `\`)) +} + +func sameWindowsExecutable(configured, actual string) bool { + return normalizeWindowsExecutablePath(configured) == normalizeWindowsExecutablePath(actual) +} +``` + +- [ ] **Step 4: Run forced tests and commit** + +Run: `cd agent && go test -race ./internal/watchdog -run TestForcedRecovery -count=1` + +Expected: PASS with exact ordered operations. + +```bash +git add agent/internal/watchdog/recovery_windows_logic.go agent/internal/watchdog/recovery_windows_logic_test.go +git commit -m "fix(watchdog): validate the SCM process before forced recovery" +``` + +--- + +### Task 4: Concrete Windows SCM and process adapters + +**Files:** +- Replace: `agent/internal/watchdog/recovery_windows.go:1-71` +- Create: `agent/internal/watchdog/recovery_windows_test.go` + +**Interfaces:** +- Implements: `windowsRecoveryBackend` and `watchedProcess`. +- Uses: `mgr.Service.Query().ProcessId`, `mgr.Service.Config().BinaryPathName`, `windows.OpenProcess`, `QueryFullProcessImageName`, `TerminateProcess`, and `WaitForSingleObject`. + +- [ ] **Step 1: Write native Windows executable-path tests** + +```go +//go:build windows + +func TestConfiguredExecutablePath(t *testing.T) { + tests := []struct{ command, want string }{ + {`"C:\Program Files\Breeze\breeze-agent.exe" run`, `C:\Program Files\Breeze\breeze-agent.exe`}, + {`C:\Breeze\breeze-agent.exe run`, `C:\Breeze\breeze-agent.exe`}, + } + for _, tt := range tests { + got, err := configuredExecutablePath(tt.command) + if err != nil || !strings.EqualFold(got, tt.want) { + t.Fatalf("configuredExecutablePath(%q)=%q,%v want %q", tt.command, got, err, tt.want) + } + } +} +``` + +Add `TestWindowsWatchedProcessImageAndAlive`, `TestWindowsWatchedProcessWaitTimeout`, `TestWindowsWatchedProcessWaitCanceled`, `TestWindowsWatchedProcessCloseIsIdempotent`, and `TestWindowsWatchedProcessImagePathGrowsBuffer`. Open the current process without invoking `Terminate`; require a nonempty image, `Alive()==(true,nil)`, a bounded wait timeout, cancellation well before a long timeout, and two successful `Close` calls. The buffer test injects `ERROR_INSUFFICIENT_BUFFER` until the allocated UTF-16 buffer grows and then succeeds. + +- [ ] **Step 2: Run on Windows and verify failures** + +Run on Windows: `cd agent && go test -race ./internal/watchdog -run 'TestConfiguredExecutablePath|TestWindowsWatchedProcess' -count=1` + +Expected: FAIL because the concrete adapter functions do not exist. + +- [ ] **Step 3: Implement SCM status/config mapping** + +`Query` maps `svc.Status.State` and returns `int(status.ProcessId)`. `ConfiguredBinaryPath` calls `Service.Config`, decomposes `BinaryPathName` with `windows.DecomposeCommandLine`, rejects an empty or non-absolute first argument, canonicalizes it with `windows.FullPath`, and returns only that executable path. `Start` and `Stop` return wrapped SCM errors. `NewRecoveryManager` constructs `osServiceController` with this Windows backend/controller; Linux and Darwin keep their build-tagged structured controllers. + +- [ ] **Step 4: Implement verified process handles** + +Open with: + +```go +access := uint32(windows.PROCESS_QUERY_LIMITED_INFORMATION | windows.PROCESS_TERMINATE | windows.SYNCHRONIZE) +handle, err := windows.OpenProcess(access, false, uint32(pid)) +``` + +`ImagePath` calls `QueryFullProcessImageName` with a buffer that doubles from `MAX_PATH` up to the 32,767-character Windows limit on `ERROR_INSUFFICIENT_BUFFER`. `Terminate` uses `TerminateProcess`. `Alive` uses zero-timeout `WaitForSingleObject`: `WAIT_TIMEOUT` means alive, `WAIT_OBJECT_0` means exited, and every other result/error fails closed. `Wait(ctx, timeout)` polls `WaitForSingleObject` in short bounded slices (capped at 100ms), checking `ctx.Done()` between slices, and requires `WAIT_OBJECT_0` before the deadline. `Close` is idempotent; process-handle methods serialize access so close cannot race an in-flight query/wait. + +- [ ] **Step 5: Run native/cross-compile verification and commit** + +Run on Windows: `cd agent && go test -race ./internal/watchdog -count=1` + +Run elsewhere: `cd agent && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c -o /tmp/watchdog.test.exe ./internal/watchdog` + +Expected: native tests PASS and cross-compilation succeeds. + +```bash +git add agent/internal/watchdog/recovery_windows.go agent/internal/watchdog/recovery_windows_test.go +git commit -m "feat(watchdog): add verified Windows SCM recovery backend" +``` + +--- + +### Task 5: Main-loop verification and structured journals + +**Files:** +- Modify: `agent/cmd/breeze-watchdog/main.go:188-556,745-790` +- Modify: `agent/cmd/breeze-watchdog/main_test.go` +- Modify: `agent/cmd/breeze-watchdog/failover_dispatch_test.go` +- Modify: `agent/internal/watchdog/integration_test.go` + +**Interfaces:** +- Consumes: `RecoveryRequest` and `RecoveryResult`. +- Produces: `recoveryJournalFields(RecoveryResult, error) map[string]any`. +- Enters pending heartbeat verification only for `RecoveryDispositionVerifyHeartbeat` and immediately enters the existing failover path for `RecoveryDispositionFailover`. +- Produces: a watchdog-run cancellation context driven independently by SCM stop or process signals and threaded through every recovery request. + +- [ ] **Step 1: Write journal-field and pending-verification tests** + +```go +func TestRecoveryJournalFieldsSeparateStateAndSCMPIDs(t *testing.T) { + result := watchdog.RecoveryResult{ + Action: watchdog.RecoveryActionForced, Phase: "verify_running", + InitialState: "running", FinalState: "running", StateFilePID: 50, + OldPID: 100, NewPID: 200, ActionTaken: true, Elapsed: 1500 * time.Millisecond, + } + fields := recoveryJournalFields(result, nil) + if fields["state_file_pid"] != 50 || fields["old_scm_pid"] != 100 || fields["new_scm_pid"] != 200 || fields["elapsed_ms"] != int64(1500) { + t.Fatalf("fields=%v", fields) + } +} + +func TestObservationDoesNotEnterPendingVerification(t *testing.T) { + result := watchdog.RecoveryResult{Action: watchdog.RecoveryActionObserve, ActionTaken: false, Disposition: watchdog.RecoveryDispositionNone} + if shouldVerifyRecovery(result) { + t.Fatal("observation-only result entered heartbeat verification") + } +} +``` + +Add `TestTerminalRecoveryDispositionEntersFailoverImmediately`, `TestFailoverStartAgentUsesEnsureStartIntent`, and `TestFailoverRestartAgentUsesRestartIntent`. The terminal test expects no next recovery attempt; the command tests prove `start_agent` cannot select forced attempt 2 even when the manager already has one attempt and `restart_agent` deliberately selects verified graceful restart after reset. + +Add `TestSCMStopCancelsInFlightRecoveryBeforeServiceStopDeadline`: inject a controller blocked in a cancellable transition wait, start `runWatchdog` with a stop channel, close the channel after the controller reaches its barrier, and require `runWatchdog` to return promptly with no later start/terminate operation. Run it under `-race`. + +- [ ] **Step 2: Run tests and verify missing helpers** + +Run: `cd agent && go test -race ./cmd/breeze-watchdog -run 'TestRecoveryJournalFields|TestObservationDoesNotEnter' -count=1` + +Expected: FAIL because structured journal mapping is not wired. + +- [ ] **Step 3: Replace bool-based recovery dispatch** + +```go +result, err := recovery.Attempt(watchdog.RecoveryRequest{ + StateFilePID: pid, + Intent: watchdog.RecoveryIntentUnhealthy, + Context: runCtx, +}) +fields := recoveryJournalFields(result, err) +if err != nil { + journal.Log(watchdog.LevelError, "recovery.failed", fields) +} else if result.ActionTaken { + journal.Log(watchdog.LevelInfo, "recovery.attempt_dispatched", fields) +} else { + journal.Log(watchdog.LevelInfo, "recovery.observed_transition", fields) +} +if result.Disposition == watchdog.RecoveryDispositionFailover { + pendingVerify = nil + wd.HandleEvent(watchdog.EventRecoveryExhausted) +} else if err == nil && result.Disposition == watchdog.RecoveryDispositionVerifyHeartbeat { + pendingVerify = &struct{ startedAt time.Time }{startedAt: time.Now()} +} +``` + +Replace the current same-goroutine `sigChan`/`stopCh` cases with a cancellation context whose cause records signal versus SCM stop. Tiny forwarding goroutines watch the signal and stop channels independently of the main state loop, cancel `runCtx`, and exit when it is done. The main loop selects on `runCtx.Done()`. Because recovery currently executes synchronously inside that loop, this independent cancellation path is required: after `Attempt` returns `RecoveryFailureCanceled`, log the shutdown cause, close IPC, and return without feeding a recovery/failover event. Thread `runCtx` through failover-dispatch helpers too, so `start_agent` and `restart_agent` requests use the same cancellation boundary. + +The fields include intent, action, disposition, phase, initial/final state, state-file PID, old/new SCM PID, elapsed milliseconds, failure class, and action-taken. The terminal disposition uses the existing `EventRecoveryExhausted` transition and clears pending verification. Map failover commands exactly as follows: + +```go +case "restart_agent": + recovery.Reset() + result, err := recovery.Attempt(watchdog.RecoveryRequest{Intent: watchdog.RecoveryIntentRestart, Context: runCtx}) +case "start_agent": + result, err := recovery.Attempt(watchdog.RecoveryRequest{Intent: watchdog.RecoveryIntentEnsureStart, Context: runCtx}) +``` + +Do not infer command intent from attempt count. Adapt integration harness calls to pass explicit `RecoveryIntentUnhealthy`. + +- [ ] **Step 4: Run watchdog packages with race detection** + +Run: `cd agent && go test -race ./internal/watchdog ./cmd/breeze-watchdog -count=1` + +Expected: PASS; heartbeat verification/flap tests remain green. + +- [ ] **Step 5: Run full agent and Windows verification** + +Run: `cd agent && go test -race ./...` + +Run on Windows: `cd agent && go test -race ./internal/watchdog ./cmd/breeze-watchdog -count=1` + +Expected: PASS with no race report. + +- [ ] **Step 6: Commit main-loop integration** + +```bash +git add agent/cmd/breeze-watchdog/main.go agent/cmd/breeze-watchdog/main_test.go agent/cmd/breeze-watchdog/failover_dispatch_test.go agent/internal/watchdog/integration_test.go +git commit -m "fix(watchdog): journal and verify Windows recovery outcomes" +``` + +## Plan Completion Gate + +- A 15-second `StopPending` interval never causes premature `Start`. +- A 35-second stop timeout returns a typed failure and does not call `Start`. +- Transitional observation consumes no attempt or 24-hour restart record. +- Forced recovery uses and revalidates the fresh SCM PID. +- Image/PID mismatch, exit timeout, stopped timeout, and start timeout fail closed. +- Ownership/PID/image uncertainty exhausts recovery and enters failover without repeating forced attempt 2. +- SCM control races and automatic SCM restart are observed without competing `Stop`/`Start` calls. +- SCM stop or process signal cancels an in-flight transition/process wait promptly and does not leave the watchdog service stuck stopping. +- A successful forced recovery has a new live SCM PID and then enters heartbeat verification. +- Journal fields distinguish state-file PID from SCM old/new PIDs. +- Native Windows tests, watchdog race tests, and `cd agent && go test -race ./...` pass. diff --git a/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md b/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md index fbfe0c90e8..d3fef0221a 100644 --- a/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md +++ b/docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md @@ -1,7 +1,7 @@ # Windows agent and helper lifecycle durability **Date:** 2026-07-14 -**Status:** Design approved, awaiting spec review +**Status:** Approved **Scope:** Windows session helpers, RDS support, main-agent exclusivity, watchdog recovery, and diagnostics **Branch:** `ToddHebebrand/helper-sessions-in-remote-desktop` @@ -235,7 +235,9 @@ and a process creation time predating the current agent owner. Only the full Windows `run` path acquires an exclusive, no-sharing file handle before config, IPC, heartbeat, or collector initialization. The lock file lives -in Breeze's hardened machine-wide ProgramData directory. Helper, +in a private SYSTEM/Administrators-owned `run` child of Breeze's hardened +machine-wide ProgramData directory. Directory and file ownership are verified +in addition to the protected DACL. Helper, service-management, enrollment, and diagnostic subcommands do not acquire it. The protected parent directory is the security boundary: an unprivileged local @@ -277,8 +279,10 @@ Forced attempt: the agent state-file PID for destructive action. 2. Verify that the PID is the SCM-owned BreezeAgent process and that its image path matches the configured service binary. -3. Terminate it, wait for process exit, and wait for SCM state `Stopped`. -4. Start the service and verify `Running` with a new live PID. +3. Terminate it, wait for process exit, and observe SCM. If SCM reaches + `Stopped`, start explicitly. If configured SCM failure recovery has already + entered `StartPending` or `Running`, do not issue a competing start. +4. In either branch, verify `Running` with a new live PID and matching image. `RecoveryManager.Attempt` will accept a richer recovery result so journal entries identify the phase, old/new PID, service states, elapsed time, and @@ -338,8 +342,9 @@ rate-limited. Support collection can then distinguish: a startup marker; it does not continue without exclusivity. - Watchdog stop timeout leaves the service untouched in its observed state and advances to the separately verified forced attempt after cooldown. -- PID validation failure blocks forced termination and enters failover rather - than risking termination of an unrelated process. +- PID validation failure blocks forced termination, returns a terminal recovery + disposition, and enters failover rather than retrying attempt 2 or risking + termination of an unrelated process. ## Compatibility and rollout From 89d72d520287bfba94bdf59080c594a5e15b03e7 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 14:08:45 -0600 Subject: [PATCH 03/47] refactor(agent): share Windows helper eligibility rules --- agent/internal/sessionbroker/helper_key.go | 40 +++++++++++++++++++ .../internal/sessionbroker/helper_key_test.go | 34 ++++++++++++++++ agent/internal/sessionbroker/lifecycle.go | 38 ++++++++---------- 3 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 agent/internal/sessionbroker/helper_key.go create mode 100644 agent/internal/sessionbroker/helper_key_test.go diff --git a/agent/internal/sessionbroker/helper_key.go b/agent/internal/sessionbroker/helper_key.go new file mode 100644 index 0000000000..88dfb7b7cc --- /dev/null +++ b/agent/internal/sessionbroker/helper_key.go @@ -0,0 +1,40 @@ +package sessionbroker + +import ( + "fmt" + "strconv" +) + +type HelperKey struct { + WindowsSessionID uint32 + Role string +} + +func (k HelperKey) String() string { + return fmt.Sprintf("%d-%s", k.WindowsSessionID, k.Role) +} + +func helperRoleDesired(s DetectedSession, role string) bool { + if s.Session == "0" || s.Type == "services" { + return false + } + switch role { + case "system": + return s.State == "active" || s.State == "connected" + case "user": + return s.State == "active" + default: + return false + } +} + +func helperKeyFromDetected(s DetectedSession, role string) (HelperKey, bool) { + if !helperRoleDesired(s, role) { + return HelperKey{}, false + } + id, err := strconv.ParseUint(s.Session, 10, 32) + if err != nil || id == 0 { + return HelperKey{}, false + } + return HelperKey{WindowsSessionID: uint32(id), Role: role}, true +} diff --git a/agent/internal/sessionbroker/helper_key_test.go b/agent/internal/sessionbroker/helper_key_test.go new file mode 100644 index 0000000000..2ebba20615 --- /dev/null +++ b/agent/internal/sessionbroker/helper_key_test.go @@ -0,0 +1,34 @@ +package sessionbroker + +import "testing" + +func TestHelperRoleDesired(t *testing.T) { + tests := []struct { + name string + s DetectedSession + role string + want bool + }{ + {"system active", DetectedSession{Session: "7", State: "active", Type: "rdp"}, "system", true}, + {"system connected", DetectedSession{Session: "7", State: "connected", Type: "rdp"}, "system", true}, + {"user active", DetectedSession{Session: "7", State: "active", Type: "rdp"}, "user", true}, + {"user connected", DetectedSession{Session: "7", State: "connected", Type: "rdp"}, "user", false}, + {"session zero", DetectedSession{Session: "0", State: "active", Type: "rdp"}, "system", false}, + {"services", DetectedSession{Session: "8", State: "active", Type: "services"}, "system", false}, + {"disconnected", DetectedSession{Session: "8", State: "disconnected", Type: "rdp"}, "system", false}, + {"unknown role", DetectedSession{Session: "8", State: "active", Type: "rdp"}, "assist", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := helperRoleDesired(tt.s, tt.role); got != tt.want { + t.Fatalf("helperRoleDesired() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHelperKeyFromDetectedRejectsInvalidSession(t *testing.T) { + if _, ok := helperKeyFromDetected(DetectedSession{Session: "not-a-number", State: "active", Type: "rdp"}, "user"); ok { + t.Fatal("invalid Windows session unexpectedly produced a key") + } +} diff --git a/agent/internal/sessionbroker/lifecycle.go b/agent/internal/sessionbroker/lifecycle.go index ce68f336e4..8e5f90dbd2 100644 --- a/agent/internal/sessionbroker/lifecycle.go +++ b/agent/internal/sessionbroker/lifecycle.go @@ -145,34 +145,28 @@ func (m *HelperLifecycleManager) reconcile() { currentKeys := make(map[string]bool, len(sessions)*2) for _, s := range sessions { - // Skip Session 0 (services) and non-interactive types. - if s.Session == "0" || s.Type == "services" { - continue - } - - // Only target active or connected (lock screen after reboot) sessions. - if s.State != "active" && s.State != "connected" { - continue - } - // Spawn SYSTEM helper if missing, reset retry tracking when connected. - systemKey := s.Session + "-system" - currentKeys[systemKey] = true - if m.broker.HasHelperForWinSessionRole(s.Session, "system") { - m.resetTracked(systemKey) - } else { - m.spawnWithRetry(s.Session, "system") + if systemKey, desired := helperKeyFromDetected(s, "system"); desired { + trackKey := systemKey.String() + currentKeys[trackKey] = true + if m.broker.HasHelperForWinSessionRole(s.Session, systemKey.Role) { + m.resetTracked(trackKey) + } else { + m.spawnWithRetry(s.Session, systemKey.Role) + } } // Spawn user-token helper if missing. Only for active sessions // (user is actually logged in); "connected" means lock screen // where WTSQueryUserToken may fail. - userKey := s.Session + "-user" - currentKeys[userKey] = true - if m.broker.HasHelperForWinSessionRole(s.Session, "user") { - m.resetTracked(userKey) - } else if s.State == "active" { - m.spawnWithRetry(s.Session, "user") + if userKey, desired := helperKeyFromDetected(s, "user"); desired { + trackKey := userKey.String() + currentKeys[trackKey] = true + if m.broker.HasHelperForWinSessionRole(s.Session, userKey.Role) { + m.resetTracked(trackKey) + } else { + m.spawnWithRetry(s.Session, userKey.Role) + } } } From 67f91714d78e9207633fd3819bd919c1129da4ff Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 14:26:09 -0600 Subject: [PATCH 04/47] fix(agent): scope helper admission to Windows sessions --- agent/internal/sessionbroker/broker.go | 215 +++++++---- .../sessionbroker/broker_admission.go | 310 ++++++++++++++++ .../sessionbroker/broker_admission_test.go | 349 ++++++++++++++++++ .../internal/sessionbroker/broker_windows.go | 3 + agent/internal/sessionbroker/lifecycle.go | 10 + 5 files changed, 821 insertions(+), 66 deletions(-) create mode 100644 agent/internal/sessionbroker/broker_admission.go create mode 100644 agent/internal/sessionbroker/broker_admission_test.go diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index 1e5890f861..d504ed0d57 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -250,13 +250,22 @@ type Broker struct { rateLimiter *ipc.RateLimiter startTime time.Time // broker creation time, used for watchdog uptime - mu timedRWMutex - sessions map[string]*Session // sessionID -> Session - byIdentity map[string][]*Session // identity key -> Sessions (UID string on Unix, SID on Windows) - staleHelpers map[string][]int // winSessionID -> PIDs of disconnected helpers - consoleUser string // macOS: current console user ("loginwindow" at login screen) - backup *backupHelper // backup helper process and session - closed bool + mu timedRWMutex + sessions map[string]*Session // sessionID -> Session + byIdentity map[string][]*Session // identity key -> Sessions (UID string on Unix, SID on Windows) + staleHelpers map[string][]int // winSessionID -> PIDs of disconnected helpers + desiredHelperKeys map[HelperKey]struct{} + helperByKey map[HelperKey]*Session + helperByAuthKey map[AuthenticatedHelperKey]*Session + helperReservations map[uint64]*helperAuthReservation + helperKeyReservations map[HelperKey]uint64 + helperAuthReservations map[AuthenticatedHelperKey]uint64 + identityReservations map[string]int + helperReservedVictims map[*Session]uint64 + nextHelperReservationID uint64 + consoleUser string // macOS: current console user ("loginwindow" at login screen) + backup *backupHelper // backup helper process and session + closed bool // snap is an atomically updated snapshot of sessions/byIdentity/consoleUser. // Updated under b.mu.Lock() on every mutation. Read-only hot paths use @@ -291,15 +300,23 @@ type Broker struct { // New creates a new session broker. func New(socketPath string, onMessage MessageHandler) *Broker { b := &Broker{ - socketPath: socketPath, - rateLimiter: ipc.NewRateLimiter(RateLimitAttempts, RateLimitWindow), - startTime: time.Now(), - sessions: make(map[string]*Session), - byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), - onMessage: onMessage, - consoleSessionIDFn: GetConsoleSessionID, - goos: runtime.GOOS, + socketPath: socketPath, + rateLimiter: ipc.NewRateLimiter(RateLimitAttempts, RateLimitWindow), + startTime: time.Now(), + sessions: make(map[string]*Session), + byIdentity: make(map[string][]*Session), + staleHelpers: make(map[string][]int), + desiredHelperKeys: make(map[HelperKey]struct{}), + helperByKey: make(map[HelperKey]*Session), + helperByAuthKey: make(map[AuthenticatedHelperKey]*Session), + helperReservations: make(map[uint64]*helperAuthReservation), + helperKeyReservations: make(map[HelperKey]uint64), + helperAuthReservations: make(map[AuthenticatedHelperKey]uint64), + identityReservations: make(map[string]int), + helperReservedVictims: make(map[*Session]uint64), + onMessage: onMessage, + consoleSessionIDFn: GetConsoleSessionID, + goos: runtime.GOOS, } b.selfHashes = b.computeAllowedHashes() b.publishSnapshotLocked() // initialise with empty maps @@ -990,6 +1007,11 @@ func (b *Broker) HasHelperForWinSession(winSessionID string) bool { func (b *Broker) HasHelperForWinSessionRole(winSessionID, role string) bool { b.mu.RLock() defer b.mu.RUnlock() + if id, err := strconv.ParseUint(winSessionID, 10, 32); err == nil { + if owner := b.helperByKey[HelperKey{WindowsSessionID: uint32(id), Role: role}]; owner != nil { + return true + } + } for _, s := range b.sessions { if s.WinSessionID == winSessionID && s.HelperRole == role { return true @@ -1273,7 +1295,8 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } - identityKey := creds.IdentityKey() + verifiedWinSessionID := peerWinSessionID(creds.PID) + identityKey := admissionIdentityKey(creds.IdentityKey(), verifiedWinSessionID, b.goos) // Step 2: Rate limit check (per identity: UID on Unix, SID on Windows) if !b.rateLimiter.Allow(identityKey) { @@ -1282,13 +1305,17 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } - // Step 3: Check max connections per identity. If the cap is hit, try to - // evict a single stranded session (idle > EvictIdleThreshold) so a - // reconnecting helper isn't permanently locked out by a dead predecessor. - // This is the cheap pre-auth reject path; the register step below - // re-runs the same check under a held write lock as the authoritative - // decision. See issue #443. - if !b.admitOrEvict(identityKey) { + // Step 3: Check max connections per identity. Windows defers any idle + // victim eviction to the authoritative registration step so a failed + // handshake cannot evict an existing helper. Unix retains its existing + // admit-or-evict behavior. + var preAuthAdmitted bool + if b.goos == "windows" { + preAuthAdmitted = b.canAdmitWithoutEviction(identityKey) + } else { + preAuthAdmitted = b.admitOrEvict(identityKey) + } + if !preAuthAdmitted { b.mu.RLock() identityCount := len(b.byIdentity[identityKey]) b.mu.RUnlock() @@ -1506,8 +1533,8 @@ func (b *Broker) handleConnection(rawConn net.Conn) { // assist/user roles to the active console session. On non-Windows / failure // this is "" and the console binding is inert (Unix path returns early). verifiedWinSession := "" - if vsid := peerWinSessionID(creds.PID); vsid != 0 { - verifiedWinSession = fmt.Sprintf("%d", vsid) + if verifiedWinSessionID != 0 { + verifiedWinSession = fmt.Sprintf("%d", verifiedWinSessionID) } consoleWinSession := b.ConsoleSessionID() @@ -1537,6 +1564,32 @@ func (b *Broker) handleConnection(rawConn net.Conn) { scopes := b.grantScopes(helperRole, authReq, runtime.GOOS, creds.BinaryPath) + var helperReservation *helperAuthReservation + if isWindowsLifecycleRole(b.goos, helperRole) { + helperKey := HelperKey{WindowsSessionID: verifiedWinSessionID, Role: helperRole} + helperReservation, err = b.reserveWindowsHelper(identityKey, creds.SID, helperKey) + if err != nil { + log.Warn("Windows helper admission rejected", + "identity", identityKey, + "sessionId", authReq.SessionID, + "helperKey", helperKey.String(), + "error", err.Error(), + ) + _ = conn.SendTyped(env.ID, ipc.TypeAuthResponse, ipc.AuthResponse{ + Accepted: false, + Reason: err.Error(), + Permanent: errors.Is(err, errDuplicateHelperKey) || errors.Is(err, errHelperKeyNotDesired), + }) + conn.Close() + return + } + defer func() { + if helperReservation != nil { + b.releaseWindowsHelper(helperReservation) + } + }() + } + // Send auth response authResp := ipc.AuthResponse{ Accepted: true, @@ -1582,45 +1635,59 @@ func (b *Broker) handleConnection(rawConn net.Conn) { session.WinSessionID = fmt.Sprintf("%d", authReq.WinSessionID) } - // Register session. Re-run tryAdmitLocked under the write lock: this - // is the authoritative cap check. Without it, two concurrent admits - // for the same identity could both pass admitOrEvict (or both see - // room after one of them evicted), then both append here and push the - // count past MaxConnectionsPerIdentity. Also captures any victim so we - // can Close() it outside the lock below. - b.mu.Lock() - admitted, victim := b.tryAdmitLocked(identityKey) - if !admitted { - b.mu.Unlock() - log.Warn("max connections exceeded at register (admit race)", - "identity", identityKey, - "sessionId", authReq.SessionID, - ) - conn.Close() - return - } - b.sessions[authReq.SessionID] = session - b.byIdentity[identityKey] = append(b.byIdentity[identityKey], session) - // Track backup helper session for direct access - if helperRole == backupipc.HelperRoleBackup { - if b.backup == nil { - b.backup = &backupHelper{} - } - b.backup.session = session - } - b.publishSnapshotLocked() - registerOnClosed := b.onSessionClosed - b.mu.Unlock() - - if victim != nil { - if err := victim.Close(); err != nil { - log.Error("error closing evicted session at register", - "sessionId", victim.SessionID, + if helperReservation != nil { + if err := b.commitWindowsHelper(helperReservation, session); err != nil { + log.Warn("Windows helper admission changed before commit", + "identity", identityKey, + "sessionId", authReq.SessionID, "error", err.Error(), ) + conn.Close() + return } - if registerOnClosed != nil { - registerOnClosed(victim) + helperReservation = nil + } else { + // Register non-lifecycle sessions with the existing path. Re-run + // tryAdmitLocked under the write lock: this is the authoritative cap + // check for Unix, assist, watchdog, and backup helpers. + // + // Without it, two concurrent admits for the same identity could both + // pass the pre-auth check, then push the count past the cap. Also captures + // any victim so Close and observer callbacks run outside b.mu. + b.mu.Lock() + admitted, victim := b.tryAdmitLocked(identityKey) + if !admitted { + b.mu.Unlock() + log.Warn("max connections exceeded at register (admit race)", + "identity", identityKey, + "sessionId", authReq.SessionID, + ) + conn.Close() + return + } + b.sessions[authReq.SessionID] = session + b.byIdentity[identityKey] = append(b.byIdentity[identityKey], session) + // Track backup helper session for direct access + if helperRole == backupipc.HelperRoleBackup { + if b.backup == nil { + b.backup = &backupHelper{} + } + b.backup.session = session + } + b.publishSnapshotLocked() + registerOnClosed := b.onSessionClosed + b.mu.Unlock() + + if victim != nil { + if err := victim.Close(); err != nil { + log.Error("error closing evicted session at register", + "sessionId", victim.SessionID, + "error", err.Error(), + ) + } + if registerOnClosed != nil { + registerOnClosed(victim) + } } } @@ -1663,7 +1730,10 @@ func (b *Broker) handleConnection(rawConn net.Conn) { // and records the PID as stale. Caller must hold b.mu.Lock(). Does NOT // Close() the session, publish a snapshot, or fire onSessionClosed; the // caller is responsible for those. -func (b *Broker) removeSessionMapsLocked(session *Session) { +func (b *Broker) removeSessionMapsLocked(session *Session) bool { + if b.sessions[session.SessionID] != session { + return false + } delete(b.sessions, session.SessionID) key := session.IdentityKey @@ -1677,6 +1747,16 @@ func (b *Broker) removeSessionMapsLocked(session *Session) { if len(b.byIdentity[key]) == 0 { delete(b.byIdentity, key) } + for helperKey, owner := range b.helperByKey { + if owner == session { + delete(b.helperByKey, helperKey) + } + } + for authKey, owner := range b.helperByAuthKey { + if owner == session { + delete(b.helperByAuthKey, authKey) + } + } // Track the PID so it can be killed before spawning a replacement // (Windows only — see trackStaleHelper; elsewhere this is a deliberate @@ -1687,16 +1767,19 @@ func (b *Broker) removeSessionMapsLocked(session *Session) { staleKey := session.WinSessionID + "-" + session.HelperRole b.trackStaleHelper(staleKey, session.PID) } + return true } func (b *Broker) removeSession(session *Session) { b.mu.Lock() - b.removeSessionMapsLocked(session) - b.publishSnapshotLocked() + removed := b.removeSessionMapsLocked(session) + if removed { + b.publishSnapshotLocked() + } onSessionClosed := b.onSessionClosed b.mu.Unlock() - if onSessionClosed != nil { + if removed && onSessionClosed != nil { onSessionClosed(session) } } diff --git a/agent/internal/sessionbroker/broker_admission.go b/agent/internal/sessionbroker/broker_admission.go new file mode 100644 index 0000000000..16c7d5abca --- /dev/null +++ b/agent/internal/sessionbroker/broker_admission.go @@ -0,0 +1,310 @@ +package sessionbroker + +import ( + "errors" + "fmt" + + "github.com/breeze-rmm/agent/internal/ipc" +) + +type AuthenticatedHelperKey struct { + PeerSID string + HelperKey +} + +type helperAuthReservation struct { + id uint64 + authKey AuthenticatedHelperKey + identityKey string + victim *Session + victimKind helperReservationVictimKind +} + +type helperReservationVictimKind uint8 + +const ( + helperReservationNoVictim helperReservationVictimKind = iota + helperReservationStaleOwner + helperReservationIdleQuota +) + +var ( + errDuplicateHelperKey = errors.New("helper already registered for Windows session and role") + errHelperKeyNotDesired = errors.New("helper Windows session and role are not currently eligible") + errMaxConnectionsPerIdentity = errors.New("too many connections for identity") + errHelperReservationInvalid = errors.New("helper admission reservation is no longer valid") + errBrokerClosed = errors.New("session broker is closed") +) + +func admissionIdentityKey(base string, peerSession uint32, goos string) string { + if goos != "windows" { + return base + } + return fmt.Sprintf("windows:%s:session:%d", base, peerSession) +} + +func isWindowsLifecycleRole(goos, role string) bool { + return goos == "windows" && (role == ipc.HelperRoleSystem || role == ipc.HelperRoleUser) +} + +// UpdateDesiredHelperKeys replaces the detector-backed admission snapshot. +// The input is copied because reconciliation owns and reuses its local map. +func (b *Broker) UpdateDesiredHelperKeys(desired map[HelperKey]struct{}) { + copyDesired := make(map[HelperKey]struct{}, len(desired)) + for key := range desired { + copyDesired[key] = struct{}{} + } + + b.mu.Lock() + b.desiredHelperKeys = copyDesired + for _, reservation := range b.helperReservations { + if _, ok := copyDesired[reservation.authKey.HelperKey]; !ok { + b.releaseWindowsHelperLocked(reservation) + } + } + b.mu.Unlock() +} + +// reserveWindowsHelper atomically reserves both a Windows session/role key and +// a slot in the session-aware identity quota. It never publishes or evicts a +// session; commitWindowsHelper is the only publication point. +func (b *Broker) reserveWindowsHelper(identityKey, peerSID string, key HelperKey) (*helperAuthReservation, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.closed { + return nil, errBrokerClosed + } + if _, desired := b.desiredHelperKeys[key]; !desired { + return nil, errHelperKeyNotDesired + } + if _, pending := b.helperKeyReservations[key]; pending { + return nil, errDuplicateHelperKey + } + + authKey := AuthenticatedHelperKey{PeerSID: peerSID, HelperKey: key} + if _, pending := b.helperAuthReservations[authKey]; pending { + return nil, errDuplicateHelperKey + } + + var victim *Session + victimKind := helperReservationNoVictim + if owner := b.helperByKey[key]; owner != nil { + if !helperOwnerReplaceable(owner) { + return nil, errDuplicateHelperKey + } + victim = owner + victimKind = helperReservationStaleOwner + } + if owner := b.helperByAuthKey[authKey]; owner != nil && owner != victim { + if !helperOwnerReplaceable(owner) { + return nil, errDuplicateHelperKey + } + victim = owner + victimKind = helperReservationStaleOwner + } + + if b.effectiveIdentityCountLocked(identityKey, victim) >= MaxConnectionsPerIdentity { + // A stale logical owner under another SID does not free capacity in this + // identity bucket. Reject rather than selecting two victims for one admit. + if victim != nil && victim.IdentityKey != identityKey { + return nil, errMaxConnectionsPerIdentity + } + victim = b.idleQuotaVictimLocked(identityKey) + if victim == nil { + return nil, errMaxConnectionsPerIdentity + } + victimKind = helperReservationIdleQuota + } + + b.nextHelperReservationID++ + reservation := &helperAuthReservation{ + id: b.nextHelperReservationID, + authKey: authKey, + identityKey: identityKey, + victim: victim, + victimKind: victimKind, + } + b.helperReservations[reservation.id] = reservation + b.helperKeyReservations[key] = reservation.id + b.helperAuthReservations[authKey] = reservation.id + b.identityReservations[identityKey]++ + if victim != nil { + b.helperReservedVictims[victim] = reservation.id + } + return reservation, nil +} + +func helperOwnerReplaceable(owner *Session) bool { + // Task 4 adds a kernel-bound peer process handle to this decision. Until + // that handle exists, PID polling would be vulnerable to PID reuse, so a + // broker-closed session is the sole authoritative stale signal. + return owner.IsClosed() +} + +func (b *Broker) effectiveIdentityCountLocked(identityKey string, prospectiveVictim *Session) int { + count := len(b.byIdentity[identityKey]) + b.identityReservations[identityKey] + for victim := range b.helperReservedVictims { + if victim.IdentityKey == identityKey && b.sessions[victim.SessionID] == victim { + count-- + } + } + if prospectiveVictim != nil && prospectiveVictim.IdentityKey == identityKey { + if _, alreadyReserved := b.helperReservedVictims[prospectiveVictim]; !alreadyReserved && b.sessions[prospectiveVictim.SessionID] == prospectiveVictim { + count-- + } + } + return count +} + +func (b *Broker) idleQuotaVictimLocked(identityKey string) *Session { + var victim *Session + var oldest = EvictIdleThreshold + for _, session := range b.byIdentity[identityKey] { + if _, reserved := b.helperReservedVictims[session]; reserved { + continue + } + idle := session.IdleDuration() + if idle > oldest { + victim = session + oldest = idle + } + } + return victim +} + +// canAdmitWithoutEviction is the cheap pre-auth capacity check. It deliberately +// leaves any idle candidate registered so a failed handshake cannot evict a +// healthy predecessor; reservation/commit performs the authoritative decision. +func (b *Broker) canAdmitWithoutEviction(identityKey string) bool { + b.mu.RLock() + defer b.mu.RUnlock() + if b.effectiveIdentityCountLocked(identityKey, nil) < MaxConnectionsPerIdentity { + return true + } + return b.idleQuotaVictimLocked(identityKey) != nil +} + +// commitWindowsHelper revalidates and atomically publishes a reserved helper. +// Any displaced session is closed and observed only after b.mu is released. +func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session *Session) error { + if reservation == nil || session == nil { + return errHelperReservationInvalid + } + + b.mu.Lock() + current, exists := b.helperReservations[reservation.id] + if !exists || current != reservation { + _, desired := b.desiredHelperKeys[reservation.authKey.HelperKey] + b.mu.Unlock() + if !desired { + return errHelperKeyNotDesired + } + return errHelperReservationInvalid + } + failLocked := func(err error) error { + b.releaseWindowsHelperLocked(reservation) + b.mu.Unlock() + return err + } + + if b.closed { + return failLocked(errBrokerClosed) + } + if _, desired := b.desiredHelperKeys[reservation.authKey.HelperKey]; !desired { + return failLocked(errHelperKeyNotDesired) + } + if session.IdentityKey != reservation.identityKey || + session.WinSessionID != fmt.Sprintf("%d", reservation.authKey.WindowsSessionID) || + session.HelperRole != reservation.authKey.Role { + return failLocked(errHelperReservationInvalid) + } + if existing := b.sessions[session.SessionID]; existing != nil && existing != reservation.victim { + return failLocked(errHelperReservationInvalid) + } + if owner := b.helperByKey[reservation.authKey.HelperKey]; owner != nil && owner != reservation.victim { + return failLocked(errDuplicateHelperKey) + } + if owner := b.helperByAuthKey[reservation.authKey]; owner != nil && owner != reservation.victim { + return failLocked(errDuplicateHelperKey) + } + + if reservation.victim != nil { + if b.helperReservedVictims[reservation.victim] != reservation.id { + return failLocked(errHelperReservationInvalid) + } + switch reservation.victimKind { + case helperReservationStaleOwner: + if b.helperByKey[reservation.authKey.HelperKey] != reservation.victim || !helperOwnerReplaceable(reservation.victim) { + return failLocked(errDuplicateHelperKey) + } + case helperReservationIdleQuota: + if b.sessions[reservation.victim.SessionID] != reservation.victim || + reservation.victim.IdentityKey != reservation.identityKey || + reservation.victim.IdleDuration() <= EvictIdleThreshold { + return failLocked(errMaxConnectionsPerIdentity) + } + default: + return failLocked(errHelperReservationInvalid) + } + } + if b.effectiveIdentityCountLocked(reservation.identityKey, nil) > MaxConnectionsPerIdentity { + return failLocked(errMaxConnectionsPerIdentity) + } + + victim := reservation.victim + if victim != nil { + b.removeSessionMapsLocked(victim) + } + b.releaseWindowsHelperLocked(reservation) + b.sessions[session.SessionID] = session + b.byIdentity[reservation.identityKey] = append(b.byIdentity[reservation.identityKey], session) + b.helperByAuthKey[reservation.authKey] = session + b.helperByKey[reservation.authKey.HelperKey] = session + b.publishSnapshotLocked() + onClosed := b.onSessionClosed + b.mu.Unlock() + + if victim != nil { + if err := victim.Close(); err != nil { + log.Error("error closing evicted Windows helper session", + "sessionId", victim.SessionID, + "error", err.Error(), + ) + } + if onClosed != nil { + onClosed(victim) + } + } + return nil +} + +func (b *Broker) releaseWindowsHelper(reservation *helperAuthReservation) { + if reservation == nil { + return + } + b.mu.Lock() + b.releaseWindowsHelperLocked(reservation) + b.mu.Unlock() +} + +func (b *Broker) releaseWindowsHelperLocked(reservation *helperAuthReservation) { + if b.helperReservations[reservation.id] != reservation { + return + } + delete(b.helperReservations, reservation.id) + if b.helperKeyReservations[reservation.authKey.HelperKey] == reservation.id { + delete(b.helperKeyReservations, reservation.authKey.HelperKey) + } + if b.helperAuthReservations[reservation.authKey] == reservation.id { + delete(b.helperAuthReservations, reservation.authKey) + } + if remaining := b.identityReservations[reservation.identityKey] - 1; remaining > 0 { + b.identityReservations[reservation.identityKey] = remaining + } else { + delete(b.identityReservations, reservation.identityKey) + } + if reservation.victim != nil && b.helperReservedVictims[reservation.victim] == reservation.id { + delete(b.helperReservedVictims, reservation.victim) + } +} diff --git a/agent/internal/sessionbroker/broker_admission_test.go b/agent/internal/sessionbroker/broker_admission_test.go new file mode 100644 index 0000000000..886565f72d --- /dev/null +++ b/agent/internal/sessionbroker/broker_admission_test.go @@ -0,0 +1,349 @@ +package sessionbroker + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/breeze-rmm/agent/internal/backupipc" + "github.com/breeze-rmm/agent/internal/ipc" +) + +func TestWindowsPreAuthCapacityCheckDoesNotEvict(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + identity := admissionIdentityKey("S-1-5-18", 7, "windows") + var clients []*ipc.Conn + defer func() { + for _, client := range clients { + _ = client.Close() + } + }() + + b.mu.Lock() + for i := 0; i < MaxConnectionsPerIdentity; i++ { + session, client := newPairedSession(t, fmt.Sprintf("existing-%d", i), identity) + clients = append(clients, client) + session.LastSeen = time.Now() + if i == 0 { + session.LastSeen = time.Now().Add(-(EvictIdleThreshold + time.Minute)) + } + b.sessions[session.SessionID] = session + b.byIdentity[identity] = append(b.byIdentity[identity], session) + } + b.publishSnapshotLocked() + b.mu.Unlock() + + if !b.canAdmitWithoutEviction(identity) { + t.Fatal("idle victim should make eventual admission possible") + } + if got := len(b.byIdentity[identity]); got != MaxConnectionsPerIdentity { + t.Fatalf("pre-auth capacity check evicted a session: got %d, want %d", got, MaxConnectionsPerIdentity) + } + for _, session := range b.byIdentity[identity] { + if session.IsClosed() { + t.Fatal("pre-auth capacity check closed an existing session") + } + } +} + +func TestAdmissionIdentityKeyWindowsIncludesSession(t *testing.T) { + a := admissionIdentityKey("S-1-5-18", 7, "windows") + b := admissionIdentityKey("S-1-5-18", 8, "windows") + if a == b { + t.Fatalf("distinct RDS sessions shared key %q", a) + } + if got := admissionIdentityKey("1000", 7, "linux"); got != "1000" { + t.Fatalf("Unix key changed: %q", got) + } +} + +func TestReserveWindowsHelperAllowsOnlyOnePerHelperKey(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + + var wg sync.WaitGroup + errs := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := b.reserveWindowsHelper("windows:S-1-5-18:session:7", "S-1-5-18", key) + errs <- err + }() + } + wg.Wait() + close(errs) + + duplicates := 0 + for err := range errs { + if errors.Is(err, errDuplicateHelperKey) { + duplicates++ + } + } + if duplicates != 1 || len(b.helperReservations) != 1 || len(b.sessions) != 0 { + t.Fatalf("duplicates=%d reservations=%d sessions=%d", duplicates, len(b.helperReservations), len(b.sessions)) + } +} + +func TestReserveWindowsHelperRejectsUndesiredWindowsKey(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + _, err := b.reserveWindowsHelper("windows:S-1-5-18:session:7", "S-1-5-18", HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem}) + if !errors.Is(err, errHelperKeyNotDesired) { + t.Fatalf("err=%v, want errHelperKeyNotDesired", err) + } +} + +func TestUpdateDesiredHelperKeysCopiesInput(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + desired := map[HelperKey]struct{}{key: {}} + b.UpdateDesiredHelperKeys(desired) + delete(desired, key) + + reservation, err := b.reserveWindowsHelper(admissionIdentityKey("S-1-5-18", 7, "windows"), "S-1-5-18", key) + if err != nil { + t.Fatalf("caller mutation changed desired snapshot: %v", err) + } + b.releaseWindowsHelper(reservation) +} + +func TestReservationIsInvisibleUntilCommit(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + identity := admissionIdentityKey("S-1-5-18", key.WindowsSessionID, "windows") + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + + reservation, err := b.reserveWindowsHelper(identity, "S-1-5-18", key) + if err != nil { + t.Fatalf("reserveWindowsHelper: %v", err) + } + if got := len(b.AllSessions()); got != 0 { + t.Fatalf("AllSessions after reserve = %d, want 0", got) + } + if b.HasHelperForWinSessionRole("7", ipc.HelperRoleSystem) { + t.Fatal("reservation was visible as a registered helper") + } + + session, client := newPairedSession(t, "system-session-7", identity) + defer client.Close() + session.WinSessionID = "7" + session.HelperRole = ipc.HelperRoleSystem + session.conn.SetSessionKey([]byte("01234567890123456789012345678901")) + if err := b.commitWindowsHelper(reservation, session); err != nil { + t.Fatalf("commitWindowsHelper: %v", err) + } + + if got := len(b.AllSessions()); got != 1 { + t.Fatalf("AllSessions after commit = %d, want 1", got) + } + if !b.HasHelperForWinSessionRole("7", ipc.HelperRoleSystem) { + t.Fatal("committed helper was not visible") + } +} + +func TestReleaseReservationAfterAcceptedWriteFailure(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + identity := admissionIdentityKey("S-1-5-18", 7, "windows") + var existing []*Session + var clients []*ipc.Conn + defer func() { + for _, client := range clients { + _ = client.Close() + } + }() + b.mu.Lock() + for i := 0; i < MaxConnectionsPerIdentity; i++ { + session, client := newPairedSession(t, fmt.Sprintf("assist-existing-%d", i), identity) + clients = append(clients, client) + session.WinSessionID = "7" + session.HelperRole = ipc.HelperRoleAssist + session.LastSeen = time.Now() + if i == 0 { + session.LastSeen = time.Now().Add(-(EvictIdleThreshold + time.Minute)) + } + existing = append(existing, session) + b.sessions[session.SessionID] = session + b.byIdentity[identity] = append(b.byIdentity[identity], session) + } + b.publishSnapshotLocked() + b.mu.Unlock() + + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + reservation, err := b.reserveWindowsHelper(identity, "S-1-5-18", key) + if err != nil { + t.Fatalf("reserveWindowsHelper: %v", err) + } + if reservation.victim != existing[0] { + t.Fatalf("reserved victim = %p, want idle session %p", reservation.victim, existing[0]) + } + b.releaseWindowsHelper(reservation) + + if len(b.helperReservations) != 0 || len(b.helperKeyReservations) != 0 || len(b.identityReservations) != 0 { + t.Fatalf("reservation state leaked: reservations=%d logical=%d identity=%d", + len(b.helperReservations), len(b.helperKeyReservations), len(b.identityReservations)) + } + for _, session := range existing { + if session.IsClosed() { + t.Fatalf("release closed existing session %q", session.SessionID) + } + if got := b.SessionByID(session.SessionID); got != session { + t.Fatalf("existing session %q was evicted: got %p, want %p", session.SessionID, got, session) + } + } +} + +func TestTwentySystemSIDsAcrossWindowsSessionsHaveIndependentAdmission(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + desired := make(map[HelperKey]struct{}) + for sessionID := uint32(1); sessionID <= 20; sessionID++ { + desired[HelperKey{WindowsSessionID: sessionID, Role: ipc.HelperRoleSystem}] = struct{}{} + } + for i := 0; i < MaxConnectionsPerIdentity+1; i++ { + desired[HelperKey{WindowsSessionID: 99, Role: fmt.Sprintf("quota-%d", i)}] = struct{}{} + } + b.UpdateDesiredHelperKeys(desired) + + for sessionID := uint32(1); sessionID <= 20; sessionID++ { + key := HelperKey{WindowsSessionID: sessionID, Role: ipc.HelperRoleSystem} + identity := admissionIdentityKey("S-1-5-18", sessionID, "windows") + if _, err := b.reserveWindowsHelper(identity, "S-1-5-18", key); err != nil { + t.Fatalf("session %d reservation: %v", sessionID, err) + } + } + + quotaIdentity := admissionIdentityKey("S-1-5-18", 99, "windows") + for i := 0; i < MaxConnectionsPerIdentity; i++ { + key := HelperKey{WindowsSessionID: 99, Role: fmt.Sprintf("quota-%d", i)} + if _, err := b.reserveWindowsHelper(quotaIdentity, "S-1-5-18", key); err != nil { + t.Fatalf("quota reservation %d: %v", i, err) + } + } + key := HelperKey{WindowsSessionID: 99, Role: fmt.Sprintf("quota-%d", MaxConnectionsPerIdentity)} + if _, err := b.reserveWindowsHelper(quotaIdentity, "S-1-5-18", key); !errors.Is(err, errMaxConnectionsPerIdentity) { + t.Fatalf("sixth reservation err=%v, want errMaxConnectionsPerIdentity", err) + } +} + +func TestUnixAndNonLifecycleRolesBypassWindowsLogicalReservation(t *testing.T) { + tests := []struct { + name string + goos string + role string + }{ + {name: "unix system", goos: "linux", role: ipc.HelperRoleSystem}, + {name: "assist", goos: "windows", role: ipc.HelperRoleAssist}, + {name: "watchdog", goos: "windows", role: ipc.HelperRoleWatchdog}, + {name: "backup", goos: "windows", role: backupipc.HelperRoleBackup}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if isWindowsLifecycleRole(tt.goos, tt.role) { + t.Fatalf("%s/%s unexpectedly requires Windows logical reservation", tt.goos, tt.role) + } + }) + } +} + +func TestLiveHelperOwnerRejectsDifferentSID(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + + owner, client := newPairedSession(t, "owner", admissionIdentityKey("S-1-5-18", 7, "windows")) + defer client.Close() + owner.WinSessionID = "7" + owner.HelperRole = ipc.HelperRoleSystem + b.mu.Lock() + b.sessions[owner.SessionID] = owner + b.byIdentity[owner.IdentityKey] = []*Session{owner} + b.helperByKey[key] = owner + b.helperByAuthKey[AuthenticatedHelperKey{PeerSID: "S-1-5-18", HelperKey: key}] = owner + b.publishSnapshotLocked() + b.mu.Unlock() + + _, err := b.reserveWindowsHelper(admissionIdentityKey("S-1-5-21-100", 7, "windows"), "S-1-5-21-100", key) + if !errors.Is(err, errDuplicateHelperKey) { + t.Fatalf("err=%v, want errDuplicateHelperKey", err) + } +} + +func TestClosedHelperOwnerCanBeReplacedAtCommit(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + identity := admissionIdentityKey("S-1-5-18", 7, "windows") + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + + owner, ownerClient := newPairedSession(t, "closed-owner", identity) + defer ownerClient.Close() + owner.WinSessionID = "7" + owner.HelperRole = ipc.HelperRoleSystem + b.mu.Lock() + b.sessions[owner.SessionID] = owner + b.byIdentity[identity] = []*Session{owner} + b.helperByKey[key] = owner + b.helperByAuthKey[AuthenticatedHelperKey{PeerSID: "S-1-5-18", HelperKey: key}] = owner + b.publishSnapshotLocked() + b.mu.Unlock() + if err := owner.Close(); err != nil { + t.Fatalf("close owner: %v", err) + } + + reservation, err := b.reserveWindowsHelper(identity, "S-1-5-18", key) + if err != nil { + t.Fatalf("reserve replacement: %v", err) + } + replacement, replacementClient := newPairedSession(t, "replacement", identity) + defer replacementClient.Close() + replacement.WinSessionID = "7" + replacement.HelperRole = ipc.HelperRoleSystem + if err := b.commitWindowsHelper(reservation, replacement); err != nil { + t.Fatalf("commit replacement: %v", err) + } + if got := b.SessionByID(replacement.SessionID); got != replacement { + t.Fatalf("replacement was not published: got %p, want %p", got, replacement) + } + if got := b.SessionByID(owner.SessionID); got != nil { + t.Fatalf("closed owner remains published: %p", got) + } +} + +func TestDesiredKeyRemovalInvalidatesReservation(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + identity := admissionIdentityKey("S-1-5-18", 7, "windows") + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + reservation, err := b.reserveWindowsHelper(identity, "S-1-5-18", key) + if err != nil { + t.Fatalf("reserveWindowsHelper: %v", err) + } + + b.UpdateDesiredHelperKeys(nil) + if len(b.helperReservations) != 0 || len(b.identityReservations) != 0 { + t.Fatalf("invalidated reservation leaked: reservations=%d identity=%d", len(b.helperReservations), len(b.identityReservations)) + } + + session, client := newPairedSession(t, "invalidated", identity) + defer client.Close() + session.WinSessionID = "7" + session.HelperRole = ipc.HelperRoleSystem + if err := b.commitWindowsHelper(reservation, session); !errors.Is(err, errHelperKeyNotDesired) { + t.Fatalf("commit err=%v, want errHelperKeyNotDesired", err) + } + if b.SessionByID(session.SessionID) != nil { + t.Fatal("invalidated reservation published a session") + } +} diff --git a/agent/internal/sessionbroker/broker_windows.go b/agent/internal/sessionbroker/broker_windows.go index f6423cbd89..94038b76fd 100644 --- a/agent/internal/sessionbroker/broker_windows.go +++ b/agent/internal/sessionbroker/broker_windows.go @@ -34,6 +34,9 @@ func (b *Broker) setupSocket() error { // peerWinSessionID returns the Windows session ID for the given process, // verified by the kernel via ProcessIdToSessionId. Returns 0 on failure. func peerWinSessionID(pid int) uint32 { + if pid <= 0 { + return 0 + } var sessionID uint32 if err := windows.ProcessIdToSessionId(uint32(pid), &sessionID); err != nil { return 0 diff --git a/agent/internal/sessionbroker/lifecycle.go b/agent/internal/sessionbroker/lifecycle.go index 8e5f90dbd2..656144c160 100644 --- a/agent/internal/sessionbroker/lifecycle.go +++ b/agent/internal/sessionbroker/lifecycle.go @@ -143,6 +143,16 @@ func (m *HelperLifecycleManager) reconcile() { } currentKeys := make(map[string]bool, len(sessions)*2) + desiredKeys := make(map[HelperKey]struct{}, len(sessions)*2) + for _, s := range sessions { + if key, desired := helperKeyFromDetected(s, "system"); desired { + desiredKeys[key] = struct{}{} + } + if key, desired := helperKeyFromDetected(s, "user"); desired { + desiredKeys[key] = struct{}{} + } + } + m.broker.UpdateDesiredHelperKeys(desiredKeys) for _, s := range sessions { // Spawn SYSTEM helper if missing, reset retry tracking when connected. From 64e40af45d8528ce94bd4642e9b5429a417dd34a Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 14:43:40 -0600 Subject: [PATCH 05/47] fix(agent): preserve role-aware helper admission --- agent/internal/sessionbroker/broker.go | 150 +++++------ .../sessionbroker/broker_admission.go | 88 ++++++- .../sessionbroker/broker_admission_test.go | 236 ++++++++++++++++++ agent/internal/sessionbroker/lifecycle.go | 25 +- .../internal/sessionbroker/lifecycle_test.go | 23 ++ 5 files changed, 415 insertions(+), 107 deletions(-) diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index d504ed0d57..c35fa4f24e 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -1296,45 +1296,19 @@ func (b *Broker) handleConnection(rawConn net.Conn) { } verifiedWinSessionID := peerWinSessionID(creds.PID) - identityKey := admissionIdentityKey(creds.IdentityKey(), verifiedWinSessionID, b.goos) - - // Step 2: Rate limit check (per identity: UID on Unix, SID on Windows) - if !b.rateLimiter.Allow(identityKey) { - log.Warn("connection rate limited", "identity", identityKey, "pid", creds.PID) - sendPreAuthRejectAndClose(rawConn, ipc.PreAuthCodeRateLimited, "connection rate limited", false) - return - } - - // Step 3: Check max connections per identity. Windows defers any idle - // victim eviction to the authoritative registration step so a failed - // handshake cannot evict an existing helper. Unix retains its existing - // admit-or-evict behavior. - var preAuthAdmitted bool - if b.goos == "windows" { - preAuthAdmitted = b.canAdmitWithoutEviction(identityKey) - } else { - preAuthAdmitted = b.admitOrEvict(identityKey) - } - if !preAuthAdmitted { - b.mu.RLock() - identityCount := len(b.byIdentity[identityKey]) - b.mu.RUnlock() - log.Warn("max connections exceeded", "identity", identityKey, "count", identityCount) - sendPreAuthRejectAndClose(rawConn, ipc.PreAuthCodeMaxConnsExceeded, "too many connections for identity", false) - return - } + baseIdentityKey := creds.IdentityKey() // Wrap connection conn := ipc.NewConn(rawConn) - // Step 4: Read auth request + // Step 2: Read auth request // (Moved ahead of binary-path verification so the hash from the auth // request can serve as the authoritative binary identity signal — // Windows cross-session spawns produce process paths that don't always // match our allowlist after path normalization. See issue #387 part D.) env, err := conn.Recv() if err != nil { - log.Warn("auth request read failed", "identity", identityKey, "error", err.Error()) + log.Warn("auth request read failed", "identity", baseIdentityKey, "error", err.Error()) conn.Close() return } @@ -1352,7 +1326,46 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } - // Step 5: Verify protocol version + // Determine helper role before selecting the admission identity. Windows + // system/user helpers are scoped by kernel SID + Windows session; assist, + // watchdog, and backup retain the legacy SID bucket. Unknown roles use the + // legacy bucket for rate/cap enforcement before permanent rejection. + helperRole := authReq.HelperRole + if helperRole == "" { + helperRole = ipc.HelperRoleSystem + } + roleKnown := true + switch helperRole { + case ipc.HelperRoleSystem, ipc.HelperRoleUser, ipc.HelperRoleWatchdog, ipc.HelperRoleAssist, backupipc.HelperRoleBackup: + default: + roleKnown = false + } + identityKey := helperAdmissionIdentityKey(baseIdentityKey, verifiedWinSessionID, b.goos, helperRole) + + // Step 3: Rate and connection quotas are authoritative after the bounded auth + // request reveals the role. Lifecycle roles reserve without pre-auth + // eviction; every other role keeps the previous admit-or-evict behavior. + if !b.rateLimiter.Allow(identityKey) { + log.Warn("connection rate limited", "identity", identityKey, "pid", creds.PID) + sendPreAuthRejectAndClose(rawConn, ipc.PreAuthCodeRateLimited, "connection rate limited", false) + return + } + var preAuthAdmitted bool + if isWindowsLifecycleRole(b.goos, helperRole) { + preAuthAdmitted = b.canAdmitWithoutEviction(identityKey) + } else { + preAuthAdmitted = b.admitOrEvict(identityKey) + } + if !preAuthAdmitted { + b.mu.RLock() + identityCount := len(b.byIdentity[identityKey]) + b.mu.RUnlock() + log.Warn("max connections exceeded", "identity", identityKey, "count", identityCount) + sendPreAuthRejectAndClose(rawConn, ipc.PreAuthCodeMaxConnsExceeded, "too many connections for identity", false) + return + } + + // Step 4: Verify protocol version if authReq.ProtocolVersion != ipc.ProtocolVersion { log.Warn("protocol version mismatch", "got", authReq.ProtocolVersion, "want", ipc.ProtocolVersion) _ = conn.SendTyped(env.ID, ipc.TypeAuthResponse, ipc.AuthResponse{ @@ -1363,14 +1376,24 @@ func (b *Broker) handleConnection(rawConn net.Conn) { conn.Close() return } + if !roleKnown { + log.Warn("unknown helper role", "role", helperRole, "identity", identityKey, "pid", creds.PID) + _ = conn.SendTyped(env.ID, ipc.TypeAuthResponse, ipc.AuthResponse{ + Accepted: false, + Reason: "unknown helper role", + Permanent: true, + }) + conn.Close() + return + } - // Step 6: Verify identity — SID on Windows, UID on Unix. + // Step 5: Verify identity — SID on Windows, UID on Unix. // The watchdog role is exempt from identity claim validation: it runs // as SYSTEM but its IPCClient doesn't self-report a SID or a usable // UID (Go's os.Getuid() returns -1 on Windows → uint32 overflow). // The kernel-verified creds from GetPeerCredentials (step 1) are // sufficient — a caller can't fake them on a named pipe / Unix socket. - if authReq.HelperRole != ipc.HelperRoleWatchdog { + if helperRole != ipc.HelperRoleWatchdog { if runtime.GOOS == "windows" { if authReq.SID == "" { log.Warn("auth missing SID on Windows", "pid", creds.PID) @@ -1406,7 +1429,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { } } - // Step 7: Verify binary path and hash from kernel-resolved peer metadata. + // Step 6: Verify binary path and hash from kernel-resolved peer metadata. // Do not trust authReq.BinaryHash: any local peer can self-report it. if strings.TrimSpace(creds.BinaryPath) == "" { log.Warn("rejecting helper connection: peer binary path unresolved", @@ -1437,7 +1460,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } - // Step 8: Verify binary hash — reject helpers if no allowed helper hash could be loaded. + // Step 7: Verify binary hash — reject helpers if no allowed helper hash could be loaded. if len(b.selfHashes) == 0 { log.Error("rejecting helper connection: helper binary hash allowlist unavailable", "identity", identityKey, @@ -1487,7 +1510,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } - // Step 9: Reject duplicate session IDs + // Step 8: Reject duplicate session IDs b.mu.RLock() if _, exists := b.sessions[authReq.SessionID]; exists { b.mu.RUnlock() @@ -1509,25 +1532,6 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } - // Determine helper role and scopes. Default to "system" for backward compat - // with helpers that don't send the role field. - helperRole := authReq.HelperRole - if helperRole == "" { - helperRole = ipc.HelperRoleSystem - } - switch helperRole { - case ipc.HelperRoleSystem, ipc.HelperRoleUser, ipc.HelperRoleWatchdog, ipc.HelperRoleAssist, backupipc.HelperRoleBackup: - default: - log.Warn("unknown helper role", "role", helperRole, "identity", identityKey, "pid", creds.PID) - _ = conn.SendTyped(env.ID, ipc.TypeAuthResponse, ipc.AuthResponse{ - Accepted: false, - Reason: "unknown helper role", - Permanent: true, - }) - conn.Close() - return - } - // Kernel-verify the peer's Windows session id (from peer PID, via // ProcessIdToSessionId) BEFORE the role gate so the gate can bind the // assist/user roles to the active console session. On non-Windows / failure @@ -1538,7 +1542,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { } consoleWinSession := b.ConsoleSessionID() - // Step 10: Validate role matches peer identity to prevent privilege escalation. + // Step 9: Validate role matches peer identity to prevent privilege escalation. // On Windows, SYSTEM helpers must run as SYSTEM (S-1-5-18), and user/assist // helpers must NOT run as SYSTEM. This prevents a non-SYSTEM process from // claiming system role to get desktop scopes, or SYSTEM from claiming user @@ -1647,17 +1651,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { } helperReservation = nil } else { - // Register non-lifecycle sessions with the existing path. Re-run - // tryAdmitLocked under the write lock: this is the authoritative cap - // check for Unix, assist, watchdog, and backup helpers. - // - // Without it, two concurrent admits for the same identity could both - // pass the pre-auth check, then push the count past the cap. Also captures - // any victim so Close and observer callbacks run outside b.mu. - b.mu.Lock() - admitted, victim := b.tryAdmitLocked(identityKey) - if !admitted { - b.mu.Unlock() + if err := b.registerNonLifecycleSession(identityKey, helperRole, session); err != nil { log.Warn("max connections exceeded at register (admit race)", "identity", identityKey, "sessionId", authReq.SessionID, @@ -1665,30 +1659,6 @@ func (b *Broker) handleConnection(rawConn net.Conn) { conn.Close() return } - b.sessions[authReq.SessionID] = session - b.byIdentity[identityKey] = append(b.byIdentity[identityKey], session) - // Track backup helper session for direct access - if helperRole == backupipc.HelperRoleBackup { - if b.backup == nil { - b.backup = &backupHelper{} - } - b.backup.session = session - } - b.publishSnapshotLocked() - registerOnClosed := b.onSessionClosed - b.mu.Unlock() - - if victim != nil { - if err := victim.Close(); err != nil { - log.Error("error closing evicted session at register", - "sessionId", victim.SessionID, - "error", err.Error(), - ) - } - if registerOnClosed != nil { - registerOnClosed(victim) - } - } } log.Info("user helper connected", diff --git a/agent/internal/sessionbroker/broker_admission.go b/agent/internal/sessionbroker/broker_admission.go index 16c7d5abca..86560de99d 100644 --- a/agent/internal/sessionbroker/broker_admission.go +++ b/agent/internal/sessionbroker/broker_admission.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/breeze-rmm/agent/internal/backupipc" "github.com/breeze-rmm/agent/internal/ipc" ) @@ -43,6 +44,13 @@ func admissionIdentityKey(base string, peerSession uint32, goos string) string { return fmt.Sprintf("windows:%s:session:%d", base, peerSession) } +func helperAdmissionIdentityKey(base string, peerSession uint32, goos, role string) string { + if !isWindowsLifecycleRole(goos, role) { + return base + } + return admissionIdentityKey(base, peerSession, goos) +} + func isWindowsLifecycleRole(goos, role string) bool { return goos == "windows" && (role == ipc.HelperRoleSystem || role == ipc.HelperRoleUser) } @@ -65,6 +73,34 @@ func (b *Broker) UpdateDesiredHelperKeys(desired map[HelperKey]struct{}) { b.mu.Unlock() } +// refreshDesiredHelperKeys obtains one detector snapshot and publishes every +// eligible lifecycle key from it before callers inspect broker connections or +// spawn helpers. +func (b *Broker) refreshDesiredHelperKeys(detector SessionDetector) ([]DetectedSession, error) { + sessions, err := detector.ListSessions() + if err != nil { + return nil, err + } + desired := make(map[HelperKey]struct{}, len(sessions)*2) + for _, session := range sessions { + if key, ok := helperKeyFromDetected(session, ipc.HelperRoleSystem); ok { + desired[key] = struct{}{} + } + if key, ok := helperKeyFromDetected(session, ipc.HelperRoleUser); ok { + desired[key] = struct{}{} + } + } + b.UpdateDesiredHelperKeys(desired) + return sessions, nil +} + +func (b *Broker) helperKeyDesired(key HelperKey) bool { + b.mu.RLock() + defer b.mu.RUnlock() + _, desired := b.desiredHelperKeys[key] + return desired +} + // reserveWindowsHelper atomically reserves both a Windows session/role key and // a slot in the session-aware identity quota. It never publishes or evicts a // session; commitWindowsHelper is the only publication point. @@ -185,6 +221,41 @@ func (b *Broker) canAdmitWithoutEviction(identityKey string) bool { return b.idleQuotaVictimLocked(identityKey) != nil } +// registerNonLifecycleSession preserves the original authoritative +// registration path for Unix and Windows assist/watchdog/backup helpers. +func (b *Broker) registerNonLifecycleSession(identityKey, helperRole string, session *Session) error { + b.mu.Lock() + admitted, victim := b.tryAdmitLocked(identityKey) + if !admitted { + b.mu.Unlock() + return errMaxConnectionsPerIdentity + } + b.sessions[session.SessionID] = session + b.byIdentity[identityKey] = append(b.byIdentity[identityKey], session) + if helperRole == backupipc.HelperRoleBackup { + if b.backup == nil { + b.backup = &backupHelper{} + } + b.backup.session = session + } + b.publishSnapshotLocked() + onClosed := b.onSessionClosed + b.mu.Unlock() + + if victim != nil { + if err := victim.Close(); err != nil { + log.Error("error closing evicted session at register", + "sessionId", victim.SessionID, + "error", err.Error(), + ) + } + if onClosed != nil { + onClosed(victim) + } + } + return nil +} + // commitWindowsHelper revalidates and atomically publishes a reserved helper. // Any displaced session is closed and observed only after b.mu is released. func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session *Session) error { @@ -229,6 +300,8 @@ func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session return failLocked(errDuplicateHelperKey) } + victim := reservation.victim + projectedIdentityCount := b.effectiveIdentityCountLocked(reservation.identityKey, nil) if reservation.victim != nil { if b.helperReservedVictims[reservation.victim] != reservation.id { return failLocked(errHelperReservationInvalid) @@ -239,8 +312,16 @@ func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session return failLocked(errDuplicateHelperKey) } case helperReservationIdleQuota: - if b.sessions[reservation.victim.SessionID] != reservation.victim || - reservation.victim.IdentityKey != reservation.identityKey || + victimPresent := b.sessions[reservation.victim.SessionID] == reservation.victim + if !victimPresent { + // effectiveIdentityCountLocked does not subtract a removed victim. + victim = nil + } else if projectedIdentityCount+1 <= MaxConnectionsPerIdentity { + // Another disconnect relaxed the quota after reserve. Keep the idle + // session and account for it in the final projected count. + victim = nil + projectedIdentityCount++ + } else if reservation.victim.IdentityKey != reservation.identityKey || reservation.victim.IdleDuration() <= EvictIdleThreshold { return failLocked(errMaxConnectionsPerIdentity) } @@ -248,11 +329,10 @@ func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session return failLocked(errHelperReservationInvalid) } } - if b.effectiveIdentityCountLocked(reservation.identityKey, nil) > MaxConnectionsPerIdentity { + if projectedIdentityCount > MaxConnectionsPerIdentity { return failLocked(errMaxConnectionsPerIdentity) } - victim := reservation.victim if victim != nil { b.removeSessionMapsLocked(victim) } diff --git a/agent/internal/sessionbroker/broker_admission_test.go b/agent/internal/sessionbroker/broker_admission_test.go index 886565f72d..546f1445bc 100644 --- a/agent/internal/sessionbroker/broker_admission_test.go +++ b/agent/internal/sessionbroker/broker_admission_test.go @@ -1,6 +1,7 @@ package sessionbroker import ( + "context" "errors" "fmt" "sync" @@ -11,6 +12,65 @@ import ( "github.com/breeze-rmm/agent/internal/ipc" ) +type admissionTestDetector struct { + sessions []DetectedSession + err error +} + +func (d admissionTestDetector) ListSessions() ([]DetectedSession, error) { + return d.sessions, d.err +} + +func (d admissionTestDetector) WatchSessions(context.Context) <-chan SessionEvent { + return make(chan SessionEvent) +} + +func TestRefreshDesiredHelperKeysPublishesCompleteSnapshot(t *testing.T) { + b := New("test", nil) + detector := admissionTestDetector{sessions: []DetectedSession{ + {Session: "7", State: "active"}, + {Session: "8", State: "connected"}, + {Session: "0", State: "active", Type: "services"}, + }} + + sessions, err := b.refreshDesiredHelperKeys(detector) + if err != nil { + t.Fatalf("refreshDesiredHelperKeys: %v", err) + } + if len(sessions) != 3 { + t.Fatalf("refreshed sessions = %d, want 3", len(sessions)) + } + + want := map[HelperKey]struct{}{ + {WindowsSessionID: 7, Role: ipc.HelperRoleSystem}: {}, + {WindowsSessionID: 7, Role: ipc.HelperRoleUser}: {}, + {WindowsSessionID: 8, Role: ipc.HelperRoleSystem}: {}, + } + b.mu.RLock() + defer b.mu.RUnlock() + if len(b.desiredHelperKeys) != len(want) { + t.Fatalf("desired keys = %v, want %v", b.desiredHelperKeys, want) + } + for key := range want { + if _, ok := b.desiredHelperKeys[key]; !ok { + t.Fatalf("desired snapshot missing %v", key) + } + } +} + +func TestDesiredSnapshotRejectsSCMKeyAbsentFromDetector(t *testing.T) { + b := New("test", nil) + _, err := b.refreshDesiredHelperKeys(admissionTestDetector{sessions: []DetectedSession{ + {Session: "8", State: "active"}, + }}) + if err != nil { + t.Fatalf("refreshDesiredHelperKeys: %v", err) + } + if b.helperKeyDesired(HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem}) { + t.Fatal("SCM event key absent from detector snapshot was considered desired") + } +} + func TestWindowsPreAuthCapacityCheckDoesNotEvict(t *testing.T) { b := New("test", nil) b.goos = "windows" @@ -60,6 +120,91 @@ func TestAdmissionIdentityKeyWindowsIncludesSession(t *testing.T) { } } +func TestWindowsAdmissionIdentityAndRateBucketsAreRoleAware(t *testing.T) { + b := New("test", nil) + systemSID := "S-1-5-18" + userSID := "S-1-5-21-100" + systemKey := helperAdmissionIdentityKey(systemSID, 7, "windows", ipc.HelperRoleSystem) + userKey := helperAdmissionIdentityKey(systemSID, 7, "windows", ipc.HelperRoleUser) + otherSessionSystemKey := helperAdmissionIdentityKey(systemSID, 8, "windows", ipc.HelperRoleSystem) + assistKey := helperAdmissionIdentityKey(userSID, 7, "windows", ipc.HelperRoleAssist) + watchdogKey := helperAdmissionIdentityKey(systemSID, 7, "windows", ipc.HelperRoleWatchdog) + backupKey := helperAdmissionIdentityKey(systemSID, 7, "windows", backupipc.HelperRoleBackup) + + if systemKey != userKey { + t.Fatalf("lifecycle roles in one Windows session split identity buckets: %q != %q", systemKey, userKey) + } + if systemKey == otherSessionSystemKey { + t.Fatalf("system helpers in distinct Windows sessions shared %q", systemKey) + } + for role, pair := range map[string][2]string{ + ipc.HelperRoleAssist: {assistKey, userSID}, + ipc.HelperRoleWatchdog: {watchdogKey, systemSID}, + backupipc.HelperRoleBackup: {backupKey, systemSID}, + } { + if pair[0] != pair[1] { + t.Fatalf("%s identity key = %q, want legacy SID %q", role, pair[0], pair[1]) + } + } + + for i := 0; i < RateLimitAttempts; i++ { + if !b.rateLimiter.Allow(watchdogKey) { + t.Fatalf("legacy bucket rejected attempt %d before limit", i+1) + } + } + if b.rateLimiter.Allow(backupKey) { + t.Fatal("backup did not share the legacy SID rate-limit bucket") + } + if !b.rateLimiter.Allow(assistKey) { + t.Fatal("assist did not retain its independent user-SID rate-limit bucket") + } + if !b.rateLimiter.Allow(systemKey) || !b.rateLimiter.Allow(otherSessionSystemKey) { + t.Fatal("session-scoped lifecycle rate-limit buckets were not independent of the legacy SID bucket") + } +} + +func TestWindowsNonLifecycleRegistrationUsesLegacyIdentityAndQuota(t *testing.T) { + roles := []string{ipc.HelperRoleAssist, ipc.HelperRoleWatchdog, backupipc.HelperRoleBackup} + for _, role := range roles { + t.Run(role, func(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + base := "S-1-5-18" + var clients []*ipc.Conn + defer func() { + for _, client := range clients { + _ = client.Close() + } + }() + + for i := 0; i < MaxConnectionsPerIdentity; i++ { + session, client := newPairedSession(t, fmt.Sprintf("%s-%d", role, i), base) + clients = append(clients, client) + session.HelperRole = role + session.WinSessionID = "7" + if err := b.registerNonLifecycleSession(base, role, session); err != nil { + t.Fatalf("register %d: %v", i, err) + } + } + if got := len(b.byIdentity[base]); got != MaxConnectionsPerIdentity { + t.Fatalf("legacy identity registrations = %d, want %d", got, MaxConnectionsPerIdentity) + } + if len(b.helperByKey) != 0 || len(b.helperReservations) != 0 { + t.Fatalf("non-lifecycle role entered logical reservation state: owners=%d reservations=%d", + len(b.helperByKey), len(b.helperReservations)) + } + + overLimit, client := newPairedSession(t, role+"-over-limit", base) + clients = append(clients, client) + overLimit.HelperRole = role + overLimit.WinSessionID = "7" + if err := b.registerNonLifecycleSession(base, role, overLimit); !errors.Is(err, errMaxConnectionsPerIdentity) { + t.Fatalf("over-limit registration err=%v, want errMaxConnectionsPerIdentity", err) + } + }) + } +} + func TestReserveWindowsHelperAllowsOnlyOnePerHelperKey(t *testing.T) { b := New("test", nil) b.goos = "windows" @@ -202,6 +347,69 @@ func TestReleaseReservationAfterAcceptedWriteFailure(t *testing.T) { } } +func TestCommitKeepsReservedIdleVictimWhenQuotaRelaxes(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + identity := admissionIdentityKey("S-1-5-18", 7, "windows") + var existing []*Session + var clients []*ipc.Conn + defer func() { + for _, client := range clients { + _ = client.Close() + } + }() + + b.mu.Lock() + for i := 0; i < MaxConnectionsPerIdentity; i++ { + session, client := newPairedSession(t, fmt.Sprintf("quota-existing-%d", i), identity) + clients = append(clients, client) + session.WinSessionID = "7" + session.HelperRole = ipc.HelperRoleAssist + session.LastSeen = time.Now() + if i == 0 { + session.LastSeen = time.Now().Add(-(EvictIdleThreshold + time.Minute)) + } + existing = append(existing, session) + b.sessions[session.SessionID] = session + b.byIdentity[identity] = append(b.byIdentity[identity], session) + } + b.publishSnapshotLocked() + b.mu.Unlock() + + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + reservation, err := b.reserveWindowsHelper(identity, "S-1-5-18", key) + if err != nil { + t.Fatalf("reserveWindowsHelper: %v", err) + } + if reservation.victim != existing[0] { + t.Fatalf("reserved victim = %p, want %p", reservation.victim, existing[0]) + } + + if err := existing[1].Close(); err != nil { + t.Fatalf("close disconnected session: %v", err) + } + b.removeSession(existing[1]) + + replacement, client := newPairedSession(t, "quota-replacement", identity) + clients = append(clients, client) + replacement.WinSessionID = "7" + replacement.HelperRole = ipc.HelperRoleSystem + if err := b.commitWindowsHelper(reservation, replacement); err != nil { + t.Fatalf("commitWindowsHelper: %v", err) + } + + if existing[0].IsClosed() { + t.Fatal("commit closed the idle victim after quota pressure disappeared") + } + if got := b.SessionByID(existing[0].SessionID); got != existing[0] { + t.Fatalf("idle victim was unnecessarily evicted: got %p, want %p", got, existing[0]) + } + if got := len(b.byIdentity[identity]); got != MaxConnectionsPerIdentity { + t.Fatalf("identity sessions after relaxed commit = %d, want %d", got, MaxConnectionsPerIdentity) + } +} + func TestTwentySystemSIDsAcrossWindowsSessionsHaveIndependentAdmission(t *testing.T) { b := New("test", nil) b.goos = "windows" @@ -347,3 +555,31 @@ func TestDesiredKeyRemovalInvalidatesReservation(t *testing.T) { t.Fatal("invalidated reservation published a session") } } + +func TestCommitAfterBrokerShutdownReleasesReservationWithoutPublishing(t *testing.T) { + b := New("test", nil) + b.goos = "windows" + key := HelperKey{WindowsSessionID: 7, Role: ipc.HelperRoleSystem} + identity := admissionIdentityKey("S-1-5-18", 7, "windows") + b.UpdateDesiredHelperKeys(map[HelperKey]struct{}{key: {}}) + reservation, err := b.reserveWindowsHelper(identity, "S-1-5-18", key) + if err != nil { + t.Fatalf("reserveWindowsHelper: %v", err) + } + b.Close() + + session, client := newPairedSession(t, "shutdown-candidate", identity) + defer client.Close() + session.WinSessionID = "7" + session.HelperRole = ipc.HelperRoleSystem + if err := b.commitWindowsHelper(reservation, session); !errors.Is(err, errBrokerClosed) { + t.Fatalf("commit err=%v, want errBrokerClosed", err) + } + if len(b.helperReservations) != 0 || len(b.identityReservations) != 0 { + t.Fatalf("shutdown commit leaked reservation state: reservations=%d identity=%d", + len(b.helperReservations), len(b.identityReservations)) + } + if b.SessionByID(session.SessionID) != nil { + t.Fatal("shutdown commit published candidate session") + } +} diff --git a/agent/internal/sessionbroker/lifecycle.go b/agent/internal/sessionbroker/lifecycle.go index 656144c160..5c3759eebd 100644 --- a/agent/internal/sessionbroker/lifecycle.go +++ b/agent/internal/sessionbroker/lifecycle.go @@ -136,23 +136,13 @@ func (m *HelperLifecycleManager) Start(ctx context.Context) { // reconcile ensures both SYSTEM and user helpers are running in every eligible session. func (m *HelperLifecycleManager) reconcile() { - sessions, err := m.detector.ListSessions() + sessions, err := m.broker.refreshDesiredHelperKeys(m.detector) if err != nil { log.Warn("lifecycle: failed to list sessions", "error", err.Error()) return } currentKeys := make(map[string]bool, len(sessions)*2) - desiredKeys := make(map[HelperKey]struct{}, len(sessions)*2) - for _, s := range sessions { - if key, desired := helperKeyFromDetected(s, "system"); desired { - desiredKeys[key] = struct{}{} - } - if key, desired := helperKeyFromDetected(s, "user"); desired { - desiredKeys[key] = struct{}{} - } - } - m.broker.UpdateDesiredHelperKeys(desiredKeys) for _, s := range sessions { // Spawn SYSTEM helper if missing, reset retry tracking when connected. @@ -213,11 +203,20 @@ func (m *HelperLifecycleManager) handleSCMEvent(evt SCMSessionEvent) { ts.fatalExitUntil = time.Time{} } m.mu.Unlock() + if _, err := m.broker.refreshDesiredHelperKeys(m.detector); err != nil { + log.Warn("lifecycle: failed to refresh desired helpers for session event", + "sessionId", sessionID, + "error", err.Error(), + ) + return + } - if !m.broker.HasHelperForWinSessionRole(sessionID, "system") { + systemKey := HelperKey{WindowsSessionID: evt.SessionID, Role: "system"} + if m.broker.helperKeyDesired(systemKey) && !m.broker.HasHelperForWinSessionRole(sessionID, systemKey.Role) { m.spawnWithRetry(sessionID, "system") } - if !m.broker.HasHelperForWinSessionRole(sessionID, "user") { + userKey := HelperKey{WindowsSessionID: evt.SessionID, Role: "user"} + if m.broker.helperKeyDesired(userKey) && !m.broker.HasHelperForWinSessionRole(sessionID, userKey.Role) { m.spawnWithRetry(sessionID, "user") } case wtsSessionLogoff, wtsSessionTerminate: diff --git a/agent/internal/sessionbroker/lifecycle_test.go b/agent/internal/sessionbroker/lifecycle_test.go index 35243c3837..144b29e9de 100644 --- a/agent/internal/sessionbroker/lifecycle_test.go +++ b/agent/internal/sessionbroker/lifecycle_test.go @@ -7,6 +7,29 @@ import ( "time" ) +func TestHandleSCMEventDoesNotSpawnBeforeDetectorPublishesEventKey(t *testing.T) { + broker := New(`\\.\pipe\test-scm-desired-order-`+t.Name(), nil) + m := NewHelperLifecycleManager(broker, nil) + m.detector = admissionTestDetector{sessions: []DetectedSession{ + {Session: "8", State: "active"}, + }} + + m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionLogon, SessionID: 7}) + + if broker.helperKeyDesired(HelperKey{WindowsSessionID: 7, Role: "system"}) || + broker.helperKeyDesired(HelperKey{WindowsSessionID: 7, Role: "user"}) { + t.Fatal("SCM event key absent from detector was published as desired") + } + m.mu.Lock() + defer m.mu.Unlock() + if _, spawned := m.tracked["7-system"]; spawned { + t.Fatal("system helper spawned before detector published eligibility") + } + if _, spawned := m.tracked["7-user"]; spawned { + t.Fatal("user helper spawned before detector published eligibility") + } +} + // --------------------------------------------------------------------------- // Priority 4: fatalExitUntil suppression in spawnWithRetry // --------------------------------------------------------------------------- From 935a71963e620cef8051bbf41744e67c595a74fa Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 14:54:49 -0600 Subject: [PATCH 06/47] fix(agent): authenticate user helpers in matching RDP sessions --- agent/internal/sessionbroker/broker.go | 67 ++++---- agent/internal/sessionbroker/broker_test.go | 81 +++++++++ .../sessionbroker/broker_windows_test.go | 6 +- .../console_session_gate_test.go | 156 +++++------------- 4 files changed, 154 insertions(+), 156 deletions(-) diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index c35fa4f24e..3c3b99c684 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -1533,29 +1533,30 @@ func (b *Broker) handleConnection(rawConn net.Conn) { } // Kernel-verify the peer's Windows session id (from peer PID, via - // ProcessIdToSessionId) BEFORE the role gate so the gate can bind the - // assist/user roles to the active console session. On non-Windows / failure - // this is "" and the console binding is inert (Unix path returns early). + // ProcessIdToSessionId) before the role gate. System/user helpers must claim + // that exact interactive session; assist remains bound to the active console. + // On non-Windows this session gate is inert. verifiedWinSession := "" if verifiedWinSessionID != 0 { verifiedWinSession = fmt.Sprintf("%d", verifiedWinSessionID) } + claimedWinSession := fmt.Sprintf("%d", authReq.WinSessionID) consoleWinSession := b.ConsoleSessionID() // Step 9: Validate role matches peer identity to prevent privilege escalation. // On Windows, SYSTEM helpers must run as SYSTEM (S-1-5-18), and user/assist // helpers must NOT run as SYSTEM. This prevents a non-SYSTEM process from // claiming system role to get desktop scopes, or SYSTEM from claiming user - // role. The watchdog must also run as root/SYSTEM. Additionally, assist/user - // are bound to the active console session so a co-logged-in user on a - // multi-user host can't register them from another session (#1009). The - // decision is factored into roleIdentityRejection so the gate can be + // role. The watchdog must also run as root/SYSTEM. System/user claims must + // equal the kernel-derived interactive session, while assist stays bound to + // the active console session (#1009). The decision is factored into + // roleIdentityRejection so the gate can be // unit-tested with an injected peer-cred SID/UID and session ids (none of // which can be faked over a pipe). - if reason, rejected := roleIdentityRejection(helperRole, creds.SID, creds.UID, verifiedWinSession, consoleWinSession, runtime.GOOS); rejected { + if reason, rejected := roleIdentityRejection(helperRole, creds.SID, creds.UID, verifiedWinSession, claimedWinSession, consoleWinSession, runtime.GOOS); rejected { log.Warn("role/identity mismatch", "reason", reason, "role", helperRole, "sid", creds.SID, "uid", creds.UID, - "peerWinSession", verifiedWinSession, "consoleWinSession", consoleWinSession, + "peerWinSession", verifiedWinSession, "claimedWinSession", claimedWinSession, "consoleWinSession", consoleWinSession, "pid", creds.PID, "binaryKind", authReq.BinaryKind) _ = conn.SendTyped(env.ID, ipc.TypeAuthResponse, ipc.AuthResponse{ Accepted: false, @@ -1624,11 +1625,12 @@ func (b *Broker) handleConnection(rawConn net.Conn) { // Use the kernel-verified Windows session ID (computed above from the peer // PID) instead of trusting the self-reported value, preventing - // session-jumping attacks. Falls back to the self-reported value only when - // the kernel lookup failed (verifiedWinSession == ""). + // session-jumping attacks. System/user helpers cannot reach this point when + // the kernel lookup failed or disagrees with the authenticated claim. Other + // roles retain the legacy fallback when no kernel session is available. if verifiedWinSession != "" { session.WinSessionID = verifiedWinSession - if verifiedWinSession != fmt.Sprintf("%d", authReq.WinSessionID) { + if verifiedWinSession != claimedWinSession { log.Warn("WinSessionID mismatch — using kernel-verified value", "reported", authReq.WinSessionID, "verified", verifiedWinSession, @@ -1636,7 +1638,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { ) } } else { - session.WinSessionID = fmt.Sprintf("%d", authReq.WinSessionID) + session.WinSessionID = claimedWinSession } if helperReservation != nil { @@ -1871,22 +1873,19 @@ const systemSID = "S-1-5-18" // permanent. It returns ("", false) when the role/identity pairing is allowed. // // peerWinSession is the kernel-verified Windows session id of the peer (from -// ProcessIdToSessionId) and consoleWinSession is the active console session id. -// On Windows the assist/user roles are additionally bound to the active console -// session: a co-logged-in non-SYSTEM user on a multi-user host (RDS/terminal -// server) running the genuine allowlisted Helper from a NON-console session must -// not be able to register as assist/user — otherwise it would obtain the device -// helper token and intercept run_as_user scripts meant for the console operator -// (#1009). The SYSTEM-capture gate is unchanged (still SID-only). The console -// binding does not apply on Unix (single interactive session; the macOS desktop -// helper authenticates as user-role from the GUI/loginwindow session). +// ProcessIdToSessionId), claimedWinSession is the authenticated numeric claim, +// and consoleWinSession is the active console session id. Windows system/user +// helpers require a nonzero peer session that exactly matches their claim, so +// legitimate RDP helpers are admitted without trusting self-reported routing. +// Assist remains console-bound for its cross-user token capability (#1009). +// Session binding does not apply on Unix. // // Pure and OS-parameterized so the privilege-escalation gate can be unit-tested // with an injected SID/UID and session ids — a real peer-cred SID and // kernel-verified session id can't be forged over a named pipe / Unix socket, // so end-to-end pipe tests can only exercise the current test process's own // identity. -func roleIdentityRejection(role, sid string, uid uint32, peerWinSession, consoleWinSession, goos string) (reason string, rejected bool) { +func roleIdentityRejection(role, sid string, uid uint32, peerWinSession, claimedWinSession, consoleWinSession, goos string) (reason string, rejected bool) { if goos == "windows" { switch { case role == ipc.HelperRoleSystem && sid != systemSID: @@ -1898,23 +1897,17 @@ func roleIdentityRejection(role, sid string, uid uint32, peerWinSession, console case role == ipc.HelperRoleWatchdog && sid != systemSID: return "watchdog role requires SYSTEM identity", true } - // Positive console-session assertion for the cross-user roles. An unknown - // console session — "" (lookup failed) or "0" (the Session-0 services - // sentinel / WTS-failure value; Broker.ConsoleSessionID normalizes it to - // "", but the raw value is rejected here too so this pure gate is correct - // in isolation) — is treated as "no match" so we fail closed rather than - // admit an arbitrary session. - consoleUnknown := consoleWinSession == "" || consoleWinSession == "0" - switch role { - case ipc.HelperRoleAssist: - if consoleUnknown || peerWinSession != consoleWinSession { - return "assist role requires the active console session", true + if role == ipc.HelperRoleUser || role == ipc.HelperRoleSystem { + if peerWinSession == "" || peerWinSession == "0" { + return role + " role requires an interactive peer session", true } - case ipc.HelperRoleUser: - if consoleUnknown || peerWinSession != consoleWinSession { - return "user role requires the active console session", true + if peerWinSession != claimedWinSession { + return role + " role session claim does not match peer token", true } } + if role == ipc.HelperRoleAssist && (consoleWinSession == "" || consoleWinSession == "0" || peerWinSession != consoleWinSession) { + return "assist role requires the active console session", true + } return "", false } // Unix: watchdog and system-role helpers must run as root. The macOS diff --git a/agent/internal/sessionbroker/broker_test.go b/agent/internal/sessionbroker/broker_test.go index f9128d9aca..7be40aef3a 100644 --- a/agent/internal/sessionbroker/broker_test.go +++ b/agent/internal/sessionbroker/broker_test.go @@ -98,6 +98,30 @@ func TestSessionForUserPrefersMostRecentUserSession(t *testing.T) { } } +func TestSessionForUserSelectsRDPHelper(t *testing.T) { + now := time.Now() + + consoleSession, consoleClient := newTestUserSession(t, "console-helper", "alice", now.Add(-20*time.Minute)) + defer consoleClient.Close() + consoleSession.WinSessionID = "1" + rdpSession, rdpClient := newTestUserSession(t, "rdp-helper", "alice", now.Add(-time.Minute)) + defer rdpClient.Close() + rdpSession.WinSessionID = "7" + + b := &Broker{ + sessions: map[string]*Session{ + consoleSession.SessionID: consoleSession, + rdpSession.SessionID: rdpSession, + }, + byIdentity: make(map[string][]*Session), + staleHelpers: make(map[string][]int), + } + + if got := b.SessionForUser("alice"); got != rdpSession { + t.Fatalf("SessionForUser returned %v, want RDP helper %q", got, rdpSession.SessionID) + } +} + func TestLaunchProcessViaUserHelperBroadcastsToAllUserSessions(t *testing.T) { now := time.Now() @@ -220,6 +244,63 @@ func TestLaunchProcessViaUserHelperForSessionTargetsMatchingHelper(t *testing.T) } } +func TestLaunchProcessViaUserHelperForSessionTargetsMatchingRDPHelper(t *testing.T) { + now := time.Now() + + consoleSession, consoleClient := newTestUserSession(t, "console-helper", "alice", now.Add(-10*time.Minute)) + rdpSession, rdpClient := newTestUserSession(t, "rdp-helper", "alice", now.Add(-time.Minute)) + consoleSession.WinSessionID = "1" + rdpSession.WinSessionID = "7" + defer consoleSession.Close() + defer rdpSession.Close() + defer consoleClient.Close() + defer rdpClient.Close() + + go consoleSession.RecvLoop(func(*Session, *ipc.Envelope) {}) + go rdpSession.RecvLoop(func(*Session, *ipc.Envelope) {}) + + b := &Broker{ + sessions: map[string]*Session{ + consoleSession.SessionID: consoleSession, + rdpSession.SessionID: rdpSession, + }, + byIdentity: make(map[string][]*Session), + staleHelpers: make(map[string][]int), + } + + seen := make(chan string, 1) + startResponder := func(label string, client *ipc.Conn) { + t.Helper() + go func() { + client.SetReadDeadline(time.Now().Add(5 * time.Second)) + env, err := client.Recv() + if err != nil { + return + } + seen <- label + respPayload, _ := json.Marshal(ipc.LaunchProcessResult{OK: true, PID: 4242}) + if err := client.Send(&ipc.Envelope{ID: env.ID, Type: ipc.TypeLaunchResult, Payload: respPayload}); err != nil { + t.Errorf("send %s launch response: %v", label, err) + } + }() + } + startResponder("console", consoleClient) + startResponder("rdp", rdpClient) + + if err := b.LaunchProcessViaUserHelperForSession("7", "/usr/local/bin/breeze-helper"); err != nil { + t.Fatalf("LaunchProcessViaUserHelperForSession: %v", err) + } + + select { + case got := <-seen: + if got != "rdp" { + t.Fatalf("targeted launch reached %s helper, want rdp", got) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for targeted RDP helper launch") + } +} + // Previously the reaper skipped capture-capable sessions. Issue #443: a // streaming helper Touches on every frame, so a genuinely idle capture // session over IdleTimeout is stranded and must be reaped. diff --git a/agent/internal/sessionbroker/broker_windows_test.go b/agent/internal/sessionbroker/broker_windows_test.go index 43ec730e68..b80686ee32 100644 --- a/agent/internal/sessionbroker/broker_windows_test.go +++ b/agent/internal/sessionbroker/broker_windows_test.go @@ -523,9 +523,9 @@ func TestAssistRoleRejectsSystemSID(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - // Console session matches peer session so the SID gate (not the - // console-session binding) is the assertion under test here. - reason, rejected := roleIdentityRejection(tc.role, tc.sid, 0, "1", "1", "windows") + // Claimed and console sessions match the peer session so the SID + // gate (not either session binding) is the assertion under test here. + reason, rejected := roleIdentityRejection(tc.role, tc.sid, 0, "1", "1", "1", "windows") if rejected != tc.wantReject { t.Fatalf("rejected = %v, want %v (reason %q)", rejected, tc.wantReject, reason) } diff --git a/agent/internal/sessionbroker/console_session_gate_test.go b/agent/internal/sessionbroker/console_session_gate_test.go index a14c16e35d..71feffe8ce 100644 --- a/agent/internal/sessionbroker/console_session_gate_test.go +++ b/agent/internal/sessionbroker/console_session_gate_test.go @@ -8,128 +8,26 @@ import ( "github.com/breeze-rmm/agent/internal/ipc" ) -// TestRoleIdentityRejectionConsoleSessionBinding verifies the positive -// console-session assertion added for the assist/user IPC roles (#1009): on a -// multi-user Windows host, a non-SYSTEM peer in a non-console session that -// claims assist/user must be rejected, while the same role from the active -// console session is accepted. The SYSTEM/watchdog paths must be unchanged. -// -// The comparison is pure (peer WinSessionID vs active console session id), so it -// is fully table-testable on darwin without any Windows APIs. -func TestRoleIdentityRejectionConsoleSessionBinding(t *testing.T) { +func TestRoleIdentityRejection(t *testing.T) { const nonSystemSID = "S-1-5-21-1-2-3-1001" - cases := []struct { - name string - role string - sid string - peerWinSession string - consoleSession string - wantReason string - wantReject bool + tests := []struct { + name, role, sid, peer, claimed, console, wantReason string + wantReject bool }{ - { - name: "assist from console session accepted", - role: ipc.HelperRoleAssist, - sid: nonSystemSID, - peerWinSession: "1", - consoleSession: "1", - wantReject: false, - }, - { - name: "assist from non-console session rejected", - role: ipc.HelperRoleAssist, - sid: nonSystemSID, - peerWinSession: "3", - consoleSession: "1", - wantReason: "assist role requires the active console session", - wantReject: true, - }, - { - name: "user from console session accepted", - role: ipc.HelperRoleUser, - sid: nonSystemSID, - peerWinSession: "2", - consoleSession: "2", - wantReject: false, - }, - { - name: "user from non-console session rejected", - role: ipc.HelperRoleUser, - sid: nonSystemSID, - peerWinSession: "2", - consoleSession: "1", - wantReason: "user role requires the active console session", - wantReject: true, - }, - { - // SYSTEM is gated by SID only and must be unaffected by the - // console-session binding regardless of which session it runs in. - name: "system role unaffected by console session", - role: ipc.HelperRoleSystem, - sid: systemSID, - peerWinSession: "0", - consoleSession: "1", - wantReject: false, - }, - { - name: "watchdog role unaffected by console session", - role: ipc.HelperRoleWatchdog, - sid: systemSID, - peerWinSession: "0", - consoleSession: "1", - wantReject: false, - }, - { - // Existing SID gate still fires before the console check. - name: "assist as SYSTEM still rejected on SID", - role: ipc.HelperRoleAssist, - sid: systemSID, - peerWinSession: "1", - consoleSession: "1", - wantReason: "assist role requires non-SYSTEM identity", - wantReject: true, - }, - { - // Defensive: an unknown/empty console session id must not silently - // admit assist/user from an arbitrary session. - name: "assist rejected when console session unknown", - role: ipc.HelperRoleAssist, - sid: nonSystemSID, - peerWinSession: "1", - consoleSession: "", - wantReason: "assist role requires the active console session", - wantReject: true, - }, - { - // Fail-closed: GetConsoleSessionID() returns "0" when - // WTSGetActiveConsoleSessionId fails (API returns 0xFFFFFFFF). A peer - // whose kernel session is also "0" (Session-0 services) must NOT be - // admitted just because peer == console == "0" — "0" is not a valid - // interactive console session (#1009 review fail-closed hole). - name: "assist rejected when console lookup failed (session 0 sentinel)", - role: ipc.HelperRoleAssist, - sid: nonSystemSID, - peerWinSession: "0", - consoleSession: "0", - wantReason: "assist role requires the active console session", - wantReject: true, - }, - { - name: "user rejected when console lookup failed (session 0 sentinel)", - role: ipc.HelperRoleUser, - sid: nonSystemSID, - peerWinSession: "0", - consoleSession: "0", - wantReason: "user role requires the active console session", - wantReject: true, - }, + {"RDP user matching kernel session", ipc.HelperRoleUser, nonSystemSID, "7", "7", "1", "", false}, + {"RDP user session mismatch", ipc.HelperRoleUser, nonSystemSID, "7", "8", "1", "user role session claim does not match peer token", true}, + {"RDP user unknown peer session", ipc.HelperRoleUser, nonSystemSID, "0", "7", "1", "user role requires an interactive peer session", true}, + {"SYSTEM cannot claim user", ipc.HelperRoleUser, systemSID, "7", "7", "1", "user role requires non-SYSTEM identity", true}, + {"non-SYSTEM cannot claim system", ipc.HelperRoleSystem, nonSystemSID, "7", "7", "1", "system role requires SYSTEM identity", true}, + {"SYSTEM matching RDP session", ipc.HelperRoleSystem, systemSID, "7", "7", "1", "", false}, + {"assist remains console bound", ipc.HelperRoleAssist, nonSystemSID, "7", "7", "1", "assist role requires the active console session", true}, } - for _, tc := range cases { + for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { reason, rejected := roleIdentityRejection( - tc.role, tc.sid, 0, tc.peerWinSession, tc.consoleSession, "windows", + tc.role, tc.sid, 0, tc.peer, tc.claimed, tc.console, "windows", ) if rejected != tc.wantReject { t.Fatalf("rejected = %v, want %v (reason %q)", rejected, tc.wantReject, reason) @@ -148,13 +46,39 @@ func TestRoleIdentityRejectionConsoleSessionBinding(t *testing.T) { // must NOT cause a rejection on goos != windows. func TestRoleIdentityRejectionUnixUnchangedByConsoleBinding(t *testing.T) { reason, rejected := roleIdentityRejection( - ipc.HelperRoleUser, "", 1000, "99", "1", "darwin", + ipc.HelperRoleUser, "", 1000, "99", "7", "1", "darwin", ) if rejected { t.Fatalf("unix user role unexpectedly rejected: reason=%q", reason) } } +func TestPreferredRunAsUserSessionIgnoresRDPHelper(t *testing.T) { + now := time.Now() + + consoleUser, consoleClient := newTestUserSession(t, "console-user", "alice", now.Add(-20*time.Minute)) + defer consoleClient.Close() + consoleUser.WinSessionID = "1" + + rdpUser, rdpClient := newTestUserSession(t, "rdp-user", "alice", now.Add(-time.Minute)) + defer rdpClient.Close() + rdpUser.WinSessionID = "7" + + b := &Broker{ + sessions: map[string]*Session{ + consoleUser.SessionID: consoleUser, + rdpUser.SessionID: rdpUser, + }, + byIdentity: make(map[string][]*Session), + staleHelpers: make(map[string][]int), + consoleSessionIDFn: func() string { return "1" }, + } + + if got := b.preferredRunAsUserSessionForOS("windows"); got != consoleUser { + t.Fatalf("preferredRunAsUserSessionForOS(windows) = %v, want physical-console helper %q", got, consoleUser.SessionID) + } +} + // TestPreferredRunAsUserSessionFiltersByConsoleSession verifies that // run_as_user routing only ever selects a helper in the active console session, // never a co-logged-in user's helper in another session (#1009). A newer helper From d413bded2ea2109e80413b0c98226c56eaf9338c Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 15:31:06 -0600 Subject: [PATCH 07/47] fix(agent): own helper processes by session and role --- agent/internal/heartbeat/heartbeat.go | 74 +++- agent/internal/heartbeat/heartbeat_test.go | 70 +++ agent/internal/sessionbroker/broker.go | 241 ++++++++++- .../sessionbroker/broker_admission.go | 10 + .../sessionbroker/broker_lifecycle_test.go | 231 ++++++++++ agent/internal/sessionbroker/broker_unix.go | 2 + .../internal/sessionbroker/broker_windows.go | 2 + agent/internal/sessionbroker/lifecycle.go | 391 ++--------------- .../internal/sessionbroker/lifecycle_core.go | 404 ++++++++++++++++++ .../sessionbroker/lifecycle_registry.go | 252 +++++++++++ .../sessionbroker/lifecycle_registry_test.go | 275 ++++++++++++ .../internal/sessionbroker/lifecycle_stub.go | 20 +- .../internal/sessionbroker/lifecycle_test.go | 381 +++-------------- .../sessionbroker/peer_process_other.go | 5 + .../sessionbroker/peer_process_windows.go | 71 +++ .../peer_process_windows_test.go | 32 ++ agent/internal/sessionbroker/session.go | 22 + agent/internal/sessionbroker/spawner_stub.go | 7 +- .../internal/sessionbroker/spawner_windows.go | 64 ++- 19 files changed, 1832 insertions(+), 722 deletions(-) create mode 100644 agent/internal/heartbeat/heartbeat_test.go create mode 100644 agent/internal/sessionbroker/broker_lifecycle_test.go create mode 100644 agent/internal/sessionbroker/lifecycle_core.go create mode 100644 agent/internal/sessionbroker/lifecycle_registry.go create mode 100644 agent/internal/sessionbroker/lifecycle_registry_test.go create mode 100644 agent/internal/sessionbroker/peer_process_other.go create mode 100644 agent/internal/sessionbroker/peer_process_windows.go create mode 100644 agent/internal/sessionbroker/peer_process_windows_test.go diff --git a/agent/internal/heartbeat/heartbeat.go b/agent/internal/heartbeat/heartbeat.go index dce85c608d..43d527d717 100644 --- a/agent/internal/heartbeat/heartbeat.go +++ b/agent/internal/heartbeat/heartbeat.go @@ -189,12 +189,20 @@ type Heartbeat struct { helperToken string // retained copy of the helper-scoped token for connect-time pushes helperTokenMu sync.RWMutex sessionBroker *sessionbroker.Broker + helperLifecycle *sessionbroker.HelperLifecycleManager + lifecycleCancel context.CancelFunc isService bool isHeadless bool scmSessionCh chan sessionbroker.SCMSessionEvent // fed by SCM handler helperFinder func(targetSession string) *sessionbroker.Session spawnHelper func(targetSession string) error killStaleHelpers func(staleKey string) + + // Shutdown seams keep lifecycle ordering directly testable without opening + // sockets or spawning Windows processes. Production leaves these nil. + stopBrokerAcceptingAndWait func(context.Context) error + stopHelperLifecycleAndWait func(context.Context) error + closeSessionBroker func() // pamFindSession / pamRequestDialog default to the real broker methods in // RunPamFlow when nil; overridden in pam_flow_test.go. pamFindSession func(capability, targetWinSession string) *sessionbroker.Session @@ -853,8 +861,11 @@ func (h *Heartbeat) Start() { // and early-boot edge cases. if h.scmSessionCh != nil && h.sessionBroker != nil { ctx, cancel := context.WithCancel(context.Background()) - go func() { <-h.stopChan; cancel() }() lm := sessionbroker.NewHelperLifecycleManager(h.sessionBroker, h.scmSessionCh) + h.mu.Lock() + h.helperLifecycle = lm + h.lifecycleCancel = cancel + h.mu.Unlock() go lm.Start(ctx) } @@ -1091,13 +1102,65 @@ func (h *Heartbeat) DrainAndWait(ctx context.Context) { func (h *Heartbeat) Stop() { h.stopOnce.Do(func() { - if h.rebootMgr != nil { - h.rebootMgr.Stop() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if h.stopBrokerAcceptingAndWait != nil { + if err := h.stopBrokerAcceptingAndWait(ctx); err != nil { + log.Warn("session broker pre-auth drain timed out", "error", err.Error()) + } + } else if h.sessionBroker != nil { + if err := h.sessionBroker.StopAcceptingAndWait(ctx); err != nil { + log.Warn("session broker pre-auth drain timed out", "error", err.Error()) + } } - // Stop backup helper if running + + if h.stopHelperLifecycleAndWait != nil { + if err := h.stopHelperLifecycleAndWait(ctx); err != nil { + log.Warn("helper lifecycle shutdown timed out", "error", err.Error()) + } + } else { + h.mu.Lock() + lifecycle := h.helperLifecycle + lifecycleCancel := h.lifecycleCancel + h.mu.Unlock() + if lifecycleCancel != nil { + lifecycleCancel() + } + if lifecycle != nil { + lifecycleStopped := make(chan struct{}) + go func() { + lifecycle.Stop() + close(lifecycleStopped) + }() + select { + case <-lifecycleStopped: + case <-ctx.Done(): + log.Warn("helper lifecycle process cleanup timed out") + } + select { + case <-lifecycle.Done(): + case <-ctx.Done(): + log.Warn("helper lifecycle reconcile loop did not stop before deadline") + } + } + } + if h.sessionBroker != nil { h.sessionBroker.StopBackupHelper() } + if h.closeSessionBroker != nil { + h.closeSessionBroker() + } else if h.sessionBroker != nil { + h.sessionBroker.Close() + } + + if h.stopChan != nil { + close(h.stopChan) + } + if h.rebootMgr != nil { + h.rebootMgr.Stop() + } if h.monitor != nil { h.monitor.Stop() } @@ -1111,9 +1174,6 @@ func (h *Heartbeat) Stop() { if h.tunnelMgr != nil { h.tunnelMgr.Stop() } - // Close stopChan first — this signals broker.Listen() to call broker.Close() - // internally. The broker's Close() is idempotent via its closed flag. - close(h.stopChan) }) } diff --git a/agent/internal/heartbeat/heartbeat_test.go b/agent/internal/heartbeat/heartbeat_test.go new file mode 100644 index 0000000000..261b370508 --- /dev/null +++ b/agent/internal/heartbeat/heartbeat_test.go @@ -0,0 +1,70 @@ +package heartbeat + +import ( + "context" + "reflect" + "sync" + "testing" + "time" +) + +func TestHeartbeatStopOrdersBrokerBeforeLifecycleAndWaitsForReap(t *testing.T) { + var mu sync.Mutex + var order []string + appendOrder := func(step string) { + mu.Lock() + order = append(order, step) + mu.Unlock() + } + lifecycleEntered := make(chan struct{}) + releaseReap := make(chan struct{}) + h := &Heartbeat{ + stopChan: make(chan struct{}), + stopBrokerAcceptingAndWait: func(context.Context) error { + appendOrder("broker-stop-accepting") + return nil + }, + stopHelperLifecycleAndWait: func(context.Context) error { + appendOrder("lifecycle-stop") + close(lifecycleEntered) + <-releaseReap + appendOrder("lifecycle-reaped") + return nil + }, + closeSessionBroker: func() { + appendOrder("broker-close") + }, + } + + stopped := make(chan struct{}) + go func() { + h.Stop() + close(stopped) + }() + <-lifecycleEntered + select { + case <-stopped: + t.Fatal("Heartbeat.Stop returned before lifecycle reap completed") + default: + } + mu.Lock() + beforeRelease := append([]string(nil), order...) + mu.Unlock() + if !reflect.DeepEqual(beforeRelease, []string{"broker-stop-accepting", "lifecycle-stop"}) { + t.Fatalf("shutdown order before reap release = %v", beforeRelease) + } + + close(releaseReap) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("Heartbeat.Stop did not finish after lifecycle reaped") + } + mu.Lock() + got := append([]string(nil), order...) + mu.Unlock() + want := []string{"broker-stop-accepting", "lifecycle-stop", "lifecycle-reaped", "broker-close"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("shutdown order = %v, want %v", got, want) + } +} diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index 3c3b99c684..0c0e741fff 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -1,6 +1,7 @@ package sessionbroker import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -249,6 +250,7 @@ type Broker struct { listener net.Listener rateLimiter *ipc.RateLimiter startTime time.Time // broker creation time, used for watchdog uptime + listenerMu sync.Mutex mu timedRWMutex sessions map[string]*Session // sessionID -> Session @@ -263,10 +265,17 @@ type Broker struct { identityReservations map[string]int helperReservedVictims map[*Session]uint64 nextHelperReservationID uint64 + lifecycleObservers map[uint64]sessionLifecycleObserver + nextLifecycleObserverID uint64 consoleUser string // macOS: current console user ("loginwindow" at login screen) backup *backupHelper // backup helper process and session closed bool + acceptMu sync.Mutex + acceptStopped bool + preAuthConns map[net.Conn]bool // true once verified auth is being published + preAuthHandlers sync.WaitGroup + // snap is an atomically updated snapshot of sessions/byIdentity/consoleUser. // Updated under b.mu.Lock() on every mutation. Read-only hot paths use // snap.Load() instead of acquiring b.mu.RLock(), eliminating reader starvation @@ -314,6 +323,8 @@ func New(socketPath string, onMessage MessageHandler) *Broker { helperAuthReservations: make(map[AuthenticatedHelperKey]uint64), identityReservations: make(map[string]int), helperReservedVictims: make(map[*Session]uint64), + lifecycleObservers: make(map[uint64]sessionLifecycleObserver), + preAuthConns: make(map[net.Conn]bool), onMessage: onMessage, consoleSessionIDFn: GetConsoleSessionID, goos: runtime.GOOS, @@ -323,6 +334,47 @@ func New(socketPath string, onMessage MessageHandler) *Broker { return b } +type sessionLifecycleObserver struct { + authenticated func(*Session) + closed func(*Session) +} + +func (b *Broker) AddSessionLifecycleObserver(authenticated, closed func(*Session)) (remove func()) { + b.mu.Lock() + b.nextLifecycleObserverID++ + id := b.nextLifecycleObserverID + b.lifecycleObservers[id] = sessionLifecycleObserver{authenticated: authenticated, closed: closed} + b.mu.Unlock() + var once sync.Once + return func() { + once.Do(func() { + b.mu.Lock() + delete(b.lifecycleObservers, id) + b.mu.Unlock() + }) + } +} + +func (b *Broker) lifecycleAuthenticatedCallbacksLocked() []func(*Session) { + callbacks := make([]func(*Session), 0, len(b.lifecycleObservers)) + for _, observer := range b.lifecycleObservers { + if observer.authenticated != nil { + callbacks = append(callbacks, observer.authenticated) + } + } + return callbacks +} + +func (b *Broker) lifecycleClosedCallbacksLocked() []func(*Session) { + callbacks := make([]func(*Session), 0, len(b.lifecycleObservers)) + for _, observer := range b.lifecycleObservers { + if observer.closed != nil { + callbacks = append(callbacks, observer.closed) + } + } + return callbacks +} + // snapshotSessions returns the sessions map and consoleUser via the atomic // snapshot if available, falling back to a locked *copy* for Broker instances // that were not created via New() (e.g., test fixtures that construct Broker{} directly). @@ -387,10 +439,14 @@ func (b *Broker) SetSessionAuthenticatedHandler(handler SessionAuthenticatedHand func (b *Broker) fireSessionAuthenticated(session *Session) { b.mu.RLock() handler := b.onSessionAuthed + callbacks := b.lifecycleAuthenticatedCallbacksLocked() b.mu.RUnlock() if handler != nil { handler(session) } + for _, callback := range callbacks { + callback(session) + } } // SetConsoleUser updates the current macOS console user. When set to @@ -418,20 +474,18 @@ func (b *Broker) Listen(stopChan <-chan struct{}) error { go b.idleReaper(stopChan) // Accept loop + listener := b.currentListener() go func() { for { - conn, err := b.listener.Accept() + conn, err := listener.Accept() if err != nil { - b.mu.RLock() - closed := b.closed - b.mu.RUnlock() - if closed { + if b.acceptingStopped() { return } log.Warn("accept error", "error", err.Error()) continue } - go b.handleConnection(conn) + b.startAcceptedConnection(conn) } }() @@ -442,6 +496,10 @@ func (b *Broker) Listen(stopChan <-chan struct{}) error { // Close shuts down the broker and all sessions. func (b *Broker) Close() { + ctx, cancel := context.WithTimeout(context.Background(), HandshakeTimeout) + _ = b.StopAcceptingAndWait(ctx) + cancel() + b.mu.Lock() if b.closed { b.mu.Unlock() @@ -458,10 +516,6 @@ func (b *Broker) Close() { s.Close() } - if b.listener != nil { - b.listener.Close() - } - // Clean up socket file on Unix if runtime.GOOS != "windows" { os.Remove(b.socketPath) @@ -470,6 +524,96 @@ func (b *Broker) Close() { log.Info("session broker closed") } +func (b *Broker) LifecycleHelperKeys() []HelperKey { + b.mu.RLock() + defer b.mu.RUnlock() + keys := make([]HelperKey, 0, len(b.helperByKey)) + for key := range b.helperByKey { + keys = append(keys, key) + } + return keys +} + +func (b *Broker) currentListener() net.Listener { + b.listenerMu.Lock() + defer b.listenerMu.Unlock() + return b.listener +} + +func (b *Broker) acceptingStopped() bool { + b.acceptMu.Lock() + defer b.acceptMu.Unlock() + return b.acceptStopped +} + +func (b *Broker) startAcceptedConnection(conn net.Conn) bool { + b.acceptMu.Lock() + if b.acceptStopped { + b.acceptMu.Unlock() + _ = conn.Close() + return false + } + b.preAuthConns[conn] = false + b.preAuthHandlers.Add(1) + b.acceptMu.Unlock() + go func() { + defer b.finishPreAuth(conn) + b.handleConnection(conn) + }() + return true +} + +func (b *Broker) beginConnectionPublication(conn net.Conn) bool { + b.acceptMu.Lock() + defer b.acceptMu.Unlock() + if b.acceptStopped { + return false + } + if _, tracked := b.preAuthConns[conn]; !tracked { + return false + } + b.preAuthConns[conn] = true + return true +} + +func (b *Broker) finishPreAuth(conn net.Conn) { + b.acceptMu.Lock() + if _, tracked := b.preAuthConns[conn]; tracked { + delete(b.preAuthConns, conn) + b.preAuthHandlers.Done() + } + b.acceptMu.Unlock() +} + +func (b *Broker) StopAcceptingAndWait(ctx context.Context) error { + b.acceptMu.Lock() + b.acceptStopped = true + connections := make([]net.Conn, 0, len(b.preAuthConns)) + for conn, publishing := range b.preAuthConns { + if !publishing { + connections = append(connections, conn) + } + } + b.acceptMu.Unlock() + if listener := b.currentListener(); listener != nil { + _ = listener.Close() + } + for _, conn := range connections { + _ = conn.Close() + } + done := make(chan struct{}) + go func() { + b.preAuthHandlers.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // SessionForUser returns the first active session for the given username. func (b *Broker) SessionForUser(username string) *Session { b.mu.RLock() @@ -1267,6 +1411,7 @@ func (b *Broker) admitOrEvict(identityKey string) bool { b.publishSnapshotLocked() } onClosed := b.onSessionClosed + callbacks := b.lifecycleClosedCallbacksLocked() b.mu.Unlock() if victim != nil { @@ -1279,6 +1424,9 @@ func (b *Broker) admitOrEvict(identityKey string) bool { if onClosed != nil { onClosed(victim) } + for _, callback := range callbacks { + callback(victim) + } } return admitted } @@ -1568,6 +1716,24 @@ func (b *Broker) handleConnection(rawConn net.Conn) { } scopes := b.grantScopes(helperRole, authReq, runtime.GOOS, creds.BinaryPath) + ownedProcess, err := openOwnedPeerProcess(uint32(creds.PID)) + if err != nil { + log.Warn("failed to retain authenticated peer process handle", "pid", creds.PID, "error", err.Error()) + _ = conn.SendTyped(env.ID, ipc.TypeAuthResponse, ipc.AuthResponse{ + Accepted: false, + Reason: "peer process handle unavailable", + Permanent: true, + }) + conn.Close() + return + } + peerProcessRef := newOwnedPeerProcessRef(ownedProcess) + peerProcessPublished := false + defer func() { + if !peerProcessPublished { + _ = peerProcessRef.close() + } + }() var helperReservation *helperAuthReservation if isWindowsLifecycleRole(b.goos, helperRole) { @@ -1606,6 +1772,10 @@ func (b *Broker) handleConnection(rawConn net.Conn) { conn.Close() return } + if !b.beginConnectionPublication(rawConn) { + conn.Close() + return + } // Set session key for HMAC validation conn.SetSessionKey(sessionKey) @@ -1622,6 +1792,7 @@ func (b *Broker) handleConnection(rawConn net.Conn) { session.BinaryKind = ipc.HelperBinaryUserHelper } session.DesktopContext = authReq.DesktopContext + session.peerProcess = peerProcessRef // Use the kernel-verified Windows session ID (computed above from the peer // PID) instead of trusting the self-reported value, preventing @@ -1662,6 +1833,8 @@ func (b *Broker) handleConnection(rawConn net.Conn) { return } } + peerProcessPublished = true + b.finishPreAuth(rawConn) log.Info("user helper connected", "identity", identityKey, @@ -1743,16 +1916,60 @@ func (b *Broker) removeSessionMapsLocked(session *Session) bool { } func (b *Broker) removeSession(session *Session) { + _ = b.closeSession(session) +} + +func (b *Broker) closeSession(session *Session) error { + b.mu.Lock() + removed := b.removeSessionMapsLocked(session) + if removed { + b.publishSnapshotLocked() + } + onSessionClosed := b.onSessionClosed + callbacks := b.lifecycleClosedCallbacksLocked() + b.mu.Unlock() + + closeErr := session.closeTransportAndPeer() + if removed { + if onSessionClosed != nil { + onSessionClosed(session) + } + for _, callback := range callbacks { + callback(session) + } + } + return closeErr +} + +func (b *Broker) TerminateHelperKey(key HelperKey) { b.mu.Lock() + session := b.helperByKey[key] + if session == nil { + b.mu.Unlock() + return + } + claim := session.peerProcess.claimTermination() removed := b.removeSessionMapsLocked(session) if removed { b.publishSnapshotLocked() } onSessionClosed := b.onSessionClosed + callbacks := b.lifecycleClosedCallbacksLocked() b.mu.Unlock() - if removed && onSessionClosed != nil { - onSessionClosed(session) + if claim != nil { + if err := claim.terminateAndClose(); err != nil { + log.Debug("failed to terminate helper process", "helperKey", key.String(), "pid", session.PID, "error", err.Error()) + } + } + _ = session.closeTransportAndPeer() + if removed { + if onSessionClosed != nil { + onSessionClosed(session) + } + for _, callback := range callbacks { + callback(session) + } } } diff --git a/agent/internal/sessionbroker/broker_admission.go b/agent/internal/sessionbroker/broker_admission.go index 86560de99d..2e0a5ec8f7 100644 --- a/agent/internal/sessionbroker/broker_admission.go +++ b/agent/internal/sessionbroker/broker_admission.go @@ -230,6 +230,7 @@ func (b *Broker) registerNonLifecycleSession(identityKey, helperRole string, ses b.mu.Unlock() return errMaxConnectionsPerIdentity } + session.broker = b b.sessions[session.SessionID] = session b.byIdentity[identityKey] = append(b.byIdentity[identityKey], session) if helperRole == backupipc.HelperRoleBackup { @@ -240,6 +241,7 @@ func (b *Broker) registerNonLifecycleSession(identityKey, helperRole string, ses } b.publishSnapshotLocked() onClosed := b.onSessionClosed + callbacks := b.lifecycleClosedCallbacksLocked() b.mu.Unlock() if victim != nil { @@ -252,6 +254,9 @@ func (b *Broker) registerNonLifecycleSession(identityKey, helperRole string, ses if onClosed != nil { onClosed(victim) } + for _, callback := range callbacks { + callback(victim) + } } return nil } @@ -337,12 +342,14 @@ func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session b.removeSessionMapsLocked(victim) } b.releaseWindowsHelperLocked(reservation) + session.broker = b b.sessions[session.SessionID] = session b.byIdentity[reservation.identityKey] = append(b.byIdentity[reservation.identityKey], session) b.helperByAuthKey[reservation.authKey] = session b.helperByKey[reservation.authKey.HelperKey] = session b.publishSnapshotLocked() onClosed := b.onSessionClosed + callbacks := b.lifecycleClosedCallbacksLocked() b.mu.Unlock() if victim != nil { @@ -355,6 +362,9 @@ func (b *Broker) commitWindowsHelper(reservation *helperAuthReservation, session if onClosed != nil { onClosed(victim) } + for _, callback := range callbacks { + callback(victim) + } } return nil } diff --git a/agent/internal/sessionbroker/broker_lifecycle_test.go b/agent/internal/sessionbroker/broker_lifecycle_test.go new file mode 100644 index 0000000000..339d598250 --- /dev/null +++ b/agent/internal/sessionbroker/broker_lifecycle_test.go @@ -0,0 +1,231 @@ +package sessionbroker + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/breeze-rmm/agent/internal/ipc" +) + +type fakeOwnedPeerProcess struct { + pid uint32 + + mu sync.Mutex + alive bool + terminated int + closed int + claimed chan struct{} + release chan struct{} +} + +func newFakeOwnedPeerProcess(pid uint32) *fakeOwnedPeerProcess { + return &fakeOwnedPeerProcess{pid: pid, alive: true} +} + +func (p *fakeOwnedPeerProcess) ProcessID() uint32 { return p.pid } +func (p *fakeOwnedPeerProcess) Alive() (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.alive, nil +} +func (p *fakeOwnedPeerProcess) Terminate() error { + if p.claimed != nil { + close(p.claimed) + <-p.release + } + p.mu.Lock() + p.terminated++ + p.alive = false + p.mu.Unlock() + return nil +} +func (p *fakeOwnedPeerProcess) Close() error { + p.mu.Lock() + p.closed++ + p.mu.Unlock() + return nil +} +func (p *fakeOwnedPeerProcess) counts() (terminated, closed int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.terminated, p.closed +} + +func newOwnedSession(t *testing.T, broker *Broker, key HelperKey, process ownedPeerProcess) *Session { + t.Helper() + server, client := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + session := NewSession(ipc.NewConn(server), 0, "sid", "user", "", "session", nil) + session.PID = int(process.ProcessID()) + session.WinSessionID = "7" + session.HelperRole = key.Role + session.peerProcess = newOwnedPeerProcessRef(process) + session.broker = broker + broker.mu.Lock() + broker.sessions[session.SessionID] = session + broker.byIdentity[session.IdentityKey] = []*Session{session} + broker.helperByKey[key] = session + broker.publishSnapshotLocked() + broker.mu.Unlock() + return session +} + +func TestLifecycleObserversAreAdditiveAndRunAfterUnlock(t *testing.T) { + b := New("observer-"+t.Name(), nil) + defer b.Close() + session := &Session{SessionID: "s"} + primary := make(chan struct{}, 1) + observer := make(chan struct{}, 1) + b.SetSessionAuthenticatedHandler(func(*Session) { primary <- struct{}{} }) + remove := b.AddSessionLifecycleObserver(func(*Session) { + _ = b.SessionCount() + observer <- struct{}{} + }, nil) + defer remove() + + b.fireSessionAuthenticated(session) + select { + case <-primary: + case <-time.After(time.Second): + t.Fatal("primary authentication handler did not run") + } + select { + case <-observer: + case <-time.After(time.Second): + t.Fatal("lifecycle observer did not run") + } +} + +func TestSessionCloseReleasesOwnedPeerProcessOnce(t *testing.T) { + b := New("peer-close-"+t.Name(), nil) + proc := newFakeOwnedPeerProcess(5100) + session := newOwnedSession(t, b, HelperKey{WindowsSessionID: 7, Role: "system"}, proc) + + if err := session.Close(); err != nil { + t.Fatal(err) + } + if err := session.Close(); err != nil { + t.Fatal(err) + } + terminated, closed := proc.counts() + if terminated != 0 || closed != 1 { + t.Fatalf("terminate=%d close=%d, want 0 and 1", terminated, closed) + } +} + +func TestUnexpectedDisconnectReleasesOwnedPeerProcess(t *testing.T) { + b := New("peer-disconnect-"+t.Name(), nil) + proc := newFakeOwnedPeerProcess(5200) + session := newOwnedSession(t, b, HelperKey{WindowsSessionID: 7, Role: "user"}, proc) + + b.removeSession(session) + terminated, closed := proc.counts() + if terminated != 0 || closed != 1 { + t.Fatalf("terminate=%d close=%d, want 0 and 1", terminated, closed) + } +} + +func TestConcurrentTerminateAndSessionCloseTerminatesAndConsumesPeerHandleOnce(t *testing.T) { + b := New("peer-race-"+t.Name(), nil) + proc := newFakeOwnedPeerProcess(5300) + proc.claimed = make(chan struct{}) + proc.release = make(chan struct{}) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + session := newOwnedSession(t, b, key, proc) + + terminated := make(chan struct{}) + go func() { + b.TerminateHelperKey(key) + close(terminated) + }() + <-proc.claimed + closed := make(chan struct{}) + go func() { + _ = session.Close() + close(closed) + }() + close(proc.release) + <-terminated + <-closed + + terminateCount, closeCount := proc.counts() + if terminateCount != 1 || closeCount != 1 { + t.Fatalf("terminate=%d close=%d, want 1 each", terminateCount, closeCount) + } +} + +func TestBrokerStopAcceptingAndWaitUnblocksStalledPreAuthConnection(t *testing.T) { + b := New("preauth-"+t.Name(), nil) + server, client := net.Pipe() + defer client.Close() + if !b.startAcceptedConnection(server) { + t.Fatal("failed to register accepted connection") + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := b.StopAcceptingAndWait(ctx); err != nil { + t.Fatalf("StopAcceptingAndWait: %v", err) + } +} + +func TestLifecycleStopTerminatesScheduledLogicalHelperWithoutTrackedSpawn(t *testing.T) { + b := New("scheduled-stop-"+t.Name(), nil) + proc := newFakeOwnedPeerProcess(5400) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + newOwnedSession(t, b, key, proc) + m := newHelperLifecycleManager(b, fakeLifecycleDetector{}, nil, &fakeHelperSpawner{}) + m.gracePeriod = 0 + m.finalWait = 0 + + m.Stop() + + terminateCount, closeCount := proc.counts() + if terminateCount != 1 || closeCount != 1 { + t.Fatalf("scheduled helper terminate=%d close=%d, want 1 each", terminateCount, closeCount) + } +} + +func TestBrokerStopAcceptingRetainsConnectionDuringAuthenticatedPublication(t *testing.T) { + b := New("publishing-"+t.Name(), nil) + server, client := net.Pipe() + defer server.Close() + defer client.Close() + b.acceptMu.Lock() + b.preAuthConns[server] = false + b.preAuthHandlers.Add(1) + b.acceptMu.Unlock() + if !b.beginConnectionPublication(server) { + t.Fatal("failed to enter authenticated publication state") + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + stopped := make(chan error, 1) + go func() { stopped <- b.StopAcceptingAndWait(ctx) }() + select { + case err := <-stopped: + t.Fatalf("StopAcceptingAndWait returned before publication finished: %v", err) + case <-time.After(20 * time.Millisecond): + } + b.finishPreAuth(server) + if err := <-stopped; err != nil { + t.Fatal(err) + } + + writeDone := make(chan error, 1) + go func() { + _, err := server.Write([]byte{1}) + writeDone <- err + }() + buffer := make([]byte, 1) + if _, err := client.Read(buffer); err != nil { + t.Fatalf("publishing connection was closed: %v", err) + } + if err := <-writeDone; err != nil { + t.Fatalf("write after publication: %v", err) + } +} diff --git a/agent/internal/sessionbroker/broker_unix.go b/agent/internal/sessionbroker/broker_unix.go index 09f49e4287..c183ef5dfc 100644 --- a/agent/internal/sessionbroker/broker_unix.go +++ b/agent/internal/sessionbroker/broker_unix.go @@ -34,7 +34,9 @@ func (b *Broker) setupSocket() error { return fmt.Errorf("chmod %s: %w", b.socketPath, err) } + b.listenerMu.Lock() b.listener = listener + b.listenerMu.Unlock() return nil } diff --git a/agent/internal/sessionbroker/broker_windows.go b/agent/internal/sessionbroker/broker_windows.go index 94038b76fd..3ca33210ef 100644 --- a/agent/internal/sessionbroker/broker_windows.go +++ b/agent/internal/sessionbroker/broker_windows.go @@ -26,7 +26,9 @@ func (b *Broker) setupSocket() error { return fmt.Errorf("listen pipe %s: %w", b.socketPath, err) } + b.listenerMu.Lock() b.listener = listener + b.listenerMu.Unlock() log.Info("named pipe listener created", "pipe", b.socketPath) return nil } diff --git a/agent/internal/sessionbroker/lifecycle.go b/agent/internal/sessionbroker/lifecycle.go index 5c3759eebd..9a12f1e338 100644 --- a/agent/internal/sessionbroker/lifecycle.go +++ b/agent/internal/sessionbroker/lifecycle.go @@ -4,392 +4,85 @@ package sessionbroker import ( "context" - "fmt" - "strconv" - "sync" "time" ) -// HelperLifecycleManager proactively spawns user helpers into eligible Windows -// sessions so remote desktop is available instantly after reboot, without -// waiting for an on-demand desktop_start command. -// -// It receives session-change events pushed by the SCM (SERVICE_CONTROL_SESSIONCHANGE) -// for instant reaction, and runs a slow reconcile tick as a safety net for edge -// cases (missed events during early boot, helper crash detection). -type HelperLifecycleManager struct { - broker *Broker - detector SessionDetector - scmCh <-chan SCMSessionEvent - mu sync.Mutex - tracked map[string]*trackedSession // "winSessionID-role" -> state -} - -type trackedSession struct { - spawnedAt time.Time - retryCount int - lastFailure time.Time - - // fatalExitUntil suppresses respawns until this time when the helper - // exited with a fatal exit code (typically 2, meaning permanent auth - // rejection). Cleared on SCM session change events. - fatalExitUntil time.Time -} - -// SCMSessionEvent is a session-change notification forwarded from the Windows -// Service Control Manager (SERVICE_CONTROL_SESSIONCHANGE). -type SCMSessionEvent struct { - // EventType is the WTS event constant (e.g. WTS_SESSION_LOGON = 0x5). - EventType uint32 - // SessionID is the Windows session ID from WTSSESSION_NOTIFICATION. - SessionID uint32 -} - const ( - // initialDelay lets WTS settle after service start / reboot. - initialDelay = 3 * time.Second - - // reconcileInterval is the safety-net tick. Kept slow because the SCM - // hook handles the fast path; this only catches helper crashes and - // edge cases. + initialDelay = 3 * time.Second reconcileInterval = 30 * time.Second - // maxBackoff caps the exponential retry delay. - maxBackoff = 30 * time.Second - - // maxSpawnRetries stops retrying a session that is permanently broken. - // Tracking resets when the session disappears and reappears in WTS. - maxSpawnRetries = 10 - - // fatalCooldown is how long a session+role is blocked from respawn - // after a helper exits with a fatal exit code (2 — permanent auth - // rejection). Cleared early on SCM session change events. - fatalCooldown = 10 * time.Minute + wtsSessionDisconnect = 0x4 + wtsSessionLogon = 0x5 + wtsSessionLogoff = 0x6 + wtsSessionLock = 0x7 + wtsSessionUnlock = 0x8 + wtsSessionCreate = 0xa + wtsSessionTerminate = 0xb +) - // helperFatalExitCode is the exit code a helper uses to signal - // permanent rejection. Must match main.go's os.Exit(2). - helperFatalExitCode = 2 +type directHelperSpawner struct{} - // helperPanicExitCode is the exit code the helper uses from its - // top-level panic recovery. Must match main.go's os.Exit(3). Treated - // as transient — no fatal cooldown. - helperPanicExitCode = 3 +func (directHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { + if key.Role == "user" { + return SpawnUserHelperInSession(key.WindowsSessionID) + } + return SpawnHelperInSession(key.WindowsSessionID) +} - // WTS event type constants (matching windows.WTS_SESSION_*). - wtsSessionLogon = 0x5 - wtsSessionLogoff = 0x6 - wtsSessionLock = 0x7 - wtsSessionUnlock = 0x8 - wtsSessionCreate = 0xa - wtsSessionTerminate = 0xb -) +func (directHelperSpawner) Close() error { return nil } -// NewHelperLifecycleManager creates a lifecycle manager driven by SCM session -// events. Pass nil for scmCh to fall back to reconcile-only mode (useful for -// testing or non-service contexts). func NewHelperLifecycleManager(broker *Broker, scmCh <-chan SCMSessionEvent) *HelperLifecycleManager { - return &HelperLifecycleManager{ - broker: broker, - detector: NewSessionDetector(), - scmCh: scmCh, - tracked: make(map[string]*trackedSession), - } + return newHelperLifecycleManager(broker, NewSessionDetector(), scmCh, directHelperSpawner{}) } -// Start begins the reconciliation loop. It blocks until ctx is cancelled. func (m *HelperLifecycleManager) Start(ctx context.Context) { - // Let WTS settle post-reboot before first reconcile. + defer m.finishStart() select { case <-time.After(initialDelay): case <-ctx.Done(): return + case <-m.stopCh: + return } - - // Initial reconcile catches sessions that appeared before the SCM hook - // was registered (or during the initial delay). m.reconcile() - ticker := time.NewTicker(reconcileInterval) defer ticker.Stop() - - // If no SCM channel was provided, use a nil channel (never receives) - // and rely solely on the reconcile tick. - scmCh := m.scmCh - if scmCh == nil { - scmCh = make(<-chan SCMSessionEvent) - } - for { select { case <-ctx.Done(): return - case evt, ok := <-scmCh: - if !ok { + case <-m.stopCh: + return + case event, ok := <-m.scmCh: + if !ok && m.scmCh != nil { return } - m.handleSCMEvent(evt) + if ok { + m.handleSCMEvent(event) + } case <-ticker.C: m.reconcile() } } } -// reconcile ensures both SYSTEM and user helpers are running in every eligible session. -func (m *HelperLifecycleManager) reconcile() { - sessions, err := m.broker.refreshDesiredHelperKeys(m.detector) - if err != nil { - log.Warn("lifecycle: failed to list sessions", "error", err.Error()) +func (m *HelperLifecycleManager) handleSCMEvent(event SCMSessionEvent) { + if event.SessionID == 0 { return } - - currentKeys := make(map[string]bool, len(sessions)*2) - - for _, s := range sessions { - // Spawn SYSTEM helper if missing, reset retry tracking when connected. - if systemKey, desired := helperKeyFromDetected(s, "system"); desired { - trackKey := systemKey.String() - currentKeys[trackKey] = true - if m.broker.HasHelperForWinSessionRole(s.Session, systemKey.Role) { - m.resetTracked(trackKey) - } else { - m.spawnWithRetry(s.Session, systemKey.Role) - } - } - - // Spawn user-token helper if missing. Only for active sessions - // (user is actually logged in); "connected" means lock screen - // where WTSQueryUserToken may fail. - if userKey, desired := helperKeyFromDetected(s, "user"); desired { - trackKey := userKey.String() - currentKeys[trackKey] = true - if m.broker.HasHelperForWinSessionRole(s.Session, userKey.Role) { - m.resetTracked(trackKey) - } else { - m.spawnWithRetry(s.Session, userKey.Role) - } - } - } - - // Clean up tracking for sessions that no longer exist. - m.mu.Lock() - for key := range m.tracked { - if !currentKeys[key] { - delete(m.tracked, key) - } - } - m.mu.Unlock() -} - -// handleSCMEvent reacts to a SERVICE_CONTROL_SESSIONCHANGE notification -// pushed from the SCM handler in service_windows.go. -func (m *HelperLifecycleManager) handleSCMEvent(evt SCMSessionEvent) { - sessionID := fmt.Sprintf("%d", evt.SessionID) - - // Skip Session 0. - if evt.SessionID == 0 { - return - } - - switch evt.EventType { + systemKey := HelperKey{WindowsSessionID: event.SessionID, Role: "system"} + userKey := HelperKey{WindowsSessionID: event.SessionID, Role: "user"} + switch event.EventType { case wtsSessionLogon, wtsSessionUnlock, wtsSessionCreate: - // A session change means the environment changed — worth retrying - // a previously-fatal helper. Clear the fatal cooldown for both - // roles in this session. - m.mu.Lock() - if ts, ok := m.tracked[sessionID+"-system"]; ok { - ts.fatalExitUntil = time.Time{} - } - if ts, ok := m.tracked[sessionID+"-user"]; ok { - ts.fatalExitUntil = time.Time{} - } - m.mu.Unlock() - if _, err := m.broker.refreshDesiredHelperKeys(m.detector); err != nil { - log.Warn("lifecycle: failed to refresh desired helpers for session event", - "sessionId", sessionID, - "error", err.Error(), - ) - return - } - - systemKey := HelperKey{WindowsSessionID: evt.SessionID, Role: "system"} - if m.broker.helperKeyDesired(systemKey) && !m.broker.HasHelperForWinSessionRole(sessionID, systemKey.Role) { - m.spawnWithRetry(sessionID, "system") - } - userKey := HelperKey{WindowsSessionID: evt.SessionID, Role: "user"} - if m.broker.helperKeyDesired(userKey) && !m.broker.HasHelperForWinSessionRole(sessionID, userKey.Role) { - m.spawnWithRetry(sessionID, "user") - } + m.registry.clearFatal(event.SessionID) + m.reconcile() + case wtsSessionDisconnect: + m.removeDesired(userKey) + m.stopKey(userKey) + m.reconcile() case wtsSessionLogoff, wtsSessionTerminate: - m.mu.Lock() - delete(m.tracked, sessionID+"-system") - delete(m.tracked, sessionID+"-user") - m.mu.Unlock() - } -} - -// resetTracked clears retry state for a tracked session. Called when the -// helper is confirmed connected via IPC, proving the spawn succeeded. -func (m *HelperLifecycleManager) resetTracked(trackKey string) { - m.mu.Lock() - defer m.mu.Unlock() - if ts, ok := m.tracked[trackKey]; ok { - ts.retryCount = 0 - ts.lastFailure = time.Time{} - } -} - -// spawnWithRetry spawns a helper with the given role into the Windows session, -// respecting exponential backoff and a max retry cap on repeated failures. -func (m *HelperLifecycleManager) spawnWithRetry(winSessionID, role string) { - trackKey := winSessionID + "-" + role - - m.mu.Lock() - ts, exists := m.tracked[trackKey] - if !exists { - ts = &trackedSession{} - m.tracked[trackKey] = ts - } - - // Respect fatal-exit cooldown from a previous helper that exited with - // exit code 2 (permanent rejection). Cleared on SCM session changes. - if now := time.Now(); !ts.fatalExitUntil.IsZero() && now.Before(ts.fatalExitUntil) { - m.mu.Unlock() - return - } - - // Give up after too many failures. The tracking entry is cleaned up - // when the session disappears from WTS, so retries reset naturally - // if the user logs out and back in. - if ts.retryCount >= maxSpawnRetries { - m.mu.Unlock() - return - } - - // Check backoff: 2s, 4s, 8s, 16s, 30s cap. - if ts.retryCount > 0 { - backoff := time.Duration(1< maxBackoff { - backoff = maxBackoff - } - if time.Since(ts.lastFailure) < backoff { - m.mu.Unlock() - return - } - } - m.mu.Unlock() - - sessionNum, err := strconv.ParseUint(winSessionID, 10, 32) - if err != nil { - log.Warn("lifecycle: invalid session ID", "winSessionID", winSessionID, "error", err.Error()) - return - } - - // Kill stale helpers for this specific role before respawning. - m.broker.KillStaleHelpers(winSessionID + "-" + role) - - // Spawn the appropriate helper type. - var spawned *SpawnedHelper - switch role { - case "user": - spawned, err = SpawnUserHelperInSession(uint32(sessionNum)) - default: - spawned, err = SpawnHelperInSession(uint32(sessionNum)) - } - - // Count every spawn attempt toward the retry cap, not just errors. - // A "successful" CreateProcessAsUser that crashes before connecting to - // IPC is still a failure from the lifecycle perspective. The counter - // resets only when the helper actually connects (resetTracked in reconcile) - // or the session disappears from WTS. - m.mu.Lock() - ts.retryCount++ - ts.lastFailure = time.Now() - m.mu.Unlock() - - if err != nil { - if ts.retryCount >= maxSpawnRetries { - log.Error("lifecycle: giving up on session after max retries", - "winSessionID", winSessionID, - "role", role, - "retryCount", ts.retryCount, - ) - } else { - log.Warn("lifecycle: failed to spawn helper", - "winSessionID", winSessionID, - "role", role, - "retryCount", ts.retryCount, - "error", err.Error(), - ) - } - return - } - - m.mu.Lock() - ts.spawnedAt = time.Now() - m.mu.Unlock() - - log.Info("proactively spawned helper in session", "winSessionID", winSessionID, "role", role) - - // Start a goroutine to wait on the helper process. When it exits, apply - // the fatal-cooldown policy if appropriate. SpawnedHelper.Wait() closes - // the handle on return. - if spawned != nil { - go m.watchHelperExit(trackKey, winSessionID, role, spawned) + m.removeDesired(systemKey, userKey) + m.stopKey(userKey) + m.stopKey(systemKey) } } - -// watchHelperExit blocks on the helper process and, when it exits, sets the -// fatal cooldown on the tracked entry if the helper exited with the fatal -// exit code (2 — permanent rejection). -func (m *HelperLifecycleManager) watchHelperExit(trackKey, winSessionID, role string, spawned *SpawnedHelper) { - exitCode, err := spawned.Wait() - if err != nil { - log.Warn("lifecycle: wait on helper process failed", - "winSessionID", winSessionID, - "role", role, - "pid", spawned.PID, - "trackKey", trackKey, - "error", err.Error(), - ) - return - } - - if exitCode == helperFatalExitCode { - m.mu.Lock() - ts, ok := m.tracked[trackKey] - if ok { - ts.fatalExitUntil = time.Now().Add(fatalCooldown) - } - m.mu.Unlock() - log.Warn("lifecycle: helper exited with fatal code, skipping respawn", - "winSessionID", winSessionID, - "role", role, - "pid", spawned.PID, - "exitCode", exitCode, - "cooldown", fatalCooldown.String(), - ) - return - } - - if exitCode == helperPanicExitCode { - // Panic, not permanent rejection. No fatal cooldown — the helper - // caught the panic at the top level, logged the stack trace, and - // exited with code 3. Respawn normally so the next desk-start can - // retry; if the panic reproduces the on-disk log and the shipped - // error will tell us what's broken. - log.Warn("lifecycle: helper panicked (exit code 3), will respawn", - "winSessionID", winSessionID, - "role", role, - "pid", spawned.PID, - "exitCode", exitCode, - ) - return - } - - log.Debug("lifecycle: helper exited", - "winSessionID", winSessionID, - "role", role, - "pid", spawned.PID, - "exitCode", exitCode, - ) -} diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go new file mode 100644 index 0000000000..f7d33c2192 --- /dev/null +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -0,0 +1,404 @@ +package sessionbroker + +import ( + "fmt" + "strconv" + "sync" + "time" +) + +const ( + maxBackoff = 30 * time.Second + maxSpawnRetries = 10 + fatalCooldown = 10 * time.Minute + helperFatalExitCode = 2 + helperPanicExitCode = 3 +) + +type SCMSessionEvent struct { + EventType uint32 + SessionID uint32 +} + +type helperProcess interface { + ProcessID() uint32 + ExecutablePath() string + Alive() bool + Terminate() error + Wait() (int, error) + Close() error +} + +type helperSpawner interface { + Spawn(HelperKey) (helperProcess, error) + Close() error +} + +type ownedPeerProcess interface { + ProcessID() uint32 + Alive() (bool, error) + Terminate() error + Close() error +} + +type ownedPeerProcessState uint8 + +const ( + peerProcessActive ownedPeerProcessState = iota + peerProcessTerminationClaimed + peerProcessConsumed +) + +type ownedPeerProcessRef struct { + mu sync.Mutex + process ownedPeerProcess + state ownedPeerProcessState +} + +func newOwnedPeerProcessRef(process ownedPeerProcess) *ownedPeerProcessRef { + if process == nil { + return nil + } + return &ownedPeerProcessRef{process: process, state: peerProcessActive} +} + +type ownedPeerTerminationClaim struct { + ref *ownedPeerProcessRef + process ownedPeerProcess +} + +func (r *ownedPeerProcessRef) claimTermination() *ownedPeerTerminationClaim { + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + if r.state != peerProcessActive || r.process == nil { + return nil + } + r.state = peerProcessTerminationClaimed + return &ownedPeerTerminationClaim{ref: r, process: r.process} +} + +func (c *ownedPeerTerminationClaim) terminateAndClose() error { + if c == nil || c.process == nil { + return nil + } + terminateErr := c.process.Terminate() + closeErr := c.process.Close() + c.ref.mu.Lock() + c.ref.state = peerProcessConsumed + c.ref.process = nil + c.ref.mu.Unlock() + if terminateErr != nil { + return terminateErr + } + return closeErr +} + +func (r *ownedPeerProcessRef) close() error { + if r == nil { + return nil + } + r.mu.Lock() + if r.state != peerProcessActive || r.process == nil { + r.mu.Unlock() + return nil + } + process := r.process + r.process = nil + r.state = peerProcessConsumed + r.mu.Unlock() + return process.Close() +} + +type HelperLifecycleManager struct { + broker *Broker + detector SessionDetector + scmCh <-chan SCMSessionEvent + spawner helperSpawner + registry *helperRegistry + + mu sync.Mutex + desired map[HelperKey]bool + stopping bool + observerRemove func() + stopOnce sync.Once + stopCh chan struct{} + done chan struct{} + doneOnce sync.Once + gracePeriod time.Duration + finalWait time.Duration +} + +func newHelperLifecycleManager(broker *Broker, detector SessionDetector, scmCh <-chan SCMSessionEvent, spawner helperSpawner) *HelperLifecycleManager { + m := &HelperLifecycleManager{ + broker: broker, + detector: detector, + scmCh: scmCh, + spawner: spawner, + registry: newHelperRegistry(), + desired: make(map[HelperKey]bool), + done: make(chan struct{}), + stopCh: make(chan struct{}), + gracePeriod: 2 * time.Second, + finalWait: 2 * time.Second, + } + if broker != nil { + m.observerRemove = broker.AddSessionLifecycleObserver(m.sessionAuthenticated, m.sessionClosed) + } + return m +} + +func (m *HelperLifecycleManager) Done() <-chan struct{} { return m.done } + +func (m *HelperLifecycleManager) finishStart() { + m.doneOnce.Do(func() { close(m.done) }) +} + +func (m *HelperLifecycleManager) reconcile() { + if m.detector == nil || m.broker == nil || m.spawner == nil { + return + } + sessions, err := m.detector.ListSessions() + if err != nil { + log.Warn("lifecycle: failed to list sessions", "error", err.Error()) + return + } + desired := make(map[HelperKey]bool, len(sessions)*2) + for _, session := range sessions { + if key, ok := helperKeyFromDetected(session, "system"); ok { + desired[key] = true + } + if key, ok := helperKeyFromDetected(session, "user"); ok { + desired[key] = true + } + } + + m.mu.Lock() + if m.stopping { + m.mu.Unlock() + return + } + previousDesired := m.desired + m.desired = cloneDesired(desired) + m.publishDesired(desired) + m.mu.Unlock() + + for key := range previousDesired { + if !desired[key] { + m.stopKey(key) + } + } + for _, key := range m.registry.keys() { + if !desired[key] { + m.stopKey(key) + } + } + for key := range desired { + m.spawnKey(key) + } +} + +func cloneDesired(source map[HelperKey]bool) map[HelperKey]bool { + copyMap := make(map[HelperKey]bool, len(source)) + for key, desired := range source { + copyMap[key] = desired + } + return copyMap +} + +func (m *HelperLifecycleManager) publishDesired(desired map[HelperKey]bool) { + if m.broker == nil { + return + } + snapshot := make(map[HelperKey]struct{}, len(desired)) + for key, wanted := range desired { + if wanted { + snapshot[key] = struct{}{} + } + } + m.broker.UpdateDesiredHelperKeys(snapshot) +} + +func (m *HelperLifecycleManager) spawnKey(key HelperKey) { + m.mu.Lock() + if m.stopping || !m.desired[key] { + m.mu.Unlock() + return + } + generation, reserved := m.registry.reserve(key, time.Now()) + m.mu.Unlock() + if !reserved { + return + } + process, err := m.spawner.Spawn(key) + if err != nil { + m.registry.noteSpawnFailure(key, generation) + log.Warn("lifecycle: failed to spawn helper", "helperKey", key.String(), "error", err.Error()) + return + } + mode := key.Role + "-helper" + entry, attached := m.registry.attachReserved(key, generation, process, mode) + if !attached { + _ = process.Terminate() + go m.watchDetachedProcess(key, generation, process) + return + } + go m.watchProcess(entry) + log.Info("proactively spawned helper in session", "helperKey", key.String(), "pid", process.ProcessID()) +} + +func (m *HelperLifecycleManager) watchProcess(entry *trackedHelper) { + exitCode, err := entry.process.Wait() + if err != nil { + log.Warn("lifecycle: wait on helper process failed", "helperKey", entry.key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + } + _ = entry.process.Close() + m.registry.noteExit(entry.key, entry.generation, exitCode) +} + +func (m *HelperLifecycleManager) watchDetachedProcess(key HelperKey, generation uint64, process helperProcess) { + exitCode, _ := process.Wait() + _ = process.Close() + m.registry.noteExit(key, generation, exitCode) +} + +func (m *HelperLifecycleManager) stopSession(sessionID uint32) { + m.removeDesired(HelperKey{WindowsSessionID: sessionID, Role: "system"}, HelperKey{WindowsSessionID: sessionID, Role: "user"}) + m.stopKey(HelperKey{WindowsSessionID: sessionID, Role: "user"}) + m.stopKey(HelperKey{WindowsSessionID: sessionID, Role: "system"}) +} + +func (m *HelperLifecycleManager) removeDesired(keys ...HelperKey) { + m.mu.Lock() + for _, key := range keys { + delete(m.desired, key) + } + desired := cloneDesired(m.desired) + m.publishDesired(desired) + m.mu.Unlock() +} + +func (m *HelperLifecycleManager) stopKey(key HelperKey) { + if m.broker != nil { + m.broker.TerminateHelperKey(key) + } + entry := m.registry.beginStop(key) + if entry == nil { + return + } + if waitDone(entry.done, m.gracePeriod) { + m.registry.detach(key, entry.generation) + return + } + if entry.process != nil && entry.process.Alive() { + if err := entry.process.Terminate(); err != nil { + log.Warn("lifecycle: failed to terminate helper", "helperKey", key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + } + } + if waitDone(entry.done, m.finalWait) { + m.registry.detach(key, entry.generation) + return + } + m.registry.detach(key, entry.generation) + log.Warn("lifecycle: helper process did not reap before shutdown deadline", "helperKey", key.String(), "pid", processID(entry.process)) +} + +func processID(process helperProcess) uint32 { + if process == nil { + return 0 + } + return process.ProcessID() +} + +func waitDone(done <-chan struct{}, timeout time.Duration) bool { + if done == nil { + return true + } + if timeout <= 0 { + select { + case <-done: + return true + default: + return false + } + } + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-done: + return true + case <-timer.C: + return false + } +} + +func (m *HelperLifecycleManager) Stop() { + m.stopOnce.Do(func() { + close(m.stopCh) + m.mu.Lock() + m.stopping = true + m.desired = make(map[HelperKey]bool) + m.publishDesired(nil) + m.mu.Unlock() + keys := make(map[HelperKey]struct{}) + for _, key := range m.registry.keys() { + keys[key] = struct{}{} + } + if m.broker != nil { + for _, key := range m.broker.LifecycleHelperKeys() { + keys[key] = struct{}{} + } + } + var stopWG sync.WaitGroup + for key := range keys { + key := key + stopWG.Add(1) + go func() { + defer stopWG.Done() + m.stopKey(key) + }() + } + stopWG.Wait() + if m.spawner != nil { + if err := m.spawner.Close(); err != nil { + log.Warn("lifecycle: failed to close helper spawner", "error", err.Error()) + } + } + if m.observerRemove != nil { + m.observerRemove() + } + }) +} + +func (m *HelperLifecycleManager) sessionAuthenticated(session *Session) { + key, ok := helperKeyFromSession(session) + if !ok { + return + } + m.registry.markConnected(key, uint32(session.PID), session) +} + +func (m *HelperLifecycleManager) sessionClosed(session *Session) { + key, ok := helperKeyFromSession(session) + if !ok { + return + } + m.registry.markSessionClosed(key, session) +} + +func helperKeyFromSession(session *Session) (HelperKey, bool) { + if session == nil || (session.HelperRole != "system" && session.HelperRole != "user") { + return HelperKey{}, false + } + id, err := strconv.ParseUint(session.WinSessionID, 10, 32) + if err != nil || id == 0 { + return HelperKey{}, false + } + return HelperKey{WindowsSessionID: uint32(id), Role: session.HelperRole}, true +} + +func (m *HelperLifecycleManager) String() string { + return fmt.Sprintf("HelperLifecycleManager(tracked=%d)", m.registry.len()) +} diff --git a/agent/internal/sessionbroker/lifecycle_registry.go b/agent/internal/sessionbroker/lifecycle_registry.go new file mode 100644 index 0000000000..42cd623887 --- /dev/null +++ b/agent/internal/sessionbroker/lifecycle_registry.go @@ -0,0 +1,252 @@ +package sessionbroker + +import ( + "sync" + "time" +) + +type helperState string + +const ( + helperStarting helperState = "starting" + helperConnected helperState = "connected" + helperStopping helperState = "stopping" + helperExited helperState = "exited" +) + +type trackedHelper struct { + key HelperKey + process helperProcess + generation uint64 + state helperState + launchedAt time.Time + retryCount int + lastFailure time.Time + fatalExitUntil time.Time + executablePath string + commandMode string + done chan struct{} + exitCode int + brokerSession *Session + doneOnce sync.Once +} + +type helperRegistry struct { + mu sync.Mutex + next uint64 + current map[HelperKey]*trackedHelper + generation map[uint64]*trackedHelper +} + +func newHelperRegistry() *helperRegistry { + return &helperRegistry{ + current: make(map[HelperKey]*trackedHelper), + generation: make(map[uint64]*trackedHelper), + } +} + +func (r *helperRegistry) newEntryLocked(key HelperKey, previous *trackedHelper) *trackedHelper { + r.next++ + entry := &trackedHelper{ + key: key, + generation: r.next, + state: helperStarting, + done: make(chan struct{}), + exitCode: -1, + } + if previous != nil { + entry.retryCount = previous.retryCount + entry.lastFailure = previous.lastFailure + entry.fatalExitUntil = previous.fatalExitUntil + } + r.current[key] = entry + r.generation[entry.generation] = entry + return entry +} + +func (r *helperRegistry) reserve(key HelperKey, now time.Time) (uint64, bool) { + r.mu.Lock() + defer r.mu.Unlock() + previous := r.current[key] + if previous != nil { + if previous.state == helperStarting || previous.state == helperStopping { + return 0, false + } + if previous.process != nil && previous.process.Alive() { + return 0, false + } + if !previous.fatalExitUntil.IsZero() && now.Before(previous.fatalExitUntil) { + return 0, false + } + if previous.retryCount >= maxSpawnRetries { + return 0, false + } + if previous.retryCount > 0 { + backoff := time.Duration(1< maxBackoff { + backoff = maxBackoff + } + if now.Sub(previous.lastFailure) < backoff { + return 0, false + } + } + } + entry := r.newEntryLocked(key, previous) + entry.retryCount++ + entry.lastFailure = now + return entry.generation, true +} + +func (r *helperRegistry) attach(key HelperKey, process helperProcess, executablePath, commandMode string) uint64 { + r.mu.Lock() + entry := r.newEntryLocked(key, r.current[key]) + entry.process = process + entry.launchedAt = time.Now() + entry.executablePath = executablePath + entry.commandMode = commandMode + r.mu.Unlock() + return entry.generation +} + +func (r *helperRegistry) attachReserved(key HelperKey, generation uint64, process helperProcess, commandMode string) (*trackedHelper, bool) { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil || entry.generation != generation || entry.state != helperStarting { + return nil, false + } + entry.process = process + entry.launchedAt = time.Now() + entry.executablePath = process.ExecutablePath() + entry.commandMode = commandMode + return entry, true +} + +func (r *helperRegistry) noteSpawnFailure(key HelperKey, generation uint64) { + r.mu.Lock() + entry := r.generation[generation] + if entry != nil && entry.key == key { + entry.state = helperExited + entry.doneOnce.Do(func() { close(entry.done) }) + delete(r.generation, generation) + } + r.mu.Unlock() +} + +func (r *helperRegistry) noteExit(key HelperKey, generation uint64, exitCode int) { + r.mu.Lock() + entry := r.generation[generation] + if entry == nil || entry.key != key { + r.mu.Unlock() + return + } + entry.exitCode = exitCode + entry.state = helperExited + entry.brokerSession = nil + if exitCode == helperFatalExitCode { + entry.fatalExitUntil = time.Now().Add(fatalCooldown) + } + entry.doneOnce.Do(func() { close(entry.done) }) + delete(r.generation, generation) + if current := r.current[key]; current != entry { + // A newer generation owns the one-process slot. + } else { + r.current[key] = entry + } + r.mu.Unlock() +} + +func (r *helperRegistry) detach(key HelperKey, generation uint64) bool { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil || entry.generation != generation { + return false + } + if entry.state != helperStopping && entry.process != nil && entry.process.Alive() { + return false + } + delete(r.current, key) + return true +} + +func (r *helperRegistry) beginStop(key HelperKey) *trackedHelper { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil { + return nil + } + if entry.state == helperExited { + delete(r.current, key) + return nil + } + if entry.state == helperStopping { + return nil + } + entry.state = helperStopping + return entry +} + +func (r *helperRegistry) markConnected(key HelperKey, pid uint32, session *Session) { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil || entry.process == nil || entry.process.ProcessID() != pid || entry.state == helperStopping { + return + } + entry.state = helperConnected + entry.retryCount = 0 + entry.lastFailure = time.Time{} + entry.brokerSession = session +} + +func (r *helperRegistry) markSessionClosed(key HelperKey, session *Session) { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil || entry.brokerSession != session { + return + } + entry.brokerSession = nil + if entry.process != nil && entry.process.Alive() { + entry.state = helperStarting + } else { + entry.state = helperExited + } +} + +func (r *helperRegistry) clearFatal(sessionID uint32) { + r.mu.Lock() + defer r.mu.Unlock() + for _, role := range []string{"system", "user"} { + if entry := r.current[HelperKey{WindowsSessionID: sessionID, Role: role}]; entry != nil { + entry.fatalExitUntil = time.Time{} + } + } +} + +func (r *helperRegistry) processID(key HelperKey) uint32 { + r.mu.Lock() + defer r.mu.Unlock() + if entry := r.current[key]; entry != nil && entry.process != nil { + return entry.process.ProcessID() + } + return 0 +} + +func (r *helperRegistry) keys() []HelperKey { + r.mu.Lock() + defer r.mu.Unlock() + keys := make([]HelperKey, 0, len(r.current)) + for key, entry := range r.current { + if entry.state != helperExited { + keys = append(keys, key) + } + } + return keys +} + +func (r *helperRegistry) len() int { + return len(r.keys()) +} diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go new file mode 100644 index 0000000000..2556a0ba4e --- /dev/null +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -0,0 +1,275 @@ +package sessionbroker + +import ( + "context" + "sync" + "testing" + "time" +) + +type fakeLifecycleDetector struct { + sessions []DetectedSession +} + +func (d fakeLifecycleDetector) ListSessions() ([]DetectedSession, error) { + return append([]DetectedSession(nil), d.sessions...), nil +} + +func (d fakeLifecycleDetector) WatchSessions(context.Context) <-chan SessionEvent { + return make(chan SessionEvent) +} + +type fakeHelperProcess struct { + pid uint32 + path string + + mu sync.Mutex + alive bool + exitCode int + exited chan struct{} + exitOnce sync.Once + waitCount int + terminateCount int + closeCount int +} + +func newFakeHelperProcess(pid uint32) *fakeHelperProcess { + return &fakeHelperProcess{ + pid: pid, + path: "breeze-user-helper.exe", + alive: true, + exited: make(chan struct{}), + } +} + +func (p *fakeHelperProcess) ProcessID() uint32 { return p.pid } +func (p *fakeHelperProcess) ExecutablePath() string { return p.path } + +func (p *fakeHelperProcess) Alive() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.alive +} + +func (p *fakeHelperProcess) Terminate() error { + p.mu.Lock() + p.terminateCount++ + p.mu.Unlock() + p.markExited(1) + return nil +} + +func (p *fakeHelperProcess) Wait() (int, error) { + p.mu.Lock() + p.waitCount++ + p.mu.Unlock() + <-p.exited + p.mu.Lock() + defer p.mu.Unlock() + return p.exitCode, nil +} + +func (p *fakeHelperProcess) Close() error { + p.mu.Lock() + p.closeCount++ + p.mu.Unlock() + return nil +} + +func (p *fakeHelperProcess) markExited(code int) { + p.exitOnce.Do(func() { + p.mu.Lock() + p.alive = false + p.exitCode = code + p.mu.Unlock() + close(p.exited) + }) +} + +func (p *fakeHelperProcess) counts() (wait, terminate, close int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.waitCount, p.terminateCount, p.closeCount +} + +type fakeHelperSpawner struct { + mu sync.Mutex + processes []helperProcess + spawned map[HelperKey]int + closed int +} + +func (s *fakeHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.spawned == nil { + s.spawned = make(map[HelperKey]int) + } + s.spawned[key]++ + if len(s.processes) == 0 { + return newFakeHelperProcess(uint32(5000 + s.spawned[key])), nil + } + proc := s.processes[0] + s.processes = s.processes[1:] + return proc, nil +} + +func (s *fakeHelperSpawner) Close() error { + s.mu.Lock() + s.closed++ + s.mu.Unlock() + return nil +} + +func (s *fakeHelperSpawner) SpawnCount(key HelperKey) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.spawned[key] +} + +func newLifecycleHarness(t *testing.T, sessions []DetectedSession, spawner helperSpawner) *HelperLifecycleManager { + t.Helper() + b := New("lifecycle-"+t.Name(), nil) + m := newHelperLifecycleManager(b, fakeLifecycleDetector{sessions: sessions}, nil, spawner) + m.gracePeriod = 5 * time.Millisecond + m.finalWait = 100 * time.Millisecond + t.Cleanup(func() { + m.Stop() + b.Close() + }) + return m +} + +func TestReconcileDoesNotRespawnWhilePreAuthProcessLives(t *testing.T) { + proc := newFakeHelperProcess(4100) + spawner := &fakeHelperSpawner{processes: []helperProcess{proc}} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) + + m.reconcile() + m.reconcile() + + if got := spawner.SpawnCount(HelperKey{WindowsSessionID: 7, Role: "system"}); got != 1 { + t.Fatalf("system spawn count = %d, want 1", got) + } +} + +func TestStaleExitCannotClearReplacement(t *testing.T) { + oldProc := newFakeHelperProcess(4100) + newProc := newFakeHelperProcess(4200) + m := newLifecycleHarness(t, nil, &fakeHelperSpawner{}) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + oldGeneration := m.registry.attach(key, oldProc, "breeze-user-helper.exe", "system-helper") + oldProc.markExited(0) + if !m.registry.detach(key, oldGeneration) { + t.Fatal("failed to detach observably exited generation") + } + m.registry.attach(key, newProc, "breeze-user-helper.exe", "system-helper") + m.registry.noteExit(key, oldGeneration, 0) + if got := m.registry.processID(key); got != 4200 { + t.Fatalf("replacement PID = %d, want 4200", got) + } +} + +func TestStopSessionTerminatesBothRoles(t *testing.T) { + m := newLifecycleHarness(t, nil, &fakeHelperSpawner{}) + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "system"}, newFakeHelperProcess(1), "helper", "system-helper") + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "user"}, newFakeHelperProcess(2), "helper", "user-helper") + m.stopSession(7) + if got := m.registry.len(); got != 0 { + t.Fatalf("registry len = %d, want 0", got) + } +} + +func TestConcurrentReconcileStopAndExitOwnsHandleOnce(t *testing.T) { + proc := newFakeHelperProcess(4300) + spawner := &fakeHelperSpawner{processes: []helperProcess{proc}} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + + var reconcileWG sync.WaitGroup + for range 16 { + reconcileWG.Add(1) + go func() { + defer reconcileWG.Done() + m.reconcile() + }() + } + reconcileWG.Wait() + + var raceWG sync.WaitGroup + raceWG.Add(2) + go func() { + defer raceWG.Done() + m.stopKey(key) + }() + go func() { + defer raceWG.Done() + proc.markExited(0) + }() + raceWG.Wait() + + deadline := time.Now().Add(time.Second) + for { + wait, _, closeCount := proc.counts() + if wait == 1 && closeCount == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("process ownership counts wait=%d close=%d, want 1 each", wait, closeCount) + } + time.Sleep(time.Millisecond) + } + if got := spawner.SpawnCount(key); got != 1 { + t.Fatalf("system spawn count = %d, want 1", got) + } +} + +func TestConcurrentStopKeyHasSingleTerminationOwner(t *testing.T) { + m := newLifecycleHarness(t, nil, &fakeHelperSpawner{}) + m.gracePeriod = 20 * time.Millisecond + m.finalWait = 0 + key := HelperKey{WindowsSessionID: 7, Role: "system"} + process := newFakeHelperProcess(4400) + m.registry.attach(key, process, "helper", "system-helper") + + var wg sync.WaitGroup + for range 16 { + wg.Add(1) + go func() { + defer wg.Done() + m.stopKey(key) + }() + } + wg.Wait() + + _, terminateCount, _ := process.counts() + if terminateCount != 1 { + t.Fatalf("terminate count = %d, want 1", terminateCount) + } +} + +func TestRegistryBeginStopGrantsOneOwnerPerGeneration(t *testing.T) { + registry := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "system"} + registry.attach(key, newFakeHelperProcess(4500), "helper", "system-helper") + if first := registry.beginStop(key); first == nil { + t.Fatal("first beginStop did not claim generation") + } + if second := registry.beginStop(key); second != nil { + t.Fatal("second beginStop claimed an already-stopping generation") + } + if got := registry.processID(key); got != 4500 { + t.Fatalf("stopping generation PID = %d, want 4500", got) + } +} + +func TestLifecycleStopEndsStartLoop(t *testing.T) { + m := NewHelperLifecycleManager(New("stop-loop-"+t.Name(), nil), nil) + go m.Start(context.Background()) + m.Stop() + select { + case <-m.Done(): + case <-time.After(time.Second): + t.Fatal("Start loop did not finish after Stop") + } +} diff --git a/agent/internal/sessionbroker/lifecycle_stub.go b/agent/internal/sessionbroker/lifecycle_stub.go index 6ee7577ec4..fae90921bc 100644 --- a/agent/internal/sessionbroker/lifecycle_stub.go +++ b/agent/internal/sessionbroker/lifecycle_stub.go @@ -4,22 +4,16 @@ package sessionbroker import "context" -// SCMSessionEvent is a no-op placeholder on non-Windows platforms. -type SCMSessionEvent struct { - EventType uint32 - SessionID uint32 -} - -// HelperLifecycleManager is a no-op on non-Windows platforms. -// Helper spawning is only needed for Windows services running in Session 0. -type HelperLifecycleManager struct{} - // NewHelperLifecycleManager returns a no-op lifecycle manager on non-Windows. -func NewHelperLifecycleManager(_ *Broker, _ <-chan SCMSessionEvent) *HelperLifecycleManager { - return &HelperLifecycleManager{} +func NewHelperLifecycleManager(broker *Broker, scmCh <-chan SCMSessionEvent) *HelperLifecycleManager { + return newHelperLifecycleManager(broker, NewSessionDetector(), scmCh, nil) } // Start is a no-op on non-Windows platforms. func (m *HelperLifecycleManager) Start(ctx context.Context) { - <-ctx.Done() + defer m.finishStart() + select { + case <-ctx.Done(): + case <-m.stopCh: + } } diff --git a/agent/internal/sessionbroker/lifecycle_test.go b/agent/internal/sessionbroker/lifecycle_test.go index 144b29e9de..9e0dcb5d8e 100644 --- a/agent/internal/sessionbroker/lifecycle_test.go +++ b/agent/internal/sessionbroker/lifecycle_test.go @@ -7,364 +7,81 @@ import ( "time" ) -func TestHandleSCMEventDoesNotSpawnBeforeDetectorPublishesEventKey(t *testing.T) { - broker := New(`\\.\pipe\test-scm-desired-order-`+t.Name(), nil) - m := NewHelperLifecycleManager(broker, nil) - m.detector = admissionTestDetector{sessions: []DetectedSession{ - {Session: "8", State: "active"}, - }} - - m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionLogon, SessionID: 7}) - - if broker.helperKeyDesired(HelperKey{WindowsSessionID: 7, Role: "system"}) || - broker.helperKeyDesired(HelperKey{WindowsSessionID: 7, Role: "user"}) { - t.Fatal("SCM event key absent from detector was published as desired") - } - m.mu.Lock() - defer m.mu.Unlock() - if _, spawned := m.tracked["7-system"]; spawned { - t.Fatal("system helper spawned before detector published eligibility") - } - if _, spawned := m.tracked["7-user"]; spawned { - t.Fatal("user helper spawned before detector published eligibility") - } -} - -// --------------------------------------------------------------------------- -// Priority 4: fatalExitUntil suppression in spawnWithRetry -// --------------------------------------------------------------------------- - -// TestSpawnWithRetrySkipsWhenFatalCooldownActive verifies that spawnWithRetry -// does nothing when the tracked session has a fatalExitUntil set to a future -// time. It verifies the suppression by checking that the retryCount in the -// tracked entry remains unchanged after the call. -func TestSpawnWithRetrySkipsWhenFatalCooldownActive(t *testing.T) { - t.Parallel() - - // Build a minimal broker that can be passed to NewHelperLifecycleManager. - // We don't start a listener — we just need the struct. - broker := New(`\\.\pipe\test-lifecycle-fatal-`+t.Name(), nil) - defer broker.Close() - - m := NewHelperLifecycleManager(broker, nil) - - winSessionID := "7" - role := "system" - trackKey := winSessionID + "-" + role - - // Pre-seed the tracked map with a fatalExitUntil in the future. - future := time.Now().Add(fatalCooldown) // 10 minutes from now - m.mu.Lock() - m.tracked[trackKey] = &trackedSession{ - fatalExitUntil: future, - retryCount: 0, - } - m.mu.Unlock() - - // spawnWithRetry should detect the cooldown and return without spawning. - m.spawnWithRetry(winSessionID, role) - - // Verify: retryCount should still be 0 (no spawn attempt was made). - // If a spawn was attempted, retryCount would be incremented to 1. - m.mu.Lock() - ts := m.tracked[trackKey] - m.mu.Unlock() - - if ts == nil { - t.Fatal("tracked entry was unexpectedly deleted") - } - if ts.retryCount != 0 { - t.Errorf("retryCount = %d, want 0 (spawnWithRetry should have returned early due to fatalExitUntil)", ts.retryCount) - } - if ts.fatalExitUntil != future { - t.Error("fatalExitUntil was modified unexpectedly") - } +func newWindowsLifecycleHarness(t *testing.T, sessions []DetectedSession) (*HelperLifecycleManager, *fakeHelperSpawner) { + t.Helper() + b := New(`\\.\pipe\lifecycle-`+t.Name(), nil) + spawner := &fakeHelperSpawner{} + m := newHelperLifecycleManager(b, fakeLifecycleDetector{sessions: sessions}, nil, spawner) + m.gracePeriod = 0 + m.finalWait = 100 * time.Millisecond + t.Cleanup(func() { + m.Stop() + b.Close() + }) + return m, spawner } -// TestSpawnWithRetryProceedsWhenFatalCooldownExpired verifies that spawnWithRetry -// proceeds past the cooldown check when fatalExitUntil is in the past. -// We set it 1 nanosecond in the past so that time.Now().Before(ts.fatalExitUntil) -// is false. The spawn attempt will fail (no real session 7 exists on a test machine), -// but the key behavioral assertion is that retryCount is incremented — proving -// the early-return guard was NOT triggered. -func TestSpawnWithRetryProceedsWhenFatalCooldownExpired(t *testing.T) { - t.Parallel() - - broker := New(`\\.\pipe\test-lifecycle-expired-`+t.Name(), nil) - defer broker.Close() - - m := NewHelperLifecycleManager(broker, nil) - - winSessionID := "8" - role := "system" - trackKey := winSessionID + "-" + role - - // Set fatalExitUntil 1 second in the past — cooldown has expired. - past := time.Now().Add(-1 * time.Second) - m.mu.Lock() - m.tracked[trackKey] = &trackedSession{ - fatalExitUntil: past, - retryCount: 0, - } - m.mu.Unlock() - - // spawnWithRetry should proceed past the cooldown check. - // The spawn will likely fail (session 8 probably doesn't exist), but that's OK. - m.spawnWithRetry(winSessionID, role) - - // retryCount incremented → the guard was not triggered. - m.mu.Lock() - ts := m.tracked[trackKey] - m.mu.Unlock() +func TestHandleSCMEventDoesNotSpawnBeforeDetectorPublishesEventKey(t *testing.T) { + m, spawner := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "8", State: "active"}}) + m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionLogon, SessionID: 7}) - if ts == nil { - t.Fatal("tracked entry was unexpectedly deleted") + if got := spawner.SpawnCount(HelperKey{WindowsSessionID: 7, Role: "system"}); got != 0 { + t.Fatalf("system spawn count = %d, want 0", got) } - if ts.retryCount == 0 { - t.Error("retryCount = 0, want > 0 (spawnWithRetry should have proceeded past expired cooldown)") + if got := spawner.SpawnCount(HelperKey{WindowsSessionID: 7, Role: "user"}); got != 0 { + t.Fatalf("user spawn count = %d, want 0", got) } } -// TestSpawnWithRetryProceedsWhenFatalCooldownZero verifies that spawnWithRetry -// proceeds when fatalExitUntil is zero (never set). -func TestSpawnWithRetryProceedsWhenFatalCooldownZero(t *testing.T) { - t.Parallel() - - broker := New(`\\.\pipe\test-lifecycle-zero-`+t.Name(), nil) - defer broker.Close() +func TestHandleSCMDisconnectStopsUserAndRetainsSystem(t *testing.T) { + m, _ := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "7", State: "connected", Type: "rdp"}}) + system := newFakeHelperProcess(6100) + user := newFakeHelperProcess(6200) + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "system"}, system, "helper", "system-helper") + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "user"}, user, "helper", "user-helper") + m.desired[HelperKey{WindowsSessionID: 7, Role: "system"}] = true + m.desired[HelperKey{WindowsSessionID: 7, Role: "user"}] = true - m := NewHelperLifecycleManager(broker, nil) + m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionDisconnect, SessionID: 7}) - winSessionID := "9" - role := "system" - trackKey := winSessionID + "-" + role - - // No pre-seed — trackedSession will be created fresh with zero fatalExitUntil. - m.spawnWithRetry(winSessionID, role) - - m.mu.Lock() - ts := m.tracked[trackKey] - m.mu.Unlock() - - if ts == nil { - t.Fatal("tracked entry was not created") + if !system.Alive() { + t.Fatal("system helper was stopped on disconnect") } - // retryCount should be 1 — the spawn attempt was made (may have failed, - // but the lifecycle counter was incremented). - if ts.retryCount == 0 { - t.Error("retryCount = 0, want > 0 (spawnWithRetry should have attempted spawn with zero fatalExitUntil)") + if user.Alive() { + t.Fatal("user helper remained alive on disconnect") } } -// --------------------------------------------------------------------------- -// Priority 5: handleSCMEvent clears fatalExitUntil -// --------------------------------------------------------------------------- - -// TestHandleSCMEventClearsFatalCooldown verifies that session-change events that -// indicate the environment changed favorably (logon, unlock, create) clear the -// fatalExitUntil field so a cooling-down helper can try again. -func TestHandleSCMEventClearsFatalCooldown(t *testing.T) { - t.Parallel() - - clearingEvents := []struct { - name string - eventType uint32 - }{ - {"WTS_SESSION_LOGON", wtsSessionLogon}, - {"WTS_SESSION_UNLOCK", wtsSessionUnlock}, - {"WTS_SESSION_CREATE", wtsSessionCreate}, - } - - for _, evt := range clearingEvents { - evt := evt - t.Run(evt.name, func(t *testing.T) { - t.Parallel() - - broker := New(`\\.\pipe\test-scm-`+evt.name+t.Name(), nil) - defer broker.Close() - - m := NewHelperLifecycleManager(broker, nil) - - sessionID := uint32(5) - sessionIDStr := "5" - - // Pre-seed both roles with future fatalExitUntil. - future := time.Now().Add(fatalCooldown) - m.mu.Lock() - m.tracked[sessionIDStr+"-system"] = &trackedSession{fatalExitUntil: future} - m.tracked[sessionIDStr+"-user"] = &trackedSession{fatalExitUntil: future} - m.mu.Unlock() - - // Deliver the event. - m.handleSCMEvent(SCMSessionEvent{ - EventType: evt.eventType, - SessionID: sessionID, - }) - - // Both roles should now have fatalExitUntil cleared (zero). - m.mu.Lock() - tsSystem := m.tracked[sessionIDStr+"-system"] - tsUser := m.tracked[sessionIDStr+"-user"] - m.mu.Unlock() - - if tsSystem != nil && !tsSystem.fatalExitUntil.IsZero() { - t.Errorf("%s: system role fatalExitUntil not cleared, still %v", evt.name, tsSystem.fatalExitUntil) - } - if tsUser != nil && !tsUser.fatalExitUntil.IsZero() { - t.Errorf("%s: user role fatalExitUntil not cleared, still %v", evt.name, tsUser.fatalExitUntil) - } - }) - } -} - -// TestHandleSCMEventLogoffDeletesTracking verifies that logoff/terminate events -// remove the tracked entries entirely (not just clear fatalExitUntil). -func TestHandleSCMEventLogoffDeletesTracking(t *testing.T) { - t.Parallel() - - deletingEvents := []struct { - name string - eventType uint32 - }{ - {"WTS_SESSION_LOGOFF", wtsSessionLogoff}, - {"WTS_SESSION_TERMINATE", wtsSessionTerminate}, - } - - for _, evt := range deletingEvents { - evt := evt - t.Run(evt.name, func(t *testing.T) { - t.Parallel() - - broker := New(`\\.\pipe\test-scm-delete-`+evt.name+t.Name(), nil) - defer broker.Close() - - m := NewHelperLifecycleManager(broker, nil) - - sessionID := uint32(6) - sessionIDStr := "6" - - m.mu.Lock() - m.tracked[sessionIDStr+"-system"] = &trackedSession{retryCount: 3} - m.tracked[sessionIDStr+"-user"] = &trackedSession{retryCount: 2} - m.mu.Unlock() - - m.handleSCMEvent(SCMSessionEvent{ - EventType: evt.eventType, - SessionID: sessionID, - }) - - m.mu.Lock() - _, systemExists := m.tracked[sessionIDStr+"-system"] - _, userExists := m.tracked[sessionIDStr+"-user"] - m.mu.Unlock() - - if systemExists { - t.Errorf("%s: system role tracking entry not deleted", evt.name) - } - if userExists { - t.Errorf("%s: user role tracking entry not deleted", evt.name) +func TestHandleSCMLogoffAndTerminateStopBothRoles(t *testing.T) { + for _, eventType := range []uint32{wtsSessionLogoff, wtsSessionTerminate} { + t.Run(string(rune(eventType)), func(t *testing.T) { + m, _ := newWindowsLifecycleHarness(t, nil) + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "system"}, newFakeHelperProcess(1), "helper", "system-helper") + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "user"}, newFakeHelperProcess(2), "helper", "user-helper") + m.handleSCMEvent(SCMSessionEvent{EventType: eventType, SessionID: 7}) + if got := m.registry.len(); got != 0 { + t.Fatalf("registry len = %d, want 0", got) } }) } } -// TestHandleSCMEventSkipsSessionZero verifies that session 0 (services) is -// ignored by handleSCMEvent — it should not create any tracked entries. func TestHandleSCMEventSkipsSessionZero(t *testing.T) { - t.Parallel() - - broker := New(`\\.\pipe\test-scm-session0-`+t.Name(), nil) - defer broker.Close() - - m := NewHelperLifecycleManager(broker, nil) - - // Deliver a logon event for session 0. - m.handleSCMEvent(SCMSessionEvent{ - EventType: wtsSessionLogon, - SessionID: 0, - }) - - m.mu.Lock() - _, exists0Sys := m.tracked["0-system"] - _, exists0User := m.tracked["0-user"] - total := len(m.tracked) - m.mu.Unlock() - - if exists0Sys || exists0User { - t.Error("session 0 should be ignored by handleSCMEvent but tracking entries were created") - } - if total != 0 { - t.Errorf("expected 0 tracked entries after session-0 event, got %d", total) - } -} - -// TestHandleSCMEventLockDoesNotClearCooldown verifies that a lock event (which -// does NOT indicate a favorable environment change) does NOT clear fatalExitUntil. -// Only logon/unlock/create trigger the clear. -func TestHandleSCMEventLockDoesNotClearCooldown(t *testing.T) { - t.Parallel() - - broker := New(`\\.\pipe\test-scm-lock-`+t.Name(), nil) - defer broker.Close() - - m := NewHelperLifecycleManager(broker, nil) - - sessionID := uint32(3) - sessionIDStr := "3" - future := time.Now().Add(fatalCooldown) - - m.mu.Lock() - m.tracked[sessionIDStr+"-system"] = &trackedSession{fatalExitUntil: future} - m.mu.Unlock() - - // Deliver a lock event — this is NOT in the clearing switch cases. - m.handleSCMEvent(SCMSessionEvent{ - EventType: wtsSessionLock, - SessionID: sessionID, - }) - - m.mu.Lock() - ts := m.tracked[sessionIDStr+"-system"] - m.mu.Unlock() - - // fatalExitUntil should remain set — lock event doesn't clear it. - if ts != nil && ts.fatalExitUntil.IsZero() { - t.Error("lock event cleared fatalExitUntil, but only logon/unlock/create should clear it") + m, spawner := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "0", State: "active", Type: "services"}}) + m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionLogon, SessionID: 0}) + if got := spawner.SpawnCount(HelperKey{WindowsSessionID: 0, Role: "system"}); got != 0 { + t.Fatalf("session-zero spawn count = %d, want 0", got) } } -// --------------------------------------------------------------------------- -// Priority 6: helperFatalExitCode consistency guard -// --------------------------------------------------------------------------- - -// TestFatalExitCodeConsistency is a compile-time-ish guard that ensures the -// exit code used by main.go's os.Exit(2) matches the constant recognized by -// lifecycle.go's watchHelperExit goroutine. A drift between these two values -// would silently break the fatal-cooldown mechanism. func TestFatalExitCodeConsistency(t *testing.T) { - // The helper in main.go uses os.Exit(2) as the fatal exit signal. - // The lifecycle manager must recognize that exact value. - const expectedFatalExitCode = 2 - if helperFatalExitCode != expectedFatalExitCode { - t.Errorf("helperFatalExitCode = %d; if this is intentional, update main.go os.Exit call to match %d", - helperFatalExitCode, expectedFatalExitCode) + if helperFatalExitCode != 2 { + t.Fatalf("helperFatalExitCode = %d, want 2", helperFatalExitCode) } } -// TestPanicExitCodeConsistency mirrors TestFatalExitCodeConsistency for the -// panic-recovery path added in PR #450. main.go's top-level panic defer uses -// os.Exit(3) to signal "transient panic, respawn normally"; lifecycle.go's -// watchHelperExit branches on this value to skip the permanent-reject -// cooldown. A drift here would silently send every helper panic into the -// 10-minute lockout meant for genuine permanent rejection. func TestPanicExitCodeConsistency(t *testing.T) { - const expectedPanicExitCode = 3 - if helperPanicExitCode != expectedPanicExitCode { - t.Errorf("helperPanicExitCode = %d; if this is intentional, update main.go os.Exit call to match %d", - helperPanicExitCode, expectedPanicExitCode) - } - if helperPanicExitCode == helperFatalExitCode { - t.Errorf("helperPanicExitCode (%d) must differ from helperFatalExitCode (%d)", - helperPanicExitCode, helperFatalExitCode) + if helperPanicExitCode != 3 || helperPanicExitCode == helperFatalExitCode { + t.Fatalf("panic exit code = %d, fatal = %d", helperPanicExitCode, helperFatalExitCode) } } diff --git a/agent/internal/sessionbroker/peer_process_other.go b/agent/internal/sessionbroker/peer_process_other.go new file mode 100644 index 0000000000..3d6b337067 --- /dev/null +++ b/agent/internal/sessionbroker/peer_process_other.go @@ -0,0 +1,5 @@ +//go:build !windows + +package sessionbroker + +func openOwnedPeerProcess(uint32) (ownedPeerProcess, error) { return nil, nil } diff --git a/agent/internal/sessionbroker/peer_process_windows.go b/agent/internal/sessionbroker/peer_process_windows.go new file mode 100644 index 0000000000..9bdb55105a --- /dev/null +++ b/agent/internal/sessionbroker/peer_process_windows.go @@ -0,0 +1,71 @@ +//go:build windows + +package sessionbroker + +import ( + "fmt" + "sync" + + "golang.org/x/sys/windows" +) + +const windowsProcessStillActive = 259 + +type windowsOwnedPeerProcess struct { + pid uint32 + mu sync.Mutex + handle windows.Handle +} + +func openOwnedPeerProcess(pid uint32) (ownedPeerProcess, error) { + handle, err := windows.OpenProcess( + windows.PROCESS_QUERY_LIMITED_INFORMATION|windows.SYNCHRONIZE|windows.PROCESS_TERMINATE, + false, + pid, + ) + if err != nil { + return nil, fmt.Errorf("OpenProcess(%d): %w", pid, err) + } + return &windowsOwnedPeerProcess{pid: pid, handle: handle}, nil +} + +func (p *windowsOwnedPeerProcess) ProcessID() uint32 { return p.pid } + +func (p *windowsOwnedPeerProcess) Alive() (bool, error) { + p.mu.Lock() + handle := p.handle + p.mu.Unlock() + if handle == 0 { + return false, nil + } + var code uint32 + if err := windows.GetExitCodeProcess(handle, &code); err != nil { + return false, err + } + return code == windowsProcessStillActive, nil +} + +func (p *windowsOwnedPeerProcess) Terminate() error { + p.mu.Lock() + handle := p.handle + p.mu.Unlock() + if handle == 0 { + return nil + } + alive, err := p.Alive() + if err != nil || !alive { + return err + } + return windows.TerminateProcess(handle, 1) +} + +func (p *windowsOwnedPeerProcess) Close() error { + p.mu.Lock() + handle := p.handle + p.handle = 0 + p.mu.Unlock() + if handle == 0 { + return nil + } + return windows.CloseHandle(handle) +} diff --git a/agent/internal/sessionbroker/peer_process_windows_test.go b/agent/internal/sessionbroker/peer_process_windows_test.go new file mode 100644 index 0000000000..a706f9a611 --- /dev/null +++ b/agent/internal/sessionbroker/peer_process_windows_test.go @@ -0,0 +1,32 @@ +//go:build windows + +package sessionbroker + +import ( + "os" + "testing" +) + +func TestOpenOwnedPeerProcessRetainsKernelBoundHandle(t *testing.T) { + pid := uint32(os.Getpid()) + process, err := openOwnedPeerProcess(pid) + if err != nil { + t.Fatal(err) + } + if process.ProcessID() != pid { + t.Fatalf("process PID = %d, want %d", process.ProcessID(), pid) + } + alive, err := process.Alive() + if err != nil { + t.Fatal(err) + } + if !alive { + t.Fatal("current process handle reported non-live") + } + if err := process.Close(); err != nil { + t.Fatal(err) + } + if err := process.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } +} diff --git a/agent/internal/sessionbroker/session.go b/agent/internal/sessionbroker/session.go index ccf63e0c5c..fd7fbe1c25 100644 --- a/agent/internal/sessionbroker/session.go +++ b/agent/internal/sessionbroker/session.go @@ -48,6 +48,9 @@ type Session struct { // from the helper in response to a broker-initiated keepalive ping. // Read/written atomically so the keepalive goroutine doesn't need s.mu. lastPongAt atomic.Int64 + + broker *Broker + peerProcess *ownedPeerProcessRef } // NewSession creates a new session for a verified user helper connection. @@ -261,6 +264,22 @@ func (s *Session) HasScope(scope string) bool { // Close closes the underlying connection and cancels all pending commands. func (s *Session) Close() error { + if s.broker != nil { + return s.broker.closeSession(s) + } + return s.closeTransportAndPeer() +} + +func (s *Session) closeTransportAndPeer() error { + transportErr := s.closeTransport() + peerErr := s.peerProcess.close() + if transportErr != nil { + return transportErr + } + return peerErr +} + +func (s *Session) closeTransport() error { s.mu.Lock() if s.closed { s.mu.Unlock() @@ -277,6 +296,9 @@ func (s *Session) Close() error { close(done) } + if s.conn == nil { + return nil + } return s.conn.Close() } diff --git a/agent/internal/sessionbroker/spawner_stub.go b/agent/internal/sessionbroker/spawner_stub.go index 36252dbe0c..0a5f34a9fc 100644 --- a/agent/internal/sessionbroker/spawner_stub.go +++ b/agent/internal/sessionbroker/spawner_stub.go @@ -19,7 +19,12 @@ type SpawnedHelper struct { } // Close is a no-op on non-Windows platforms. -func (s *SpawnedHelper) Close() {} +func (s *SpawnedHelper) Close() error { return nil } + +func (s *SpawnedHelper) ProcessID() uint32 { return s.PID } +func (s *SpawnedHelper) ExecutablePath() string { return s.BinaryPath } +func (s *SpawnedHelper) Alive() bool { return false } +func (s *SpawnedHelper) Terminate() error { return nil } // Wait is a no-op on non-Windows platforms. Returns (exitCode=-1, nil). func (s *SpawnedHelper) Wait() (int, error) { return -1, nil } diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index c674bc5769..ff219ef913 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -4,6 +4,7 @@ package sessionbroker import ( "fmt" + "sync" "unsafe" "golang.org/x/sys/windows" @@ -22,15 +23,57 @@ type SpawnedHelper struct { PID uint32 Handle windows.Handle BinaryPath string + mu sync.Mutex } // Close releases the duplicated process handle. Safe to call more than once. -func (s *SpawnedHelper) Close() { - if s == nil || s.Handle == 0 { - return +func (s *SpawnedHelper) Close() error { + if s == nil { + return nil } - _ = windows.CloseHandle(s.Handle) + s.mu.Lock() + defer s.mu.Unlock() + if s.Handle == 0 { + return nil + } + err := windows.CloseHandle(s.Handle) s.Handle = 0 + return err +} + +func (s *SpawnedHelper) ProcessID() uint32 { return s.PID } +func (s *SpawnedHelper) ExecutablePath() string { return s.BinaryPath } + +func (s *SpawnedHelper) Alive() bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Handle == 0 { + return false + } + var exitCode uint32 + return windows.GetExitCodeProcess(s.Handle, &exitCode) == nil && exitCode == windowsProcessStillActive +} + +func (s *SpawnedHelper) Terminate() error { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Handle == 0 { + return nil + } + var exitCode uint32 + if err := windows.GetExitCodeProcess(s.Handle, &exitCode); err != nil { + return err + } + if exitCode != windowsProcessStillActive { + return nil + } + return windows.TerminateProcess(s.Handle, 1) } // Wait blocks until the spawned helper process exits and returns its exit @@ -38,11 +81,16 @@ func (s *SpawnedHelper) Close() { // automatically after Wait returns so callers do not need to call Close // in the normal path. func (s *SpawnedHelper) Wait() (int, error) { - if s == nil || s.Handle == 0 { + if s == nil { + return -1, fmt.Errorf("SpawnedHelper: no handle") + } + s.mu.Lock() + handle := s.Handle + s.mu.Unlock() + if handle == 0 { return -1, fmt.Errorf("SpawnedHelper: no handle") } - defer s.Close() - event, err := windows.WaitForSingleObject(s.Handle, windows.INFINITE) + event, err := windows.WaitForSingleObject(handle, windows.INFINITE) if err != nil { return -1, fmt.Errorf("WaitForSingleObject: %w", err) } @@ -50,7 +98,7 @@ func (s *SpawnedHelper) Wait() (int, error) { return -1, fmt.Errorf("WaitForSingleObject: unexpected event %d", event) } var exitCode uint32 - if err := windows.GetExitCodeProcess(s.Handle, &exitCode); err != nil { + if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { return -1, fmt.Errorf("GetExitCodeProcess: %w", err) } return int(exitCode), nil From 8e9b8198de02bad31b62fc12a71886c0fd6b2a39 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 15:59:36 -0600 Subject: [PATCH 08/47] fix(agent): close helper lifecycle ownership races --- .../heartbeat/handlers_desktop_helper.go | 33 +---- .../heartbeat/handlers_desktop_helper_test.go | 33 +++-- agent/internal/heartbeat/heartbeat.go | 83 +++++++---- agent/internal/heartbeat/heartbeat_test.go | 82 +++++++++++ agent/internal/sessionbroker/backup_test.go | 4 - agent/internal/sessionbroker/broker.go | 139 +++++++++--------- .../sessionbroker/broker_leak_test.go | 9 -- .../sessionbroker/broker_lifecycle_test.go | 118 ++++++++++++++- .../broker_stale_helpers_test.go | 36 ----- agent/internal/sessionbroker/broker_test.go | 13 -- agent/internal/sessionbroker/broker_unix.go | 6 +- .../internal/sessionbroker/broker_windows.go | 6 +- .../console_session_gate_test.go | 8 - .../internal/sessionbroker/lifecycle_core.go | 58 +++++++- .../sessionbroker/lifecycle_registry.go | 2 +- .../sessionbroker/lifecycle_registry_test.go | 130 +++++++++++++++- .../internal/sessionbroker/spawner_windows.go | 5 +- 17 files changed, 518 insertions(+), 247 deletions(-) delete mode 100644 agent/internal/sessionbroker/broker_stale_helpers_test.go diff --git a/agent/internal/heartbeat/handlers_desktop_helper.go b/agent/internal/heartbeat/handlers_desktop_helper.go index 09dec09aa2..37c4a671b2 100644 --- a/agent/internal/heartbeat/handlers_desktop_helper.go +++ b/agent/internal/heartbeat/handlers_desktop_helper.go @@ -55,20 +55,6 @@ func (h *Heartbeat) spawnDesktopHelper(targetSession string) error { return h.spawnHelperForDesktop(targetSession) } -func (h *Heartbeat) killDesktopStaleHelpers(targetSession string) { - if targetSession == "" { - return - } - staleKey := targetSession + "-" + ipc.HelperRoleSystem - if h.killStaleHelpers != nil { - h.killStaleHelpers(staleKey) - return - } - if h.sessionBroker != nil { - h.sessionBroker.KillStaleHelpers(staleKey) - } -} - func (h *Heartbeat) rememberDesktopOwner(desktopSessionID, helperSessionID string) { if desktopSessionID == "" || helperSessionID == "" { return @@ -574,26 +560,11 @@ func (h *Heartbeat) spawnHelperForDesktop(targetSession string) error { } } - sessionNum, err := sessionbroker.ParseWindowsSessionIDForHeartbeat(targetSession) + _, err := sessionbroker.ParseWindowsSessionIDForHeartbeat(targetSession) if err != nil { return fmt.Errorf("invalid session ID %q: %w", targetSession, err) } - - // Kill any stale helpers from previous sessions in this Windows session - // to release DXGI Desktop Duplication locks before spawning a new one. - h.killDesktopStaleHelpers(targetSession) - - // The heartbeat path spawns a one-off helper; we don't track its exit - // code here. Release the handle immediately — the lifecycle manager, - // which does respect exit codes, owns the canonical spawn path. - helper, err := sessionbroker.SpawnHelperInSession(sessionNum) - if err != nil { - return err - } - if helper != nil { - helper.Close() - } - return nil + return fmt.Errorf("no lifecycle-owned helper is connected for Windows session %s; waiting for lifecycle reconciliation", targetSession) } // findGUIUserUIDs returns the UIDs of users with a loginwindow process (macOS). diff --git a/agent/internal/heartbeat/handlers_desktop_helper_test.go b/agent/internal/heartbeat/handlers_desktop_helper_test.go index 0459f9f799..6bf93e802f 100644 --- a/agent/internal/heartbeat/handlers_desktop_helper_test.go +++ b/agent/internal/heartbeat/handlers_desktop_helper_test.go @@ -2,6 +2,10 @@ package heartbeat import ( "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" "testing" "time" @@ -11,6 +15,20 @@ import ( "github.com/breeze-rmm/agent/internal/sessionbroker" ) +func TestWindowsDesktopDemandNeverSpawnsOutsideLifecycleRegistry(t *testing.T) { + _, testFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + source, err := os.ReadFile(filepath.Join(filepath.Dir(testFile), "handlers_desktop_helper.go")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(source), "sessionbroker.SpawnHelperInSession") { + t.Fatal("desktop demand directly spawns a helper without lifecycle ownership") + } +} + func TestStartDesktopViaHelperPreservesTargetSessionOnRetry(t *testing.T) { serverConn1, clientConn1 := createTestSocketPair(t) serverIPC1 := ipc.NewConn(serverConn1) @@ -211,21 +229,6 @@ func TestStartDesktopViaHelperDoesNotReuseWrongTargetSessionHelper(t *testing.T) } } -func TestKillDesktopStaleHelpersUsesRoleScopedKey(t *testing.T) { - var seen string - h := &Heartbeat{ - killStaleHelpers: func(staleKey string) { - seen = staleKey - }, - } - - h.killDesktopStaleHelpers("42") - - if seen != "42-system" { - t.Fatalf("stale helper key = %q, want 42-system", seen) - } -} - func TestHandleStopDesktopUsesRecordedOwner(t *testing.T) { ownerServerConn, ownerClientConn := createTestSocketPair(t) ownerServerIPC := ipc.NewConn(ownerServerConn) diff --git a/agent/internal/heartbeat/heartbeat.go b/agent/internal/heartbeat/heartbeat.go index 43d527d717..db5bdb7a2f 100644 --- a/agent/internal/heartbeat/heartbeat.go +++ b/agent/internal/heartbeat/heartbeat.go @@ -143,6 +143,11 @@ type Command struct { Payload map[string]any `json:"payload"` } +type helperLifecycleController interface { + Stop() + Done() <-chan struct{} +} + type Heartbeat struct { config *config.Config secureToken *secmem.SecureString @@ -186,17 +191,17 @@ type Heartbeat struct { lastPatchUpdate time.Time // stamped at startup; gate then re-runs every PatchScanIntervalHours // User session helper (IPC) - helperToken string // retained copy of the helper-scoped token for connect-time pushes - helperTokenMu sync.RWMutex - sessionBroker *sessionbroker.Broker - helperLifecycle *sessionbroker.HelperLifecycleManager - lifecycleCancel context.CancelFunc - isService bool - isHeadless bool - scmSessionCh chan sessionbroker.SCMSessionEvent // fed by SCM handler - helperFinder func(targetSession string) *sessionbroker.Session - spawnHelper func(targetSession string) error - killStaleHelpers func(staleKey string) + helperToken string // retained copy of the helper-scoped token for connect-time pushes + helperTokenMu sync.RWMutex + sessionBroker *sessionbroker.Broker + helperLifecycle helperLifecycleController + lifecycleCancel context.CancelFunc + shutdownTimeout time.Duration + isService bool + isHeadless bool + scmSessionCh chan sessionbroker.SCMSessionEvent // fed by SCM handler + helperFinder func(targetSession string) *sessionbroker.Session + spawnHelper func(targetSession string) error // Shutdown seams keep lifecycle ordering directly testable without opening // sockets or spawning Windows processes. Production leaves these nil. @@ -843,30 +848,47 @@ func checkUpdateMarker() bool { return true } -func (h *Heartbeat) Start() { - // Start session broker for user helpers - if h.sessionBroker != nil { - go h.sessionBroker.Listen(h.stopChan) - h.startDarwinDesktopWatcher() +func bootstrapThenListen(bootstrap func() error, listen func()) error { + if bootstrap != nil { + if err := bootstrap(); err != nil { + return err + } } - if h.sessionCol != nil { - h.sessionCol.Start(h.stopChan) + if listen != nil { + listen() } + return nil +} +func (h *Heartbeat) Start() { // Proactively spawn helpers into user sessions so remote desktop works // instantly after reboot (Windows service only). The SCM session event // channel (created in constructor) is fed by the service handler // (service_windows.go) for instant notification; the lifecycle manager // also runs a slow reconcile tick as a safety net for helper crashes // and early-boot edge cases. + var lifecycle *sessionbroker.HelperLifecycleManager if h.scmSessionCh != nil && h.sessionBroker != nil { ctx, cancel := context.WithCancel(context.Background()) - lm := sessionbroker.NewHelperLifecycleManager(h.sessionBroker, h.scmSessionCh) + lifecycle = sessionbroker.NewHelperLifecycleManager(h.sessionBroker, h.scmSessionCh) h.mu.Lock() - h.helperLifecycle = lm + h.helperLifecycle = lifecycle h.lifecycleCancel = cancel h.mu.Unlock() - go lm.Start(ctx) + if err := bootstrapThenListen(lifecycle.Bootstrap, func() { + go h.sessionBroker.Listen(h.stopChan) + }); err != nil { + log.Error("helper lifecycle bootstrap failed; refusing broker listener without desired state", "error", err.Error()) + } + go lifecycle.Start(ctx) + } else if h.sessionBroker != nil { + go h.sessionBroker.Listen(h.stopChan) + } + if h.sessionBroker != nil { + h.startDarwinDesktopWatcher() + } + if h.sessionCol != nil { + h.sessionCol.Start(h.stopChan) } // Jitter: random delay before first heartbeat to avoid thundering herd @@ -1102,7 +1124,11 @@ func (h *Heartbeat) DrainAndWait(ctx context.Context) { func (h *Heartbeat) Stop() { h.stopOnce.Do(func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + shutdownTimeout := h.shutdownTimeout + if shutdownTimeout <= 0 { + shutdownTimeout = 5 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() if h.stopBrokerAcceptingAndWait != nil { @@ -1128,16 +1154,9 @@ func (h *Heartbeat) Stop() { lifecycleCancel() } if lifecycle != nil { - lifecycleStopped := make(chan struct{}) - go func() { - lifecycle.Stop() - close(lifecycleStopped) - }() - select { - case <-lifecycleStopped: - case <-ctx.Done(): - log.Warn("helper lifecycle process cleanup timed out") - } + // Stop bounds its own cleanup work. Keep this synchronous so broker + // close cannot overlap a still-running lifecycle cleanup goroutine. + lifecycle.Stop() select { case <-lifecycle.Done(): case <-ctx.Done(): diff --git a/agent/internal/heartbeat/heartbeat_test.go b/agent/internal/heartbeat/heartbeat_test.go index 261b370508..4fc7407dca 100644 --- a/agent/internal/heartbeat/heartbeat_test.go +++ b/agent/internal/heartbeat/heartbeat_test.go @@ -2,12 +2,55 @@ package heartbeat import ( "context" + "errors" "reflect" "sync" "testing" "time" ) +type blockingLifecycleShutdown struct { + entered chan struct{} + release chan struct{} + done chan struct{} +} + +func (l *blockingLifecycleShutdown) Stop() { + close(l.entered) + <-l.release + close(l.done) +} + +func (l *blockingLifecycleShutdown) Done() <-chan struct{} { return l.done } + +func TestBootstrapHelperLifecycleBeforeBrokerListen(t *testing.T) { + var order []string + err := bootstrapThenListen(func() error { + order = append(order, "bootstrap") + return nil + }, func() { + order = append(order, "listen") + }) + if err != nil { + t.Fatal(err) + } + if want := []string{"bootstrap", "listen"}; !reflect.DeepEqual(order, want) { + t.Fatalf("startup order = %v, want %v", order, want) + } +} + +func TestBootstrapFailureRefusesBrokerListen(t *testing.T) { + wantErr := errors.New("detector unavailable") + listened := false + err := bootstrapThenListen(func() error { return wantErr }, func() { listened = true }) + if !errors.Is(err, wantErr) { + t.Fatalf("bootstrapThenListen error = %v, want %v", err, wantErr) + } + if listened { + t.Fatal("broker listened without authoritative lifecycle desired state") + } +} + func TestHeartbeatStopOrdersBrokerBeforeLifecycleAndWaitsForReap(t *testing.T) { var mu sync.Mutex var order []string @@ -68,3 +111,42 @@ func TestHeartbeatStopOrdersBrokerBeforeLifecycleAndWaitsForReap(t *testing.T) { t.Fatalf("shutdown order = %v, want %v", got, want) } } + +func TestHeartbeatTimeoutNeverOverlapsLifecycleCleanupWithBrokerClose(t *testing.T) { + lifecycle := &blockingLifecycleShutdown{ + entered: make(chan struct{}), + release: make(chan struct{}), + done: make(chan struct{}), + } + brokerClosed := make(chan struct{}) + h := &Heartbeat{ + stopChan: make(chan struct{}), + helperLifecycle: lifecycle, + shutdownTimeout: 5 * time.Millisecond, + stopBrokerAcceptingAndWait: func(context.Context) error { return nil }, + closeSessionBroker: func() { close(brokerClosed) }, + } + stopped := make(chan struct{}) + go func() { + h.Stop() + close(stopped) + }() + <-lifecycle.entered + time.Sleep(20 * time.Millisecond) + select { + case <-brokerClosed: + t.Fatal("broker closed while lifecycle cleanup was still running") + default: + } + close(lifecycle.release) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("Heartbeat.Stop did not finish after lifecycle cleanup") + } + select { + case <-brokerClosed: + default: + t.Fatal("broker was not closed after lifecycle cleanup") + } +} diff --git a/agent/internal/sessionbroker/backup_test.go b/agent/internal/sessionbroker/backup_test.go index 6cbeefa964..6a727a826e 100644 --- a/agent/internal/sessionbroker/backup_test.go +++ b/agent/internal/sessionbroker/backup_test.go @@ -11,7 +11,6 @@ func TestSetClearBackupSession(t *testing.T) { b := &Broker{ sessions: make(map[string]*Session), byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } s := &Session{SessionID: "backup-test"} @@ -41,7 +40,6 @@ func TestStopBackupHelper_NilBroker(t *testing.T) { b := &Broker{ sessions: make(map[string]*Session), byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } // Should not panic when backup is nil b.StopBackupHelper() @@ -51,7 +49,6 @@ func TestForwardBackupCommand_NotConnected(t *testing.T) { b := &Broker{ sessions: make(map[string]*Session), byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } _, err := b.ForwardBackupCommand("cmd-1", "backup_run", nil, 5e9) if err == nil { @@ -75,7 +72,6 @@ func TestGetOrSpawnBackupHelper_ExistingSession(t *testing.T) { b := &Broker{ sessions: make(map[string]*Session), byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } // Pre-set a backup session diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index 0c0e741fff..f26c9b0ecc 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -250,12 +250,10 @@ type Broker struct { listener net.Listener rateLimiter *ipc.RateLimiter startTime time.Time // broker creation time, used for watchdog uptime - listenerMu sync.Mutex mu timedRWMutex sessions map[string]*Session // sessionID -> Session byIdentity map[string][]*Session // identity key -> Sessions (UID string on Unix, SID on Windows) - staleHelpers map[string][]int // winSessionID -> PIDs of disconnected helpers desiredHelperKeys map[HelperKey]struct{} helperByKey map[HelperKey]*Session helperByAuthKey map[AuthenticatedHelperKey]*Session @@ -271,10 +269,11 @@ type Broker struct { backup *backupHelper // backup helper process and session closed bool - acceptMu sync.Mutex - acceptStopped bool - preAuthConns map[net.Conn]bool // true once verified auth is being published - preAuthHandlers sync.WaitGroup + acceptMu sync.Mutex + acceptStopped bool + preAuthConns map[net.Conn]bool // true once verified auth is being published + preAuthHandlers sync.WaitGroup + beforePreAuthRead func() // test barrier; set before Listen/startAcceptedConnection // snap is an atomically updated snapshot of sessions/byIdentity/consoleUser. // Updated under b.mu.Lock() on every mutation. Read-only hot paths use @@ -314,7 +313,6 @@ func New(socketPath string, onMessage MessageHandler) *Broker { startTime: time.Now(), sessions: make(map[string]*Session), byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), desiredHelperKeys: make(map[HelperKey]struct{}), helperByKey: make(map[HelperKey]*Session), helperByAuthKey: make(map[AuthenticatedHelperKey]*Session), @@ -437,13 +435,23 @@ func (b *Broker) SetSessionAuthenticatedHandler(handler SessionAuthenticatedHand // fireSessionAuthenticated invokes the on-authenticated handler if set. func (b *Broker) fireSessionAuthenticated(session *Session) { + b.firePrimarySessionAuthenticated(session) + b.fireLifecycleSessionAuthenticated(session) +} + +func (b *Broker) firePrimarySessionAuthenticated(session *Session) { b.mu.RLock() handler := b.onSessionAuthed - callbacks := b.lifecycleAuthenticatedCallbacksLocked() b.mu.RUnlock() if handler != nil { handler(session) } +} + +func (b *Broker) fireLifecycleSessionAuthenticated(session *Session) { + b.mu.RLock() + callbacks := b.lifecycleAuthenticatedCallbacksLocked() + b.mu.RUnlock() for _, callback := range callbacks { callback(session) } @@ -534,12 +542,46 @@ func (b *Broker) LifecycleHelperKeys() []HelperKey { return keys } +// HasHelperKeyOwner reports whether an authenticated helper currently owns the +// logical Windows session/role key. Scheduled helpers may own a key without a +// proactive lifecycle registry entry. +func (b *Broker) HasHelperKeyOwner(key HelperKey) bool { + b.mu.RLock() + defer b.mu.RUnlock() + return b.helperByKey[key] != nil +} + +func (b *Broker) whileHelperKeyOwnedBy(key HelperKey, session *Session, fn func()) bool { + b.mu.RLock() + defer b.mu.RUnlock() + if session == nil || b.helperByKey[key] != session { + return false + } + fn() + return true +} + func (b *Broker) currentListener() net.Listener { - b.listenerMu.Lock() - defer b.listenerMu.Unlock() + b.acceptMu.Lock() + defer b.acceptMu.Unlock() return b.listener } +func (b *Broker) publishListener(listener net.Listener) bool { + if listener == nil { + return false + } + b.acceptMu.Lock() + if b.acceptStopped { + b.acceptMu.Unlock() + _ = listener.Close() + return false + } + b.listener = listener + b.acceptMu.Unlock() + return true +} + func (b *Broker) acceptingStopped() bool { b.acceptMu.Lock() defer b.acceptMu.Unlock() @@ -588,6 +630,8 @@ func (b *Broker) finishPreAuth(conn net.Conn) { func (b *Broker) StopAcceptingAndWait(ctx context.Context) error { b.acceptMu.Lock() b.acceptStopped = true + listener := b.listener + b.listener = nil connections := make([]net.Conn, 0, len(b.preAuthConns)) for conn, publishing := range b.preAuthConns { if !publishing { @@ -595,7 +639,7 @@ func (b *Broker) StopAcceptingAndWait(ctx context.Context) error { } } b.acceptMu.Unlock() - if listener := b.currentListener(); listener != nil { + if listener != nil { _ = listener.Close() } for _, conn := range connections { @@ -1432,6 +1476,9 @@ func (b *Broker) admitOrEvict(identityKey string) bool { } func (b *Broker) handleConnection(rawConn net.Conn) { + if b.beforePreAuthRead != nil { + b.beforePreAuthRead() + } // Set handshake deadline rawConn.SetDeadline(time.Now().Add(HandshakeTimeout)) @@ -1847,11 +1894,11 @@ func (b *Broker) handleConnection(rawConn net.Conn) { "desktopContext", session.DesktopContext, ) - // Notify the on-authenticated handler now that the session is fully - // registered (admitted, appended, scopes assigned). Fired outside the - // broker mutex and in a goroutine so a slow handler (e.g. one that pushes - // the helper token over IPC) can't block the accept loop or hold b.mu. - go b.fireSessionAuthenticated(session) + // Lifecycle ownership must be published synchronously before RecvLoop can + // fail and emit the corresponding close callback. The primary application + // handler remains asynchronous because it may perform slow IPC work. + b.fireLifecycleSessionAuthenticated(session) + go b.firePrimarySessionAuthenticated(session) // Keepalive: send periodic pings and close the session if pongs stop // arriving. Without this, a wedged helper (e.g. a capture process killed @@ -1871,10 +1918,9 @@ func (b *Broker) handleConnection(rawConn net.Conn) { log.Info("user helper disconnected", "uid", session.UID, "sessionId", session.SessionID) } -// removeSessionMapsLocked removes session from b.sessions and b.byIdentity -// and records the PID as stale. Caller must hold b.mu.Lock(). Does NOT -// Close() the session, publish a snapshot, or fire onSessionClosed; the -// caller is responsible for those. +// removeSessionMapsLocked removes session from b.sessions and b.byIdentity. +// Caller must hold b.mu.Lock(). Does NOT Close() the session, publish a +// snapshot, or fire onSessionClosed; the caller is responsible for those. func (b *Broker) removeSessionMapsLocked(session *Session) bool { if b.sessions[session.SessionID] != session { return false @@ -1902,16 +1948,6 @@ func (b *Broker) removeSessionMapsLocked(session *Session) bool { delete(b.helperByAuthKey, authKey) } } - - // Track the PID so it can be killed before spawning a replacement - // (Windows only — see trackStaleHelper; elsewhere this is a deliberate - // no-op). Don't kill here — the process may still be serving an active - // desktop session. Key includes role so SYSTEM and user helper stale - // PIDs are tracked separately. - if session.PID > 0 { - staleKey := session.WinSessionID + "-" + session.HelperRole - b.trackStaleHelper(staleKey, session.PID) - } return true } @@ -1973,49 +2009,6 @@ func (b *Broker) TerminateHelperKey(key HelperKey) { } } -// trackStaleHelper records a disconnected helper PID for later cleanup. -// Called under b.mu lock. -// -// Windows-only: the sole drain path is KillStaleHelpers, whose callers are all -// Windows-gated (it exists to release DXGI Desktop Duplication locks before -// respawning a helper). On macOS/Linux nothing ever drains this map, so -// tracking there grows unbounded for the life of the daemon (issue #2387). -// macOS handles helper teardown differently: live sessions are closed at -// logout via CloseSessionsByDesktopContext, and launchd `kickstart -k` kills -// any lingering instance on respawn. -func (b *Broker) trackStaleHelper(winSessionID string, pid int) { - if runtime.GOOS != "windows" { - return - } - b.staleHelpers[winSessionID] = append(b.staleHelpers[winSessionID], pid) -} - -// KillStaleHelpers kills any disconnected helper processes for the given -// Windows session. Call this before spawning a new helper to release DXGI -// Desktop Duplication locks held by orphaned processes. -// -// Windows-only contract: trackStaleHelper only records PIDs on Windows, so on -// any other platform this is always a no-op against an empty map — do not add -// non-Windows callers expecting it to find anything. -func (b *Broker) KillStaleHelpers(winSessionID string) { - b.mu.Lock() - pids := b.staleHelpers[winSessionID] - delete(b.staleHelpers, winSessionID) - b.mu.Unlock() - - for _, pid := range pids { - if proc, err := os.FindProcess(pid); err == nil { - if err := proc.Kill(); err != nil { - log.Debug("failed to kill stale userhelper (may have already exited)", - "pid", pid, "error", err.Error()) - } else { - log.Info("killed stale userhelper before respawn", - "pid", pid, "winSessionID", winSessionID) - } - } - } -} - // CloseSessionsByDesktopContext closes all sessions with the given desktop // context (e.g., "user_session"). Used on macOS to tear down stale helpers // after a logout event. Returns the number of sessions closed. diff --git a/agent/internal/sessionbroker/broker_leak_test.go b/agent/internal/sessionbroker/broker_leak_test.go index e826ff7cb3..682747e876 100644 --- a/agent/internal/sessionbroker/broker_leak_test.go +++ b/agent/internal/sessionbroker/broker_leak_test.go @@ -67,7 +67,6 @@ func TestKeepaliveReapsStrandedSession(t *testing.T) { b := &Broker{ sessions: map[string]*Session{session.SessionID: session}, byIdentity: map[string][]*Session{session.IdentityKey: {session}}, - staleHelpers: make(map[string][]int), } done := make(chan struct{}) @@ -124,7 +123,6 @@ func TestKeepaliveKeepsHealthySessionAlive(t *testing.T) { b := &Broker{ sessions: map[string]*Session{session.SessionID: session}, byIdentity: map[string][]*Session{session.IdentityKey: {session}}, - staleHelpers: make(map[string][]int), } keepaliveDone := make(chan struct{}) @@ -190,7 +188,6 @@ func TestMaybeStartKeepalive_WatchdogSessionStaysAlive(t *testing.T) { b := &Broker{ sessions: map[string]*Session{session.SessionID: session}, byIdentity: map[string][]*Session{session.IdentityKey: {session}}, - staleHelpers: make(map[string][]int), } b.maybeStartKeepalive(session, ipc.HelperRoleWatchdog) @@ -228,7 +225,6 @@ func TestMaybeStartKeepalive_SystemSessionIsReaped(t *testing.T) { b := &Broker{ sessions: map[string]*Session{session.SessionID: session}, byIdentity: map[string][]*Session{session.IdentityKey: {session}}, - staleHelpers: make(map[string][]int), } b.maybeStartKeepalive(session, ipc.HelperRoleSystem) @@ -264,7 +260,6 @@ func TestAdmitOrEvictEvictsIdleSession(t *testing.T) { b := &Broker{ sessions: make(map[string]*Session), byIdentity: map[string][]*Session{identity: sessions}, - staleHelpers: make(map[string][]int), } for _, s := range sessions { b.sessions[s.SessionID] = s @@ -301,7 +296,6 @@ func TestAdmitOrEvictRejectsWhenAllRecent(t *testing.T) { b := &Broker{ sessions: make(map[string]*Session), byIdentity: map[string][]*Session{identity: sessions}, - staleHelpers: make(map[string][]int), } for _, s := range sessions { b.sessions[s.SessionID] = s @@ -334,7 +328,6 @@ func TestKeepaliveClosesAfterRepeatedSendFailures(t *testing.T) { b := &Broker{ sessions: map[string]*Session{session.SessionID: session}, byIdentity: map[string][]*Session{session.IdentityKey: {session}}, - staleHelpers: make(map[string][]int), } done := make(chan struct{}) @@ -373,7 +366,6 @@ func TestRecvLoopDispatchesPongToNotePong(t *testing.T) { b := &Broker{ sessions: map[string]*Session{session.SessionID: session}, byIdentity: map[string][]*Session{session.IdentityKey: {session}}, - staleHelpers: make(map[string][]int), } recvDone := make(chan struct{}) @@ -417,7 +409,6 @@ func TestAdmitOrEvictUnderCapAdmits(t *testing.T) { b := &Broker{ sessions: map[string]*Session{s.SessionID: s}, byIdentity: map[string][]*Session{identity: {s}}, - staleHelpers: make(map[string][]int), } if !b.admitOrEvict(identity) { diff --git a/agent/internal/sessionbroker/broker_lifecycle_test.go b/agent/internal/sessionbroker/broker_lifecycle_test.go index 339d598250..8cabf38a56 100644 --- a/agent/internal/sessionbroker/broker_lifecycle_test.go +++ b/agent/internal/sessionbroker/broker_lifecycle_test.go @@ -3,6 +3,11 @@ package sessionbroker import ( "context" "net" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" "sync" "testing" "time" @@ -10,6 +15,20 @@ import ( "github.com/breeze-rmm/agent/internal/ipc" ) +func TestBrokerLifecycleCleanupNeverReopensRecordedPID(t *testing.T) { + _, testFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + source, err := os.ReadFile(filepath.Join(filepath.Dir(testFile), "broker.go")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(source), "os.FindProcess") { + t.Fatal("broker lifecycle cleanup reopens a later process by recorded PID") + } +} + type fakeOwnedPeerProcess struct { pid uint32 @@ -21,6 +40,59 @@ type fakeOwnedPeerProcess struct { release chan struct{} } +type closeTrackingListener struct { + closed chan struct{} + closeOnce sync.Once +} + +func newCloseTrackingListener() *closeTrackingListener { + return &closeTrackingListener{closed: make(chan struct{})} +} + +func (l *closeTrackingListener) Accept() (net.Conn, error) { + <-l.closed + return nil, net.ErrClosed +} + +func (l *closeTrackingListener) Close() error { + l.closeOnce.Do(func() { close(l.closed) }) + return nil +} + +func (l *closeTrackingListener) Addr() net.Addr { return &net.UnixAddr{Name: "test", Net: "unix"} } + +func TestListenerCreatedDuringStopIsRefusedAndClosed(t *testing.T) { + b := New("listener-race-"+t.Name(), nil) + listener := newCloseTrackingListener() + created := make(chan struct{}) + releasePublication := make(chan struct{}) + published := make(chan bool, 1) + go func() { + close(created) + <-releasePublication + published <- b.publishListener(listener) + }() + <-created + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := b.StopAcceptingAndWait(ctx); err != nil { + t.Fatal(err) + } + close(releasePublication) + if <-published { + t.Fatal("listener was published after acceptance stopped") + } + select { + case <-listener.closed: + default: + t.Fatal("listener created after stop was not closed") + } + if got := b.currentListener(); got != nil { + t.Fatalf("current listener = %T, want nil", got) + } +} + func newFakeOwnedPeerProcess(pid uint32) *fakeOwnedPeerProcess { return &fakeOwnedPeerProcess{pid: pid, alive: true} } @@ -99,6 +171,35 @@ func TestLifecycleObserversAreAdditiveAndRunAfterUnlock(t *testing.T) { } } +func TestImmediateDisconnectCannotNotifyLifecycleCloseBeforeAuthenticated(t *testing.T) { + b := New("observer-order-"+t.Name(), nil) + proc := newFakeOwnedPeerProcess(5090) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + session := newOwnedSession(t, b, key, proc) + var mu sync.Mutex + var order []string + b.AddSessionLifecycleObserver(func(*Session) { + mu.Lock() + order = append(order, "authenticated") + mu.Unlock() + }, func(*Session) { + mu.Lock() + order = append(order, "closed") + mu.Unlock() + }) + + b.fireLifecycleSessionAuthenticated(session) + b.removeSession(session) // models RecvLoop returning immediately after publication + + mu.Lock() + got := append([]string(nil), order...) + mu.Unlock() + want := []string{"authenticated", "closed"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("lifecycle observer order = %v, want %v", got, want) + } +} + func TestSessionCloseReleasesOwnedPeerProcessOnce(t *testing.T) { b := New("peer-close-"+t.Name(), nil) proc := newFakeOwnedPeerProcess(5100) @@ -159,15 +260,30 @@ func TestConcurrentTerminateAndSessionCloseTerminatesAndConsumesPeerHandleOnce(t func TestBrokerStopAcceptingAndWaitUnblocksStalledPreAuthConnection(t *testing.T) { b := New("preauth-"+t.Name(), nil) + authBlocked := make(chan struct{}) + releaseAuth := make(chan struct{}) + b.beforePreAuthRead = func() { + close(authBlocked) + <-releaseAuth + } server, client := net.Pipe() defer client.Close() if !b.startAcceptedConnection(server) { t.Fatal("failed to register accepted connection") } + <-authBlocked ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - if err := b.StopAcceptingAndWait(ctx); err != nil { + stopped := make(chan error, 1) + go func() { stopped <- b.StopAcceptingAndWait(ctx) }() + select { + case err := <-stopped: + t.Fatalf("StopAcceptingAndWait returned while auth handler was blocked: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(releaseAuth) + if err := <-stopped; err != nil { t.Fatalf("StopAcceptingAndWait: %v", err) } } diff --git a/agent/internal/sessionbroker/broker_stale_helpers_test.go b/agent/internal/sessionbroker/broker_stale_helpers_test.go deleted file mode 100644 index 5f4ee59cda..0000000000 --- a/agent/internal/sessionbroker/broker_stale_helpers_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package sessionbroker - -import ( - "runtime" - "testing" -) - -// TestTrackStaleHelperGatedToWindows is the regression test for the -// staleHelpers part of issue #2387: the only drain path (KillStaleHelpers) is -// Windows-only, so tracking on other platforms grows the map unbounded for -// the life of the daemon. -func TestTrackStaleHelperGatedToWindows(t *testing.T) { - b := &Broker{staleHelpers: make(map[string][]int)} - - b.mu.Lock() - b.trackStaleHelper("1-user", 4242) - b.trackStaleHelper("1-user", 4243) - b.trackStaleHelper("2-system", 4244) - b.mu.Unlock() - - if runtime.GOOS == "windows" { - if got := len(b.staleHelpers["1-user"]); got != 2 { - t.Fatalf("windows: expected 2 tracked PIDs for key 1-user, got %d", got) - } - if got := len(b.staleHelpers["2-system"]); got != 1 { - t.Fatalf("windows: expected 1 tracked PID for key 2-system, got %d", got) - } - return - } - - // Non-Windows: nothing may accumulate — there is no drain path. - if got := len(b.staleHelpers); got != 0 { - t.Fatalf("non-windows: staleHelpers must stay empty (no drain path exists), got %d keys: %v", - got, b.staleHelpers) - } -} diff --git a/agent/internal/sessionbroker/broker_test.go b/agent/internal/sessionbroker/broker_test.go index 7be40aef3a..e504b65828 100644 --- a/agent/internal/sessionbroker/broker_test.go +++ b/agent/internal/sessionbroker/broker_test.go @@ -89,7 +89,6 @@ func TestSessionForUserPrefersMostRecentUserSession(t *testing.T) { userSessionNew.SessionID: userSessionNew, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } got := b.SessionForUser("alice") @@ -114,7 +113,6 @@ func TestSessionForUserSelectsRDPHelper(t *testing.T) { rdpSession.SessionID: rdpSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } if got := b.SessionForUser("alice"); got != rdpSession { @@ -141,7 +139,6 @@ func TestLaunchProcessViaUserHelperBroadcastsToAllUserSessions(t *testing.T) { newerSession.SessionID: newerSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } seen := make(chan string, 2) @@ -208,7 +205,6 @@ func TestLaunchProcessViaUserHelperForSessionTargetsMatchingHelper(t *testing.T) sessionB.SessionID: sessionB, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } seen := make(chan string, 1) @@ -265,7 +261,6 @@ func TestLaunchProcessViaUserHelperForSessionTargetsMatchingRDPHelper(t *testing rdpSession.SessionID: rdpSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } seen := make(chan string, 1) @@ -316,7 +311,6 @@ func TestReapIdleSessionsReapsStrandedCaptureSessions(t *testing.T) { session.SessionID: session, }, byIdentity: map[string][]*Session{session.IdentityKey: []*Session{session}}, - staleHelpers: make(map[string][]int), } b.reapIdleSessions() @@ -430,7 +424,6 @@ func TestPreferredSessionWithScopePrefersNewestUserHelper(t *testing.T) { newerUser.SessionID: newerUser, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } got := b.PreferredSessionWithScope("run_as_user") @@ -467,7 +460,6 @@ func TestPreferredDesktopSession_LoginWindowConsole_PrefersLoginWindowHelper(t * loginSession.SessionID: loginSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } // Without console user set, user_session wins (existing behavior). @@ -512,7 +504,6 @@ func TestPreferredDesktopSession_LoggedInConsole_PrefersUserSession(t *testing.T loginSession.SessionID: loginSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } // With a real user logged in, user_session should still win. @@ -542,7 +533,6 @@ func TestPreferredDesktopSession_LoginWindowConsole_OnlyLoginHelpers(t *testing. userSession.SessionID: userSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } b.SetConsoleUser("loginwindow") @@ -585,7 +575,6 @@ func TestPreferredDesktopSession_LoginWindow_DeterministicRegardlessOfOrder(t *t loginSession.SessionID: loginSession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } b.SetConsoleUser("loginwindow") @@ -627,7 +616,6 @@ func TestCloseSessionsByDesktopContext(t *testing.T) { notifySession.SessionID: notifySession, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } closed := b.CloseSessionsByDesktopContext(ipc.DesktopContextUserSession) @@ -689,7 +677,6 @@ func TestCloseSessionsByDesktopContext_MultipleMatches(t *testing.T) { loginSess.SessionID: loginSess, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), } closed := b.CloseSessionsByDesktopContext(ipc.DesktopContextUserSession) diff --git a/agent/internal/sessionbroker/broker_unix.go b/agent/internal/sessionbroker/broker_unix.go index c183ef5dfc..bebda8e847 100644 --- a/agent/internal/sessionbroker/broker_unix.go +++ b/agent/internal/sessionbroker/broker_unix.go @@ -34,9 +34,9 @@ func (b *Broker) setupSocket() error { return fmt.Errorf("chmod %s: %w", b.socketPath, err) } - b.listenerMu.Lock() - b.listener = listener - b.listenerMu.Unlock() + if !b.publishListener(listener) { + return fmt.Errorf("broker acceptance already stopped") + } return nil } diff --git a/agent/internal/sessionbroker/broker_windows.go b/agent/internal/sessionbroker/broker_windows.go index 3ca33210ef..b0403e9165 100644 --- a/agent/internal/sessionbroker/broker_windows.go +++ b/agent/internal/sessionbroker/broker_windows.go @@ -26,9 +26,9 @@ func (b *Broker) setupSocket() error { return fmt.Errorf("listen pipe %s: %w", b.socketPath, err) } - b.listenerMu.Lock() - b.listener = listener - b.listenerMu.Unlock() + if !b.publishListener(listener) { + return fmt.Errorf("broker acceptance already stopped") + } log.Info("named pipe listener created", "pipe", b.socketPath) return nil } diff --git a/agent/internal/sessionbroker/console_session_gate_test.go b/agent/internal/sessionbroker/console_session_gate_test.go index 71feffe8ce..e3050eb5ee 100644 --- a/agent/internal/sessionbroker/console_session_gate_test.go +++ b/agent/internal/sessionbroker/console_session_gate_test.go @@ -70,7 +70,6 @@ func TestPreferredRunAsUserSessionIgnoresRDPHelper(t *testing.T) { rdpUser.SessionID: rdpUser, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } @@ -103,7 +102,6 @@ func TestPreferredRunAsUserSessionFiltersByConsoleSession(t *testing.T) { otherUser.SessionID: otherUser, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } @@ -140,7 +138,6 @@ func TestPreferredRunAsUserSessionNoConsoleHelper(t *testing.T) { otherUser.SessionID: otherUser, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } @@ -177,7 +174,6 @@ func TestPreferredRunAsUserSessionWarnsWhenConsoleBindingSuppressesDelivery(t *t otherUser.SessionID: otherUser, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } @@ -226,7 +222,6 @@ func TestPreferredRunAsUserSessionWarnCountsAllExcludedHelpers(t *testing.T) { c.SessionID: c, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } @@ -259,7 +254,6 @@ func TestPreferredRunAsUserSessionWarnsWhenConsoleLookupFailed(t *testing.T) { helper.SessionID: helper, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), // "0" is the WTSGetActiveConsoleSessionId failure / Session-0 sentinel, // normalized to "" by ConsoleSessionID(). consoleSessionIDFn: func() string { return "0" }, @@ -288,7 +282,6 @@ func TestPreferredRunAsUserSessionNoWarnWhenNoUserHelper(t *testing.T) { b := &Broker{ sessions: map[string]*Session{}, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } @@ -325,7 +318,6 @@ func TestPreferredRunAsUserSessionNoWarnOnConsoleMatch(t *testing.T) { otherUser.SessionID: otherUser, }, byIdentity: make(map[string][]*Session), - staleHelpers: make(map[string][]int), consoleSessionIDFn: func() string { return "1" }, } diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index f7d33c2192..f77c1ea573 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -156,14 +156,13 @@ func (m *HelperLifecycleManager) finishStart() { m.doneOnce.Do(func() { close(m.done) }) } -func (m *HelperLifecycleManager) reconcile() { - if m.detector == nil || m.broker == nil || m.spawner == nil { - return +func (m *HelperLifecycleManager) detectedDesired() (map[HelperKey]bool, error) { + if m.detector == nil { + return map[HelperKey]bool{}, nil } sessions, err := m.detector.ListSessions() if err != nil { - log.Warn("lifecycle: failed to list sessions", "error", err.Error()) - return + return nil, err } desired := make(map[HelperKey]bool, len(sessions)*2) for _, session := range sessions { @@ -174,6 +173,39 @@ func (m *HelperLifecycleManager) reconcile() { desired[key] = true } } + return desired, nil +} + +// Bootstrap publishes one detector snapshot without spawning. Heartbeat calls +// this before the broker starts accepting helpers so scheduled helpers can +// authenticate against authoritative desired state during startup. +func (m *HelperLifecycleManager) Bootstrap() error { + if m.broker == nil { + return nil + } + desired, err := m.detectedDesired() + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.stopping { + return nil + } + m.desired = cloneDesired(desired) + m.publishDesired(desired) + return nil +} + +func (m *HelperLifecycleManager) reconcile() { + if m.detector == nil || m.broker == nil || m.spawner == nil { + return + } + desired, err := m.detectedDesired() + if err != nil { + log.Warn("lifecycle: failed to list sessions", "error", err.Error()) + return + } m.mu.Lock() if m.stopping { @@ -227,6 +259,10 @@ func (m *HelperLifecycleManager) spawnKey(key HelperKey) { m.mu.Unlock() return } + if m.broker != nil && m.broker.HasHelperKeyOwner(key) { + m.mu.Unlock() + return + } generation, reserved := m.registry.reserve(key, time.Now()) m.mu.Unlock() if !reserved { @@ -301,8 +337,9 @@ func (m *HelperLifecycleManager) stopKey(key HelperKey) { m.registry.detach(key, entry.generation) return } - m.registry.detach(key, entry.generation) - log.Warn("lifecycle: helper process did not reap before shutdown deadline", "helperKey", key.String(), "pid", processID(entry.process)) + if !m.registry.detach(key, entry.generation) { + log.Warn("lifecycle: helper process did not reap before shutdown deadline; retaining live stopping generation", "helperKey", key.String(), "pid", processID(entry.process)) + } } func processID(process helperProcess) uint32 { @@ -377,7 +414,12 @@ func (m *HelperLifecycleManager) sessionAuthenticated(session *Session) { if !ok { return } - m.registry.markConnected(key, uint32(session.PID), session) + if m.broker == nil { + return + } + m.broker.whileHelperKeyOwnedBy(key, session, func() { + m.registry.markConnected(key, uint32(session.PID), session) + }) } func (m *HelperLifecycleManager) sessionClosed(session *Session) { diff --git a/agent/internal/sessionbroker/lifecycle_registry.go b/agent/internal/sessionbroker/lifecycle_registry.go index 42cd623887..6685eefd65 100644 --- a/agent/internal/sessionbroker/lifecycle_registry.go +++ b/agent/internal/sessionbroker/lifecycle_registry.go @@ -163,7 +163,7 @@ func (r *helperRegistry) detach(key HelperKey, generation uint64) bool { if entry == nil || entry.generation != generation { return false } - if entry.state != helperStopping && entry.process != nil && entry.process.Alive() { + if entry.process != nil && entry.process.Alive() { return false } delete(r.current, key) diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index 2556a0ba4e..76e03abf09 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -7,6 +7,39 @@ import ( "time" ) +func TestSessionAuthenticatedMarksOnlyCurrentBrokerOwnerConnected(t *testing.T) { + b := New("current-owner-"+t.Name(), nil) + m := newHelperLifecycleManager(b, fakeLifecycleDetector{}, nil, &fakeHelperSpawner{}) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + proc := newFakeHelperProcess(4090) + m.registry.attach(key, proc, "helper", "system-helper") + session := &Session{PID: int(proc.pid), WinSessionID: "7", HelperRole: "system"} + t.Cleanup(func() { + proc.markExited(0) + m.Stop() + b.Close() + }) + + m.sessionAuthenticated(session) + m.registry.mu.Lock() + stateWithoutOwner := m.registry.current[key].state + m.registry.mu.Unlock() + if stateWithoutOwner == helperConnected { + t.Fatal("non-owning authenticated callback marked registry connected") + } + + b.mu.Lock() + b.helperByKey[key] = session + b.mu.Unlock() + m.sessionAuthenticated(session) + m.registry.mu.Lock() + stateWithOwner := m.registry.current[key].state + m.registry.mu.Unlock() + if stateWithOwner != helperConnected { + t.Fatalf("current owner state = %q, want connected", stateWithOwner) + } +} + type fakeLifecycleDetector struct { sessions []DetectedSession } @@ -31,14 +64,16 @@ type fakeHelperProcess struct { waitCount int terminateCount int closeCount int + terminateExits bool } func newFakeHelperProcess(pid uint32) *fakeHelperProcess { return &fakeHelperProcess{ - pid: pid, - path: "breeze-user-helper.exe", - alive: true, - exited: make(chan struct{}), + pid: pid, + path: "breeze-user-helper.exe", + alive: true, + exited: make(chan struct{}), + terminateExits: true, } } @@ -54,11 +89,43 @@ func (p *fakeHelperProcess) Alive() bool { func (p *fakeHelperProcess) Terminate() error { p.mu.Lock() p.terminateCount++ + exits := p.terminateExits p.mu.Unlock() - p.markExited(1) + if exits { + p.markExited(1) + } return nil } +func TestTimedOutStopRetainsLiveGenerationAndSuppressesReplacement(t *testing.T) { + key := HelperKey{WindowsSessionID: 7, Role: "system"} + proc := newFakeHelperProcess(4075) + proc.terminateExits = false + spawner := &fakeHelperSpawner{byKey: map[HelperKey]helperProcess{key: proc}} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) + m.gracePeriod = time.Millisecond + m.finalWait = time.Millisecond + m.reconcile() + m.registry.mu.Lock() + m.registry.current[key].lastFailure = time.Now().Add(-time.Minute) + m.registry.mu.Unlock() + + started := time.Now() + m.stopKey(key) + if elapsed := time.Since(started); elapsed > 100*time.Millisecond { + t.Fatalf("bounded stop took %v", elapsed) + } + if got := m.registry.processID(key); got != proc.pid { + t.Fatalf("live stopping generation PID = %d, want %d", got, proc.pid) + } + m.spawnKey(key) + if got := spawner.SpawnCount(key); got != 1 { + t.Fatalf("spawn count with live stopping generation = %d, want 1", got) + } + + proc.markExited(1) +} + func (p *fakeHelperProcess) Wait() (int, error) { p.mu.Lock() p.waitCount++ @@ -95,6 +162,7 @@ func (p *fakeHelperProcess) counts() (wait, terminate, close int) { type fakeHelperSpawner struct { mu sync.Mutex processes []helperProcess + byKey map[HelperKey]helperProcess spawned map[HelperKey]int closed int } @@ -106,6 +174,10 @@ func (s *fakeHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { s.spawned = make(map[HelperKey]int) } s.spawned[key]++ + if proc := s.byKey[key]; proc != nil { + delete(s.byKey, key) + return proc, nil + } if len(s.processes) == 0 { return newFakeHelperProcess(uint32(5000 + s.spawned[key])), nil } @@ -114,6 +186,50 @@ func (s *fakeHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { return proc, nil } +func TestLifecycleBootstrapPublishesDesiredKeysBeforeSpawning(t *testing.T) { + b := New("bootstrap-"+t.Name(), nil) + spawner := &fakeHelperSpawner{} + m := newHelperLifecycleManager(b, fakeLifecycleDetector{sessions: []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}}, nil, spawner) + t.Cleanup(func() { + m.Stop() + b.Close() + }) + + if err := m.Bootstrap(); err != nil { + t.Fatalf("Bootstrap: %v", err) + } + for _, role := range []string{"system", "user"} { + key := HelperKey{WindowsSessionID: 7, Role: role} + if !b.helperKeyDesired(key) { + t.Fatalf("%s key was not published by bootstrap", role) + } + if got := spawner.SpawnCount(key); got != 0 { + t.Fatalf("%s spawn count during bootstrap = %d, want 0", role, got) + } + } +} + +func TestReconcileDoesNotSpawnWhenScheduledHelperOwnsLogicalKey(t *testing.T) { + b := New("scheduled-owner-"+t.Name(), nil) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + proc := newFakeOwnedPeerProcess(4050) + newOwnedSession(t, b, key, proc) + spawner := &fakeHelperSpawner{} + m := newHelperLifecycleManager(b, fakeLifecycleDetector{sessions: []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}}, nil, spawner) + m.gracePeriod = 0 + m.finalWait = 0 + t.Cleanup(func() { + m.Stop() + b.Close() + }) + + m.reconcile() + + if got := spawner.SpawnCount(key); got != 0 { + t.Fatalf("scheduled owner duplicate spawn count = %d, want 0", got) + } +} + func (s *fakeHelperSpawner) Close() error { s.mu.Lock() s.closed++ @@ -182,9 +298,9 @@ func TestStopSessionTerminatesBothRoles(t *testing.T) { func TestConcurrentReconcileStopAndExitOwnsHandleOnce(t *testing.T) { proc := newFakeHelperProcess(4300) - spawner := &fakeHelperSpawner{processes: []helperProcess{proc}} - m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) key := HelperKey{WindowsSessionID: 7, Role: "system"} + spawner := &fakeHelperSpawner{byKey: map[HelperKey]helperProcess{key: proc}} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) var reconcileWG sync.WaitGroup for range 16 { diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index ff219ef913..b13f49239c 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -77,9 +77,8 @@ func (s *SpawnedHelper) Terminate() error { } // Wait blocks until the spawned helper process exits and returns its exit -// code. Returns -1 + error on failure. The process handle is released -// automatically after Wait returns so callers do not need to call Close -// in the normal path. +// code. Returns -1 + error on failure. Wait does not release the process +// handle; the lifecycle watcher calls Close after Wait returns. func (s *SpawnedHelper) Wait() (int, error) { if s == nil { return -1, fmt.Errorf("SpawnedHelper: no handle") From 7a7f029fcb452102b47e4344a4087c6447cb2c62 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 16:19:47 -0600 Subject: [PATCH 09/47] fix(agent): close lifecycle handoff races --- agent/internal/sessionbroker/broker.go | 35 ++++++++-- .../sessionbroker/broker_lifecycle_test.go | 25 +++++++ agent/internal/sessionbroker/broker_unix.go | 16 ++--- .../internal/sessionbroker/broker_windows.go | 11 ++- .../internal/sessionbroker/lifecycle_core.go | 19 +++++ .../sessionbroker/lifecycle_registry_test.go | 70 +++++++++++++++++++ 6 files changed, 152 insertions(+), 24 deletions(-) diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index f26c9b0ecc..c002f5b91c 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -269,11 +269,12 @@ type Broker struct { backup *backupHelper // backup helper process and session closed bool - acceptMu sync.Mutex - acceptStopped bool - preAuthConns map[net.Conn]bool // true once verified auth is being published - preAuthHandlers sync.WaitGroup - beforePreAuthRead func() // test barrier; set before Listen/startAcceptedConnection + acceptMu sync.Mutex + acceptStopped bool + preAuthConns map[net.Conn]bool // true once verified auth is being published + preAuthHandlers sync.WaitGroup + beforePreAuthRead func() // test barrier; set before Listen/startAcceptedConnection + afterListenerPublished func() // test barrier; set before Listen // snap is an atomically updated snapshot of sessions/byIdentity/consoleUser. // Updated under b.mu.Lock() on every mutation. Read-only hot paths use @@ -472,9 +473,20 @@ func (b *Broker) SetConsoleUser(username string) { // Listen starts the IPC listener. Blocks until stopChan is closed. func (b *Broker) Listen(stopChan <-chan struct{}) error { - if err := b.setupSocket(); err != nil { + listener, err := b.setupSocket() + if err != nil { return fmt.Errorf("sessionbroker: setup socket: %w", err) } + return b.listenOn(listener, stopChan) +} + +func (b *Broker) listenOn(listener net.Listener, stopChan <-chan struct{}) error { + if !b.publishListener(listener) { + return nil + } + if b.afterListenerPublished != nil { + b.afterListenerPublished() + } log.Info("session broker listening", "path", b.socketPath) @@ -482,7 +494,6 @@ func (b *Broker) Listen(stopChan <-chan struct{}) error { go b.idleReaper(stopChan) // Accept loop - listener := b.currentListener() go func() { for { conn, err := listener.Accept() @@ -551,6 +562,16 @@ func (b *Broker) HasHelperKeyOwner(key HelperKey) bool { return b.helperByKey[key] != nil } +func (b *Broker) helperKeyOwnerPID(key HelperKey) (uint32, bool) { + b.mu.RLock() + defer b.mu.RUnlock() + owner := b.helperByKey[key] + if owner == nil || owner.PID <= 0 { + return 0, owner != nil + } + return uint32(owner.PID), true +} + func (b *Broker) whileHelperKeyOwnedBy(key HelperKey, session *Session, fn func()) bool { b.mu.RLock() defer b.mu.RUnlock() diff --git a/agent/internal/sessionbroker/broker_lifecycle_test.go b/agent/internal/sessionbroker/broker_lifecycle_test.go index 8cabf38a56..439774a901 100644 --- a/agent/internal/sessionbroker/broker_lifecycle_test.go +++ b/agent/internal/sessionbroker/broker_lifecycle_test.go @@ -93,6 +93,31 @@ func TestListenerCreatedDuringStopIsRefusedAndClosed(t *testing.T) { } } +func TestListenStopBetweenPublicationAndAcceptCaptureDoesNotPanic(t *testing.T) { + b := New("listener-handoff-"+t.Name(), nil) + listener := newCloseTrackingListener() + stopChan := make(chan struct{}) + b.afterListenerPublished = func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := b.StopAcceptingAndWait(ctx); err != nil { + t.Fatalf("StopAcceptingAndWait: %v", err) + } + close(stopChan) + } + + done := make(chan error, 1) + go func() { done <- b.listenOn(listener, stopChan) }() + select { + case err := <-done: + if err != nil { + t.Fatalf("Listen: %v", err) + } + case <-time.After(time.Second): + t.Fatal("Listen did not return after stop between publication and accept capture") + } +} + func newFakeOwnedPeerProcess(pid uint32) *fakeOwnedPeerProcess { return &fakeOwnedPeerProcess{pid: pid, alive: true} } diff --git a/agent/internal/sessionbroker/broker_unix.go b/agent/internal/sessionbroker/broker_unix.go index bebda8e847..b3a2244539 100644 --- a/agent/internal/sessionbroker/broker_unix.go +++ b/agent/internal/sessionbroker/broker_unix.go @@ -9,35 +9,31 @@ import ( "path/filepath" ) -func (b *Broker) setupSocket() error { +func (b *Broker) setupSocket() (net.Listener, error) { // Remove stale socket file os.Remove(b.socketPath) // Ensure directory exists dir := filepath.Dir(b.socketPath) if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("mkdir %s: %w", dir, err) + return nil, fmt.Errorf("mkdir %s: %w", dir, err) } if err := os.Chmod(dir, 0755); err != nil { - return fmt.Errorf("chmod %s: %w", dir, err) + return nil, fmt.Errorf("chmod %s: %w", dir, err) } listener, err := net.Listen("unix", b.socketPath) if err != nil { - return fmt.Errorf("listen %s: %w", b.socketPath, err) + return nil, fmt.Errorf("listen %s: %w", b.socketPath, err) } // Allow normal user helpers to traverse the directory and connect to the // socket. Peer credential verification and binary checks remain the gate. if err := os.Chmod(b.socketPath, 0660); err != nil { listener.Close() - return fmt.Errorf("chmod %s: %w", b.socketPath, err) + return nil, fmt.Errorf("chmod %s: %w", b.socketPath, err) } - - if !b.publishListener(listener) { - return fmt.Errorf("broker acceptance already stopped") - } - return nil + return listener, nil } // peerWinSessionID is a no-op on non-Windows platforms. diff --git a/agent/internal/sessionbroker/broker_windows.go b/agent/internal/sessionbroker/broker_windows.go index b0403e9165..37cf272c71 100644 --- a/agent/internal/sessionbroker/broker_windows.go +++ b/agent/internal/sessionbroker/broker_windows.go @@ -4,6 +4,7 @@ package sessionbroker import ( "fmt" + "net" "github.com/Microsoft/go-winio" "golang.org/x/sys/windows" @@ -14,7 +15,7 @@ import ( // excludes service accounts, batch jobs, and network logons. const pipeSecurity = "D:P(A;;GA;;;SY)(A;;GRGW;;;IU)" -func (b *Broker) setupSocket() error { +func (b *Broker) setupSocket() (net.Listener, error) { cfg := &winio.PipeConfig{ SecurityDescriptor: pipeSecurity, InputBufferSize: 64 * 1024, @@ -23,14 +24,10 @@ func (b *Broker) setupSocket() error { listener, err := winio.ListenPipe(b.socketPath, cfg) if err != nil { - return fmt.Errorf("listen pipe %s: %w", b.socketPath, err) - } - - if !b.publishListener(listener) { - return fmt.Errorf("broker acceptance already stopped") + return nil, fmt.Errorf("listen pipe %s: %w", b.socketPath, err) } log.Info("named pipe listener created", "pipe", b.socketPath) - return nil + return listener, nil } // peerWinSessionID returns the Windows session ID for the given process, diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index f77c1ea573..7724a31efb 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -282,6 +282,12 @@ func (m *HelperLifecycleManager) spawnKey(key HelperKey) { return } go m.watchProcess(entry) + if m.broker != nil { + if ownerPID, owned := m.broker.helperKeyOwnerPID(key); owned && ownerPID != process.ProcessID() { + m.stopTrackedKey(key) + return + } + } log.Info("proactively spawned helper in session", "helperKey", key.String(), "pid", process.ProcessID()) } @@ -320,6 +326,10 @@ func (m *HelperLifecycleManager) stopKey(key HelperKey) { if m.broker != nil { m.broker.TerminateHelperKey(key) } + m.stopTrackedKey(key) +} + +func (m *HelperLifecycleManager) stopTrackedKey(key HelperKey) { entry := m.registry.beginStop(key) if entry == nil { return @@ -417,9 +427,18 @@ func (m *HelperLifecycleManager) sessionAuthenticated(session *Session) { if m.broker == nil { return } + rollbackProactive := false m.broker.whileHelperKeyOwnedBy(key, session, func() { + trackedPID := m.registry.processID(key) + if trackedPID != 0 && trackedPID != uint32(session.PID) { + rollbackProactive = true + return + } m.registry.markConnected(key, uint32(session.PID), session) }) + if rollbackProactive { + m.stopTrackedKey(key) + } } func (m *HelperLifecycleManager) sessionClosed(session *Session) { diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index 76e03abf09..9ca8f5d368 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -167,6 +167,76 @@ type fakeHelperSpawner struct { closed int } +type blockingHelperSpawner struct { + process helperProcess + entered chan struct{} + release chan struct{} +} + +func (s *blockingHelperSpawner) Spawn(HelperKey) (helperProcess, error) { + close(s.entered) + <-s.release + return s.process, nil +} + +func (*blockingHelperSpawner) Close() error { return nil } + +func TestScheduledHelperPublishedDuringSpawnRollsBackProactiveProcess(t *testing.T) { + b := New("scheduled-spawn-race-"+t.Name(), nil) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + proactive := newFakeHelperProcess(4060) + spawner := &blockingHelperSpawner{ + process: proactive, + entered: make(chan struct{}), + release: make(chan struct{}), + } + m := newHelperLifecycleManager(b, fakeLifecycleDetector{}, nil, spawner) + m.gracePeriod = 0 + m.finalWait = 100 * time.Millisecond + m.mu.Lock() + m.desired[key] = true + m.mu.Unlock() + t.Cleanup(func() { + proactive.markExited(1) + m.Stop() + b.Close() + }) + + spawnDone := make(chan struct{}) + go func() { + m.spawnKey(key) + close(spawnDone) + }() + <-spawner.entered // owner check and registry reservation have completed + + scheduledProcess := newFakeOwnedPeerProcess(6060) + scheduledSession := newOwnedSession(t, b, key, scheduledProcess) + b.fireLifecycleSessionAuthenticated(scheduledSession) + close(spawner.release) + <-spawnDone + + deadline := time.Now().Add(time.Second) + for { + _, terminated, closed := proactive.counts() + if terminated == 1 && closed == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("proactive rollback counts terminate=%d close=%d, want 1 each", terminated, closed) + } + time.Sleep(time.Millisecond) + } + if proactive.Alive() { + t.Fatal("proactive duplicate remains alive after scheduled owner publication") + } + if !b.HasHelperKeyOwner(key) { + t.Fatal("scheduled helper lost logical ownership during proactive rollback") + } + if terminated, closed := scheduledProcess.counts(); terminated != 0 || closed != 0 { + t.Fatalf("scheduled owner terminate=%d close=%d, want 0 each", terminated, closed) + } +} + func (s *fakeHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { s.mu.Lock() defer s.mu.Unlock() From 8c8118142026db785e0b0bb1a5e7ec401214eda6 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 16:29:07 -0600 Subject: [PATCH 10/47] fix(agent): retain system helper on RDP disconnect --- agent/internal/sessionbroker/helper_key.go | 2 +- .../internal/sessionbroker/helper_key_test.go | 4 +- .../sessionbroker/lifecycle_registry_test.go | 48 +++++++++++++++++++ .../internal/sessionbroker/lifecycle_test.go | 2 +- 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/agent/internal/sessionbroker/helper_key.go b/agent/internal/sessionbroker/helper_key.go index 88dfb7b7cc..7f52d66435 100644 --- a/agent/internal/sessionbroker/helper_key.go +++ b/agent/internal/sessionbroker/helper_key.go @@ -20,7 +20,7 @@ func helperRoleDesired(s DetectedSession, role string) bool { } switch role { case "system": - return s.State == "active" || s.State == "connected" + return s.State == "active" || s.State == "connected" || (s.Type == "rdp" && s.State == "disconnected") case "user": return s.State == "active" default: diff --git a/agent/internal/sessionbroker/helper_key_test.go b/agent/internal/sessionbroker/helper_key_test.go index 2ebba20615..a907a611d1 100644 --- a/agent/internal/sessionbroker/helper_key_test.go +++ b/agent/internal/sessionbroker/helper_key_test.go @@ -15,7 +15,9 @@ func TestHelperRoleDesired(t *testing.T) { {"user connected", DetectedSession{Session: "7", State: "connected", Type: "rdp"}, "user", false}, {"session zero", DetectedSession{Session: "0", State: "active", Type: "rdp"}, "system", false}, {"services", DetectedSession{Session: "8", State: "active", Type: "services"}, "system", false}, - {"disconnected", DetectedSession{Session: "8", State: "disconnected", Type: "rdp"}, "system", false}, + {"system disconnected rdp", DetectedSession{Session: "8", State: "disconnected", Type: "rdp"}, "system", true}, + {"user disconnected rdp", DetectedSession{Session: "8", State: "disconnected", Type: "rdp"}, "user", false}, + {"system disconnected non-rdp", DetectedSession{Session: "8", State: "disconnected", Type: "console"}, "system", false}, {"unknown role", DetectedSession{Session: "8", State: "active", Type: "rdp"}, "assist", false}, } for _, tt := range tests { diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index 9ca8f5d368..a11ad0a81d 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -339,6 +339,54 @@ func TestReconcileDoesNotRespawnWhilePreAuthProcessLives(t *testing.T) { } } +func TestSCMDisconnectThenReconcileRetainsSystemAndStopsUserForDisconnectedRDP(t *testing.T) { + b := New("disconnect-policy-"+t.Name(), nil) + detector := &fakeLifecycleDetector{sessions: []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}} + systemKey := HelperKey{WindowsSessionID: 7, Role: "system"} + userKey := HelperKey{WindowsSessionID: 7, Role: "user"} + system := newFakeHelperProcess(4110) + user := newFakeHelperProcess(4120) + spawner := &fakeHelperSpawner{byKey: map[HelperKey]helperProcess{ + systemKey: system, + userKey: user, + }} + m := newHelperLifecycleManager(b, detector, nil, spawner) + m.gracePeriod = 0 + m.finalWait = 100 * time.Millisecond + t.Cleanup(func() { + system.markExited(0) + user.markExited(0) + m.Stop() + b.Close() + }) + + m.reconcile() + detector.sessions = []DetectedSession{{Session: "7", State: "disconnected", Type: "rdp"}} + + // This is the WTS_SESSION_DISCONNECT lifecycle sequence: stop the user + // role immediately, then reconcile against the detector's real state. + m.removeDesired(userKey) + m.stopKey(userKey) + m.reconcile() + + m.mu.Lock() + systemDesired := m.desired[systemKey] + userDesired := m.desired[userKey] + m.mu.Unlock() + if !systemDesired { + t.Fatal("SYSTEM helper became undesired after disconnected-RDP reconcile") + } + if userDesired { + t.Fatal("user helper remained desired after disconnected-RDP reconcile") + } + if !system.Alive() { + t.Fatal("SYSTEM helper was stopped after disconnected-RDP reconcile") + } + if user.Alive() { + t.Fatal("user helper remained alive after disconnected-RDP reconcile") + } +} + func TestStaleExitCannotClearReplacement(t *testing.T) { oldProc := newFakeHelperProcess(4100) newProc := newFakeHelperProcess(4200) diff --git a/agent/internal/sessionbroker/lifecycle_test.go b/agent/internal/sessionbroker/lifecycle_test.go index 9e0dcb5d8e..cad6f7884e 100644 --- a/agent/internal/sessionbroker/lifecycle_test.go +++ b/agent/internal/sessionbroker/lifecycle_test.go @@ -34,7 +34,7 @@ func TestHandleSCMEventDoesNotSpawnBeforeDetectorPublishesEventKey(t *testing.T) } func TestHandleSCMDisconnectStopsUserAndRetainsSystem(t *testing.T) { - m, _ := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "7", State: "connected", Type: "rdp"}}) + m, _ := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "7", State: "disconnected", Type: "rdp"}}) system := newFakeHelperProcess(6100) user := newFakeHelperProcess(6200) m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "system"}, system, "helper", "system-helper") From 3fe734f1d34ffcaaad5cddfe9e9baa7f7dc8a7c6 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 17:06:30 -0600 Subject: [PATCH 11/47] fix(agent): contain Windows helpers in a kill-on-close job --- .../sessionbroker/helper_job_windows.go | 68 +++ .../sessionbroker/helper_job_windows_test.go | 476 ++++++++++++++++++ agent/internal/sessionbroker/lifecycle.go | 33 +- .../internal/sessionbroker/spawner_windows.go | 385 ++++++++++---- 4 files changed, 858 insertions(+), 104 deletions(-) create mode 100644 agent/internal/sessionbroker/helper_job_windows.go create mode 100644 agent/internal/sessionbroker/helper_job_windows_test.go diff --git a/agent/internal/sessionbroker/helper_job_windows.go b/agent/internal/sessionbroker/helper_job_windows.go new file mode 100644 index 0000000000..f409e0a20b --- /dev/null +++ b/agent/internal/sessionbroker/helper_job_windows.go @@ -0,0 +1,68 @@ +//go:build windows + +package sessionbroker + +import ( + "errors" + "fmt" + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +// helperJob owns every helper process created by the lifecycle manager. The +// kill-on-close limit gives service shutdown a final kernel-enforced cleanup +// boundary even when an individual helper does not exit cooperatively. +type helperJob struct { + mu sync.Mutex + handle windows.Handle +} + +func newHelperJob() (*helperJob, error) { + handle, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("CreateJobObject: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + handle, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ); err != nil { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("SetInformationJobObject: %w", err) + } + return &helperJob{handle: handle}, nil +} + +func (j *helperJob) Assign(process windows.Handle) error { + if j == nil { + return errors.New("helper job is closed") + } + j.mu.Lock() + defer j.mu.Unlock() + if j.handle == 0 { + return errors.New("helper job is closed") + } + if err := windows.AssignProcessToJobObject(j.handle, process); err != nil { + return fmt.Errorf("AssignProcessToJobObject: %w", err) + } + return nil +} + +func (j *helperJob) Close() error { + if j == nil { + return nil + } + j.mu.Lock() + defer j.mu.Unlock() + if j.handle == 0 { + return nil + } + err := windows.CloseHandle(j.handle) + j.handle = 0 + return err +} diff --git a/agent/internal/sessionbroker/helper_job_windows_test.go b/agent/internal/sessionbroker/helper_job_windows_test.go new file mode 100644 index 0000000000..a983178d2a --- /dev/null +++ b/agent/internal/sessionbroker/helper_job_windows_test.go @@ -0,0 +1,476 @@ +//go:build windows + +package sessionbroker + +import ( + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +type helperJobTestProcess struct { + Handle windows.Handle + Thread windows.Handle + PID uint32 + Path string +} + +func startSuspendedTestProcess(t *testing.T) *helperJobTestProcess { + t.Helper() + exePath, err := os.Executable() + if err != nil { + t.Fatal(err) + } + args := []string{exePath, "-test.run=^TestHelperJobChildProcess$", "--", "breeze-helper-job-child"} + for i := range args { + args[i] = windows.EscapeArg(args[i]) + } + cmdLine, err := windows.UTF16PtrFromString(strings.Join(args, " ")) + if err != nil { + t.Fatal(err) + } + si := windows.StartupInfo{Cb: uint32(unsafe.Sizeof(windows.StartupInfo{}))} + var pi windows.ProcessInformation + if err := windows.CreateProcess( + nil, + cmdLine, + nil, + nil, + false, + windows.CREATE_SUSPENDED|windows.CREATE_NO_WINDOW, + nil, + nil, + &si, + &pi, + ); err != nil { + t.Fatal(err) + } + return &helperJobTestProcess{ + Handle: pi.Process, + Thread: pi.Thread, + PID: pi.ProcessId, + Path: exePath, + } +} + +func (p *helperJobTestProcess) Close() { + if p == nil { + return + } + if p.Handle != 0 { + _ = windows.TerminateProcess(p.Handle, 1) + } + if p.Thread != 0 { + _ = windows.CloseHandle(p.Thread) + p.Thread = 0 + } + if p.Handle != 0 { + _ = windows.CloseHandle(p.Handle) + p.Handle = 0 + } +} + +func resumeTestProcess(t *testing.T, thread windows.Handle) { + t.Helper() + if _, err := windows.ResumeThread(thread); err != nil { + t.Fatal(err) + } +} + +func TestHelperJobChildProcess(t *testing.T) { + if len(os.Args) == 0 || os.Args[len(os.Args)-1] != "breeze-helper-job-child" { + return + } + for { + time.Sleep(time.Hour) + } +} + +func TestClosingHelperJobTerminatesAssignedProcess(t *testing.T) { + job, err := newHelperJob() + if err != nil { + t.Fatal(err) + } + proc := startSuspendedTestProcess(t) + defer proc.Close() + if err := job.Assign(proc.Handle); err != nil { + t.Fatal(err) + } + resumeTestProcess(t, proc.Thread) + if err := job.Close(); err != nil { + t.Fatal(err) + } + if event, err := windows.WaitForSingleObject(proc.Handle, 5_000); err != nil || event != windows.WAIT_OBJECT_0 { + t.Fatalf("process survived job close: event=%d err=%v", event, err) + } +} + +func TestSpawnedHelperTerminateIsIdempotent(t *testing.T) { + proc := startSuspendedTestProcess(t) + defer proc.Close() + helper := &SpawnedHelper{PID: proc.PID, Handle: proc.Handle, BinaryPath: proc.Path} + resumeTestProcess(t, proc.Thread) + if err := helper.Terminate(); err != nil { + t.Fatal(err) + } + if err := helper.Terminate(); err != nil { + t.Fatal(err) + } + if event, err := windows.WaitForSingleObject(proc.Handle, 5_000); err != nil || event != windows.WAIT_OBJECT_0 { + t.Fatalf("process survived Terminate: event=%d err=%v", event, err) + } +} + +func TestSpawnedHelperWaitOwnsDuplicateAcrossConcurrentClose(t *testing.T) { + waitStarted := make(chan struct{}) + allowExit := make(chan struct{}) + var mu sync.Mutex + var closed []windows.Handle + helper := &SpawnedHelper{ + Handle: windows.Handle(77), + ops: &spawnedHelperOps{ + duplicateProcessHandle: func(handle windows.Handle) (windows.Handle, error) { + if handle != windows.Handle(77) { + t.Errorf("duplicated handle = %d, want 77", handle) + } + return windows.Handle(88), nil + }, + waitForSingleObject: func(handle windows.Handle, _ uint32) (uint32, error) { + if handle != windows.Handle(88) { + t.Errorf("wait handle = %d, want duplicate 88", handle) + } + close(waitStarted) + <-allowExit + return windows.WAIT_OBJECT_0, nil + }, + getExitCodeProcess: func(handle windows.Handle, exitCode *uint32) error { + if handle != windows.Handle(88) { + t.Errorf("exit-code handle = %d, want duplicate 88", handle) + } + *exitCode = 0 + return nil + }, + closeHandle: func(handle windows.Handle) error { + mu.Lock() + defer mu.Unlock() + closed = append(closed, handle) + return nil + }, + }, + } + waitDone := make(chan error, 1) + go func() { + _, err := helper.Wait() + waitDone <- err + }() + <-waitStarted + if err := helper.Close(); err != nil { + t.Fatal(err) + } + close(allowExit) + if err := <-waitDone; err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if fmt.Sprint(closed) != fmt.Sprint([]windows.Handle{77, 88}) { + t.Fatalf("closed handles = %v, want original then duplicate", closed) + } +} + +type fakeHelperJob struct { + mu sync.Mutex + assignErr error + assignCalls []windows.Handle + closeCalls int + closed chan struct{} +} + +func (j *fakeHelperJob) Assign(process windows.Handle) error { + j.mu.Lock() + defer j.mu.Unlock() + j.assignCalls = append(j.assignCalls, process) + return j.assignErr +} + +func (j *fakeHelperJob) Close() error { + j.mu.Lock() + defer j.mu.Unlock() + j.closeCalls++ + if j.closed != nil && j.closeCalls == 1 { + close(j.closed) + } + return nil +} + +func (j *fakeHelperJob) counts() (assigns, closes int) { + j.mu.Lock() + defer j.mu.Unlock() + return len(j.assignCalls), j.closeCalls +} + +func fakeSuspendedHelper() *suspendedHelper { + return &suspendedHelper{ + process: windows.Handle(101), + thread: windows.Handle(102), + pid: 4100, + binaryPath: `C:\Program Files\Breeze\breeze-user-helper.exe`, + } +} + +func TestWindowsHelperSpawnerAssignFailureTerminatesSuspendedProcess(t *testing.T) { + assignErr := errors.New("assign failed") + job := &fakeHelperJob{assignErr: assignErr} + var mu sync.Mutex + var resumed []windows.Handle + var terminated []windows.Handle + var closed []windows.Handle + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + createSuspended: func(HelperKey) (*suspendedHelper, error) { + return fakeSuspendedHelper(), nil + }, + resumeThread: func(handle windows.Handle) (uint32, error) { + mu.Lock() + defer mu.Unlock() + resumed = append(resumed, handle) + return 1, nil + }, + terminateProcess: func(handle windows.Handle, _ uint32) error { + mu.Lock() + defer mu.Unlock() + terminated = append(terminated, handle) + return nil + }, + closeHandle: func(handle windows.Handle) error { + mu.Lock() + defer mu.Unlock() + closed = append(closed, handle) + return nil + }, + }) + + process, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "system"}) + if process != nil { + t.Fatalf("process = %#v, want nil", process) + } + if !errors.Is(err, assignErr) || !strings.Contains(err.Error(), "assign helper to job") { + t.Fatalf("Spawn error = %v, want wrapped assignment error", err) + } + mu.Lock() + defer mu.Unlock() + if len(resumed) != 0 { + t.Fatalf("resumed handles = %v, want none", resumed) + } + if fmt.Sprint(terminated) != fmt.Sprint([]windows.Handle{101}) { + t.Fatalf("terminated handles = %v, want [101]", terminated) + } + if fmt.Sprint(closed) != fmt.Sprint([]windows.Handle{102, 101}) { + t.Fatalf("closed handles = %v, want [102 101]", closed) + } +} + +func TestWindowsHelperSpawnerResumeFailureTerminatesSuspendedProcess(t *testing.T) { + resumeErr := errors.New("resume failed") + job := &fakeHelperJob{} + var mu sync.Mutex + var terminated []windows.Handle + var closed []windows.Handle + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + createSuspended: func(HelperKey) (*suspendedHelper, error) { + return fakeSuspendedHelper(), nil + }, + resumeThread: func(windows.Handle) (uint32, error) { + return 0, resumeErr + }, + terminateProcess: func(handle windows.Handle, _ uint32) error { + mu.Lock() + defer mu.Unlock() + terminated = append(terminated, handle) + return nil + }, + closeHandle: func(handle windows.Handle) error { + mu.Lock() + defer mu.Unlock() + closed = append(closed, handle) + return nil + }, + }) + + process, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "user"}) + if process != nil { + t.Fatalf("process = %#v, want nil", process) + } + if !errors.Is(err, resumeErr) || !strings.Contains(err.Error(), "resume helper") { + t.Fatalf("Spawn error = %v, want wrapped resume error", err) + } + if assigns, _ := job.counts(); assigns != 1 { + t.Fatalf("job assign calls = %d, want 1", assigns) + } + mu.Lock() + defer mu.Unlock() + if fmt.Sprint(terminated) != fmt.Sprint([]windows.Handle{101}) { + t.Fatalf("terminated handles = %v, want [101]", terminated) + } + if fmt.Sprint(closed) != fmt.Sprint([]windows.Handle{102, 101}) { + t.Fatalf("closed handles = %v, want [102 101]", closed) + } +} + +func TestWindowsHelperSpawnerSerializesSpawnThroughResumeAgainstClose(t *testing.T) { + job := &fakeHelperJob{closed: make(chan struct{})} + resumeStarted := make(chan struct{}) + allowResume := make(chan struct{}) + closeStarted := make(chan struct{}) + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + createSuspended: func(HelperKey) (*suspendedHelper, error) { + return fakeSuspendedHelper(), nil + }, + resumeThread: func(windows.Handle) (uint32, error) { + close(resumeStarted) + <-allowResume + return 1, nil + }, + terminateProcess: windows.TerminateProcess, + closeHandle: func(windows.Handle) error { return nil }, + closeStarting: func() { close(closeStarted) }, + }) + + spawnDone := make(chan error, 1) + go func() { + _, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "user"}) + spawnDone <- err + }() + <-resumeStarted + closeDone := make(chan error, 1) + go func() { closeDone <- spawner.Close() }() + <-closeStarted + select { + case <-job.closed: + t.Fatal("job closed before suspended helper was resumed") + default: + } + close(allowResume) + if err := <-spawnDone; err != nil { + t.Fatal(err) + } + if err := <-closeDone; err != nil { + t.Fatal(err) + } + if assigns, closes := job.counts(); assigns != 1 || closes != 1 { + t.Fatalf("job calls assign=%d close=%d, want 1 each", assigns, closes) + } +} + +func TestSpawnedHelperCloseDoesNotCloseStandaloneJob(t *testing.T) { + job := &fakeHelperJob{} + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{}) + helper := &SpawnedHelper{ + Handle: windows.Handle(77), + standaloneOwner: spawner, + ops: &spawnedHelperOps{ + closeHandle: func(windows.Handle) error { return nil }, + }, + } + if err := helper.Close(); err != nil { + t.Fatal(err) + } + if _, closes := job.counts(); closes != 0 { + t.Fatalf("job close calls = %d, want 0; Close must only release the process handle", closes) + } +} + +func TestStandaloneHelperJobReaperRetainsJobUntilProcessExit(t *testing.T) { + job := &fakeHelperJob{} + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{}) + waitStarted := make(chan struct{}) + allowExit := make(chan struct{}) + var closed []windows.Handle + done := make(chan struct{}) + go func() { + reapStandaloneHelperWithOps(windows.Handle(88), spawner, &spawnedHelperOps{ + waitForSingleObject: func(handle windows.Handle, _ uint32) (uint32, error) { + if handle != windows.Handle(88) { + t.Errorf("wait handle = %d, want 88", handle) + } + close(waitStarted) + <-allowExit + return windows.WAIT_OBJECT_0, nil + }, + closeHandle: func(handle windows.Handle) error { + closed = append(closed, handle) + return nil + }, + }) + close(done) + }() + <-waitStarted + if _, closes := job.counts(); closes != 0 { + t.Fatalf("job closed before process exit: close calls = %d", closes) + } + close(allowExit) + <-done + if _, closes := job.counts(); closes != 1 { + t.Fatalf("job close calls after exit = %d, want 1", closes) + } + if fmt.Sprint(closed) != fmt.Sprint([]windows.Handle{88}) { + t.Fatalf("closed wait handles = %v, want [88] exactly once", closed) + } +} + +func TestWindowsHelperSpawnerClosePreventsLaterSpawn(t *testing.T) { + job := &fakeHelperJob{} + createCalls := 0 + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + createSuspended: func(HelperKey) (*suspendedHelper, error) { + createCalls++ + return fakeSuspendedHelper(), nil + }, + }) + if err := spawner.Close(); err != nil { + t.Fatal(err) + } + if _, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "system"}); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("Spawn after Close error = %v, want closed error", err) + } + if createCalls != 0 { + t.Fatalf("create calls = %d, want 0", createCalls) + } +} + +func TestBuildWindowsHelperLifecycleManagerOwnsAndClosesSpawner(t *testing.T) { + job := &fakeHelperJob{} + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{}) + m, err := buildWindowsHelperLifecycleManager(nil, nil, func() (*windowsHelperSpawner, error) { + return spawner, nil + }) + if err != nil { + t.Fatal(err) + } + if m.spawner != spawner { + t.Fatalf("lifecycle spawner = %T %p, want %p", m.spawner, m.spawner, spawner) + } + m.Stop() + if _, closes := job.counts(); closes != 1 { + t.Fatalf("job close calls = %d, want 1", closes) + } +} + +func TestBuildWindowsHelperLifecycleManagerJobFailureReturnsNoManager(t *testing.T) { + wantErr := errors.New("job construction failed") + m, err := buildWindowsHelperLifecycleManager(nil, nil, func() (*windowsHelperSpawner, error) { + return nil, wantErr + }) + if m != nil { + t.Fatalf("manager = %#v, want nil", m) + } + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want %v", err, wantErr) + } +} diff --git a/agent/internal/sessionbroker/lifecycle.go b/agent/internal/sessionbroker/lifecycle.go index 9a12f1e338..feed02c8e8 100644 --- a/agent/internal/sessionbroker/lifecycle.go +++ b/agent/internal/sessionbroker/lifecycle.go @@ -4,6 +4,7 @@ package sessionbroker import ( "context" + "fmt" "time" ) @@ -20,19 +21,31 @@ const ( wtsSessionTerminate = 0xb ) -type directHelperSpawner struct{} - -func (directHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { - if key.Role == "user" { - return SpawnUserHelperInSession(key.WindowsSessionID) +func NewHelperLifecycleManager(broker *Broker, scmCh <-chan SCMSessionEvent) *HelperLifecycleManager { + manager, err := buildWindowsHelperLifecycleManager(broker, scmCh, newWindowsHelperSpawner) + if err != nil { + // Keep heartbeat startup operational, but disable proactive spawning. + // Reconciliation will retry on the next agent/service restart, when a + // fresh Job Object can be created before any helper process exists. + log.Error("lifecycle: failed to initialize helper Job Object", "error", err.Error()) + return newHelperLifecycleManager(broker, NewSessionDetector(), scmCh, nil) } - return SpawnHelperInSession(key.WindowsSessionID) + return manager } -func (directHelperSpawner) Close() error { return nil } - -func NewHelperLifecycleManager(broker *Broker, scmCh <-chan SCMSessionEvent) *HelperLifecycleManager { - return newHelperLifecycleManager(broker, NewSessionDetector(), scmCh, directHelperSpawner{}) +func buildWindowsHelperLifecycleManager( + broker *Broker, + scmCh <-chan SCMSessionEvent, + newSpawner func() (*windowsHelperSpawner, error), +) (*HelperLifecycleManager, error) { + spawner, err := newSpawner() + if err != nil { + return nil, fmt.Errorf("initialize Windows helper spawner: %w", err) + } + if spawner == nil { + return nil, fmt.Errorf("initialize Windows helper spawner: nil spawner") + } + return newHelperLifecycleManager(broker, NewSessionDetector(), scmCh, spawner), nil } func (m *HelperLifecycleManager) Start(ctx context.Context) { diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index b13f49239c..7f2d7cf543 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -20,25 +20,57 @@ import ( // the console-subsystem agent fallback when logging spawn outcomes — useful // when chasing reports of the logon console flash regression. type SpawnedHelper struct { - PID uint32 - Handle windows.Handle - BinaryPath string - mu sync.Mutex + PID uint32 + Handle windows.Handle + BinaryPath string + mu sync.Mutex + terminated bool + standaloneOwner *windowsHelperSpawner + ops *spawnedHelperOps } -// Close releases the duplicated process handle. Safe to call more than once. +type spawnedHelperOps struct { + duplicateProcessHandle func(windows.Handle) (windows.Handle, error) + waitForSingleObject func(windows.Handle, uint32) (uint32, error) + getExitCodeProcess func(windows.Handle, *uint32) error + closeHandle func(windows.Handle) error +} + +func defaultSpawnedHelperOps() *spawnedHelperOps { + return &spawnedHelperOps{ + duplicateProcessHandle: duplicateProcessHandle, + waitForSingleObject: windows.WaitForSingleObject, + getExitCodeProcess: windows.GetExitCodeProcess, + closeHandle: windows.CloseHandle, + } +} + +func (s *SpawnedHelper) processOps() *spawnedHelperOps { + if s.ops != nil { + return s.ops + } + return defaultSpawnedHelperOps() +} + +// Close releases the process handle. Safe to call more than once. It never +// terminates the process: the lifecycle spawner owns the shared Job Object, +// while legacy standalone helpers have a private reaper that closes their Job +// only after observing process exit. func (s *SpawnedHelper) Close() error { if s == nil { return nil } s.mu.Lock() - defer s.mu.Unlock() - if s.Handle == 0 { + handle := s.Handle + s.Handle = 0 + s.standaloneOwner = nil + ops := s.processOps() + s.mu.Unlock() + + if handle == 0 { return nil } - err := windows.CloseHandle(s.Handle) - s.Handle = 0 - return err + return ops.closeHandle(handle) } func (s *SpawnedHelper) ProcessID() uint32 { return s.PID } @@ -63,7 +95,7 @@ func (s *SpawnedHelper) Terminate() error { } s.mu.Lock() defer s.mu.Unlock() - if s.Handle == 0 { + if s.Handle == 0 || s.terminated { return nil } var exitCode uint32 @@ -71,9 +103,14 @@ func (s *SpawnedHelper) Terminate() error { return err } if exitCode != windowsProcessStillActive { + s.terminated = true return nil } - return windows.TerminateProcess(s.Handle, 1) + if err := windows.TerminateProcess(s.Handle, 1); err != nil { + return err + } + s.terminated = true + return nil } // Wait blocks until the spawned helper process exits and returns its exit @@ -84,12 +121,18 @@ func (s *SpawnedHelper) Wait() (int, error) { return -1, fmt.Errorf("SpawnedHelper: no handle") } s.mu.Lock() - handle := s.Handle - s.mu.Unlock() - if handle == 0 { + if s.Handle == 0 { + s.mu.Unlock() return -1, fmt.Errorf("SpawnedHelper: no handle") } - event, err := windows.WaitForSingleObject(handle, windows.INFINITE) + ops := s.processOps() + waitHandle, err := ops.duplicateProcessHandle(s.Handle) + s.mu.Unlock() + if err != nil { + return -1, fmt.Errorf("duplicate process handle for wait: %w", err) + } + defer ops.closeHandle(waitHandle) + event, err := ops.waitForSingleObject(waitHandle, windows.INFINITE) if err != nil { return -1, fmt.Errorf("WaitForSingleObject: %w", err) } @@ -97,62 +140,194 @@ func (s *SpawnedHelper) Wait() (int, error) { return -1, fmt.Errorf("WaitForSingleObject: unexpected event %d", event) } var exitCode uint32 - if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { + if err := ops.getExitCodeProcess(waitHandle, &exitCode); err != nil { return -1, fmt.Errorf("GetExitCodeProcess: %w", err) } return int(exitCode), nil } -// SpawnHelperInSession launches a user-helper process as SYSTEM in the -// specified Windows session. Returns a SpawnedHelper describing the child -// process, or nil + an error on failure. The caller is responsible for -// closing the returned handle. -func SpawnHelperInSession(sessionID uint32) (*SpawnedHelper, error) { - // 1. Open our own process token (SYSTEM). +func duplicateProcessHandle(handle windows.Handle) (windows.Handle, error) { + currentProcess, err := windows.GetCurrentProcess() + if err != nil { + return 0, fmt.Errorf("GetCurrentProcess: %w", err) + } + var duplicate windows.Handle + if err := windows.DuplicateHandle( + currentProcess, + handle, + currentProcess, + &duplicate, + 0, + false, + windows.DUPLICATE_SAME_ACCESS, + ); err != nil { + return 0, fmt.Errorf("DuplicateHandle: %w", err) + } + return duplicate, nil +} + +type helperJobOwner interface { + Assign(windows.Handle) error + Close() error +} + +type suspendedHelper struct { + process windows.Handle + thread windows.Handle + pid uint32 + binaryPath string + tokenSource string +} + +type windowsSpawnOps struct { + createSuspended func(HelperKey) (*suspendedHelper, error) + resumeThread func(windows.Handle) (uint32, error) + terminateProcess func(windows.Handle, uint32) error + closeHandle func(windows.Handle) error + closeStarting func() +} + +// windowsHelperSpawner is the single owner of the helper Job Object. Its +// mutex covers the complete create-suspended -> assign -> resume transaction +// and Job close, so no helper can escape during lifecycle shutdown. +type windowsHelperSpawner struct { + mu sync.Mutex + job helperJobOwner + closing bool + ops windowsSpawnOps +} + +func newWindowsHelperSpawner() (*windowsHelperSpawner, error) { + job, err := newHelperJob() + if err != nil { + return nil, err + } + return newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{}), nil +} + +func newWindowsHelperSpawnerWithJob(job helperJobOwner, ops windowsSpawnOps) *windowsHelperSpawner { + if ops.createSuspended == nil { + ops.createSuspended = createHelperSuspended + } + if ops.resumeThread == nil { + ops.resumeThread = windows.ResumeThread + } + if ops.terminateProcess == nil { + ops.terminateProcess = windows.TerminateProcess + } + if ops.closeHandle == nil { + ops.closeHandle = windows.CloseHandle + } + return &windowsHelperSpawner{job: job, ops: ops} +} + +func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { + if s == nil { + return nil, fmt.Errorf("Windows helper spawner is closed") + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closing || s.job == nil { + return nil, fmt.Errorf("Windows helper spawner is closed") + } + + pending, err := s.ops.createSuspended(key) + if err != nil { + return nil, err + } + cleanup := true + defer func() { + if cleanup { + _ = s.ops.terminateProcess(pending.process, 1) + _ = s.ops.closeHandle(pending.thread) + _ = s.ops.closeHandle(pending.process) + } + }() + if err := s.job.Assign(pending.process); err != nil { + return nil, fmt.Errorf("assign helper to job: %w", err) + } + if _, err := s.ops.resumeThread(pending.thread); err != nil { + return nil, fmt.Errorf("resume helper: %w", err) + } + _ = s.ops.closeHandle(pending.thread) + cleanup = false + + log.Info("spawned user helper in session", + "sessionId", key.WindowsSessionID, + "role", key.Role, + "pid", pending.pid, + "exe", pending.binaryPath, + "tokenSource", pending.tokenSource, + ) + return &SpawnedHelper{ + PID: pending.pid, + Handle: pending.process, + BinaryPath: pending.binaryPath, + }, nil +} + +func (s *windowsHelperSpawner) Close() error { + if s == nil { + return nil + } + if s.ops.closeStarting != nil { + s.ops.closeStarting() + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closing { + return nil + } + s.closing = true + if s.job == nil { + return nil + } + return s.job.Close() +} + +func createHelperSuspended(key HelperKey) (*suspendedHelper, error) { + if key.Role == "user" { + return createUserHelperSuspended(key.WindowsSessionID) + } + return createSystemHelperSuspended(key.WindowsSessionID) +} + +// createSystemHelperSuspended creates the SYSTEM-token helper without allowing +// its primary thread to run. The caller must assign it to the Job Object before +// resuming it. +func createSystemHelperSuspended(sessionID uint32) (*suspendedHelper, error) { var processToken windows.Token proc, err := windows.GetCurrentProcess() if err != nil { return nil, fmt.Errorf("GetCurrentProcess: %w", err) } - err = windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_QUERY, &processToken) - if err != nil { + if err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_QUERY, &processToken); err != nil { return nil, fmt.Errorf("OpenProcessToken: %w", err) } defer processToken.Close() - // 2. Duplicate as a primary token we can modify. - // SecurityImpersonation is sufficient for local DXGI desktop capture; - // SecurityDelegation is only needed for credential delegation to remote - // machines, which the helper never performs. var dupToken windows.Token - err = windows.DuplicateTokenEx( + if err := windows.DuplicateTokenEx( processToken, windows.MAXIMUM_ALLOWED, - nil, // default security attributes + nil, windows.SecurityImpersonation, windows.TokenPrimary, &dupToken, - ) - if err != nil { + ); err != nil { return nil, fmt.Errorf("DuplicateTokenEx: %w", err) } defer dupToken.Close() - // 3. Set the session ID on the duplicate token. - err = windows.SetTokenInformation( + if err := windows.SetTokenInformation( dupToken, windows.TokenSessionId, (*byte)(unsafe.Pointer(&sessionID)), uint32(unsafe.Sizeof(sessionID)), - ) - if err != nil { + ); err != nil { return nil, fmt.Errorf("SetTokenInformation(TokenSessionId=%d): %w", sessionID, err) } - // 4. Build the command line. We launch the GUI-subsystem sibling binary - // (breeze-user-helper.exe) so the kernel does not allocate a console - // window in the user session. Falls back to the agent exe if the sibling - // is missing — see userHelperExePath documentation. exePath, err := userHelperExePath() if err != nil { return nil, fmt.Errorf("userHelperExePath: %w", err) @@ -161,60 +336,44 @@ func SpawnHelperInSession(sessionID uint32) (*SpawnedHelper, error) { if err != nil { return nil, fmt.Errorf("UTF16PtrFromString: %w", err) } - - // 5. Target the interactive window station + default desktop. desktop, err := windows.UTF16PtrFromString(`winsta0\Default`) if err != nil { return nil, fmt.Errorf("UTF16PtrFromString desktop: %w", err) } - si := windows.StartupInfo{ Cb: uint32(unsafe.Sizeof(windows.StartupInfo{})), Desktop: desktop, } var pi windows.ProcessInformation - - // 6. Create the process. - err = windows.CreateProcessAsUser( + creationFlags := uint32(windows.CREATE_SUSPENDED | windows.CREATE_NO_WINDOW | windows.CREATE_UNICODE_ENVIRONMENT) + if err := windows.CreateProcessAsUser( dupToken, - nil, // lpApplicationName (use cmdLine) - cmdLine, // lpCommandLine - nil, // lpProcessAttributes - nil, // lpThreadAttributes - false, // bInheritHandles - windows.CREATE_NO_WINDOW|windows.CREATE_UNICODE_ENVIRONMENT, - nil, // lpEnvironment (inherit) - nil, // lpCurrentDirectory (inherit) + nil, + cmdLine, + nil, + nil, + false, + creationFlags, + nil, + nil, &si, &pi, - ) - if err != nil { + ); err != nil { return nil, fmt.Errorf("CreateProcessAsUser(session=%d): %w", sessionID, err) } - - // Release the thread handle (we don't need it). Keep the process handle - // so the lifecycle manager can wait on it and read the exit code. - windows.CloseHandle(pi.Thread) - - log.Info("spawned user helper in session", - "sessionId", sessionID, - "role", "system", - "pid", pi.ProcessId, - "exe", exePath, - ) - return &SpawnedHelper{PID: pi.ProcessId, Handle: pi.Process, BinaryPath: exePath}, nil + return &suspendedHelper{ + process: pi.Process, + thread: pi.Thread, + pid: pi.ProcessId, + binaryPath: exePath, + tokenSource: "system", + }, nil } -// SpawnUserHelperInSession launches a user-helper process using the logged-in -// user's token in the specified Windows session. Tries WTSQueryUserToken first, -// falls back to explorer.exe token theft for Azure AD sessions. -// This helper runs as the interactive user, enabling run_as_user script -// execution and launching the Breeze Helper Tauri app. -// -// Returns a SpawnedHelper describing the child process; the caller is -// responsible for closing the returned handle. -func SpawnUserHelperInSession(sessionID uint32) (*SpawnedHelper, error) { - // Try WTSQueryUserToken first, fall back to explorer.exe token. +// createUserHelperSuspended creates the interactive-user helper without +// allowing its primary thread to run. The caller assigns it to the Job Object +// before resume. +func createUserHelperSuspended(sessionID uint32) (*suspendedHelper, error) { dupToken, envBlock, method, err := acquireUserToken(sessionID) if err != nil { return nil, fmt.Errorf("acquire user token(session=%d): %w", sessionID, err) @@ -224,8 +383,6 @@ func SpawnUserHelperInSession(sessionID uint32) (*SpawnedHelper, error) { defer windows.DestroyEnvironmentBlock(envBlock) } - // Build command line with --role user flag. Use the GUI-subsystem sibling - // binary so no console window flashes in the user session. exePath, err := userHelperExePath() if err != nil { return nil, fmt.Errorf("userHelperExePath: %w", err) @@ -234,18 +391,16 @@ func SpawnUserHelperInSession(sessionID uint32) (*SpawnedHelper, error) { if err != nil { return nil, fmt.Errorf("UTF16PtrFromString: %w", err) } - desktop, err := windows.UTF16PtrFromString(`winsta0\Default`) if err != nil { return nil, fmt.Errorf("UTF16PtrFromString desktop: %w", err) } - si := windows.StartupInfo{ Cb: uint32(unsafe.Sizeof(windows.StartupInfo{})), Desktop: desktop, } var pi windows.ProcessInformation - + creationFlags := uint32(windows.CREATE_SUSPENDED | windows.CREATE_NO_WINDOW | windows.CREATE_UNICODE_ENVIRONMENT) if err := windows.CreateProcessAsUser( dupToken, nil, @@ -253,7 +408,7 @@ func SpawnUserHelperInSession(sessionID uint32) (*SpawnedHelper, error) { nil, nil, false, - windows.CREATE_NO_WINDOW|windows.CREATE_UNICODE_ENVIRONMENT, + creationFlags, envBlock, nil, &si, @@ -261,15 +416,57 @@ func SpawnUserHelperInSession(sessionID uint32) (*SpawnedHelper, error) { ); err != nil { return nil, fmt.Errorf("CreateProcessAsUser(session=%d, role=user): %w", sessionID, err) } + return &suspendedHelper{ + process: pi.Process, + thread: pi.Thread, + pid: pi.ProcessId, + binaryPath: exePath, + tokenSource: method, + }, nil +} + +// SpawnHelperInSession is retained for compatibility with older callers. It +// creates a private one-process Job Object and transfers that spawner to the +// returned helper so Close releases the Job after the process exits. +func SpawnHelperInSession(sessionID uint32) (*SpawnedHelper, error) { + return spawnStandaloneHelper(HelperKey{WindowsSessionID: sessionID, Role: "system"}) +} + +// SpawnUserHelperInSession is the user-token counterpart to +// SpawnHelperInSession. +func SpawnUserHelperInSession(sessionID uint32) (*SpawnedHelper, error) { + return spawnStandaloneHelper(HelperKey{WindowsSessionID: sessionID, Role: "user"}) +} + +func spawnStandaloneHelper(key HelperKey) (*SpawnedHelper, error) { + spawner, err := newWindowsHelperSpawner() + if err != nil { + return nil, err + } + process, err := spawner.Spawn(key) + if err != nil { + _ = spawner.Close() + return nil, err + } + helper := process.(*SpawnedHelper) + waitHandle, err := duplicateProcessHandle(helper.Handle) + if err != nil { + _ = helper.Terminate() + _ = helper.Close() + _ = spawner.Close() + return nil, fmt.Errorf("retain standalone helper Job ownership: %w", err) + } + helper.standaloneOwner = spawner + go reapStandaloneHelper(waitHandle, spawner) + return helper, nil +} - windows.CloseHandle(pi.Thread) +func reapStandaloneHelper(waitHandle windows.Handle, spawner *windowsHelperSpawner) { + reapStandaloneHelperWithOps(waitHandle, spawner, defaultSpawnedHelperOps()) +} - log.Info("spawned user-token helper in session", - "sessionId", sessionID, - "role", "user", - "pid", pi.ProcessId, - "exe", exePath, - "tokenSource", method, - ) - return &SpawnedHelper{PID: pi.ProcessId, Handle: pi.Process, BinaryPath: exePath}, nil +func reapStandaloneHelperWithOps(waitHandle windows.Handle, spawner *windowsHelperSpawner, ops *spawnedHelperOps) { + _, _ = ops.waitForSingleObject(waitHandle, windows.INFINITE) + _ = ops.closeHandle(waitHandle) + _ = spawner.Close() } From a2ca1842c2051a9a2aa084be769f69d895a1581e Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 17:17:07 -0600 Subject: [PATCH 12/47] fix(agent): retain helper job until confirmed exit --- .../sessionbroker/helper_job_windows_test.go | 124 ++++++++++++++++++ .../internal/sessionbroker/spawner_windows.go | 37 +++++- 2 files changed, 158 insertions(+), 3 deletions(-) diff --git a/agent/internal/sessionbroker/helper_job_windows_test.go b/agent/internal/sessionbroker/helper_job_windows_test.go index a983178d2a..f8b7629cee 100644 --- a/agent/internal/sessionbroker/helper_job_windows_test.go +++ b/agent/internal/sessionbroker/helper_job_windows_test.go @@ -424,6 +424,130 @@ func TestStandaloneHelperJobReaperRetainsJobUntilProcessExit(t *testing.T) { } } +func TestStandaloneHelperJobReaperRetriesUnconfirmedWaitResults(t *testing.T) { + waitErr := errors.New("wait failed") + tests := []struct { + name string + firstResults []struct { + event uint32 + err error + } + }{ + { + name: "wait error", + firstResults: []struct { + event uint32 + err error + }{ + {event: windows.WAIT_FAILED, err: waitErr}, + }, + }, + { + name: "unexpected events", + firstResults: []struct { + event uint32 + err error + }{ + {event: uint32(windows.WAIT_TIMEOUT)}, + {event: windows.WAIT_ABANDONED}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + job := &fakeHelperJob{} + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{}) + waitHandle := windows.Handle(88) + waitCalls := 0 + closeCalls := 0 + var sleeps []time.Duration + reapStandaloneHelperWithOps(waitHandle, spawner, &spawnedHelperOps{ + waitForSingleObject: func(handle windows.Handle, timeout uint32) (uint32, error) { + if handle != waitHandle { + t.Fatalf("wait handle = %d, want retained handle %d", handle, waitHandle) + } + if timeout != windows.INFINITE { + t.Fatalf("wait timeout = %d, want INFINITE", timeout) + } + call := waitCalls + waitCalls++ + if call < len(tt.firstResults) { + return tt.firstResults[call].event, tt.firstResults[call].err + } + return windows.WAIT_OBJECT_0, nil + }, + closeHandle: func(handle windows.Handle) error { + if handle != waitHandle { + t.Fatalf("closed handle = %d, want %d", handle, waitHandle) + } + closeCalls++ + return nil + }, + sleep: func(delay time.Duration) { + if _, closes := job.counts(); closes != 0 { + t.Fatalf("job closed before authoritative process exit: close calls = %d", closes) + } + if closeCalls != 0 { + t.Fatalf("wait handle closed before authoritative process exit: close calls = %d", closeCalls) + } + sleeps = append(sleeps, delay) + }, + }) + + wantWaits := len(tt.firstResults) + 1 + if waitCalls != wantWaits { + t.Fatalf("wait calls = %d, want %d", waitCalls, wantWaits) + } + if len(sleeps) != len(tt.firstResults) { + t.Fatalf("sleep calls = %d, want %d", len(sleeps), len(tt.firstResults)) + } + if closeCalls != 1 { + t.Fatalf("wait handle close calls = %d, want 1", closeCalls) + } + if _, closes := job.counts(); closes != 1 { + t.Fatalf("job close calls = %d, want 1", closes) + } + }) + } +} + +func TestStandaloneHelperJobReaperBoundsRetryBackoff(t *testing.T) { + job := &fakeHelperJob{} + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{}) + waitCalls := 0 + var sleeps []time.Duration + reapStandaloneHelperWithOps(windows.Handle(88), spawner, &spawnedHelperOps{ + waitForSingleObject: func(windows.Handle, uint32) (uint32, error) { + waitCalls++ + if waitCalls <= 10 { + return windows.WAIT_FAILED, errors.New("transient wait failure") + } + return windows.WAIT_OBJECT_0, nil + }, + closeHandle: func(windows.Handle) error { return nil }, + sleep: func(delay time.Duration) { + sleeps = append(sleeps, delay) + }, + }) + if len(sleeps) != 10 { + t.Fatalf("sleep calls = %d, want 10", len(sleeps)) + } + if sleeps[0] != standaloneReaperInitialBackoff { + t.Fatalf("initial backoff = %v, want %v", sleeps[0], standaloneReaperInitialBackoff) + } + for i, delay := range sleeps { + if delay > standaloneReaperMaxBackoff { + t.Fatalf("backoff[%d] = %v, exceeds cap %v", i, delay, standaloneReaperMaxBackoff) + } + if i > 0 && delay < sleeps[i-1] { + t.Fatalf("backoff decreased: %v then %v", sleeps[i-1], delay) + } + } + if sleeps[len(sleeps)-1] != standaloneReaperMaxBackoff { + t.Fatalf("final backoff = %v, want cap %v", sleeps[len(sleeps)-1], standaloneReaperMaxBackoff) + } +} + func TestWindowsHelperSpawnerClosePreventsLaterSpawn(t *testing.T) { job := &fakeHelperJob{} createCalls := 0 diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index 7f2d7cf543..d6bd70201c 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -5,6 +5,7 @@ package sessionbroker import ( "fmt" "sync" + "time" "unsafe" "golang.org/x/sys/windows" @@ -34,6 +35,7 @@ type spawnedHelperOps struct { waitForSingleObject func(windows.Handle, uint32) (uint32, error) getExitCodeProcess func(windows.Handle, *uint32) error closeHandle func(windows.Handle) error + sleep func(time.Duration) } func defaultSpawnedHelperOps() *spawnedHelperOps { @@ -42,6 +44,7 @@ func defaultSpawnedHelperOps() *spawnedHelperOps { waitForSingleObject: windows.WaitForSingleObject, getExitCodeProcess: windows.GetExitCodeProcess, closeHandle: windows.CloseHandle, + sleep: time.Sleep, } } @@ -465,8 +468,36 @@ func reapStandaloneHelper(waitHandle windows.Handle, spawner *windowsHelperSpawn reapStandaloneHelperWithOps(waitHandle, spawner, defaultSpawnedHelperOps()) } +const ( + standaloneReaperInitialBackoff = 100 * time.Millisecond + standaloneReaperMaxBackoff = 5 * time.Second +) + func reapStandaloneHelperWithOps(waitHandle windows.Handle, spawner *windowsHelperSpawner, ops *spawnedHelperOps) { - _, _ = ops.waitForSingleObject(waitHandle, windows.INFINITE) - _ = ops.closeHandle(waitHandle) - _ = spawner.Close() + backoff := standaloneReaperInitialBackoff + attempt := 0 + for { + event, err := ops.waitForSingleObject(waitHandle, windows.INFINITE) + if err == nil && event == windows.WAIT_OBJECT_0 { + _ = ops.closeHandle(waitHandle) + _ = spawner.Close() + return + } + + attempt++ + if attempt == 1 || attempt%12 == 0 { + log.Warn("standalone helper wait did not confirm process exit; retaining Job ownership", + "attempt", attempt, + "event", event, + "error", err, + ) + } + ops.sleep(backoff) + if backoff < standaloneReaperMaxBackoff { + backoff *= 2 + if backoff > standaloneReaperMaxBackoff { + backoff = standaloneReaperMaxBackoff + } + } + } } From 0e6d4f059b80a6209e7d2e29309a9ffedc79fd17 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 17:28:11 -0600 Subject: [PATCH 13/47] test(agent): cover RDS helper convergence on Windows --- .github/workflows/ci.yml | 27 ++- .../rds_lifecycle_integration_test.go | 197 ++++++++++++++++++ 2 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 agent/internal/sessionbroker/rds_lifecycle_integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9983a5d59f..e93e681aad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -696,6 +696,29 @@ jobs: path: agent/coverage.out retention-days: 7 + test-agent-windows: + name: Test Agent (Windows) + runs-on: windows-latest + needs: [lint] + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: agent/go.sum + + - name: Download Go dependencies + working-directory: agent + run: go mod download + + - name: Run Windows Go tests + working-directory: agent + run: go test -race ./internal/sessionbroker ./internal/heartbeat ./internal/agentapp ./internal/config ./internal/watchdog ./cmd/breeze-watchdog + # ─── Windows runtime smoke (#1000) ──────────────────────────────────── # CI cross-compiles the Windows agent (build-agent matrix) but never RAN # it on Windows, so runtime-only Windows regressions shipped green. This @@ -1319,7 +1342,7 @@ jobs: ci-success: name: CI Success runs-on: ubuntu-latest - needs: [lint, typecheck, test-api, test-web, test-portal, test-office-addin-core, test-excel-addin, test-word-addin, test-powerpoint-addin, test-outlook-addin, security-audit, build-api, build-web, build-portal, build-agent, test-agent, windows-runtime-smoke, integration-test, rust-check, smoke-test] + needs: [lint, typecheck, test-api, test-web, test-portal, test-office-addin-core, test-excel-addin, test-word-addin, test-powerpoint-addin, test-outlook-addin, security-audit, build-api, build-web, build-portal, build-agent, test-agent, test-agent-windows, windows-runtime-smoke, integration-test, rust-check, smoke-test] if: ${{ !cancelled() }} steps: - name: Check all jobs passed @@ -1340,6 +1363,7 @@ jobs: BUILD_PORTAL_RESULT: ${{ needs.build-portal.result }} BUILD_AGENT_RESULT: ${{ needs.build-agent.result }} TEST_AGENT_RESULT: ${{ needs.test-agent.result }} + TEST_AGENT_WINDOWS_RESULT: ${{ needs.test-agent-windows.result }} WINDOWS_RUNTIME_SMOKE_RESULT: ${{ needs.windows-runtime-smoke.result }} INTEGRATION_TEST_RESULT: ${{ needs.integration-test.result }} RUST_CHECK_RESULT: ${{ needs.rust-check.result }} @@ -1362,6 +1386,7 @@ jobs: [[ "${BUILD_PORTAL_RESULT}" != "success" ]] || \ [[ "${BUILD_AGENT_RESULT}" != "success" ]] || \ [[ "${TEST_AGENT_RESULT}" != "success" ]] || \ + [[ "${TEST_AGENT_WINDOWS_RESULT}" != "success" ]] || \ [[ "${WINDOWS_RUNTIME_SMOKE_RESULT}" != "success" ]] || \ [[ "${INTEGRATION_TEST_RESULT}" != "success" ]] || \ [[ "${RUST_CHECK_RESULT}" != "success" ]]; then diff --git a/agent/internal/sessionbroker/rds_lifecycle_integration_test.go b/agent/internal/sessionbroker/rds_lifecycle_integration_test.go new file mode 100644 index 0000000000..5dbf8feed9 --- /dev/null +++ b/agent/internal/sessionbroker/rds_lifecycle_integration_test.go @@ -0,0 +1,197 @@ +package sessionbroker + +import ( + "errors" + "fmt" + "strconv" + "sync" + "testing" + + "github.com/breeze-rmm/agent/internal/ipc" +) + +type countingHelperSpawner struct { + mu sync.Mutex + spawned map[HelperKey]int + nextPID uint32 +} + +func newCountingHelperSpawner() *countingHelperSpawner { + return &countingHelperSpawner{ + spawned: make(map[HelperKey]int), + nextPID: 7000, + } +} + +func (s *countingHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.spawned[key]++ + s.nextPID++ + return newFakeHelperProcess(s.nextPID), nil +} + +func (*countingHelperSpawner) Close() error { return nil } + +func (s *countingHelperSpawner) TotalSpawnCount() int { + s.mu.Lock() + defer s.mu.Unlock() + total := 0 + for _, count := range s.spawned { + total += count + } + return total +} + +func (s *countingHelperSpawner) Keys() []HelperKey { + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]HelperKey, 0, len(s.spawned)) + for key := range s.spawned { + keys = append(keys, key) + } + return keys +} + +func (s *countingHelperSpawner) SpawnCount(key HelperKey) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.spawned[key] +} + +func TestRDSReconcileConvergesWithoutAccumulation(t *testing.T) { + sessions := make([]DetectedSession, 0, 20) + for id := 1; id <= 20; id++ { + sessions = append(sessions, DetectedSession{Session: strconv.Itoa(id), State: "active", Type: "rdp"}) + } + spawner := newCountingHelperSpawner() + m := newLifecycleHarness(t, sessions, spawner) + for i := 0; i < 10; i++ { + m.reconcile() + } + if got := spawner.TotalSpawnCount(); got != 40 { + t.Fatalf("spawned %d helpers, want 40", got) + } + for _, key := range spawner.Keys() { + if got := spawner.SpawnCount(key); got != 1 { + t.Fatalf("%s spawned %d times, want 1", key, got) + } + } +} + +func TestRDSBrokerAdmissionAndLifecycleConvergeTwentySessions(t *testing.T) { + t.Run("distinct session identity buckets", func(t *testing.T) { + const systemSID = "S-1-5-18" + sessions := make([]DetectedSession, 0, 20) + desired := make(map[HelperKey]struct{}, 40) + for id := uint32(1); id <= 20; id++ { + sessions = append(sessions, DetectedSession{Session: strconv.FormatUint(uint64(id), 10), State: "active", Type: "rdp"}) + desired[HelperKey{WindowsSessionID: id, Role: ipc.HelperRoleSystem}] = struct{}{} + desired[HelperKey{WindowsSessionID: id, Role: ipc.HelperRoleUser}] = struct{}{} + } + + b := New("rds-integration-"+t.Name(), nil) + b.goos = "windows" + b.UpdateDesiredHelperKeys(desired) + spawner := newCountingHelperSpawner() + m := newHelperLifecycleManager(b, fakeLifecycleDetector{sessions: sessions}, nil, spawner) + var clients []*ipc.Conn + t.Cleanup(func() { + m.Stop() + b.Close() + for _, client := range clients { + _ = client.Close() + } + }) + + committed := make(map[uint32]*Session, 20) + for id := uint32(1); id <= 20; id++ { + key := HelperKey{WindowsSessionID: id, Role: ipc.HelperRoleSystem} + identity := admissionIdentityKey(systemSID, id, "windows") + reservation, err := b.reserveWindowsHelper(identity, systemSID, key) + if err != nil { + t.Fatalf("reserve SYSTEM helper for Windows session %d: %v", id, err) + } + session, client := newPairedSession(t, fmt.Sprintf("rds-system-%d", id), identity) + clients = append(clients, client) + session.WinSessionID = strconv.FormatUint(uint64(id), 10) + session.HelperRole = ipc.HelperRoleSystem + if err := b.commitWindowsHelper(reservation, session); err != nil { + t.Fatalf("commit SYSTEM helper for Windows session %d: %v", id, err) + } + committed[id] = session + } + + b.mu.RLock() + if got := len(b.byIdentity); got != 20 { + b.mu.RUnlock() + t.Fatalf("composite identity buckets = %d, want 20", got) + } + for id := uint32(1); id <= 20; id++ { + identity := admissionIdentityKey(systemSID, id, "windows") + bucket := b.byIdentity[identity] + if len(bucket) != 1 || bucket[0] != committed[id] { + b.mu.RUnlock() + t.Fatalf("Windows session %d identity bucket = %v, want only committed session", id, bucket) + } + } + b.mu.RUnlock() + + for i := 0; i < 10; i++ { + m.reconcile() + } + + if got := len(b.AllSessions()); got != 20 { + t.Fatalf("broker sessions after reconciliation = %d, want 20", got) + } + for id := uint32(1); id <= 20; id++ { + if got := b.SessionByID(committed[id].SessionID); got != committed[id] { + t.Fatalf("Windows session %d SYSTEM owner changed: got %p, want %p", id, got, committed[id]) + } + systemKey := HelperKey{WindowsSessionID: id, Role: ipc.HelperRoleSystem} + userKey := HelperKey{WindowsSessionID: id, Role: ipc.HelperRoleUser} + if got := spawner.SpawnCount(systemKey); got != 0 { + t.Fatalf("%s replacement spawn count = %d, want 0", systemKey, got) + } + if got := spawner.SpawnCount(userKey); got != 1 { + t.Fatalf("%s spawn count = %d, want 1", userKey, got) + } + } + if got := spawner.TotalSpawnCount(); got != 20 { + t.Fatalf("spawned helpers with committed SYSTEM owners = %d, want 20 user helpers", got) + } + }) + + t.Run("same session pre-auth identity bound", func(t *testing.T) { + if MaxConnectionsPerIdentity != 5 { + t.Fatalf("MaxConnectionsPerIdentity = %d, want existing bound 5", MaxConnectionsPerIdentity) + } + const systemSID = "S-1-5-18" + const windowsSessionID = uint32(7) + b := New("rds-same-session-"+t.Name(), nil) + b.goos = "windows" + t.Cleanup(b.Close) + + desired := make(map[HelperKey]struct{}, MaxConnectionsPerIdentity+1) + keys := make([]HelperKey, 0, MaxConnectionsPerIdentity+1) + for slot := 1; slot <= MaxConnectionsPerIdentity+1; slot++ { + key := HelperKey{WindowsSessionID: windowsSessionID, Role: fmt.Sprintf("pre-auth-%d", slot)} + desired[key] = struct{}{} + keys = append(keys, key) + } + b.UpdateDesiredHelperKeys(desired) + identity := admissionIdentityKey(systemSID, windowsSessionID, "windows") + for slot, key := range keys { + _, err := b.reserveWindowsHelper(identity, systemSID, key) + if slot < MaxConnectionsPerIdentity { + if err != nil { + t.Fatalf("reserve pre-auth identity slot %d: %v", slot+1, err) + } + continue + } + if !errors.Is(err, errMaxConnectionsPerIdentity) { + t.Fatalf("sixth pre-auth identity reservation err = %v, want errMaxConnectionsPerIdentity", err) + } + } + }) +} From 2fee77a83c1f323a38ef85633b44600601c9f8ae Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 17:47:20 -0600 Subject: [PATCH 14/47] feat(agent): classify process roles and helper fallback --- agent/internal/agentapp/process_role.go | 61 +++++++++++++++ agent/internal/agentapp/process_role_test.go | 73 ++++++++++++++++++ .../sessionbroker/helper_job_windows_test.go | 23 ++++++ agent/internal/sessionbroker/spawner_stub.go | 5 +- .../internal/sessionbroker/spawner_windows.go | 77 +++++++++++-------- .../internal/sessionbroker/userhelper_path.go | 43 ++++++++--- .../sessionbroker/userhelper_path_other.go | 8 +- .../sessionbroker/userhelper_path_test.go | 33 ++++++-- .../sessionbroker/userhelper_path_windows.go | 4 +- 9 files changed, 272 insertions(+), 55 deletions(-) create mode 100644 agent/internal/agentapp/process_role.go create mode 100644 agent/internal/agentapp/process_role_test.go diff --git a/agent/internal/agentapp/process_role.go b/agent/internal/agentapp/process_role.go new file mode 100644 index 0000000000..5f6a52cae4 --- /dev/null +++ b/agent/internal/agentapp/process_role.go @@ -0,0 +1,61 @@ +package agentapp + +import ( + "path/filepath" + "strings" + "time" + + "github.com/breeze-rmm/agent/internal/sessionbroker" +) + +// ProcessStartup is the non-secret diagnostic record for one agent or helper +// process startup. CompanionHelper is true only for a user-helper command that +// did not resolve to the main agent binary fallback. +type ProcessStartup struct { + Binary string `json:"binary"` + ExecutablePath string `json:"executablePath"` + PID int `json:"pid"` + ParentPID int `json:"parentPid"` + WindowsSessionID uint32 `json:"windowsSessionId"` + LaunchMode string `json:"launchMode"` + HelperRole string `json:"helperRole,omitempty"` + LifecycleKey string `json:"lifecycleKey,omitempty"` + CompanionHelper bool `json:"companionHelper"` + MainBinaryFallback bool `json:"mainBinaryFallback"` + Version string `json:"version"` + CreatedAt time.Time `json:"createdAt"` +} + +func classifyProcess(command, role, executable string, service bool) (string, bool) { + base := strings.ToLower(filepath.Base(executable)) + fallback := command == "user-helper" && base != strings.ToLower(sessionbroker.UserHelperBinaryName) + switch { + case command == "run" && service: + return "service-run", false + case command == "run": + return "console-run", false + case command == "user-helper" && role == "system": + return "system-helper", fallback + case command == "user-helper" && role == "user": + return "user-helper", fallback + default: + return "other", false + } +} + +func processStartupFields(s ProcessStartup) map[string]any { + return map[string]any{ + "binary": s.Binary, + "executablePath": s.ExecutablePath, + "pid": s.PID, + "parentPid": s.ParentPID, + "windowsSessionId": s.WindowsSessionID, + "launchMode": s.LaunchMode, + "helperRole": s.HelperRole, + "lifecycleKey": s.LifecycleKey, + "companionHelper": s.CompanionHelper, + "mainBinaryFallback": s.MainBinaryFallback, + "version": s.Version, + "createdAt": s.CreatedAt, + } +} diff --git a/agent/internal/agentapp/process_role_test.go b/agent/internal/agentapp/process_role_test.go new file mode 100644 index 0000000000..495bb5098c --- /dev/null +++ b/agent/internal/agentapp/process_role_test.go @@ -0,0 +1,73 @@ +package agentapp + +import ( + "reflect" + "testing" + "time" +) + +func TestClassifyProcess(t *testing.T) { + tests := []struct { + name, command, role, exe string + service bool + wantMode string + wantFallback bool + }{ + {"SCM main", "run", "", "breeze-agent.exe", true, "service-run", false}, + {"console main", "run", "", "breeze-agent.exe", false, "console-run", false}, + {"companion user", "user-helper", "user", "breeze-user-helper.exe", false, "user-helper", false}, + {"fallback user", "user-helper", "user", "breeze-agent.exe", false, "user-helper", true}, + {"renamed fallback user", "user-helper", "user", "breeze-agent-0.70.exe", false, "user-helper", true}, + {"companion system", "user-helper", "system", "breeze-user-helper.exe", false, "system-helper", false}, + {"status", "status", "", "breeze-agent.exe", false, "other", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, fallback := classifyProcess(tt.command, tt.role, tt.exe, tt.service) + if mode != tt.wantMode || fallback != tt.wantFallback { + t.Fatalf("got (%q,%v), want (%q,%v)", mode, fallback, tt.wantMode, tt.wantFallback) + } + }) + } +} + +func TestProcessStartupFieldsContainsOnlyDiagnosticMetadata(t *testing.T) { + createdAt := time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC) + startup := ProcessStartup{ + Binary: "breeze-agent.exe", + ExecutablePath: `C:\Program Files\Breeze\breeze-agent.exe`, + PID: 42, + ParentPID: 7, + WindowsSessionID: 3, + LaunchMode: "user-helper", + HelperRole: "user", + LifecycleKey: "3:user", + CompanionHelper: false, + MainBinaryFallback: true, + Version: "0.70.0", + CreatedAt: createdAt, + } + want := map[string]any{ + "binary": startup.Binary, + "executablePath": startup.ExecutablePath, + "pid": startup.PID, + "parentPid": startup.ParentPID, + "windowsSessionId": startup.WindowsSessionID, + "launchMode": startup.LaunchMode, + "helperRole": startup.HelperRole, + "lifecycleKey": startup.LifecycleKey, + "companionHelper": startup.CompanionHelper, + "mainBinaryFallback": startup.MainBinaryFallback, + "version": startup.Version, + "createdAt": startup.CreatedAt, + } + fields := processStartupFields(startup) + if !reflect.DeepEqual(fields, want) { + t.Fatalf("processStartupFields() = %#v, want %#v", fields, want) + } + for _, forbidden := range []string{"authToken", "token", "password", "secret"} { + if _, ok := fields[forbidden]; ok { + t.Fatalf("startup diagnostic fields contain forbidden key %q", forbidden) + } + } +} diff --git a/agent/internal/sessionbroker/helper_job_windows_test.go b/agent/internal/sessionbroker/helper_job_windows_test.go index f8b7629cee..a196a26f93 100644 --- a/agent/internal/sessionbroker/helper_job_windows_test.go +++ b/agent/internal/sessionbroker/helper_job_windows_test.go @@ -225,6 +225,29 @@ func fakeSuspendedHelper() *suspendedHelper { } } +func TestWindowsHelperSpawnerRetainsMainBinaryFallback(t *testing.T) { + job := &fakeHelperJob{} + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + createSuspended: func(HelperKey) (*suspendedHelper, error) { + pending := fakeSuspendedHelper() + pending.binaryPath = `C:\Program Files\Breeze\breeze-agent.exe` + pending.mainBinaryFallback = true + return pending, nil + }, + resumeThread: func(windows.Handle) (uint32, error) { return 1, nil }, + closeHandle: func(windows.Handle) error { return nil }, + }) + + process, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "user"}) + if err != nil { + t.Fatal(err) + } + helper := process.(*SpawnedHelper) + if !helper.MainBinaryFallback { + t.Fatal("SpawnedHelper.MainBinaryFallback = false, want true") + } +} + func TestWindowsHelperSpawnerAssignFailureTerminatesSuspendedProcess(t *testing.T) { assignErr := errors.New("assign failed") job := &fakeHelperJob{assignErr: assignErr} diff --git a/agent/internal/sessionbroker/spawner_stub.go b/agent/internal/sessionbroker/spawner_stub.go index 0a5f34a9fc..28bdc72259 100644 --- a/agent/internal/sessionbroker/spawner_stub.go +++ b/agent/internal/sessionbroker/spawner_stub.go @@ -14,8 +14,9 @@ import "fmt" // from the console-subsystem agent fallback in their telemetry. Always // empty on non-Windows builds since the stub never spawns. type SpawnedHelper struct { - PID uint32 - BinaryPath string + PID uint32 + BinaryPath string + MainBinaryFallback bool } // Close is a no-op on non-Windows platforms. diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index d6bd70201c..2688ab4d02 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -21,13 +21,14 @@ import ( // the console-subsystem agent fallback when logging spawn outcomes — useful // when chasing reports of the logon console flash regression. type SpawnedHelper struct { - PID uint32 - Handle windows.Handle - BinaryPath string - mu sync.Mutex - terminated bool - standaloneOwner *windowsHelperSpawner - ops *spawnedHelperOps + PID uint32 + Handle windows.Handle + BinaryPath string + MainBinaryFallback bool + mu sync.Mutex + terminated bool + standaloneOwner *windowsHelperSpawner + ops *spawnedHelperOps } type spawnedHelperOps struct { @@ -175,11 +176,12 @@ type helperJobOwner interface { } type suspendedHelper struct { - process windows.Handle - thread windows.Handle - pid uint32 - binaryPath string - tokenSource string + process windows.Handle + thread windows.Handle + pid uint32 + binaryPath string + mainBinaryFallback bool + tokenSource string } type windowsSpawnOps struct { @@ -194,10 +196,11 @@ type windowsSpawnOps struct { // mutex covers the complete create-suspended -> assign -> resume transaction // and Job close, so no helper can escape during lifecycle shutdown. type windowsHelperSpawner struct { - mu sync.Mutex - job helperJobOwner - closing bool - ops windowsSpawnOps + mu sync.Mutex + job helperJobOwner + closing bool + ops windowsSpawnOps + fallbackWarning helperFallbackWarningOwner } func newWindowsHelperSpawner() (*windowsHelperSpawner, error) { @@ -246,6 +249,10 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { _ = s.ops.closeHandle(pending.process) } }() + s.fallbackWarning.WarnIfFallback(ResolvedHelperExecutable{ + Path: pending.binaryPath, + MainBinaryFallback: pending.mainBinaryFallback, + }) if err := s.job.Assign(pending.process); err != nil { return nil, fmt.Errorf("assign helper to job: %w", err) } @@ -260,12 +267,14 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { "role", key.Role, "pid", pending.pid, "exe", pending.binaryPath, + "mainBinaryFallback", pending.mainBinaryFallback, "tokenSource", pending.tokenSource, ) return &SpawnedHelper{ - PID: pending.pid, - Handle: pending.process, - BinaryPath: pending.binaryPath, + PID: pending.pid, + Handle: pending.process, + BinaryPath: pending.binaryPath, + MainBinaryFallback: pending.mainBinaryFallback, }, nil } @@ -331,11 +340,11 @@ func createSystemHelperSuspended(sessionID uint32) (*suspendedHelper, error) { return nil, fmt.Errorf("SetTokenInformation(TokenSessionId=%d): %w", sessionID, err) } - exePath, err := userHelperExePath() + resolvedExe, err := userHelperExePath() if err != nil { return nil, fmt.Errorf("userHelperExePath: %w", err) } - cmdLine, err := windows.UTF16PtrFromString(buildUserHelperCmdLine(exePath, "system")) + cmdLine, err := windows.UTF16PtrFromString(buildUserHelperCmdLine(resolvedExe.Path, "system")) if err != nil { return nil, fmt.Errorf("UTF16PtrFromString: %w", err) } @@ -365,11 +374,12 @@ func createSystemHelperSuspended(sessionID uint32) (*suspendedHelper, error) { return nil, fmt.Errorf("CreateProcessAsUser(session=%d): %w", sessionID, err) } return &suspendedHelper{ - process: pi.Process, - thread: pi.Thread, - pid: pi.ProcessId, - binaryPath: exePath, - tokenSource: "system", + process: pi.Process, + thread: pi.Thread, + pid: pi.ProcessId, + binaryPath: resolvedExe.Path, + mainBinaryFallback: resolvedExe.MainBinaryFallback, + tokenSource: "system", }, nil } @@ -386,11 +396,11 @@ func createUserHelperSuspended(sessionID uint32) (*suspendedHelper, error) { defer windows.DestroyEnvironmentBlock(envBlock) } - exePath, err := userHelperExePath() + resolvedExe, err := userHelperExePath() if err != nil { return nil, fmt.Errorf("userHelperExePath: %w", err) } - cmdLine, err := windows.UTF16PtrFromString(buildUserHelperCmdLine(exePath, "user")) + cmdLine, err := windows.UTF16PtrFromString(buildUserHelperCmdLine(resolvedExe.Path, "user")) if err != nil { return nil, fmt.Errorf("UTF16PtrFromString: %w", err) } @@ -420,11 +430,12 @@ func createUserHelperSuspended(sessionID uint32) (*suspendedHelper, error) { return nil, fmt.Errorf("CreateProcessAsUser(session=%d, role=user): %w", sessionID, err) } return &suspendedHelper{ - process: pi.Process, - thread: pi.Thread, - pid: pi.ProcessId, - binaryPath: exePath, - tokenSource: method, + process: pi.Process, + thread: pi.Thread, + pid: pi.ProcessId, + binaryPath: resolvedExe.Path, + mainBinaryFallback: resolvedExe.MainBinaryFallback, + tokenSource: method, }, nil } diff --git a/agent/internal/sessionbroker/userhelper_path.go b/agent/internal/sessionbroker/userhelper_path.go index 4eb82b5528..f9add818c4 100644 --- a/agent/internal/sessionbroker/userhelper_path.go +++ b/agent/internal/sessionbroker/userhelper_path.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "sync" ) // UserHelperBinaryName is the on-disk filename of the GUI-subsystem user-helper @@ -23,7 +24,7 @@ const UserHelperBinaryName = "breeze-user-helper.exe" // build tags or os.Executable. // // - sibling present → return sibling path -// - sibling missing with fs.ErrNotExist → log Warn + return agentExe (fallback) +// - sibling missing with fs.ErrNotExist → return agentExe with fallback provenance // - any other stat error → wrap and return // // The fallback exists because some failure modes (failed build, AV @@ -37,20 +38,42 @@ const UserHelperBinaryName = "breeze-user-helper.exe" // supported install vector (msiexec /i, /fa repair, in-place upgrade via // dev_update), the fs.ErrNotExist branch should be promoted to a hard // error so the regression surfaces loudly instead of degrading silently. -// Without the fs.ErrNotExist branch any stat error would silently degrade — -// the Warn provides ops telemetry for fleet-side detection in the meantime. -func resolveUserHelperPath(agentExe string) (string, error) { +// Without the fs.ErrNotExist branch any stat error would silently degrade. +// The owning spawner emits the warning once for fleet-side detection. +// ResolvedHelperExecutable records both the selected helper path and whether +// a missing Windows companion forced selection of the main agent binary. +type ResolvedHelperExecutable struct { + Path string + MainBinaryFallback bool +} + +func resolveUserHelperPath(agentExe string) (ResolvedHelperExecutable, error) { helper := filepath.Join(filepath.Dir(agentExe), UserHelperBinaryName) _, statErr := os.Stat(helper) if statErr == nil { - return helper, nil + return ResolvedHelperExecutable{Path: helper}, nil } if errors.Is(statErr, fs.ErrNotExist) { + return ResolvedHelperExecutable{Path: agentExe, MainBinaryFallback: true}, nil + } + return ResolvedHelperExecutable{}, fmt.Errorf("stat %s: %w", helper, statErr) +} + +// helperFallbackWarningOwner limits the missing-companion warning to the +// lifetime of its owning spawner. The lifecycle manager owns one spawner; +// legacy standalone calls each own one private spawner. +type helperFallbackWarningOwner struct { + once sync.Once +} + +func (o *helperFallbackWarningOwner) WarnIfFallback(resolved ResolvedHelperExecutable) { + if o == nil || !resolved.MainBinaryFallback { + return + } + o.once.Do(func() { log.Warn("breeze-user-helper.exe missing — falling back to agent binary; console window will flash at user logon until the install is repaired", - "expectedPath", helper, - "fallbackPath", agentExe, + "expectedPath", filepath.Join(filepath.Dir(resolved.Path), UserHelperBinaryName), + "fallbackPath", resolved.Path, ) - return agentExe, nil - } - return "", fmt.Errorf("stat %s: %w", helper, statErr) + }) } diff --git a/agent/internal/sessionbroker/userhelper_path_other.go b/agent/internal/sessionbroker/userhelper_path_other.go index 6b33de9817..106b0e6040 100644 --- a/agent/internal/sessionbroker/userhelper_path_other.go +++ b/agent/internal/sessionbroker/userhelper_path_other.go @@ -8,6 +8,10 @@ import "os" // platforms. macOS and Linux launch the user-helper as a subcommand of the // agent binary (LaunchAgent / systemd user unit respectively), which does not // have the Windows console-window problem. -func userHelperExePath() (string, error) { - return os.Executable() +func userHelperExePath() (ResolvedHelperExecutable, error) { + path, err := os.Executable() + if err != nil { + return ResolvedHelperExecutable{}, err + } + return ResolvedHelperExecutable{Path: path}, nil } diff --git a/agent/internal/sessionbroker/userhelper_path_test.go b/agent/internal/sessionbroker/userhelper_path_test.go index d2baf77178..7be00172dc 100644 --- a/agent/internal/sessionbroker/userhelper_path_test.go +++ b/agent/internal/sessionbroker/userhelper_path_test.go @@ -30,8 +30,8 @@ func TestResolveUserHelperPath_PicksGUIBinaryWhenAvailable(t *testing.T) { if err != nil { t.Fatalf("resolveUserHelperPath returned unexpected error: %v", err) } - if got != helperExe { - t.Fatalf("resolveUserHelperPath = %q, want %q (sibling helper)", got, helperExe) + if got.Path != helperExe || got.MainBinaryFallback { + t.Fatalf("resolveUserHelperPath = %#v, want path %q without fallback", got, helperExe) } } @@ -41,8 +41,8 @@ func TestResolveUserHelperPath_PicksGUIBinaryWhenAvailable(t *testing.T) { // XML points at breeze-user-helper.exe but the binary itself is missing // (failed build, AV quarantine, tamper). The fallback returns the agent // path so run_as_user functionality keeps working at the cost of a visible -// console window — and the log.Warn provides the ops telemetry without -// which the silent fallback would reintroduce the bug this PR fixes. +// console window. The owning spawner uses the returned provenance to emit +// bounded ops telemetry rather than warning on every reconciliation. func TestUserHelperExePath_FallsBackToAgentWhenSiblingMissing(t *testing.T) { tmpDir := t.TempDir() agentExe := filepath.Join(tmpDir, "breeze-agent.exe") @@ -55,8 +55,29 @@ func TestUserHelperExePath_FallsBackToAgentWhenSiblingMissing(t *testing.T) { if err != nil { t.Fatalf("resolveUserHelperPath returned error on missing sibling, want nil + agent fallback: %v", err) } - if got != agentExe { - t.Fatalf("resolveUserHelperPath fallback = %q, want %q (agent path)", got, agentExe) + if got.Path != agentExe || !got.MainBinaryFallback { + t.Fatalf("resolveUserHelperPath fallback = %#v, want path %q with fallback", got, agentExe) + } +} + +func TestHelperFallbackWarningOwnerWarnsOncePerOwner(t *testing.T) { + buf := captureLogs(t) + resolved := ResolvedHelperExecutable{ + Path: filepath.Join(t.TempDir(), "breeze-agent.exe"), + MainBinaryFallback: true, + } + + var lifecycleOwner helperFallbackWarningOwner + lifecycleOwner.WarnIfFallback(resolved) + lifecycleOwner.WarnIfFallback(resolved) + + var standaloneOwner helperFallbackWarningOwner + standaloneOwner.WarnIfFallback(resolved) + standaloneOwner.WarnIfFallback(resolved) + + const warning = "breeze-user-helper.exe missing" + if got := strings.Count(buf.String(), warning); got != 2 { + t.Fatalf("warning count = %d, want one for each of two owners; logs: %s", got, buf.String()) } } diff --git a/agent/internal/sessionbroker/userhelper_path_windows.go b/agent/internal/sessionbroker/userhelper_path_windows.go index 8aaa0ee26d..f193df3bd9 100644 --- a/agent/internal/sessionbroker/userhelper_path_windows.go +++ b/agent/internal/sessionbroker/userhelper_path_windows.go @@ -17,10 +17,10 @@ import ( // fallback semantics are unit-testable on every platform the agent builds on, // not just Windows. This wrapper just supplies the agent's own executable // path and delegates. -func userHelperExePath() (string, error) { +func userHelperExePath() (ResolvedHelperExecutable, error) { agentExe, err := os.Executable() if err != nil { - return "", fmt.Errorf("os.Executable: %w", err) + return ResolvedHelperExecutable{}, fmt.Errorf("os.Executable: %w", err) } return resolveUserHelperPath(agentExe) } From bbec3d3580efde0a5a3bc733adeeb3e9cb192a35 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 17:58:09 -0600 Subject: [PATCH 15/47] fix(agent): warn before helper process creation --- agent/internal/agentapp/process_role_test.go | 2 + .../sessionbroker/helper_job_windows_test.go | 95 +++++++++++++++++-- .../internal/sessionbroker/spawner_windows.go | 44 ++++----- .../sessionbroker/userhelper_path_test.go | 30 ++++-- 4 files changed, 133 insertions(+), 38 deletions(-) diff --git a/agent/internal/agentapp/process_role_test.go b/agent/internal/agentapp/process_role_test.go index 495bb5098c..8878efd841 100644 --- a/agent/internal/agentapp/process_role_test.go +++ b/agent/internal/agentapp/process_role_test.go @@ -19,6 +19,8 @@ func TestClassifyProcess(t *testing.T) { {"fallback user", "user-helper", "user", "breeze-agent.exe", false, "user-helper", true}, {"renamed fallback user", "user-helper", "user", "breeze-agent-0.70.exe", false, "user-helper", true}, {"companion system", "user-helper", "system", "breeze-user-helper.exe", false, "system-helper", false}, + {"empty helper role", "user-helper", "", "breeze-agent.exe", false, "other", false}, + {"invalid helper role", "user-helper", "invalid", "breeze-agent.exe", false, "other", false}, {"status", "status", "", "breeze-agent.exe", false, "other", false}, } for _, tt := range tests { diff --git a/agent/internal/sessionbroker/helper_job_windows_test.go b/agent/internal/sessionbroker/helper_job_windows_test.go index a196a26f93..91ab85315b 100644 --- a/agent/internal/sessionbroker/helper_job_windows_test.go +++ b/agent/internal/sessionbroker/helper_job_windows_test.go @@ -225,13 +225,20 @@ func fakeSuspendedHelper() *suspendedHelper { } } +func fakeResolvedHelperExecutable() (ResolvedHelperExecutable, error) { + return ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-user-helper.exe`}, nil +} + func TestWindowsHelperSpawnerRetainsMainBinaryFallback(t *testing.T) { job := &fakeHelperJob{} spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ - createSuspended: func(HelperKey) (*suspendedHelper, error) { + resolveExecutable: func() (ResolvedHelperExecutable, error) { + return ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-agent.exe`, MainBinaryFallback: true}, nil + }, + createSuspended: func(_ HelperKey, resolved ResolvedHelperExecutable) (*suspendedHelper, error) { pending := fakeSuspendedHelper() - pending.binaryPath = `C:\Program Files\Breeze\breeze-agent.exe` - pending.mainBinaryFallback = true + pending.binaryPath = resolved.Path + pending.mainBinaryFallback = resolved.MainBinaryFallback return pending, nil }, resumeThread: func(windows.Handle) (uint32, error) { return 1, nil }, @@ -248,6 +255,76 @@ func TestWindowsHelperSpawnerRetainsMainBinaryFallback(t *testing.T) { } } +func TestWindowsHelperSpawnerWarnsBeforeRepeatedCreateFailures(t *testing.T) { + createErr := errors.New("CreateProcessAsUser failed") + tests := []struct { + name string + role string + resolved ResolvedHelperExecutable + wantWarnings int + }{ + { + name: "system fallback", + role: "system", + resolved: ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-agent.exe`, MainBinaryFallback: true}, + wantWarnings: 1, + }, + { + name: "user fallback", + role: "user", + resolved: ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-agent.exe`, MainBinaryFallback: true}, + wantWarnings: 1, + }, + { + name: "system companion", + role: "system", + resolved: ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-user-helper.exe`}, + }, + { + name: "user companion", + role: "user", + resolved: ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-user-helper.exe`}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := captureLogs(t) + job := &fakeHelperJob{} + resolveCalls := 0 + createCalls := 0 + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + resolveExecutable: func() (ResolvedHelperExecutable, error) { + resolveCalls++ + return tt.resolved, nil + }, + createSuspended: func(key HelperKey, resolved ResolvedHelperExecutable) (*suspendedHelper, error) { + createCalls++ + if key.Role != tt.role { + t.Fatalf("create role = %q, want %q", key.Role, tt.role) + } + if resolved != tt.resolved { + t.Fatalf("create resolved executable = %#v, want %#v", resolved, tt.resolved) + } + return nil, createErr + }, + }) + + for attempt := 0; attempt < 3; attempt++ { + process, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: tt.role}) + if process != nil || !errors.Is(err, createErr) { + t.Fatalf("Spawn attempt %d = (%#v, %v), want nil and create error", attempt, process, err) + } + } + if resolveCalls != 3 || createCalls != 3 { + t.Fatalf("calls resolve=%d create=%d, want 3 each", resolveCalls, createCalls) + } + if got := strings.Count(buf.String(), "breeze-user-helper.exe missing"); got != tt.wantWarnings { + t.Fatalf("warning count = %d, want %d; logs: %s", got, tt.wantWarnings, buf.String()) + } + }) + } +} + func TestWindowsHelperSpawnerAssignFailureTerminatesSuspendedProcess(t *testing.T) { assignErr := errors.New("assign failed") job := &fakeHelperJob{assignErr: assignErr} @@ -256,7 +333,8 @@ func TestWindowsHelperSpawnerAssignFailureTerminatesSuspendedProcess(t *testing. var terminated []windows.Handle var closed []windows.Handle spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ - createSuspended: func(HelperKey) (*suspendedHelper, error) { + resolveExecutable: fakeResolvedHelperExecutable, + createSuspended: func(HelperKey, ResolvedHelperExecutable) (*suspendedHelper, error) { return fakeSuspendedHelper(), nil }, resumeThread: func(handle windows.Handle) (uint32, error) { @@ -306,7 +384,8 @@ func TestWindowsHelperSpawnerResumeFailureTerminatesSuspendedProcess(t *testing. var terminated []windows.Handle var closed []windows.Handle spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ - createSuspended: func(HelperKey) (*suspendedHelper, error) { + resolveExecutable: fakeResolvedHelperExecutable, + createSuspended: func(HelperKey, ResolvedHelperExecutable) (*suspendedHelper, error) { return fakeSuspendedHelper(), nil }, resumeThread: func(windows.Handle) (uint32, error) { @@ -352,7 +431,8 @@ func TestWindowsHelperSpawnerSerializesSpawnThroughResumeAgainstClose(t *testing allowResume := make(chan struct{}) closeStarted := make(chan struct{}) spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ - createSuspended: func(HelperKey) (*suspendedHelper, error) { + resolveExecutable: fakeResolvedHelperExecutable, + createSuspended: func(HelperKey, ResolvedHelperExecutable) (*suspendedHelper, error) { return fakeSuspendedHelper(), nil }, resumeThread: func(windows.Handle) (uint32, error) { @@ -575,7 +655,8 @@ func TestWindowsHelperSpawnerClosePreventsLaterSpawn(t *testing.T) { job := &fakeHelperJob{} createCalls := 0 spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ - createSuspended: func(HelperKey) (*suspendedHelper, error) { + resolveExecutable: fakeResolvedHelperExecutable, + createSuspended: func(HelperKey, ResolvedHelperExecutable) (*suspendedHelper, error) { createCalls++ return fakeSuspendedHelper(), nil }, diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index 2688ab4d02..2949c5b46e 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -185,11 +185,12 @@ type suspendedHelper struct { } type windowsSpawnOps struct { - createSuspended func(HelperKey) (*suspendedHelper, error) - resumeThread func(windows.Handle) (uint32, error) - terminateProcess func(windows.Handle, uint32) error - closeHandle func(windows.Handle) error - closeStarting func() + resolveExecutable func() (ResolvedHelperExecutable, error) + createSuspended func(HelperKey, ResolvedHelperExecutable) (*suspendedHelper, error) + resumeThread func(windows.Handle) (uint32, error) + terminateProcess func(windows.Handle, uint32) error + closeHandle func(windows.Handle) error + closeStarting func() } // windowsHelperSpawner is the single owner of the helper Job Object. Its @@ -212,6 +213,9 @@ func newWindowsHelperSpawner() (*windowsHelperSpawner, error) { } func newWindowsHelperSpawnerWithJob(job helperJobOwner, ops windowsSpawnOps) *windowsHelperSpawner { + if ops.resolveExecutable == nil { + ops.resolveExecutable = userHelperExePath + } if ops.createSuspended == nil { ops.createSuspended = createHelperSuspended } @@ -237,7 +241,13 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { return nil, fmt.Errorf("Windows helper spawner is closed") } - pending, err := s.ops.createSuspended(key) + resolvedExe, err := s.ops.resolveExecutable() + if err != nil { + return nil, fmt.Errorf("resolve helper executable: %w", err) + } + s.fallbackWarning.WarnIfFallback(resolvedExe) + + pending, err := s.ops.createSuspended(key, resolvedExe) if err != nil { return nil, err } @@ -249,10 +259,6 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { _ = s.ops.closeHandle(pending.process) } }() - s.fallbackWarning.WarnIfFallback(ResolvedHelperExecutable{ - Path: pending.binaryPath, - MainBinaryFallback: pending.mainBinaryFallback, - }) if err := s.job.Assign(pending.process); err != nil { return nil, fmt.Errorf("assign helper to job: %w", err) } @@ -297,17 +303,17 @@ func (s *windowsHelperSpawner) Close() error { return s.job.Close() } -func createHelperSuspended(key HelperKey) (*suspendedHelper, error) { +func createHelperSuspended(key HelperKey, resolvedExe ResolvedHelperExecutable) (*suspendedHelper, error) { if key.Role == "user" { - return createUserHelperSuspended(key.WindowsSessionID) + return createUserHelperSuspended(key.WindowsSessionID, resolvedExe) } - return createSystemHelperSuspended(key.WindowsSessionID) + return createSystemHelperSuspended(key.WindowsSessionID, resolvedExe) } // createSystemHelperSuspended creates the SYSTEM-token helper without allowing // its primary thread to run. The caller must assign it to the Job Object before // resuming it. -func createSystemHelperSuspended(sessionID uint32) (*suspendedHelper, error) { +func createSystemHelperSuspended(sessionID uint32, resolvedExe ResolvedHelperExecutable) (*suspendedHelper, error) { var processToken windows.Token proc, err := windows.GetCurrentProcess() if err != nil { @@ -340,10 +346,6 @@ func createSystemHelperSuspended(sessionID uint32) (*suspendedHelper, error) { return nil, fmt.Errorf("SetTokenInformation(TokenSessionId=%d): %w", sessionID, err) } - resolvedExe, err := userHelperExePath() - if err != nil { - return nil, fmt.Errorf("userHelperExePath: %w", err) - } cmdLine, err := windows.UTF16PtrFromString(buildUserHelperCmdLine(resolvedExe.Path, "system")) if err != nil { return nil, fmt.Errorf("UTF16PtrFromString: %w", err) @@ -386,7 +388,7 @@ func createSystemHelperSuspended(sessionID uint32) (*suspendedHelper, error) { // createUserHelperSuspended creates the interactive-user helper without // allowing its primary thread to run. The caller assigns it to the Job Object // before resume. -func createUserHelperSuspended(sessionID uint32) (*suspendedHelper, error) { +func createUserHelperSuspended(sessionID uint32, resolvedExe ResolvedHelperExecutable) (*suspendedHelper, error) { dupToken, envBlock, method, err := acquireUserToken(sessionID) if err != nil { return nil, fmt.Errorf("acquire user token(session=%d): %w", sessionID, err) @@ -396,10 +398,6 @@ func createUserHelperSuspended(sessionID uint32) (*suspendedHelper, error) { defer windows.DestroyEnvironmentBlock(envBlock) } - resolvedExe, err := userHelperExePath() - if err != nil { - return nil, fmt.Errorf("userHelperExePath: %w", err) - } cmdLine, err := windows.UTF16PtrFromString(buildUserHelperCmdLine(resolvedExe.Path, "user")) if err != nil { return nil, fmt.Errorf("UTF16PtrFromString: %w", err) diff --git a/agent/internal/sessionbroker/userhelper_path_test.go b/agent/internal/sessionbroker/userhelper_path_test.go index 7be00172dc..4ee0747e2c 100644 --- a/agent/internal/sessionbroker/userhelper_path_test.go +++ b/agent/internal/sessionbroker/userhelper_path_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" ) @@ -67,17 +68,30 @@ func TestHelperFallbackWarningOwnerWarnsOncePerOwner(t *testing.T) { MainBinaryFallback: true, } - var lifecycleOwner helperFallbackWarningOwner - lifecycleOwner.WarnIfFallback(resolved) - lifecycleOwner.WarnIfFallback(resolved) - - var standaloneOwner helperFallbackWarningOwner - standaloneOwner.WarnIfFallback(resolved) - standaloneOwner.WarnIfFallback(resolved) + owners := []*helperFallbackWarningOwner{{}, {}} + start := make(chan struct{}) + var ready sync.WaitGroup + var calls sync.WaitGroup + const callsPerOwner = 32 + ready.Add(len(owners) * callsPerOwner) + calls.Add(len(owners) * callsPerOwner) + for _, owner := range owners { + for range callsPerOwner { + go func(owner *helperFallbackWarningOwner) { + defer calls.Done() + ready.Done() + <-start + owner.WarnIfFallback(resolved) + }(owner) + } + } + ready.Wait() + close(start) + calls.Wait() const warning = "breeze-user-helper.exe missing" if got := strings.Count(buf.String(), warning); got != 2 { - t.Fatalf("warning count = %d, want one for each of two owners; logs: %s", got, buf.String()) + t.Fatalf("warning count = %d, want one for each of two concurrent owners; logs: %s", got, buf.String()) } } From 09a495f74c637ee802754aa8fb1fe75eae159422 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 18:29:45 -0600 Subject: [PATCH 16/47] fix(agent): enforce one full Windows agent instance --- agent/internal/agentapp/main_instance.go | 23 + .../internal/agentapp/main_instance_other.go | 11 + .../agentapp/main_instance_windows.go | 382 +++++++++++++++ .../agentapp/main_instance_windows_test.go | 239 +++++++++ agent/internal/config/permissions_windows.go | 463 ++++++++++++++++++ .../config/permissions_windows_test.go | 204 ++++++++ 6 files changed, 1322 insertions(+) create mode 100644 agent/internal/agentapp/main_instance.go create mode 100644 agent/internal/agentapp/main_instance_other.go create mode 100644 agent/internal/agentapp/main_instance_windows.go create mode 100644 agent/internal/agentapp/main_instance_windows_test.go diff --git a/agent/internal/agentapp/main_instance.go b/agent/internal/agentapp/main_instance.go new file mode 100644 index 0000000000..4259ede30e --- /dev/null +++ b/agent/internal/agentapp/main_instance.go @@ -0,0 +1,23 @@ +package agentapp + +import ( + "errors" + "os" +) + +const ( + mainAgentLockFile = "agent.lock" + exitAlreadyRunning = 17 + exitInstanceGuardError = 18 +) + +var ErrMainAgentAlreadyRunning = errors.New("main agent already running") + +type mainAgentGuard interface { + Close() error +} + +var ( + acquireMainAgentGuardFn = acquireMainAgentGuard + mainAgentExitFn = os.Exit +) diff --git a/agent/internal/agentapp/main_instance_other.go b/agent/internal/agentapp/main_instance_other.go new file mode 100644 index 0000000000..c0c6ee18ed --- /dev/null +++ b/agent/internal/agentapp/main_instance_other.go @@ -0,0 +1,11 @@ +//go:build !windows + +package agentapp + +type noopMainAgentGuard struct{} + +func acquireMainAgentGuard(ProcessStartup) (mainAgentGuard, error) { + return noopMainAgentGuard{}, nil +} + +func (noopMainAgentGuard) Close() error { return nil } diff --git a/agent/internal/agentapp/main_instance_windows.go b/agent/internal/agentapp/main_instance_windows.go new file mode 100644 index 0000000000..a58dd96b06 --- /dev/null +++ b/agent/internal/agentapp/main_instance_windows.go @@ -0,0 +1,382 @@ +//go:build windows + +package agentapp + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + "unsafe" + + "github.com/breeze-rmm/agent/internal/config" + "golang.org/x/sys/windows" +) + +const windowsMainAgentLockSDDL = "O:BAG:BAD:P(A;;FA;;;SY)(A;;FA;;;BA)" + +var errMainAgentLockContended = errors.New("main-agent lock contended") + +type mainAgentLockDirectory interface { + Handle() windows.Handle + Close() error +} + +type openedMainAgentLock struct { + handle windows.Handle + directory mainAgentLockDirectory +} + +// mainAgentLockMetadata is an explicit allowlist. Future ProcessStartup fields +// do not enter ProgramData lock metadata without a deliberate security review. +type mainAgentLockMetadata struct { + Binary string `json:"binary"` + ExecutablePath string `json:"executablePath"` + PID int `json:"pid"` + ParentPID int `json:"parentPid"` + WindowsSessionID uint32 `json:"windowsSessionId"` + LaunchMode string `json:"launchMode"` + HelperRole string `json:"helperRole,omitempty"` + LifecycleKey string `json:"lifecycleKey,omitempty"` + CompanionHelper bool `json:"companionHelper"` + MainBinaryFallback bool `json:"mainBinaryFallback"` + Version string `json:"version"` + CreatedAt time.Time `json:"createdAt"` +} + +func mainAgentLockMetadataFrom(meta ProcessStartup) mainAgentLockMetadata { + return mainAgentLockMetadata{ + Binary: meta.Binary, + ExecutablePath: meta.ExecutablePath, + PID: meta.PID, + ParentPID: meta.ParentPID, + WindowsSessionID: meta.WindowsSessionID, + LaunchMode: meta.LaunchMode, + HelperRole: meta.HelperRole, + LifecycleKey: meta.LifecycleKey, + CompanionHelper: meta.CompanionHelper, + MainBinaryFallback: meta.MainBinaryFallback, + Version: meta.Version, + CreatedAt: meta.CreatedAt, + } +} + +var ( + prepareMainAgentLockDirFn = config.PrepareMainAgentLockDir + openMainAgentLockFn = openMainAgentLock + openPreparedMainAgentLockDirFn = openPreparedMainAgentLockDir + openMainAgentLockRelativeFn = openMainAgentLockRelative + getMainAgentLockInfoFn = windows.GetFileInformationByHandle + hardenAndVerifyLockHandleFn = hardenAndVerifyLockHandle +) + +type fileMainAgentGuard struct { + mu sync.Mutex + file *os.File + dir mainAgentLockDirectory +} + +func acquireMainAgentGuard(meta ProcessStartup) (mainAgentGuard, error) { + dir, err := prepareMainAgentLockDirFn() + if err != nil { + return nil, fmt.Errorf("prepare instance-lock directory: %w", err) + } + path := filepath.Join(dir, mainAgentLockFile) + opened, err := openMainAgentLockFn(path) + if err != nil { + if errors.Is(err, errMainAgentLockContended) { + return nil, ErrMainAgentAlreadyRunning + } + return nil, fmt.Errorf("open main-agent lock: %w", err) + } + h := opened.handle + f := os.NewFile(uintptr(h), path) + if f == nil { + _ = windows.CloseHandle(h) + _ = opened.directory.Close() + return nil, errors.New("wrap main-agent lock handle") + } + + var info windows.ByHandleFileInformation + if err := getMainAgentLockInfoFn(h, &info); err != nil { + _ = f.Close() + _ = opened.directory.Close() + return nil, fmt.Errorf("inspect main-agent lock: %w", err) + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + _ = f.Close() + _ = opened.directory.Close() + return nil, errors.New("refuse reparse-point main-agent lock") + } + if err := hardenAndVerifyLockHandleFn(h); err != nil { + _ = f.Close() + _ = opened.directory.Close() + return nil, fmt.Errorf("verify main-agent lock security: %w", err) + } + if err := f.Truncate(0); err != nil { + _ = f.Close() + _ = opened.directory.Close() + return nil, fmt.Errorf("truncate main-agent lock metadata: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + _ = f.Close() + _ = opened.directory.Close() + return nil, fmt.Errorf("seek main-agent lock metadata: %w", err) + } + if err := json.NewEncoder(f).Encode(mainAgentLockMetadataFrom(meta)); err != nil { + _ = f.Close() + _ = opened.directory.Close() + return nil, fmt.Errorf("write main-agent lock metadata: %w", err) + } + if err := f.Sync(); err != nil { + _ = f.Close() + _ = opened.directory.Close() + return nil, fmt.Errorf("flush main-agent lock metadata: %w", err) + } + return &fileMainAgentGuard{file: f, dir: opened.directory}, nil +} + +func openPreparedMainAgentLockDir() (mainAgentLockDirectory, error) { + return config.OpenPreparedMainAgentLockDir() +} + +func openMainAgentLock(path string) (openedMainAgentLock, error) { + if filepath.Base(path) != mainAgentLockFile { + return openedMainAgentLock{}, fmt.Errorf("unexpected main-agent lock name %q", filepath.Base(path)) + } + dir, err := openPreparedMainAgentLockDirFn() + if err != nil { + return openedMainAgentLock{}, fmt.Errorf("open verified main-agent run directory: %w", err) + } + h, err := openMainAgentLockRelativeFn(dir.Handle(), mainAgentLockFile) + if err != nil { + _ = dir.Close() + if errors.Is(err, windows.ERROR_SHARING_VIOLATION) || errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return openedMainAgentLock{}, errMainAgentLockContended + } + return openedMainAgentLock{}, fmt.Errorf("open final main-agent lock relative to verified run handle: %w", err) + } + return openedMainAgentLock{handle: h, directory: dir}, nil +} + +func openMainAgentLockRelative(runHandle windows.Handle, name string) (windows.Handle, error) { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return windows.InvalidHandle, fmt.Errorf("encode relative main-agent lock name: %w", err) + } + sa, err := privateLockSecurityAttributes() + if err != nil { + return windows.InvalidHandle, fmt.Errorf("build main-agent lock security: %w", err) + } + attrs := &windows.OBJECT_ATTRIBUTES{ + Length: uint32(unsafe.Sizeof(windows.OBJECT_ATTRIBUTES{})), + RootDirectory: runHandle, + ObjectName: objectName, + Attributes: windows.OBJ_CASE_INSENSITIVE, + SecurityDescriptor: sa.SecurityDescriptor, + } + var ( + h windows.Handle + iosb windows.IO_STATUS_BLOCK + ) + err = windows.NtCreateFile( + &h, + windows.GENERIC_READ|windows.GENERIC_WRITE|windows.READ_CONTROL|windows.WRITE_DAC|windows.WRITE_OWNER, + attrs, + &iosb, + nil, + windows.FILE_ATTRIBUTE_NORMAL, + 0, + windows.FILE_OPEN_IF, + windows.FILE_NON_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + 0, + 0, + ) + if err == windows.STATUS_SHARING_VIOLATION { + return windows.InvalidHandle, windows.ERROR_SHARING_VIOLATION + } + if err == windows.STATUS_FILE_LOCK_CONFLICT || err == windows.STATUS_LOCK_NOT_GRANTED { + return windows.InvalidHandle, windows.ERROR_LOCK_VIOLATION + } + if err != nil { + return windows.InvalidHandle, err + } + return h, nil +} + +func privateLockSecurityAttributes() (*windows.SecurityAttributes, error) { + sd, err := windows.SecurityDescriptorFromString(windowsMainAgentLockSDDL) + if err != nil { + return nil, fmt.Errorf("parse private lock security descriptor: %w", err) + } + return &windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + InheritHandle: 0, + }, nil +} + +func hardenAndVerifyLockHandle(h windows.Handle) error { + before, err := mainAgentHandleIdentity(h) + if err != nil { + return err + } + want, err := windows.SecurityDescriptorFromString(windowsMainAgentLockSDDL) + if err != nil { + return fmt.Errorf("parse lock security descriptor: %w", err) + } + owner, _, err := want.Owner() + if err != nil { + return fmt.Errorf("extract lock owner: %w", err) + } + group, _, err := want.Group() + if err != nil { + return fmt.Errorf("extract lock group: %w", err) + } + dacl, _, err := want.DACL() + if err != nil { + return fmt.Errorf("extract lock DACL: %w", err) + } + if err := windows.SetSecurityInfo( + h, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, + owner, + group, + dacl, + nil, + ); err != nil { + return fmt.Errorf("set lock owner and protected DACL: %w", err) + } + after, err := mainAgentHandleIdentity(h) + if err != nil { + return err + } + if before != after { + return errors.New("main-agent lock handle identity changed during hardening") + } + if after.attributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return errors.New("main-agent lock became a reparse point during hardening") + } + return verifyMainAgentLockSecurity(h, want) +} + +type mainAgentLockIdentity struct { + volume uint32 + indexHigh uint32 + indexLow uint32 + attributes uint32 +} + +func mainAgentHandleIdentity(h windows.Handle) (mainAgentLockIdentity, error) { + var info windows.ByHandleFileInformation + if err := getMainAgentLockInfoFn(h, &info); err != nil { + return mainAgentLockIdentity{}, fmt.Errorf("read main-agent lock identity: %w", err) + } + return mainAgentLockIdentity{ + volume: info.VolumeSerialNumber, + indexHigh: info.FileIndexHigh, + indexLow: info.FileIndexLow, + attributes: info.FileAttributes, + }, nil +} + +func verifyMainAgentLockSecurity(h windows.Handle, want *windows.SECURITY_DESCRIPTOR) error { + got, err := windows.GetSecurityInfo( + h, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return fmt.Errorf("read lock security: %w", err) + } + if got == nil { + return errors.New("lock has no security descriptor") + } + owner, _, err := got.Owner() + if err != nil { + return fmt.Errorf("read lock owner: %w", err) + } + if owner == nil || (!owner.IsWellKnown(windows.WinLocalSystemSid) && !owner.IsWellKnown(windows.WinBuiltinAdministratorsSid)) { + return fmt.Errorf("lock owner is not LocalSystem or BUILTIN\\Administrators: %v", owner) + } + group, _, err := got.Group() + if err != nil { + return fmt.Errorf("read lock group: %w", err) + } + if group == nil || !group.IsWellKnown(windows.WinBuiltinAdministratorsSid) { + return fmt.Errorf("lock group is not BUILTIN\\Administrators: %v", group) + } + return verifyProtectedExactDACL(got, want, "main-agent lock") +} + +func verifyProtectedExactDACL(got, want *windows.SECURITY_DESCRIPTOR, object string) error { + control, _, err := got.Control() + if err != nil { + return fmt.Errorf("read %s security control: %w", object, err) + } + if control&windows.SE_DACL_PROTECTED == 0 { + return fmt.Errorf("%s DACL is not protected", object) + } + gotDACL, _, err := got.DACL() + if err != nil { + return fmt.Errorf("read %s DACL: %w", object, err) + } + wantDACL, _, err := want.DACL() + if err != nil { + return fmt.Errorf("read expected %s DACL: %w", object, err) + } + if !equalWindowsACL(gotDACL, wantDACL) { + return fmt.Errorf("%s DACL does not match the private SY/BA policy", object) + } + return nil +} + +func equalWindowsACL(a, b *windows.ACL) bool { + if a == nil || b == nil { + return a == b + } + if a.AceCount != b.AceCount { + return false + } + for i := uint32(0); i < uint32(a.AceCount); i++ { + var aACE, bACE *windows.ACCESS_ALLOWED_ACE + if windows.GetAce(a, i, &aACE) != nil || windows.GetAce(b, i, &bACE) != nil { + return false + } + if aACE.Header.AceType != bACE.Header.AceType || + aACE.Header.AceFlags != bACE.Header.AceFlags || + aACE.Mask != bACE.Mask { + return false + } + aSID := (*windows.SID)(unsafe.Pointer(&aACE.SidStart)) + bSID := (*windows.SID)(unsafe.Pointer(&bACE.SidStart)) + if !aSID.Equals(bSID) { + return false + } + } + return true +} + +func (g *fileMainAgentGuard) Close() error { + g.mu.Lock() + f := g.file + dir := g.dir + g.file = nil + g.dir = nil + g.mu.Unlock() + var firstErr error + if f != nil { + firstErr = f.Close() + } + if dir != nil { + if err := dir.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} diff --git a/agent/internal/agentapp/main_instance_windows_test.go b/agent/internal/agentapp/main_instance_windows_test.go new file mode 100644 index 0000000000..84de1b2974 --- /dev/null +++ b/agent/internal/agentapp/main_instance_windows_test.go @@ -0,0 +1,239 @@ +//go:build windows + +package agentapp + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/breeze-rmm/agent/internal/config" + "golang.org/x/sys/windows" +) + +func TestMainAgentGuardExclusiveAndReacquirable(t *testing.T) { + dir := t.TempDir() + restoreMainAgentGuardSeams(t) + useMainAgentLockTestDir(t, dir) + meta := ProcessStartup{PID: os.Getpid(), LaunchMode: "console-run", CreatedAt: time.Now()} + + first, err := acquireMainAgentGuard(meta) + if err != nil { + t.Fatal(err) + } + if _, err := acquireMainAgentGuard(meta); !errors.Is(err, ErrMainAgentAlreadyRunning) { + t.Fatalf("second acquire err=%v, want ErrMainAgentAlreadyRunning", err) + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + if err := first.Close(); err != nil { + t.Fatalf("idempotent close: %v", err) + } + third, err := acquireMainAgentGuard(meta) + if err != nil { + t.Fatalf("reacquire after close: %v", err) + } + _ = third.Close() +} + +func TestStaleLockFileDoesNotBlock(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, mainAgentLockFile), []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + restoreMainAgentGuardSeams(t) + useMainAgentLockTestDir(t, dir) + guard, err := acquireMainAgentGuard(ProcessStartup{PID: os.Getpid(), LaunchMode: "console-run"}) + if err != nil { + t.Fatal(err) + } + _ = guard.Close() +} + +func TestMainAgentGuardRejectsReparseFinalObject(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, mainAgentLockFile), nil, 0o600); err != nil { + t.Fatal(err) + } + restoreMainAgentGuardSeams(t) + useMainAgentLockTestDir(t, dir) + getMainAgentLockInfoFn = func(_ windows.Handle, info *windows.ByHandleFileInformation) error { + info.FileAttributes = windows.FILE_ATTRIBUTE_REPARSE_POINT + return nil + } + + _, err := acquireMainAgentGuard(ProcessStartup{PID: os.Getpid(), LaunchMode: "console-run"}) + if err == nil { + t.Fatal("expected reparse-point security error") + } + if errors.Is(err, ErrMainAgentAlreadyRunning) { + t.Fatalf("reparse security error misclassified as duplicate agent: %v", err) + } +} + +func TestMainAgentGuardOnlyMapsFinalLockContentionErrors(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"sharing violation", windows.ERROR_SHARING_VIOLATION, true}, + {"lock violation", windows.ERROR_LOCK_VIOLATION, true}, + {"access denied", windows.ERROR_ACCESS_DENIED, false}, + {"invalid ACL", windows.ERROR_INVALID_ACL, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + restoreMainAgentGuardSeams(t) + prepareMainAgentLockDirFn = func() (string, error) { return t.TempDir(), nil } + openPreparedMainAgentLockDirFn = func() (mainAgentLockDirectory, error) { + return &testMainAgentLockDirectory{handle: windows.Handle(0x1234)}, nil + } + openMainAgentLockRelativeFn = func(windows.Handle, string) (windows.Handle, error) { + return windows.InvalidHandle, tt.err + } + + _, err := acquireMainAgentGuard(ProcessStartup{PID: os.Getpid()}) + if got := errors.Is(err, ErrMainAgentAlreadyRunning); got != tt.want { + t.Fatalf("error = %v, duplicate classification=%v, want %v", err, got, tt.want) + } + }) + } +} + +func TestMainAgentGuardDirectorySharingFailureIsSecurityError(t *testing.T) { + restoreMainAgentGuardSeams(t) + prepareMainAgentLockDirFn = func() (string, error) { return t.TempDir(), nil } + openPreparedMainAgentLockDirFn = func() (mainAgentLockDirectory, error) { + return nil, windows.ERROR_SHARING_VIOLATION + } + openMainAgentLockRelativeFn = func(windows.Handle, string) (windows.Handle, error) { + t.Fatal("final lock open must not run after directory security failure") + return windows.InvalidHandle, nil + } + + _, err := acquireMainAgentGuard(ProcessStartup{PID: os.Getpid()}) + if err == nil { + t.Fatal("expected fail-closed directory sharing error") + } + if errors.Is(err, ErrMainAgentAlreadyRunning) { + t.Fatalf("directory sharing failure misclassified as duplicate agent: %v", err) + } +} + +func TestMainAgentGuardPrepareFailureIsSecurityError(t *testing.T) { + restoreMainAgentGuardSeams(t) + wantErr := errors.New("injected directory security failure") + prepareMainAgentLockDirFn = func() (string, error) { return "", wantErr } + + _, err := acquireMainAgentGuard(ProcessStartup{PID: os.Getpid()}) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want wrapped preparation error", err) + } + if errors.Is(err, ErrMainAgentAlreadyRunning) { + t.Fatalf("directory security error misclassified as duplicate agent: %v", err) + } +} + +func TestOpenMainAgentLockUsesVerifiedRunHandle(t *testing.T) { + restoreMainAgentGuardSeams(t) + wantDirHandle := windows.Handle(0x1234) + wantLockHandle := windows.Handle(0x5678) + dir := &testMainAgentLockDirectory{handle: wantDirHandle} + openPreparedMainAgentLockDirFn = func() (mainAgentLockDirectory, error) { + return dir, nil + } + openMainAgentLockRelativeFn = func(dir windows.Handle, name string) (windows.Handle, error) { + if dir != wantDirHandle { + t.Fatalf("relative root = %#x, want verified run handle %#x", dir, wantDirHandle) + } + if name != mainAgentLockFile { + t.Fatalf("relative lock name = %q, want %q", name, mainAgentLockFile) + } + return wantLockHandle, nil + } + + got, err := openMainAgentLock(filepath.Join(`C:\ProgramData\Breeze\run`, mainAgentLockFile)) + if err != nil { + t.Fatal(err) + } + if got.handle != wantLockHandle { + t.Fatalf("lock handle = %#x, want %#x", got.handle, wantLockHandle) + } + if got.directory != dir { + t.Fatal("verified namespace owner was not retained with the lock handle") + } + if dir.closed { + t.Fatal("verified namespace was closed before guard ownership") + } +} + +func restoreMainAgentGuardSeams(t *testing.T) { + t.Helper() + oldPrepare := prepareMainAgentLockDirFn + oldOpen := openMainAgentLockFn + oldInfo := getMainAgentLockInfoFn + oldHarden := hardenAndVerifyLockHandleFn + oldOpenPreparedDir := openPreparedMainAgentLockDirFn + oldOpenRelative := openMainAgentLockRelativeFn + t.Cleanup(func() { + prepareMainAgentLockDirFn = oldPrepare + openMainAgentLockFn = oldOpen + getMainAgentLockInfoFn = oldInfo + hardenAndVerifyLockHandleFn = oldHarden + openPreparedMainAgentLockDirFn = oldOpenPreparedDir + openMainAgentLockRelativeFn = oldOpenRelative + }) + prepareMainAgentLockDirFn = config.PrepareMainAgentLockDir +} + +func useMainAgentLockTestDir(t *testing.T, dir string) { + t.Helper() + prepareMainAgentLockDirFn = func() (string, error) { return dir, nil } + openPreparedMainAgentLockDirFn = func() (mainAgentLockDirectory, error) { + path16, err := windows.UTF16PtrFromString(dir) + if err != nil { + return nil, err + } + h, err := windows.CreateFile( + path16, + windows.GENERIC_READ|windows.GENERIC_WRITE|windows.READ_CONTROL|windows.WRITE_DAC|windows.WRITE_OWNER, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_EXISTING, + windows.FILE_FLAG_OPEN_REPARSE_POINT|windows.FILE_FLAG_BACKUP_SEMANTICS, + 0, + ) + if err != nil { + return nil, err + } + return &testMainAgentLockDirectory{ + handle: h, + closeFn: func() error { + return windows.CloseHandle(h) + }, + }, nil + } +} + +type testMainAgentLockDirectory struct { + handle windows.Handle + closed bool + closeFn func() error +} + +func (d *testMainAgentLockDirectory) Handle() windows.Handle { return d.handle } + +func (d *testMainAgentLockDirectory) Close() error { + if d.closed { + return nil + } + d.closed = true + if d.closeFn != nil { + return d.closeFn() + } + return nil +} diff --git a/agent/internal/config/permissions_windows.go b/agent/internal/config/permissions_windows.go index 6dd653b91a..8698be037c 100644 --- a/agent/internal/config/permissions_windows.go +++ b/agent/internal/config/permissions_windows.go @@ -3,7 +3,10 @@ package config import ( + "errors" "fmt" + "path/filepath" + "sync" "unsafe" "golang.org/x/sys/windows" @@ -23,6 +26,9 @@ const ( windowsConfigFileSDDL = `D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FR;;;BU)` windowsSecretFileSDDL = `D:P(A;;FA;;;SY)(A;;FA;;;BA)` + windowsConfigDirCreateSDDL = "O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;;FRFX;;;BU)" + windowsAgentRunDirSDDL = "O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)" + // windowsProgramDataDirSDDL mirrors the MSI HardenProgramDataAcl action // (icacls /inheritance:r /grant:r *S-1-5-18:(OI)(CI)F *S-1-5-32-544:(OI)(CI)F): // SYSTEM and Administrators get full control, container+object inheritable, @@ -33,6 +39,463 @@ const ( windowsProgramDataDirSDDL = `D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)` ) +const fileDeleteChildAccess = 0x00000040 + +var ( + mainAgentConfigDirFn = ConfigDir + getMainAgentFileInformationFn = windows.GetFileInformationByHandle + getMainAgentSecurityInfoFn = windows.GetSecurityInfo + setMainAgentSecurityInfoFn = windows.SetSecurityInfo + removeMainAgentLockRelativeFn = removeMainAgentLockRelative + openMainAgentDirectoryFn = openMainAgentDirectory + createMainAgentDirectoryFn = createMainAgentDirectory + closeMainAgentDirectoryHandleFn = windows.CloseHandle +) + +// PrepareMainAgentLockDir returns a private, non-reparse run directory only +// after proving the ConfigDir and run path names still resolve to the handles +// whose owner and protected DACL were repaired. All security and identity +// failures are fatal; callers must not reinterpret them as lock contention. +func PrepareMainAgentLockDir() (string, error) { + configPath := mainAgentConfigDirFn() + configHandle, err := ensureMainAgentDirectory(configPath, windowsConfigDirCreateSDDL) + if err != nil { + return "", fmt.Errorf("secure main-agent config directory: %w", err) + } + defer closeMainAgentDirectoryHandleFn(configHandle) + + configIdentity, err := inspectMainAgentDirectory(configHandle, "config directory") + if err != nil { + return "", err + } + if err := hardenAndVerifyMainAgentDirectory(configHandle, windowsConfigDirCreateSDDL, "config directory"); err != nil { + return "", err + } + if err := verifyMainAgentDirectoryPath(configPath, configIdentity, "", "config directory"); err != nil { + return "", err + } + + runPath := filepath.Join(configPath, "run") + runHandle, err := ensureMainAgentDirectory(runPath, windowsAgentRunDirSDDL) + if err != nil { + return "", fmt.Errorf("secure main-agent run directory: %w", err) + } + defer closeMainAgentDirectoryHandleFn(runHandle) + + runIdentity, err := inspectMainAgentDirectory(runHandle, "run directory") + if err != nil { + return "", err + } + originalTrusted, err := mainAgentHandleHasTrustedOwner(runHandle, "run directory") + if err != nil { + return "", err + } + if err := hardenAndVerifyMainAgentDirectory(runHandle, windowsAgentRunDirSDDL, "run directory"); err != nil { + return "", err + } + if !originalTrusted { + if err := removeMainAgentLockRelativeFn(runHandle, "agent.lock"); err != nil { + return "", fmt.Errorf("sanitize hostile main-agent lock: %w", err) + } + } + if err := verifyMainAgentDirectoryPath(runPath, runIdentity, windowsAgentRunDirSDDL, "run directory"); err != nil { + return "", err + } + return runPath, nil +} + +// MainAgentLockDirectory pins the verified ConfigDir and run namespace while +// the main-agent lock is held. Handle is the run directory used as the root +// for handle-relative lock acquisition. +type MainAgentLockDirectory struct { + mu sync.Mutex + configHandle windows.Handle + runHandle windows.Handle +} + +func (d *MainAgentLockDirectory) Handle() windows.Handle { return d.runHandle } + +func (d *MainAgentLockDirectory) Close() error { + d.mu.Lock() + configHandle := d.configHandle + runHandle := d.runHandle + d.configHandle = windows.InvalidHandle + d.runHandle = windows.InvalidHandle + d.mu.Unlock() + var firstErr error + if runHandle != windows.InvalidHandle { + if err := closeMainAgentDirectoryHandleFn(runHandle); err != nil { + firstErr = err + } + } + if configHandle != windows.InvalidHandle { + if err := closeMainAgentDirectoryHandleFn(configHandle); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + +// OpenPreparedMainAgentLockDir revalidates and pins the current ConfigDir and +// its handle-relative run child without delete/write sharing. A hostile +// pre-existing rename/write handle therefore makes this fail closed, and no +// new namespace mutation handle can open until MainAgentLockDirectory.Close. +func OpenPreparedMainAgentLockDir() (*MainAgentLockDirectory, error) { + configPath := mainAgentConfigDirFn() + configHandle, err := openMainAgentDirectoryPinned(configPath) + if err != nil { + return nil, fmt.Errorf("open prepared config directory: %w", err) + } + + _, err = inspectMainAgentDirectory(configHandle, "prepared config directory") + if err != nil { + _ = closeMainAgentDirectoryHandleFn(configHandle) + return nil, err + } + if err := verifyMainAgentDirectorySecurity(configHandle, windowsConfigDirCreateSDDL, "prepared config directory"); err != nil { + _ = closeMainAgentDirectoryHandleFn(configHandle) + return nil, err + } + runHandle, err := openMainAgentChildDirectory(configHandle, "run") + if err != nil { + _ = closeMainAgentDirectoryHandleFn(configHandle) + return nil, fmt.Errorf("open run relative to verified config handle: %w", err) + } + if _, err := inspectMainAgentDirectory(runHandle, "prepared run directory"); err != nil { + _ = closeMainAgentDirectoryHandleFn(runHandle) + _ = closeMainAgentDirectoryHandleFn(configHandle) + return nil, err + } + if err := verifyMainAgentDirectorySecurity(runHandle, windowsAgentRunDirSDDL, "prepared run directory"); err != nil { + _ = closeMainAgentDirectoryHandleFn(runHandle) + _ = closeMainAgentDirectoryHandleFn(configHandle) + return nil, err + } + return &MainAgentLockDirectory{configHandle: configHandle, runHandle: runHandle}, nil +} + +func ensureMainAgentDirectory(path, sddl string) (windows.Handle, error) { + h, err := openMainAgentDirectoryFn(path) + if err == nil { + return h, nil + } + if !errors.Is(err, windows.ERROR_FILE_NOT_FOUND) && !errors.Is(err, windows.ERROR_PATH_NOT_FOUND) { + return windows.InvalidHandle, fmt.Errorf("open %s without following reparse points: %w", path, err) + } + if err := createMainAgentDirectoryFn(path, sddl); err != nil && !errors.Is(err, windows.ERROR_ALREADY_EXISTS) { + return windows.InvalidHandle, fmt.Errorf("create %s atomically with private security: %w", path, err) + } + h, err = openMainAgentDirectoryFn(path) + if err != nil { + return windows.InvalidHandle, fmt.Errorf("open newly secured %s without following reparse points: %w", path, err) + } + return h, nil +} + +func openMainAgentDirectory(path string) (windows.Handle, error) { + return openMainAgentDirectoryWithShare(path, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE) +} + +func openMainAgentDirectoryPinned(path string) (windows.Handle, error) { + return openMainAgentDirectoryWithShare(path, windows.FILE_SHARE_READ) +} + +func openMainAgentDirectoryWithShare(path string, share uint32) (windows.Handle, error) { + path16, err := windows.UTF16PtrFromString(path) + if err != nil { + return windows.InvalidHandle, err + } + return windows.CreateFile( + path16, + windows.FILE_LIST_DIRECTORY|windows.FILE_TRAVERSE|windows.FILE_READ_ATTRIBUTES|fileDeleteChildAccess|windows.READ_CONTROL|windows.WRITE_DAC|windows.WRITE_OWNER, + share, + nil, + windows.OPEN_EXISTING, + windows.FILE_FLAG_OPEN_REPARSE_POINT|windows.FILE_FLAG_BACKUP_SEMANTICS, + 0, + ) +} + +func openMainAgentChildDirectory(parent windows.Handle, name string) (windows.Handle, error) { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return windows.InvalidHandle, err + } + attrs := &windows.OBJECT_ATTRIBUTES{ + Length: uint32(unsafe.Sizeof(windows.OBJECT_ATTRIBUTES{})), + RootDirectory: parent, + ObjectName: objectName, + Attributes: windows.OBJ_CASE_INSENSITIVE, + } + var ( + h windows.Handle + iosb windows.IO_STATUS_BLOCK + ) + if err := windows.NtCreateFile( + &h, + windows.FILE_LIST_DIRECTORY|windows.FILE_TRAVERSE|windows.FILE_READ_ATTRIBUTES|fileDeleteChildAccess|windows.READ_CONTROL|windows.WRITE_DAC|windows.WRITE_OWNER|windows.SYNCHRONIZE, + attrs, + &iosb, + nil, + 0, + windows.FILE_SHARE_READ, + windows.FILE_OPEN, + windows.FILE_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT|windows.FILE_SYNCHRONOUS_IO_NONALERT, + 0, + 0, + ); err != nil { + return windows.InvalidHandle, err + } + return h, nil +} + +func createMainAgentDirectory(path, sddl string) error { + sd, err := windows.SecurityDescriptorFromString(sddl) + if err != nil { + return fmt.Errorf("parse directory security descriptor: %w", err) + } + path16, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + sa := &windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + InheritHandle: 0, + } + return windows.CreateDirectory(path16, sa) +} + +type mainAgentDirectoryIdentity struct { + volume uint32 + indexHigh uint32 + indexLow uint32 +} + +func inspectMainAgentDirectory(h windows.Handle, label string) (mainAgentDirectoryIdentity, error) { + var info windows.ByHandleFileInformation + if err := getMainAgentFileInformationFn(h, &info); err != nil { + return mainAgentDirectoryIdentity{}, fmt.Errorf("inspect %s handle: %w", label, err) + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return mainAgentDirectoryIdentity{}, fmt.Errorf("refuse reparse-point %s", label) + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY == 0 { + return mainAgentDirectoryIdentity{}, fmt.Errorf("refuse non-directory %s", label) + } + return mainAgentDirectoryIdentity{ + volume: info.VolumeSerialNumber, + indexHigh: info.FileIndexHigh, + indexLow: info.FileIndexLow, + }, nil +} + +func verifyMainAgentDirectoryPath(path string, want mainAgentDirectoryIdentity, securitySDDL, label string) error { + reopened, err := openMainAgentDirectoryFn(path) + if err != nil { + return fmt.Errorf("reopen %s for identity verification: %w", label, err) + } + defer closeMainAgentDirectoryHandleFn(reopened) + got, err := inspectMainAgentDirectory(reopened, label) + if err != nil { + return err + } + if got != want { + return fmt.Errorf("%s path identity changed during security hardening", label) + } + if securitySDDL != "" { + if err := verifyMainAgentDirectorySecurity(reopened, securitySDDL, label); err != nil { + return err + } + } + return nil +} + +func hardenAndVerifyMainAgentDirectory(h windows.Handle, sddl, label string) error { + want, err := windows.SecurityDescriptorFromString(sddl) + if err != nil { + return fmt.Errorf("parse %s security descriptor: %w", label, err) + } + owner, _, err := want.Owner() + if err != nil { + return fmt.Errorf("extract %s owner: %w", label, err) + } + group, _, err := want.Group() + if err != nil { + return fmt.Errorf("extract %s group: %w", label, err) + } + dacl, _, err := want.DACL() + if err != nil { + return fmt.Errorf("extract %s DACL: %w", label, err) + } + if err := setMainAgentSecurityInfoFn( + h, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, + owner, + group, + dacl, + nil, + ); err != nil { + return fmt.Errorf("set %s trusted owner and protected DACL through handle: %w", label, err) + } + return verifyMainAgentDirectorySecurityDescriptor(h, want, label) +} + +func verifyMainAgentDirectorySecurity(h windows.Handle, sddl, label string) error { + want, err := windows.SecurityDescriptorFromString(sddl) + if err != nil { + return fmt.Errorf("parse expected %s security descriptor: %w", label, err) + } + return verifyMainAgentDirectorySecurityDescriptor(h, want, label) +} + +func verifyMainAgentDirectorySecurityDescriptor(h windows.Handle, want *windows.SECURITY_DESCRIPTOR, label string) error { + got, err := getMainAgentSecurityInfoFn( + h, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return fmt.Errorf("read %s owner and DACL through handle: %w", label, err) + } + if got == nil { + return fmt.Errorf("%s has no security descriptor", label) + } + owner, _, err := got.Owner() + if err != nil { + return fmt.Errorf("read %s owner: %w", label, err) + } + if !trustedMainAgentOwner(owner) { + return fmt.Errorf("%s owner is not LocalSystem or BUILTIN\\Administrators: %v", label, owner) + } + group, _, err := got.Group() + if err != nil { + return fmt.Errorf("read %s group: %w", label, err) + } + if group == nil || !group.IsWellKnown(windows.WinBuiltinAdministratorsSid) { + return fmt.Errorf("%s group is not BUILTIN\\Administrators: %v", label, group) + } + control, _, err := got.Control() + if err != nil { + return fmt.Errorf("read %s DACL control: %w", label, err) + } + if control&windows.SE_DACL_PROTECTED == 0 { + return fmt.Errorf("%s DACL is not protected", label) + } + gotDACL, _, err := got.DACL() + if err != nil { + return fmt.Errorf("read %s DACL: %w", label, err) + } + wantDACL, _, err := want.DACL() + if err != nil { + return fmt.Errorf("read expected %s DACL: %w", label, err) + } + if !equalMainAgentACL(gotDACL, wantDACL) { + return fmt.Errorf("%s DACL does not match its fixed security policy", label) + } + return nil +} + +func mainAgentHandleHasTrustedOwner(h windows.Handle, label string) (bool, error) { + sd, err := getMainAgentSecurityInfoFn( + h, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return false, fmt.Errorf("read original %s owner through handle: %w", label, err) + } + if sd == nil { + return false, fmt.Errorf("original %s has no security descriptor", label) + } + owner, _, err := sd.Owner() + if err != nil { + return false, fmt.Errorf("read original %s owner: %w", label, err) + } + if _, _, err := sd.DACL(); err != nil { + return false, fmt.Errorf("inspect original %s DACL: %w", label, err) + } + return trustedMainAgentOwner(owner), nil +} + +func trustedMainAgentOwner(owner *windows.SID) bool { + return owner != nil && (owner.IsWellKnown(windows.WinLocalSystemSid) || owner.IsWellKnown(windows.WinBuiltinAdministratorsSid)) +} + +func equalMainAgentACL(a, b *windows.ACL) bool { + if a == nil || b == nil { + return a == b + } + if a.AceCount != b.AceCount { + return false + } + for i := uint32(0); i < uint32(a.AceCount); i++ { + var aACE, bACE *windows.ACCESS_ALLOWED_ACE + if windows.GetAce(a, i, &aACE) != nil || windows.GetAce(b, i, &bACE) != nil { + return false + } + if aACE.Header.AceType != bACE.Header.AceType || + aACE.Header.AceFlags != bACE.Header.AceFlags || + aACE.Mask != bACE.Mask { + return false + } + aSID := (*windows.SID)(unsafe.Pointer(&aACE.SidStart)) + bSID := (*windows.SID)(unsafe.Pointer(&bACE.SidStart)) + if !aSID.Equals(bSID) { + return false + } + } + return true +} + +func removeMainAgentLockRelative(dir windows.Handle, name string) error { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return err + } + attrs := &windows.OBJECT_ATTRIBUTES{ + Length: uint32(unsafe.Sizeof(windows.OBJECT_ATTRIBUTES{})), + RootDirectory: dir, + ObjectName: objectName, + Attributes: windows.OBJ_CASE_INSENSITIVE, + } + var ( + lock windows.Handle + iosb windows.IO_STATUS_BLOCK + ) + err = windows.NtCreateFile( + &lock, + windows.DELETE|windows.FILE_READ_ATTRIBUTES, + attrs, + &iosb, + nil, + 0, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + windows.FILE_OPEN, + windows.FILE_NON_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + 0, + 0, + ) + if err != nil { + if err == windows.STATUS_NO_SUCH_FILE || err == windows.STATUS_OBJECT_NAME_NOT_FOUND || err == windows.STATUS_OBJECT_PATH_NOT_FOUND { + return nil + } + return fmt.Errorf("open hostile lock relative to verified run handle: %w", err) + } + defer windows.CloseHandle(lock) + deleteFile := byte(1) + if err := windows.SetFileInformationByHandle( + lock, + windows.FileDispositionInfo, + (*byte)(unsafe.Pointer(&deleteFile)), + uint32(unsafe.Sizeof(deleteFile)), + ); err != nil { + return fmt.Errorf("mark hostile lock for deletion through relative handle: %w", err) + } + return nil +} + func enforceConfigDirPermissions(path string) error { return applyWindowsDACL(path, windowsConfigDirSDDL) } diff --git a/agent/internal/config/permissions_windows_test.go b/agent/internal/config/permissions_windows_test.go index 2cfbb0d8c4..c86a366910 100644 --- a/agent/internal/config/permissions_windows_test.go +++ b/agent/internal/config/permissions_windows_test.go @@ -3,6 +3,7 @@ package config import ( + "errors" "os" "path/filepath" "strings" @@ -43,6 +44,209 @@ func TestWindowsConfigDACLGrantsUsersRead(t *testing.T) { } } +func TestMainAgentDirectorySDDLs(t *testing.T) { + const wantConfig = "O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;;FRFX;;;BU)" + const wantRun = "O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)" + if windowsConfigDirCreateSDDL != wantConfig { + t.Fatalf("config creation SDDL = %q, want %q", windowsConfigDirCreateSDDL, wantConfig) + } + if windowsAgentRunDirSDDL != wantRun { + t.Fatalf("run directory SDDL = %q, want %q", windowsAgentRunDirSDDL, wantRun) + } + for name, sddl := range map[string]string{"config": wantConfig, "run": wantRun} { + if _, err := windows.SecurityDescriptorFromString(sddl); err != nil { + t.Fatalf("%s SDDL does not parse: %v", name, err) + } + } +} + +func TestPrepareMainAgentLockDirRejectsReparseParent(t *testing.T) { + dir := t.TempDir() + restoreMainAgentDirectorySeams(t, dir) + getMainAgentFileInformationFn = func(_ windows.Handle, info *windows.ByHandleFileInformation) error { + info.FileAttributes = windows.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_REPARSE_POINT + return nil + } + + _, err := PrepareMainAgentLockDir() + assertMainAgentSecurityError(t, err) +} + +func TestPrepareMainAgentLockDirRejectsUntrustedOwner(t *testing.T) { + dir := t.TempDir() + restoreMainAgentDirectorySeams(t, dir) + untrusted := mustWindowsSecurityDescriptor(t, "O:BUD:P(A;;FA;;;BU)") + setMainAgentSecurityInfoFn = func(windows.Handle, windows.SE_OBJECT_TYPE, windows.SECURITY_INFORMATION, *windows.SID, *windows.SID, *windows.ACL, *windows.ACL) error { + return nil + } + getMainAgentSecurityInfoFn = func(windows.Handle, windows.SE_OBJECT_TYPE, windows.SECURITY_INFORMATION) (*windows.SECURITY_DESCRIPTOR, error) { + return untrusted, nil + } + + _, err := PrepareMainAgentLockDir() + assertMainAgentSecurityError(t, err) +} + +func TestPrepareMainAgentLockDirSanitizesUntrustedExistingRunDir(t *testing.T) { + dir := t.TempDir() + runDir := filepath.Join(dir, "run") + if err := os.Mkdir(runDir, 0o700); err != nil { + t.Fatal(err) + } + lockPath := filepath.Join(runDir, "agent.lock") + if err := os.WriteFile(lockPath, []byte("hostile"), 0o600); err != nil { + t.Fatal(err) + } + restoreMainAgentDirectorySeams(t, dir) + trustedConfig := mustWindowsSecurityDescriptor(t, windowsConfigDirCreateSDDL) + trustedRun := mustWindowsSecurityDescriptor(t, windowsAgentRunDirSDDL) + untrusted := mustWindowsSecurityDescriptor(t, "O:BUD:P(A;;FA;;;BU)") + securityRead := 0 + getMainAgentSecurityInfoFn = func(windows.Handle, windows.SE_OBJECT_TYPE, windows.SECURITY_INFORMATION) (*windows.SECURITY_DESCRIPTOR, error) { + securityRead++ + switch securityRead { + case 1: + return trustedConfig, nil + case 2: // Existing run directory before handle-based repair. + return untrusted, nil + default: + return trustedRun, nil + } + } + removed := false + removeMainAgentLockRelativeFn = func(_ windows.Handle, name string) error { + if name != "agent.lock" { + t.Fatalf("removed relative name = %q", name) + } + removed = true + return os.Remove(lockPath) + } + + got, err := PrepareMainAgentLockDir() + if err != nil { + t.Fatal(err) + } + if got != runDir { + t.Fatalf("lock dir = %q, want %q", got, runDir) + } + if !removed { + t.Fatal("hostile existing agent.lock was not sanitized") + } + if _, err := os.Stat(lockPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("hostile lock remains: %v", err) + } +} + +func TestPrepareMainAgentLockDirDetectsParentSwap(t *testing.T) { + dir := t.TempDir() + restoreMainAgentDirectorySeams(t, dir) + infoRead := 0 + getMainAgentFileInformationFn = func(_ windows.Handle, info *windows.ByHandleFileInformation) error { + infoRead++ + info.FileAttributes = windows.FILE_ATTRIBUTE_DIRECTORY + info.VolumeSerialNumber = 7 + info.FileIndexLow = uint32(infoRead) // Reopen resolves to a different object. + return nil + } + + _, err := PrepareMainAgentLockDir() + assertMainAgentSecurityError(t, err) +} + +func TestPrepareMainAgentLockDirDACLFailureIsSecurityError(t *testing.T) { + dir := t.TempDir() + restoreMainAgentDirectorySeams(t, dir) + wantErr := errors.New("injected SetSecurityInfo failure") + setMainAgentSecurityInfoFn = func(_ windows.Handle, objectType windows.SE_OBJECT_TYPE, info windows.SECURITY_INFORMATION, _ *windows.SID, _ *windows.SID, _ *windows.ACL, _ *windows.ACL) error { + if objectType != windows.SE_FILE_OBJECT { + t.Fatalf("object type = %v", objectType) + } + const want = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION + if info != want { + t.Fatalf("security information = %#x, want %#x", info, want) + } + return wantErr + } + + _, err := PrepareMainAgentLockDir() + assertMainAgentSecurityError(t, err) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want wrapped injected error", err) + } +} + +func TestOpenPreparedMainAgentLockDirKeepsVerifiedHandleLive(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "run"), 0o700); err != nil { + t.Fatal(err) + } + restoreMainAgentDirectorySeams(t, dir) + + pinned, err := OpenPreparedMainAgentLockDir() + if err != nil { + t.Fatal(err) + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(pinned.Handle(), &info); err != nil { + t.Fatalf("returned run handle is not live: %v", err) + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY == 0 || info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + t.Fatalf("returned handle attributes = %#x", info.FileAttributes) + } + if err := pinned.Close(); err != nil { + t.Fatal(err) + } + if err := pinned.Close(); err != nil { + t.Fatalf("idempotent close: %v", err) + } +} + +func restoreMainAgentDirectorySeams(t *testing.T, dir string) { + t.Helper() + oldConfigDir := mainAgentConfigDirFn + oldFileInfo := getMainAgentFileInformationFn + oldGetSecurity := getMainAgentSecurityInfoFn + oldSetSecurity := setMainAgentSecurityInfoFn + oldRemoveLock := removeMainAgentLockRelativeFn + t.Cleanup(func() { + mainAgentConfigDirFn = oldConfigDir + getMainAgentFileInformationFn = oldFileInfo + getMainAgentSecurityInfoFn = oldGetSecurity + setMainAgentSecurityInfoFn = oldSetSecurity + removeMainAgentLockRelativeFn = oldRemoveLock + }) + mainAgentConfigDirFn = func() string { return dir } + trustedConfig := mustWindowsSecurityDescriptor(t, windowsConfigDirCreateSDDL) + trustedRun := mustWindowsSecurityDescriptor(t, windowsAgentRunDirSDDL) + securityRead := 0 + getMainAgentSecurityInfoFn = func(windows.Handle, windows.SE_OBJECT_TYPE, windows.SECURITY_INFORMATION) (*windows.SECURITY_DESCRIPTOR, error) { + securityRead++ + if securityRead == 1 { + return trustedConfig, nil + } + return trustedRun, nil + } + setMainAgentSecurityInfoFn = func(windows.Handle, windows.SE_OBJECT_TYPE, windows.SECURITY_INFORMATION, *windows.SID, *windows.SID, *windows.ACL, *windows.ACL) error { + return nil + } +} + +func mustWindowsSecurityDescriptor(t *testing.T, sddl string) *windows.SECURITY_DESCRIPTOR { + t.Helper() + sd, err := windows.SecurityDescriptorFromString(sddl) + if err != nil { + t.Fatal(err) + } + return sd +} + +func assertMainAgentSecurityError(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("expected fail-closed security error") + } +} + // TestEnforceConfigFileDACLAppliesUsersRead applies the real DACL to a temp file // and reads it back, confirming a Users (BU) ACE is present on agent.yaml and // absent on secrets.yaml. From 692864c5483bed0845ff57fc955a04d366968bee Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 18:41:42 -0600 Subject: [PATCH 17/47] fix(agent): acquire Windows instance guard before startup --- .../agentapp/instance_marker_other.go | 27 ++++ .../agentapp/instance_marker_windows.go | 24 ++++ agent/internal/agentapp/main.go | 33 ++++- agent/internal/agentapp/main_instance.go | 8 +- agent/internal/agentapp/main_test.go | 53 +++++++ agent/internal/agentapp/process_role.go | 33 +++++ agent/internal/agentapp/process_role_other.go | 15 ++ agent/internal/agentapp/process_role_test.go | 21 +++ .../internal/agentapp/process_role_windows.go | 53 +++++++ agent/internal/eventlog/eventlog.go | 3 + agent/internal/eventlog/eventlog_windows.go | 57 ++++++-- .../eventlog/eventlog_windows_test.go | 135 ++++++++++++++++++ 12 files changed, 440 insertions(+), 22 deletions(-) create mode 100644 agent/internal/agentapp/instance_marker_other.go create mode 100644 agent/internal/agentapp/instance_marker_windows.go create mode 100644 agent/internal/agentapp/process_role_other.go create mode 100644 agent/internal/agentapp/process_role_windows.go create mode 100644 agent/internal/eventlog/eventlog_windows_test.go diff --git a/agent/internal/agentapp/instance_marker_other.go b/agent/internal/agentapp/instance_marker_other.go new file mode 100644 index 0000000000..275edfd910 --- /dev/null +++ b/agent/internal/agentapp/instance_marker_other.go @@ -0,0 +1,27 @@ +//go:build !windows + +package agentapp + +import ( + "fmt" + "os" +) + +func writeInstanceGuardMarker(startup ProcessStartup, guardErr error) { + _ = writeInstanceGuardEventFn( + "BreezeAgent", + fmt.Sprintf( + "Breeze main-agent instance guard failure: pid=%d launchMode=%s error=%v", + startup.PID, + startup.LaunchMode, + guardErr, + ), + ) + fmt.Fprintf( + os.Stderr, + "Breeze main-agent instance guard failure (pid=%d mode=%s): %v\n", + startup.PID, + startup.LaunchMode, + guardErr, + ) +} diff --git a/agent/internal/agentapp/instance_marker_windows.go b/agent/internal/agentapp/instance_marker_windows.go new file mode 100644 index 0000000000..670cb5847c --- /dev/null +++ b/agent/internal/agentapp/instance_marker_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package agentapp + +import ( + "fmt" + "os" +) + +func writeInstanceGuardMarker(startup ProcessStartup, guardErr error) { + message := fmt.Sprintf( + "Breeze main-agent instance guard failure: pid=%d parentPid=%d windowsSessionId=%d launchMode=%s binary=%q version=%q error=%v", + startup.PID, + startup.ParentPID, + startup.WindowsSessionID, + startup.LaunchMode, + startup.Binary, + startup.Version, + guardErr, + ) + if err := writeInstanceGuardEventFn("BreezeAgent", message); err != nil { + fmt.Fprintf(os.Stderr, "Breeze instance-guard Event Log marker failed: %v\n", err) + } +} diff --git a/agent/internal/agentapp/main.go b/agent/internal/agentapp/main.go index 93752d3b5e..0d5e82c031 100644 --- a/agent/internal/agentapp/main.go +++ b/agent/internal/agentapp/main.go @@ -104,8 +104,9 @@ var waitForEnrollmentPollInterval = 10 * time.Second // is defined in service_seams_windows.go because its signature references // Windows-only types. var ( - startAgentFn func(*config.Config) (*agentComponents, error) = startAgent - waitForEnrollmentFn func(context.Context, string) *config.Config = waitForEnrollment + startAgentFn func(*config.Config) (*agentComponents, error) = startAgent + waitForEnrollmentFn func(context.Context, string) *config.Config = waitForEnrollment + reconcileServiceUnitIfNeededFn = reconcileServiceUnitIfNeeded ) // initBootstrapLogging initializes the logging package with stderr + @@ -908,18 +909,40 @@ func retryTCCGrant() { // - Receiving pending commands from the server via heartbeat response // - Executing commands and reporting results back to the server func runAgent() { + serviceMode := isWindowsService() + startup := currentProcessStartup("run", "", serviceMode) + guard, err := acquireMainAgentGuardFn(startup) + if err != nil { + writeInstanceGuardMarkerFn(startup, err) + if errors.Is(err, ErrMainAgentAlreadyRunning) { + fmt.Fprintf( + os.Stderr, + "Breeze main agent is already running (pid=%d session=%d mode=%s)\n", + startup.PID, + startup.WindowsSessionID, + startup.LaunchMode, + ) + mainAgentExitFn(exitAlreadyRunning) + return + } + fmt.Fprintf(os.Stderr, "Breeze main-agent instance guard failed: %v\n", err) + mainAgentExitFn(exitInstanceGuardError) + return + } + defer guard.Close() + // Self-heal the installed service unit from older installs (launchd plists on // macOS; systemd unit on Linux) after a binary-only auto-update. - reconcileServiceUnitIfNeeded() + reconcileServiceUnitIfNeededFn() // On Windows, if launched by the SCM, run under the service framework // so we report Running/Stopped status back to the SCM correctly. The // service wrapper owns its own config loading, enrollment check, and // cancellation via the SCM request channel. - if isWindowsService() { + if serviceMode { if err := runAsService(cfgFile); err != nil { log.Error("service failed", "error", err.Error()) - os.Exit(1) + mainAgentExitFn(1) } return } diff --git a/agent/internal/agentapp/main_instance.go b/agent/internal/agentapp/main_instance.go index 4259ede30e..3d6e19adb8 100644 --- a/agent/internal/agentapp/main_instance.go +++ b/agent/internal/agentapp/main_instance.go @@ -3,6 +3,8 @@ package agentapp import ( "errors" "os" + + breezeeventlog "github.com/breeze-rmm/agent/internal/eventlog" ) const ( @@ -18,6 +20,8 @@ type mainAgentGuard interface { } var ( - acquireMainAgentGuardFn = acquireMainAgentGuard - mainAgentExitFn = os.Exit + acquireMainAgentGuardFn = acquireMainAgentGuard + mainAgentExitFn = os.Exit + writeInstanceGuardMarkerFn = writeInstanceGuardMarker + writeInstanceGuardEventFn = breezeeventlog.WriteError ) diff --git a/agent/internal/agentapp/main_test.go b/agent/internal/agentapp/main_test.go index 505e918469..dd1d70835b 100644 --- a/agent/internal/agentapp/main_test.go +++ b/agent/internal/agentapp/main_test.go @@ -2,6 +2,7 @@ package agentapp import ( "context" + "errors" "os" "path/filepath" "runtime" @@ -14,6 +15,58 @@ import ( "github.com/breeze-rmm/agent/internal/config" ) +func TestRunAgentDuplicateStopsBeforeInitialization(t *testing.T) { + testRunAgentGuardFailureStopsBeforeInitialization(t, ErrMainAgentAlreadyRunning, exitAlreadyRunning) +} + +func TestRunAgentSecurityFailureStopsBeforeInitialization(t *testing.T) { + testRunAgentGuardFailureStopsBeforeInitialization(t, errors.New("lock ACL verification failed"), exitInstanceGuardError) +} + +func testRunAgentGuardFailureStopsBeforeInitialization(t *testing.T, guardErr error, wantExit int) { + t.Helper() + + origAcquire := acquireMainAgentGuardFn + origExit := mainAgentExitFn + origMarker := writeInstanceGuardMarkerFn + origWriteEvent := writeInstanceGuardEventFn + origReconcile := reconcileServiceUnitIfNeededFn + origStart := startAgentFn + t.Cleanup(func() { + acquireMainAgentGuardFn = origAcquire + mainAgentExitFn = origExit + writeInstanceGuardMarkerFn = origMarker + writeInstanceGuardEventFn = origWriteEvent + reconcileServiceUnitIfNeededFn = origReconcile + startAgentFn = origStart + }) + + reconciled, started, markerWritten, exitCode := false, false, false, 0 + acquireMainAgentGuardFn = func(ProcessStartup) (mainAgentGuard, error) { + return nil, guardErr + } + writeInstanceGuardMarkerFn = writeInstanceGuardMarker + writeInstanceGuardEventFn = func(source, message string) error { + markerWritten = true + if source != "BreezeAgent" || !strings.Contains(message, guardErr.Error()) { + t.Errorf("marker = source:%q message:%q", source, message) + } + return nil + } + mainAgentExitFn = func(code int) { exitCode = code } + reconcileServiceUnitIfNeededFn = func() { reconciled = true } + startAgentFn = func(*config.Config) (*agentComponents, error) { + started = true + return nil, nil + } + + runAgent() + + if exitCode != wantExit || reconciled || started || !markerWritten { + t.Fatalf("exit=%d, want=%d reconciled=%v started=%v marker=%v", exitCode, wantExit, reconciled, started, markerWritten) + } +} + // TestHelperWarnLimiterBudget verifies that the first `limit` calls with the // same message all emit WARN (emit=true, suppressed=0), and that the (limit+1)th // call does NOT emit a WARN when the info interval has not elapsed. diff --git a/agent/internal/agentapp/process_role.go b/agent/internal/agentapp/process_role.go index 5f6a52cae4..18bdb8d051 100644 --- a/agent/internal/agentapp/process_role.go +++ b/agent/internal/agentapp/process_role.go @@ -1,6 +1,8 @@ package agentapp import ( + "fmt" + "os" "path/filepath" "strings" "time" @@ -8,6 +10,12 @@ import ( "github.com/breeze-rmm/agent/internal/sessionbroker" ) +type platformProcessMetadata struct { + ParentPID int + WindowsSessionID uint32 + CreatedAt time.Time +} + // ProcessStartup is the non-secret diagnostic record for one agent or helper // process startup. CompanionHelper is true only for a user-helper command that // did not resolve to the main agent binary fallback. @@ -26,6 +34,31 @@ type ProcessStartup struct { CreatedAt time.Time `json:"createdAt"` } +func currentProcessStartup(command, role string, service bool) ProcessStartup { + executable, _ := os.Executable() + metadata := currentPlatformProcessMetadata() + mode, fallback := classifyProcess(command, role, executable, service) + lifecycleKey := "" + if metadata.WindowsSessionID != 0 && (role == "user" || role == "system") { + lifecycleKey = fmt.Sprintf("%d-%s", metadata.WindowsSessionID, role) + } + + return ProcessStartup{ + Binary: filepath.Base(executable), + ExecutablePath: executable, + PID: os.Getpid(), + ParentPID: metadata.ParentPID, + WindowsSessionID: metadata.WindowsSessionID, + LaunchMode: mode, + HelperRole: role, + LifecycleKey: lifecycleKey, + CompanionHelper: command == "user-helper" && !fallback, + MainBinaryFallback: fallback, + Version: version, + CreatedAt: metadata.CreatedAt, + } +} + func classifyProcess(command, role, executable string, service bool) (string, bool) { base := strings.ToLower(filepath.Base(executable)) fallback := command == "user-helper" && base != strings.ToLower(sessionbroker.UserHelperBinaryName) diff --git a/agent/internal/agentapp/process_role_other.go b/agent/internal/agentapp/process_role_other.go new file mode 100644 index 0000000000..4643646a9e --- /dev/null +++ b/agent/internal/agentapp/process_role_other.go @@ -0,0 +1,15 @@ +//go:build !windows + +package agentapp + +import ( + "os" + "time" +) + +func currentPlatformProcessMetadata() platformProcessMetadata { + return platformProcessMetadata{ + ParentPID: os.Getppid(), + CreatedAt: time.Now(), + } +} diff --git a/agent/internal/agentapp/process_role_test.go b/agent/internal/agentapp/process_role_test.go index 8878efd841..5a280db495 100644 --- a/agent/internal/agentapp/process_role_test.go +++ b/agent/internal/agentapp/process_role_test.go @@ -1,11 +1,32 @@ package agentapp import ( + "os" "reflect" "testing" "time" ) +func TestCurrentProcessStartupCapturesDiagnosticRoleMetadata(t *testing.T) { + startup := currentProcessStartup("user-helper", "user", false) + + if startup.PID != os.Getpid() { + t.Fatalf("PID = %d, want %d", startup.PID, os.Getpid()) + } + if startup.Binary == "" || startup.ExecutablePath == "" { + t.Fatalf("executable metadata missing: %+v", startup) + } + if startup.LaunchMode != "user-helper" || startup.HelperRole != "user" { + t.Fatalf("role metadata = (%q, %q)", startup.LaunchMode, startup.HelperRole) + } + if !startup.MainBinaryFallback || startup.CompanionHelper { + t.Fatalf("helper provenance = fallback:%v companion:%v", startup.MainBinaryFallback, startup.CompanionHelper) + } + if startup.Version != version || startup.CreatedAt.IsZero() { + t.Fatalf("version/creation metadata = (%q, %v)", startup.Version, startup.CreatedAt) + } +} + func TestClassifyProcess(t *testing.T) { tests := []struct { name, command, role, exe string diff --git a/agent/internal/agentapp/process_role_windows.go b/agent/internal/agentapp/process_role_windows.go new file mode 100644 index 0000000000..10133c2e19 --- /dev/null +++ b/agent/internal/agentapp/process_role_windows.go @@ -0,0 +1,53 @@ +//go:build windows + +package agentapp + +import ( + "os" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +func currentPlatformProcessMetadata() platformProcessMetadata { + pid := uint32(os.Getpid()) + metadata := platformProcessMetadata{} + + _ = windows.ProcessIdToSessionId(pid, &metadata.WindowsSessionID) + metadata.ParentPID = currentWindowsParentPID(pid) + + var creationTime, exitTime, kernelTime, userTime windows.Filetime + if err := windows.GetProcessTimes( + windows.CurrentProcess(), + &creationTime, + &exitTime, + &kernelTime, + &userTime, + ); err == nil { + metadata.CreatedAt = time.Unix(0, creationTime.Nanoseconds()) + } + + return metadata +} + +func currentWindowsParentPID(pid uint32) int { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) + if err != nil { + return 0 + } + defer windows.CloseHandle(snapshot) + + entry := windows.ProcessEntry32{Size: uint32(unsafe.Sizeof(windows.ProcessEntry32{}))} + if err := windows.Process32First(snapshot, &entry); err != nil { + return 0 + } + for { + if entry.ProcessID == pid { + return int(entry.ParentProcessID) + } + if err := windows.Process32Next(snapshot, &entry); err != nil { + return 0 + } + } +} diff --git a/agent/internal/eventlog/eventlog.go b/agent/internal/eventlog/eventlog.go index 31b62ced67..b1ae5d99cf 100644 --- a/agent/internal/eventlog/eventlog.go +++ b/agent/internal/eventlog/eventlog.go @@ -16,3 +16,6 @@ func Warning(source, message string) {} // Error writes an error event. func Error(source, message string) {} + +// WriteError is a no-op on non-Windows platforms. +func WriteError(source, message string) error { return nil } diff --git a/agent/internal/eventlog/eventlog_windows.go b/agent/internal/eventlog/eventlog_windows.go index b5b3cee076..c1fb67eae0 100644 --- a/agent/internal/eventlog/eventlog_windows.go +++ b/agent/internal/eventlog/eventlog_windows.go @@ -4,11 +4,13 @@ // the Windows Application Event Log. Registration is lazy: on first // use per source, we attempt InstallAsEventCreate, and if that fails // because the source already exists we fall back to Open. Both -// failures are silently swallowed — the package's contract is that -// logging is best-effort and never returns errors to callers. +// failures are silently swallowed by Info, Warning, and Error. WriteError +// exposes failures for bootstrap diagnostics that have no normal logger yet. package eventlog import ( + "errors" + "fmt" "sync" "golang.org/x/sys/windows/svc/eventlog" @@ -23,16 +25,22 @@ const ( ) var ( - registryMu sync.Mutex - registry = map[string]*sourceEntry{} + registryMu sync.Mutex + registry = map[string]*sourceEntry{} + installAsEventCreateFn = eventlog.InstallAsEventCreate + openEventLogFn = eventlog.Open + writeEventLogErrorFn = func(handle *eventlog.Log, eventID uint32, message string) error { + return handle.Error(eventID, message) + } ) type sourceEntry struct { - once sync.Once - log *eventlog.Log // nil if registration failed + once sync.Once + log *eventlog.Log // nil if registration/open failed + initErr error } -func lookupOrRegister(source string) *eventlog.Log { +func lookupOrRegister(source string) (*eventlog.Log, error) { registryMu.Lock() entry, ok := registry[source] if !ok { @@ -46,38 +54,57 @@ func lookupOrRegister(source string) *eventlog.Log { // Windows environments require admin to install a new event // source; if the source already exists, Install returns an // "already exists" error which we treat as benign. - _ = eventlog.InstallAsEventCreate( + installErr := installAsEventCreateFn( source, eventlog.Info|eventlog.Warning|eventlog.Error, ) // Open returns a handle usable for Info/Warning/Error regardless // of whether we just installed it or it already existed. - logHandle, openErr := eventlog.Open(source) + logHandle, openErr := openEventLogFn(source) if openErr != nil { - return // entry.log stays nil; subsequent calls are no-ops + entry.initErr = errors.Join( + wrapEventLogInitError("register", source, installErr), + fmt.Errorf("open Windows event source %q: %w", source, openErr), + ) + return } entry.log = logHandle }) - return entry.log + return entry.log, entry.initErr +} + +func wrapEventLogInitError(operation, source string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%s Windows event source %q: %w", operation, source, err) } // Info writes an informational event to the Windows Application log. func Info(source, message string) { - if handle := lookupOrRegister(source); handle != nil { + if handle, _ := lookupOrRegister(source); handle != nil { _ = handle.Info(eventIDInfo, message) } } // Warning writes a warning event. func Warning(source, message string) { - if handle := lookupOrRegister(source); handle != nil { + if handle, _ := lookupOrRegister(source); handle != nil { _ = handle.Warning(eventIDWarning, message) } } // Error writes an error event. func Error(source, message string) { - if handle := lookupOrRegister(source); handle != nil { - _ = handle.Error(eventIDError, message) + _ = WriteError(source, message) +} + +// WriteError writes an error event and reports source initialization or write +// failures to callers that need bootstrap-safe diagnostics. +func WriteError(source, message string) error { + handle, err := lookupOrRegister(source) + if err != nil { + return err } + return writeEventLogErrorFn(handle, eventIDError, message) } diff --git a/agent/internal/eventlog/eventlog_windows_test.go b/agent/internal/eventlog/eventlog_windows_test.go new file mode 100644 index 0000000000..abe9df6a5d --- /dev/null +++ b/agent/internal/eventlog/eventlog_windows_test.go @@ -0,0 +1,135 @@ +//go:build windows + +package eventlog + +import ( + "errors" + "reflect" + "testing" + + windowseventlog "golang.org/x/sys/windows/svc/eventlog" +) + +func TestWriteErrorRegistersMissingSourceBeforeOpen(t *testing.T) { + resetEventLogTestState(t) + + var calls []string + handle := &windowseventlog.Log{} + installAsEventCreateFn = func(source string, supported uint32) error { + calls = append(calls, "install") + if source != "BreezeAgent" { + t.Fatalf("install source = %q", source) + } + want := uint32(windowseventlog.Info | windowseventlog.Warning | windowseventlog.Error) + if supported != want { + t.Fatalf("supported events = %d, want %d", supported, want) + } + return nil + } + openEventLogFn = func(source string) (*windowseventlog.Log, error) { + calls = append(calls, "open") + return handle, nil + } + writeEventLogErrorFn = func(got *windowseventlog.Log, eventID uint32, message string) error { + calls = append(calls, "write") + if got != handle || eventID != eventIDError || message != "guard failure" { + t.Fatalf("write = (%p, %d, %q)", got, eventID, message) + } + return nil + } + + if err := WriteError("BreezeAgent", "guard failure"); err != nil { + t.Fatalf("WriteError() error = %v", err) + } + if want := []string{"install", "open", "write"}; !reflect.DeepEqual(calls, want) { + t.Fatalf("call order = %v, want %v", calls, want) + } +} + +func TestWriteErrorAlreadyRegisteredStillOpens(t *testing.T) { + resetEventLogTestState(t) + + var calls []string + installAsEventCreateFn = func(string, uint32) error { + calls = append(calls, "install") + return errors.New("event source registry key already exists") + } + openEventLogFn = func(string) (*windowseventlog.Log, error) { + calls = append(calls, "open") + return &windowseventlog.Log{}, nil + } + writeEventLogErrorFn = func(*windowseventlog.Log, uint32, string) error { + calls = append(calls, "write") + return nil + } + + if err := WriteError("BreezeAgent", "guard failure"); err != nil { + t.Fatalf("WriteError() error = %v", err) + } + if want := []string{"install", "open", "write"}; !reflect.DeepEqual(calls, want) { + t.Fatalf("call order = %v, want %v", calls, want) + } +} + +func TestWriteErrorReturnsRegistrationOpenFailure(t *testing.T) { + resetEventLogTestState(t) + + installErr := errors.New("registration denied") + openErr := errors.New("open denied") + installCalls, openCalls, writeCalls := 0, 0, 0 + installAsEventCreateFn = func(string, uint32) error { + installCalls++ + return installErr + } + openEventLogFn = func(string) (*windowseventlog.Log, error) { + openCalls++ + return nil, openErr + } + writeEventLogErrorFn = func(*windowseventlog.Log, uint32, string) error { + writeCalls++ + return nil + } + + for i := 0; i < 2; i++ { + err := WriteError("BreezeAgent", "guard failure") + if !errors.Is(err, installErr) || !errors.Is(err, openErr) { + t.Fatalf("WriteError() error = %v, want registration and open errors", err) + } + } + if installCalls != 1 || openCalls != 1 || writeCalls != 0 { + t.Fatalf("calls: install=%d open=%d write=%d", installCalls, openCalls, writeCalls) + } +} + +func TestWriteErrorReturnsWriteFailure(t *testing.T) { + resetEventLogTestState(t) + + writeErr := errors.New("write denied") + installAsEventCreateFn = func(string, uint32) error { return nil } + openEventLogFn = func(string) (*windowseventlog.Log, error) { return &windowseventlog.Log{}, nil } + writeEventLogErrorFn = func(*windowseventlog.Log, uint32, string) error { return writeErr } + + if err := WriteError("BreezeAgent", "guard failure"); !errors.Is(err, writeErr) { + t.Fatalf("WriteError() error = %v, want %v", err, writeErr) + } +} + +func resetEventLogTestState(t *testing.T) { + t.Helper() + + origInstall := installAsEventCreateFn + origOpen := openEventLogFn + origWriteError := writeEventLogErrorFn + registryMu.Lock() + origRegistry := registry + registry = map[string]*sourceEntry{} + registryMu.Unlock() + t.Cleanup(func() { + installAsEventCreateFn = origInstall + openEventLogFn = origOpen + writeEventLogErrorFn = origWriteError + registryMu.Lock() + registry = origRegistry + registryMu.Unlock() + }) +} From 920b59d2f6a1d1c8f9e5183107951974030b13a8 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 19:02:41 -0600 Subject: [PATCH 18/47] feat(agent): report Windows process roles and fallback mode --- .superpowers/sdd/instance-task-4-report.md | 102 ++++++++++++++++++ agent/internal/agentapp/main.go | 17 +-- agent/internal/agentapp/main_test.go | 98 +++++++++++++++++ agent/internal/agentapp/process_role.go | 45 ++++++++ .../sessionbroker/helper_job_windows_test.go | 33 ++++-- .../sessionbroker/spawner_cmdline_test.go | 18 ++++ agent/internal/sessionbroker/spawner_stub.go | 3 + .../internal/sessionbroker/spawner_windows.go | 31 +++--- 8 files changed, 315 insertions(+), 32 deletions(-) create mode 100644 .superpowers/sdd/instance-task-4-report.md diff --git a/.superpowers/sdd/instance-task-4-report.md b/.superpowers/sdd/instance-task-4-report.md new file mode 100644 index 0000000000..5e9266ba6a --- /dev/null +++ b/.superpowers/sdd/instance-task-4-report.md @@ -0,0 +1,102 @@ +# Instance Diagnostics Task 4 Report + +Base: `692864c5483bed0845ff57fc955a04d366968bee` + +## Result + +Implemented structured process-role diagnostics for the main agent and helper processes, plus retained parent-side spawn provenance. The implementation does not change helper process handles, Job Object ownership, lifecycle convergence, or RDS role policy. + +## TDD evidence + +### RED + +1. Native focused test command: + + `go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestProcessStartup|TestLogProcessStartup|TestSpawnedHelperDiagnostics|TestClassifyProcess|TestResolveUserHelperPath|TestBuildUserHelperCmdLine' -count=1` + + Exit 1. Expected missing-feature failures: + + - `undefined: logProcessStartup` + - `unknown field CommandMode in struct literal of type SpawnedHelper` + - `unknown field Role in struct literal of type SpawnedHelper` + - `unknown field WindowsSessionID in struct literal of type SpawnedHelper` + +2. Windows amd64 sessionbroker test cross-compile: + + `GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c ./internal/sessionbroker -o /tmp/breeze-task4-red/sessionbroker.test.exe` + + Exit 1. The Windows `SpawnedHelper` lacked `CommandMode`, `Role`, and `WindowsSessionID`. + +3. Cached main-startup regression test with the cache implementation removed: + + `go test ./internal/agentapp -run TestCachedMainProcessStartupUsesGuardRecord -count=1` + + Exit 1. The cache and cache accessors were absent, proving the test requires the Task 3 guard-time record rather than a later recomputation. + +### GREEN + +- Focused cache/startup tests: pass under `-race`. +- Brief-required focused command: + + `go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestProcessStartup|TestClassifyProcess|TestResolveUserHelperPath|TestBuildUserHelperCmdLine' -count=1` + + Pass. + +- Affected package suites: + + `go test -race ./internal/agentapp ./internal/sessionbroker -count=1` + + Pass. + +- Full native agent suite: + + `go test -race ./...` + + Pass with no race report. + +- Windows cross-compilation for both `amd64` and `arm64`: `agentapp`, `sessionbroker`, and `heartbeat` test binaries plus `cmd/breeze-agent` all compile successfully. + +## One-event evidence + +- There is one log implementation for the event: `log.Info("process startup", args...)` in `process_role.go`. +- Main startup has one call site, after full logging and log shipping initialization: `logProcessStartup(cachedMainProcessStartup())` in `startAgent`. +- Helper startup has one call site, after helper logging and optional shipping initialization: `logProcessStartup(currentProcessStartup("user-helper", role, false))` in `runHelperProcess`. +- The previous `starting agent` and `starting helper` messages have been removed; repository string search over the touched startup paths returns zero occurrences. +- `TestLogProcessStartupEmitsOneStructuredEvent` captures JSON logs and asserts exactly one `process startup` record. +- Parent-side spawn diagnostics remain a distinct `spawned user helper in session` record and are asserted once in the Windows spawner test. They are not a second `process startup` record. + +## Field and security checklist + +The process-startup whitelist contains only: + +- `binary` +- `executablePath` +- `pid` +- `parentPid` +- `windowsSessionId` +- `launchMode` +- `helperRole` +- `lifecycleKey` +- `companionHelper` +- `mainBinaryFallback` +- `version` +- `createdAt` + +The parent spawn diagnostic contains only process provenance needed for support: PID, actual binary path, `user-helper` command mode, explicit authenticated helper role, kernel-target Windows session ID, and typed resolver fallback status. It does not log token-source metadata. + +Tests reject `authToken`, `helperAuthToken`, and `mtlsKey`; the Windows spawn-log test also rejects token-source, certificate, tenant, and organization fields. No secret value, token value, certificate material, tenant ID, organization ID, server URL, or agent ID was added to either diagnostic. + +Helper self-diagnostics use `os.Executable()` and kernel process metadata. `WindowsSessionID` and `LifecycleKey` therefore come from the current process session, not a claimed session argument. Parent diagnostics copy `BinaryPath` and `MainBinaryFallback` from `ResolvedHelperExecutable` into `SpawnedHelper`, so provenance survives an immediate child exit. + +## Guard and lifecycle checklist + +- The Task 3 main startup record is cached before the instance guard and reused after logging becomes available. +- Only `start` and legacy `run` route through `runAgent` and acquire the main guard. +- `user-helper`, `desktop-helper`, `status`, `enroll`, and `bootstrap` do not call `runAgent` and do not acquire the main guard. +- Existing duplicate-main exit behavior and exit code 17 remain unchanged. +- No helper lifecycle, process-handle, Job Object, spawn ordering, or RDS desired-role policy code was changed. +- Windows and non-Windows `SpawnedHelper` definitions have parity for `CommandMode`, `Role`, `WindowsSessionID`, `BinaryPath`, and `MainBinaryFallback`. + +## Manual limitation + +Windows CIM/manual endpoint verification was not executed because no Windows test endpoint is available in this environment. In particular, this report does not claim live verification of SCM/console classification, the elevated duplicate exit, or CIM `SessionId`/`CommandLine` correlation. The Windows-only tests were cross-compiled, not executed locally. diff --git a/agent/internal/agentapp/main.go b/agent/internal/agentapp/main.go index 0d5e82c031..a4c810b9c3 100644 --- a/agent/internal/agentapp/main.go +++ b/agent/internal/agentapp/main.go @@ -611,11 +611,7 @@ func startAgent(cfg *config.Config) (*agentComponents, error) { } } - log.Info("starting agent", - "version", version, - "server", cfg.ServerURL, - "agentId", cfg.AgentID, - ) + logProcessStartup(cachedMainProcessStartup()) // Surface a failed systemd unit auto-heal (reconcileServiceUnitIfNeeded / // the reconcile-unit subcommand) to the fleet now that the log shipper is @@ -911,6 +907,7 @@ func retryTCCGrant() { func runAgent() { serviceMode := isWindowsService() startup := currentProcessStartup("run", "", serviceMode) + cacheMainProcessStartup(startup) guard, err := acquireMainAgentGuardFn(startup) if err != nil { writeInstanceGuardMarkerFn(startup, err) @@ -1569,15 +1566,7 @@ func runHelperProcess(name, role, context, binaryKind string) { } }() - log.Info("starting helper", - "name", name, - "version", version, - "socket", socketPath, - "pid", os.Getpid(), - "role", role, - "context", context, - "binaryKind", binaryKind, - ) + logProcessStartup(currentProcessStartup("user-helper", role, false)) // Handle shutdown signals via a done channel so multiple selects // can observe the shutdown without racing on a buffered sigChan. diff --git a/agent/internal/agentapp/main_test.go b/agent/internal/agentapp/main_test.go index dd1d70835b..21170190ad 100644 --- a/agent/internal/agentapp/main_test.go +++ b/agent/internal/agentapp/main_test.go @@ -1,6 +1,7 @@ package agentapp import ( + "bytes" "context" "errors" "os" @@ -13,8 +14,105 @@ import ( "github.com/breeze-rmm/agent/internal/collectors" "github.com/breeze-rmm/agent/internal/config" + "github.com/breeze-rmm/agent/internal/logging" ) +func TestProcessStartupFieldsContainRoleEvidenceOnly(t *testing.T) { + startup := ProcessStartup{ + Binary: "breeze-agent.exe", + ExecutablePath: `C:\Program Files\Breeze\breeze-agent.exe`, + PID: 42, + ParentPID: 4, + WindowsSessionID: 7, + LaunchMode: "user-helper", + HelperRole: "user", + LifecycleKey: "7-user", + MainBinaryFallback: true, + Version: "0.70.0", + CreatedAt: time.Unix(100, 0), + } + fields := processStartupFields(startup) + for _, key := range []string{"pid", "parentPid", "windowsSessionId", "launchMode", "helperRole", "lifecycleKey", "mainBinaryFallback"} { + if _, ok := fields[key]; !ok { + t.Fatalf("missing field %q", key) + } + } + for _, forbidden := range []string{"authToken", "helperAuthToken", "mtlsKey"} { + if _, ok := fields[forbidden]; ok { + t.Fatalf("forbidden field %q", forbidden) + } + } +} + +func TestLogProcessStartupEmitsOneStructuredEvent(t *testing.T) { + var output bytes.Buffer + logging.Init("json", "info", &output) + t.Cleanup(func() { logging.Init("text", "info", nil) }) + + startup := ProcessStartup{ + Binary: "breeze-agent.exe", + ExecutablePath: `C:\Program Files\Breeze\breeze-agent.exe`, + PID: 42, + ParentPID: 4, + WindowsSessionID: 7, + LaunchMode: "user-helper", + HelperRole: "user", + LifecycleKey: "7-user", + MainBinaryFallback: true, + Version: "0.70.0", + CreatedAt: time.Unix(100, 0), + } + logProcessStartup(startup) + + got := output.String() + if count := strings.Count(got, `"msg":"process startup"`); count != 1 { + t.Fatalf("process startup event count = %d, want 1; log=%s", count, got) + } + for _, evidence := range []string{ + `"windowsSessionId":7`, + `"launchMode":"user-helper"`, + `"helperRole":"user"`, + `"lifecycleKey":"7-user"`, + `"mainBinaryFallback":true`, + } { + if !strings.Contains(got, evidence) { + t.Fatalf("process startup log missing %s: %s", evidence, got) + } + } + for _, forbidden := range []string{"authToken", "helperAuthToken", "mtlsKey"} { + if strings.Contains(got, forbidden) { + t.Fatalf("process startup log contains forbidden field %q: %s", forbidden, got) + } + } +} + +func TestCachedMainProcessStartupUsesGuardRecord(t *testing.T) { + mainProcessStartupCache.Lock() + original := mainProcessStartupCache.startup + mainProcessStartupCache.startup = ProcessStartup{} + mainProcessStartupCache.Unlock() + t.Cleanup(func() { + mainProcessStartupCache.Lock() + mainProcessStartupCache.startup = original + mainProcessStartupCache.Unlock() + }) + + want := ProcessStartup{ + Binary: "breeze-agent.exe", + ExecutablePath: `C:\Program Files\Breeze\breeze-agent.exe`, + PID: 42, + ParentPID: 4, + WindowsSessionID: 7, + LaunchMode: "service-run", + Version: "0.70.0", + CreatedAt: time.Unix(100, 0), + } + cacheMainProcessStartup(want) + if got := cachedMainProcessStartup(); got != want { + t.Fatalf("cachedMainProcessStartup() = %+v, want %+v", got, want) + } +} + func TestRunAgentDuplicateStopsBeforeInitialization(t *testing.T) { testRunAgentGuardFailureStopsBeforeInitialization(t, ErrMainAgentAlreadyRunning, exitAlreadyRunning) } diff --git a/agent/internal/agentapp/process_role.go b/agent/internal/agentapp/process_role.go index 18bdb8d051..d8beebc973 100644 --- a/agent/internal/agentapp/process_role.go +++ b/agent/internal/agentapp/process_role.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/breeze-rmm/agent/internal/sessionbroker" @@ -34,6 +35,27 @@ type ProcessStartup struct { CreatedAt time.Time `json:"createdAt"` } +var mainProcessStartupCache struct { + sync.RWMutex + startup ProcessStartup +} + +func cacheMainProcessStartup(startup ProcessStartup) { + mainProcessStartupCache.Lock() + mainProcessStartupCache.startup = startup + mainProcessStartupCache.Unlock() +} + +func cachedMainProcessStartup() ProcessStartup { + mainProcessStartupCache.RLock() + startup := mainProcessStartupCache.startup + mainProcessStartupCache.RUnlock() + if startup.PID != 0 { + return startup + } + return currentProcessStartup("run", "", isWindowsService()) +} + func currentProcessStartup(command, role string, service bool) ProcessStartup { executable, _ := os.Executable() metadata := currentPlatformProcessMetadata() @@ -92,3 +114,26 @@ func processStartupFields(s ProcessStartup) map[string]any { "createdAt": s.CreatedAt, } } + +func logProcessStartup(startup ProcessStartup) { + fields := processStartupFields(startup) + keys := []string{ + "binary", + "executablePath", + "pid", + "parentPid", + "windowsSessionId", + "launchMode", + "helperRole", + "lifecycleKey", + "companionHelper", + "mainBinaryFallback", + "version", + "createdAt", + } + args := make([]any, 0, len(keys)*2) + for _, key := range keys { + args = append(args, key, fields[key]) + } + log.Info("process startup", args...) +} diff --git a/agent/internal/sessionbroker/helper_job_windows_test.go b/agent/internal/sessionbroker/helper_job_windows_test.go index 91ab85315b..09d7d651fe 100644 --- a/agent/internal/sessionbroker/helper_job_windows_test.go +++ b/agent/internal/sessionbroker/helper_job_windows_test.go @@ -230,22 +230,23 @@ func fakeResolvedHelperExecutable() (ResolvedHelperExecutable, error) { } func TestWindowsHelperSpawnerRetainsMainBinaryFallback(t *testing.T) { + buf := captureLogs(t) job := &fakeHelperJob{} spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ resolveExecutable: func() (ResolvedHelperExecutable, error) { return ResolvedHelperExecutable{Path: `C:\Program Files\Breeze\breeze-agent.exe`, MainBinaryFallback: true}, nil }, - createSuspended: func(_ HelperKey, resolved ResolvedHelperExecutable) (*suspendedHelper, error) { - pending := fakeSuspendedHelper() - pending.binaryPath = resolved.Path - pending.mainBinaryFallback = resolved.MainBinaryFallback - return pending, nil + createSuspended: func(_ HelperKey, _ ResolvedHelperExecutable) (*suspendedHelper, error) { + // Return deliberately stale suspended-process bookkeeping. The + // public diagnostic must retain the typed resolver provenance. + return fakeSuspendedHelper(), nil }, resumeThread: func(windows.Handle) (uint32, error) { return 1, nil }, closeHandle: func(windows.Handle) error { return nil }, }) - process, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "user"}) + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process, err := spawner.Spawn(key) if err != nil { t.Fatal(err) } @@ -253,6 +254,26 @@ func TestWindowsHelperSpawnerRetainsMainBinaryFallback(t *testing.T) { if !helper.MainBinaryFallback { t.Fatal("SpawnedHelper.MainBinaryFallback = false, want true") } + if helper.BinaryPath != `C:\Program Files\Breeze\breeze-agent.exe` { + t.Fatalf("SpawnedHelper.BinaryPath = %q, want typed resolver path", helper.BinaryPath) + } + if helper.CommandMode != "user-helper" || helper.Role != key.Role || helper.WindowsSessionID != key.WindowsSessionID { + t.Fatalf("SpawnedHelper role provenance = command:%q role:%q session:%d", helper.CommandMode, helper.Role, helper.WindowsSessionID) + } + logs := buf.String() + if count := strings.Count(logs, "spawned user helper in session"); count != 1 { + t.Fatalf("spawn diagnostic count = %d, want 1; logs: %s", count, logs) + } + for _, field := range []string{"commandMode=user-helper", "windowsSessionId=7", "role=user", "mainBinaryFallback=true"} { + if !strings.Contains(logs, field) { + t.Fatalf("spawn diagnostic missing %q; logs: %s", field, logs) + } + } + for _, forbidden := range []string{"authToken", "helperAuthToken", "mtlsKey", "tokenSource", "certificate", "tenant", "orgId"} { + if strings.Contains(logs, forbidden) { + t.Fatalf("spawn diagnostic contains forbidden field %q; logs: %s", forbidden, logs) + } + } } func TestWindowsHelperSpawnerWarnsBeforeRepeatedCreateFailures(t *testing.T) { diff --git a/agent/internal/sessionbroker/spawner_cmdline_test.go b/agent/internal/sessionbroker/spawner_cmdline_test.go index 534148669a..cf73e4c4e4 100644 --- a/agent/internal/sessionbroker/spawner_cmdline_test.go +++ b/agent/internal/sessionbroker/spawner_cmdline_test.go @@ -43,3 +43,21 @@ func TestBuildUserHelperCmdLine_AlwaysExplicitRole(t *testing.T) { }) } } + +func TestSpawnedHelperDiagnosticsRetainRoleProvenance(t *testing.T) { + helper := &SpawnedHelper{ + PID: 42, + BinaryPath: `C:\Program Files\Breeze\breeze-agent.exe`, + CommandMode: "user-helper", + Role: "user", + WindowsSessionID: 7, + MainBinaryFallback: true, + } + + if helper.CommandMode != "user-helper" || helper.Role != "user" || helper.WindowsSessionID != 7 { + t.Fatalf("spawn role provenance = command:%q role:%q session:%d", helper.CommandMode, helper.Role, helper.WindowsSessionID) + } + if helper.BinaryPath != `C:\Program Files\Breeze\breeze-agent.exe` || !helper.MainBinaryFallback { + t.Fatalf("spawn executable provenance = path:%q fallback:%v", helper.BinaryPath, helper.MainBinaryFallback) + } +} diff --git a/agent/internal/sessionbroker/spawner_stub.go b/agent/internal/sessionbroker/spawner_stub.go index 28bdc72259..5a84752ab7 100644 --- a/agent/internal/sessionbroker/spawner_stub.go +++ b/agent/internal/sessionbroker/spawner_stub.go @@ -16,6 +16,9 @@ import "fmt" type SpawnedHelper struct { PID uint32 BinaryPath string + CommandMode string + Role string + WindowsSessionID uint32 MainBinaryFallback bool } diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index 2949c5b46e..ca38e38161 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -24,6 +24,9 @@ type SpawnedHelper struct { PID uint32 Handle windows.Handle BinaryPath string + CommandMode string + Role string + WindowsSessionID uint32 MainBinaryFallback bool mu sync.Mutex terminated bool @@ -268,20 +271,24 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { _ = s.ops.closeHandle(pending.thread) cleanup = false - log.Info("spawned user helper in session", - "sessionId", key.WindowsSessionID, - "role", key.Role, - "pid", pending.pid, - "exe", pending.binaryPath, - "mainBinaryFallback", pending.mainBinaryFallback, - "tokenSource", pending.tokenSource, - ) - return &SpawnedHelper{ + helper := &SpawnedHelper{ PID: pending.pid, Handle: pending.process, - BinaryPath: pending.binaryPath, - MainBinaryFallback: pending.mainBinaryFallback, - }, nil + BinaryPath: resolvedExe.Path, + CommandMode: "user-helper", + Role: key.Role, + WindowsSessionID: key.WindowsSessionID, + MainBinaryFallback: resolvedExe.MainBinaryFallback, + } + log.Info("spawned user helper in session", + "pid", helper.PID, + "binaryPath", helper.BinaryPath, + "commandMode", helper.CommandMode, + "role", helper.Role, + "windowsSessionId", helper.WindowsSessionID, + "mainBinaryFallback", helper.MainBinaryFallback, + ) + return helper, nil } func (s *windowsHelperSpawner) Close() error { From eb2b4f393505e3aea3876ca15a1317f0b58e8d53 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 19:04:53 -0600 Subject: [PATCH 19/47] chore(agent): remove internal task report --- .superpowers/sdd/instance-task-4-report.md | 102 --------------------- 1 file changed, 102 deletions(-) delete mode 100644 .superpowers/sdd/instance-task-4-report.md diff --git a/.superpowers/sdd/instance-task-4-report.md b/.superpowers/sdd/instance-task-4-report.md deleted file mode 100644 index 5e9266ba6a..0000000000 --- a/.superpowers/sdd/instance-task-4-report.md +++ /dev/null @@ -1,102 +0,0 @@ -# Instance Diagnostics Task 4 Report - -Base: `692864c5483bed0845ff57fc955a04d366968bee` - -## Result - -Implemented structured process-role diagnostics for the main agent and helper processes, plus retained parent-side spawn provenance. The implementation does not change helper process handles, Job Object ownership, lifecycle convergence, or RDS role policy. - -## TDD evidence - -### RED - -1. Native focused test command: - - `go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestProcessStartup|TestLogProcessStartup|TestSpawnedHelperDiagnostics|TestClassifyProcess|TestResolveUserHelperPath|TestBuildUserHelperCmdLine' -count=1` - - Exit 1. Expected missing-feature failures: - - - `undefined: logProcessStartup` - - `unknown field CommandMode in struct literal of type SpawnedHelper` - - `unknown field Role in struct literal of type SpawnedHelper` - - `unknown field WindowsSessionID in struct literal of type SpawnedHelper` - -2. Windows amd64 sessionbroker test cross-compile: - - `GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c ./internal/sessionbroker -o /tmp/breeze-task4-red/sessionbroker.test.exe` - - Exit 1. The Windows `SpawnedHelper` lacked `CommandMode`, `Role`, and `WindowsSessionID`. - -3. Cached main-startup regression test with the cache implementation removed: - - `go test ./internal/agentapp -run TestCachedMainProcessStartupUsesGuardRecord -count=1` - - Exit 1. The cache and cache accessors were absent, proving the test requires the Task 3 guard-time record rather than a later recomputation. - -### GREEN - -- Focused cache/startup tests: pass under `-race`. -- Brief-required focused command: - - `go test -race ./internal/agentapp ./internal/sessionbroker -run 'TestProcessStartup|TestClassifyProcess|TestResolveUserHelperPath|TestBuildUserHelperCmdLine' -count=1` - - Pass. - -- Affected package suites: - - `go test -race ./internal/agentapp ./internal/sessionbroker -count=1` - - Pass. - -- Full native agent suite: - - `go test -race ./...` - - Pass with no race report. - -- Windows cross-compilation for both `amd64` and `arm64`: `agentapp`, `sessionbroker`, and `heartbeat` test binaries plus `cmd/breeze-agent` all compile successfully. - -## One-event evidence - -- There is one log implementation for the event: `log.Info("process startup", args...)` in `process_role.go`. -- Main startup has one call site, after full logging and log shipping initialization: `logProcessStartup(cachedMainProcessStartup())` in `startAgent`. -- Helper startup has one call site, after helper logging and optional shipping initialization: `logProcessStartup(currentProcessStartup("user-helper", role, false))` in `runHelperProcess`. -- The previous `starting agent` and `starting helper` messages have been removed; repository string search over the touched startup paths returns zero occurrences. -- `TestLogProcessStartupEmitsOneStructuredEvent` captures JSON logs and asserts exactly one `process startup` record. -- Parent-side spawn diagnostics remain a distinct `spawned user helper in session` record and are asserted once in the Windows spawner test. They are not a second `process startup` record. - -## Field and security checklist - -The process-startup whitelist contains only: - -- `binary` -- `executablePath` -- `pid` -- `parentPid` -- `windowsSessionId` -- `launchMode` -- `helperRole` -- `lifecycleKey` -- `companionHelper` -- `mainBinaryFallback` -- `version` -- `createdAt` - -The parent spawn diagnostic contains only process provenance needed for support: PID, actual binary path, `user-helper` command mode, explicit authenticated helper role, kernel-target Windows session ID, and typed resolver fallback status. It does not log token-source metadata. - -Tests reject `authToken`, `helperAuthToken`, and `mtlsKey`; the Windows spawn-log test also rejects token-source, certificate, tenant, and organization fields. No secret value, token value, certificate material, tenant ID, organization ID, server URL, or agent ID was added to either diagnostic. - -Helper self-diagnostics use `os.Executable()` and kernel process metadata. `WindowsSessionID` and `LifecycleKey` therefore come from the current process session, not a claimed session argument. Parent diagnostics copy `BinaryPath` and `MainBinaryFallback` from `ResolvedHelperExecutable` into `SpawnedHelper`, so provenance survives an immediate child exit. - -## Guard and lifecycle checklist - -- The Task 3 main startup record is cached before the instance guard and reused after logging becomes available. -- Only `start` and legacy `run` route through `runAgent` and acquire the main guard. -- `user-helper`, `desktop-helper`, `status`, `enroll`, and `bootstrap` do not call `runAgent` and do not acquire the main guard. -- Existing duplicate-main exit behavior and exit code 17 remain unchanged. -- No helper lifecycle, process-handle, Job Object, spawn ordering, or RDS desired-role policy code was changed. -- Windows and non-Windows `SpawnedHelper` definitions have parity for `CommandMode`, `Role`, `WindowsSessionID`, `BinaryPath`, and `MainBinaryFallback`. - -## Manual limitation - -Windows CIM/manual endpoint verification was not executed because no Windows test endpoint is available in this environment. In particular, this report does not claim live verification of SCM/console classification, the elevated duplicate exit, or CIM `SessionId`/`CommandLine` correlation. The Windows-only tests were cross-compiled, not executed locally. From 01e1afb074b14fe2efa1f5d4e12d67b761da77c0 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 21:49:54 -0600 Subject: [PATCH 20/47] docs(agent): plan helper lifecycle review remediation --- ...-14-helper-lifecycle-review-remediation.md | 1242 +++++++++++++++++ 1 file changed, 1242 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md diff --git a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md new file mode 100644 index 0000000000..3f95e9a551 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md @@ -0,0 +1,1242 @@ +# Helper Lifecycle Review Remediation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the eight verified review blockers in the Windows helper lifecycle layer so slices 1-5 of `2026-07-14-windows-agent-helper-lifecycle-durability-design.md` are merge-ready, before executing the watchdog slice. + +**Architecture:** Every blocker shares one root cause — the helper liveness layer collapses "I don't know" into "it's dead". The fix widens `helperProcess.Alive()` to `(bool, error)` so uncertainty is representable, then makes each of the four call sites fail *closed* (unknown ⇒ assume alive ⇒ terminate/refuse, never ⇒ spawn a duplicate). Around that root fix sit five independent repairs: restore deleted security-gate coverage, make the Windows CI job auto-discover packages, close a fail-open to SYSTEM, replace the deleted `KillStaleHelpers` deadlock-breaker with a startup timeout that finally reads `launchedAt`, and make a failed helper kill both visible and retryable. + +**Tech Stack:** Go 1.x, Windows syscalls via `golang.org/x/sys/windows`, `go test -race`, GitHub Actions (`windows-latest`). + +## Global Constraints + +- **Fail closed, always.** An unknown liveness/identity state must never result in spawning a second helper or skipping a termination. Unknown ⇒ assume the process is alive. +- **This code ships to customer machines and terminates processes as SYSTEM.** No change may widen what can be terminated. +- Every behavioral change lands with a test that fails before the fix (TDD). +- `go test -race ./...` must pass in `agent/` on darwin (host) after every task. +- `GOOS=windows go vet ./...` must stay clean — it is the only compile check for `_windows.go` files on this host. New `unsafe.Pointer` warnings are forbidden; the pre-existing ones in `internal/remote/desktop` are out of scope. +- Never edit a shipped migration; not applicable here (no DB changes). +- Do not reformat unrelated code. Stale `gofmt` alignment in four `sessionbroker` test files is pre-existing and out of scope for this plan (CI runs neither `gofmt` nor `go vet` on the agent). +- Commit after every task with the exact message given. + +## File Structure + +- `agent/internal/sessionbroker/console_session_gate_test.go` — restore the five deleted role-authorization cases (Task 1). +- `.github/workflows/ci.yml:720` — Windows job auto-discovers packages (Task 2). +- `agent/internal/sessionbroker/spawner_windows.go` — exhaustive role switch (Task 3); `Alive()` signature (Task 4). +- `agent/internal/sessionbroker/spawner_stub.go` — `Alive()` signature, non-Windows (Task 4). +- `agent/internal/sessionbroker/lifecycle_core.go` — `helperProcess` interface + `stopTrackedKey`/`reconcile` callers (Tasks 4, 5); `watchDetachedProcess` logging (Task 8). +- `agent/internal/sessionbroker/lifecycle_registry.go` — `reserve`/`detach`/`markSessionClosed` callers (Task 4); `startupExpired` + `launchedAt` wiring (Task 5). +- `agent/internal/sessionbroker/lifecycle_registry_test.go` — `fakeHelperProcess` implements the new signature (Task 4). +- `agent/internal/sessionbroker/broker.go` — `TerminateHelperKey` visibility + key retention (Task 6). +- `agent/internal/sessionbroker/broker_admission.go` — resolve the stale `helperOwnerReplaceable` comment (Task 7). +- `agent/internal/heartbeat/heartbeat.go` — bootstrap retry (Task 7). + +--- + +### Task 1: Restore deleted role-authorization gate coverage + +`main` had 10 table cases in `TestRoleIdentityRejection`; this branch has 7. Mutation testing proved four security gates in `roleIdentityRejection` now ship green when deleted. The two Session-0 sentinel cases were the institutional memory for the `#1009` fail-closed fix. + +**Files:** +- Modify: `agent/internal/sessionbroker/console_session_gate_test.go:18-24` + +**Interfaces:** +- Consumes: `roleIdentityRejection(role, sid string, uid uint32, peerWinSession, claimedWinSession, consoleWinSession, goos string) (reason string, rejected bool)` from `broker.go:2119`; `systemSID` const; `ipc.HelperRoleAssist`, `ipc.HelperRoleWatchdog`. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Add the five failing cases to the existing table** + +In `console_session_gate_test.go`, the table literal currently ends with the `"assist remains console bound"` line. Add these five cases immediately after it, inside the same `}{...}` literal: + +```go + {"assist from console session accepted", ipc.HelperRoleAssist, nonSystemSID, "1", "1", "1", "", false}, + {"assist as SYSTEM rejected on SID", ipc.HelperRoleAssist, systemSID, "1", "1", "1", "assist role requires non-SYSTEM identity", true}, + {"assist rejected when console lookup failed (session 0 sentinel)", ipc.HelperRoleAssist, nonSystemSID, "0", "0", "0", "assist role requires the active console session", true}, + {"watchdog as SYSTEM unaffected by console session", ipc.HelperRoleWatchdog, systemSID, "0", "0", "1", "", false}, + {"watchdog as non-SYSTEM rejected", ipc.HelperRoleWatchdog, nonSystemSID, "1", "1", "1", "watchdog role requires SYSTEM identity", true}, +``` + +- [ ] **Step 2: Run the test to confirm the restored cases pass against current code** + +Run: `cd agent && go test -race ./internal/sessionbroker -run TestRoleIdentityRejection -v` +Expected: PASS, 12 subtests. These cases describe behavior the code *already has* — they pass now. Their value is proven in Step 3, not here. + +If `"assist rejected when console lookup failed (session 0 sentinel)"` FAILS, stop and report: it means the `consoleWinSession == "0"` disjunct at `broker.go:2139` does not behave as the #1009 fix intends, which is a real bug rather than a test gap. + +- [ ] **Step 3: Prove each case actually guards its gate (mutation check)** + +For each mutation below, apply it to `agent/internal/sessionbroker/broker.go`, run `cd agent && go test ./internal/sessionbroker -run TestRoleIdentityRejection`, confirm **FAIL**, then revert the mutation with `git checkout agent/internal/sessionbroker/broker.go`. + +1. Delete the line `case role == ipc.HelperRoleAssist && sid == systemSID:` and its `return` (≈`:2126-2127`) → must FAIL on `"assist as SYSTEM rejected on SID"`. +2. Delete the line `case role == ipc.HelperRoleWatchdog && sid != systemSID:` and its `return` (≈`:2128-2129`) → must FAIL on `"watchdog as non-SYSTEM rejected"`. +3. In the assist console check (≈`:2139`), remove the `consoleWinSession == "0" ||` disjunct → must FAIL on `"assist rejected when console lookup failed (session 0 sentinel)"`. +4. In the same check, replace the condition with `role == ipc.HelperRoleAssist` (assist always rejected) → must FAIL on `"assist from console session accepted"`. + +All four must FAIL. Any mutation that still passes means the case does not guard its gate — fix the case before continuing. + +- [ ] **Step 4: Verify the working tree is clean of mutations** + +Run: `cd agent && git diff --stat internal/sessionbroker/broker.go` +Expected: empty output. If not, `git checkout agent/internal/sessionbroker/broker.go`. + +- [ ] **Step 5: Run the full package suite** + +Run: `cd agent && go test -race ./internal/sessionbroker` +Expected: `ok` + +- [ ] **Step 6: Commit** + +```bash +git add agent/internal/sessionbroker/console_session_gate_test.go +git commit -m "test(agent): restore deleted helper role-authorization gate cases" +``` + +--- + +### Task 2: Make the Windows CI job auto-discover packages + +`ci.yml:720` hardcodes six packages. `./internal/eventlog` is absent, so `eventlog_windows_test.go` (135 new lines, `//go:build windows`) executes nowhere: the Linux job's `./...` skips it via build tag, and the Windows job never names it. Nine other packages with Windows-tagged tests are also omitted. This silently falsifies CLAUDE.md's "New test files are auto-discovered — no CI config changes needed." + +**Files:** +- Modify: `.github/workflows/ci.yml:720` + +**Interfaces:** +- Consumes: the `test-agent-windows` job added earlier on this branch. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Confirm the gap is real before changing anything** + +Run: `grep -n 'go test -race ./internal/sessionbroker' .github/workflows/ci.yml` +Expected: line 720, with no `./internal/eventlog` in the list. + +Run: `head -1 agent/internal/eventlog/eventlog_windows_test.go` +Expected: `//go:build windows` — confirming the Linux job cannot run it. + +- [ ] **Step 2: Replace the hardcoded list with package auto-discovery** + +In `.github/workflows/ci.yml`, replace line 720: + +```yaml + run: go test -race ./internal/sessionbroker ./internal/heartbeat ./internal/agentapp ./internal/config ./internal/watchdog ./cmd/breeze-watchdog +``` + +with: + +```yaml + # Auto-discover every package so Windows-tagged tests cannot be + # silently skipped. A hardcoded list drifts: ./internal/eventlog and + # nine other packages with //go:build windows tests were omitted, so + # their tests ran on no platform at all (the Linux job's ./... skips + # them via build tag). Keeps CLAUDE.md's "new test files are + # auto-discovered" contract true on Windows. + run: go test -race ./... +``` + +- [ ] **Step 3: Verify the cross-compile still builds every test binary** + +`go test -race ./...` cannot run on this darwin host for Windows, so vet is the compile check. Run: + +`cd agent && GOOS=windows go vet ./... 2>&1 | grep -v "possible misuse of unsafe.Pointer"; echo "VET_DONE"` + +Expected: only `VET_DONE`. The filtered `unsafe.Pointer` warnings are pre-existing in `internal/remote/desktop` and out of scope. + +- [ ] **Step 4: Verify the YAML parses** + +Run: `python3 -c "import yaml,sys; yaml.safe_load(open('.github/workflows/ci.yml')); print('YAML_OK')"` +Expected: `YAML_OK` + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/ci.yml +git commit -m "ci(agent): auto-discover Windows test packages" +``` + +**Note for the executor:** widening to `./...` may surface pre-existing failures in the nine newly-included packages when CI first runs on `windows-latest`. That is the point of the change. Report any such failures rather than re-narrowing the list. + +--- + +### Task 3: Close the fail-open to SYSTEM in helper spawn + +`createHelperSuspended` routes any role that is not exactly `"user"` — including a zero-value `HelperKey{}` whose `Role` is `""` — down the SYSTEM-token path. It is unreachable today only because every key happens to flow through `helperKeyFromDetected`. That is safety by data-flow accident on a privilege boundary, and nothing tests it. + +**Files:** +- Modify: `agent/internal/sessionbroker/spawner_windows.go:313-318` +- Test: `agent/internal/sessionbroker/spawner_cmdline_test.go` + +**Interfaces:** +- Consumes: `HelperKey{WindowsSessionID uint32; Role string}` from `helper_key.go:8`; `ResolvedHelperExecutable` from `userhelper_path.go:45`; `ipc.HelperRoleSystem`/`ipc.HelperRoleUser` (`ipc/message.go:123-124`). +- Produces: `helperRoleSpawnable(role string) bool` — used by no later task, but keep the name stable. + +- [ ] **Step 1: Write the failing test** + +`createHelperSuspended` calls real Windows APIs, so it cannot be unit-tested on this host. Extract and test the *decision*. Append to `agent/internal/sessionbroker/spawner_cmdline_test.go`: + +```go +func TestHelperRoleSpawnableRejectsNonLifecycleRoles(t *testing.T) { + tests := []struct { + name string + role string + want bool + }{ + {"system role spawnable", ipc.HelperRoleSystem, true}, + {"user role spawnable", ipc.HelperRoleUser, true}, + {"zero-value role is not spawnable", "", false}, + {"wrong case is not spawnable", "User", false}, + {"assist is not a lifecycle role", ipc.HelperRoleAssist, false}, + {"watchdog is not a lifecycle role", ipc.HelperRoleWatchdog, false}, + {"unknown role is not spawnable", "banana", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := helperRoleSpawnable(tc.role); got != tc.want { + t.Fatalf("helperRoleSpawnable(%q) = %v, want %v", tc.role, got, tc.want) + } + }) + } +} +``` + +Ensure the file's import block includes `"github.com/breeze-rmm/agent/internal/ipc"`. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd agent && go test ./internal/sessionbroker -run TestHelperRoleSpawnable` +Expected: FAIL — `undefined: helperRoleSpawnable` + +- [ ] **Step 3: Add the predicate in a platform-neutral file** + +`spawner_cmdline_test.go` has no build tag, so the predicate must live in a file that builds everywhere. Add to `agent/internal/sessionbroker/helper_key.go` (imports `"github.com/breeze-rmm/agent/internal/ipc"` — add it): + +```go +// helperRoleSpawnable reports whether role is one the lifecycle manager may +// launch a process for. Only the two lifecycle roles qualify: assist and +// watchdog helpers are started by other means and must never be spawned here. +// +// This gate exists because the spawn path selects a privilege level from the +// role. Anything that is not exactly ipc.HelperRoleUser would otherwise take +// the SYSTEM-token branch, so an empty or misspelled role silently escalates. +func helperRoleSpawnable(role string) bool { + return role == ipc.HelperRoleSystem || role == ipc.HelperRoleUser +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd agent && go test ./internal/sessionbroker -run TestHelperRoleSpawnable -v` +Expected: PASS, 7 subtests. + +- [ ] **Step 5: Make the spawn switch exhaustive and fail closed** + +Replace `createHelperSuspended` at `spawner_windows.go:313-318` in full. It calls the predicate rather than duplicating the role list, so the tested predicate is the thing that actually guards the privilege boundary: + +```go +// createHelperSuspended creates the helper process for key without letting its +// primary thread run. The role selects the token privilege level, so an +// unrecognized role must never reach a spawn call: the old permissive default +// sent anything that was not exactly "user" down the SYSTEM-token path, so an +// empty or misspelled role silently escalated. +func createHelperSuspended(key HelperKey, resolvedExe ResolvedHelperExecutable) (*suspendedHelper, error) { + if !helperRoleSpawnable(key.Role) { + return nil, fmt.Errorf("refusing to spawn helper for non-lifecycle role %q", key.Role) + } + switch key.Role { + case ipc.HelperRoleUser: + return createUserHelperSuspended(key.WindowsSessionID, resolvedExe) + case ipc.HelperRoleSystem: + return createSystemHelperSuspended(key.WindowsSessionID, resolvedExe) + default: + return nil, fmt.Errorf("role %q passed helperRoleSpawnable but has no spawn path", key.Role) + } +} +``` + +Confirm `spawner_windows.go` imports both `"fmt"` and `"github.com/breeze-rmm/agent/internal/ipc"`; add whichever is missing. + +- [ ] **Step 6: Gate the platform-neutral spawn path on the same predicate** + +`createHelperSuspended` is Windows-only, so Step 5 cannot be tested on this host. Add the same guard to `spawnKey` in `lifecycle_core.go:256` — a path that *is* testable everywhere — immediately after the `m.mu.Lock()` and the `m.stopping || !m.desired[key]` check, before the reserve: + +```go + if !helperRoleSpawnable(key.Role) { + m.mu.Unlock() + log.Error("lifecycle: refusing to spawn helper for non-lifecycle role", "helperKey", key.String(), "role", key.Role) + return + } +``` + +Match the surrounding unlock style exactly — the existing early returns in `spawnKey` unlock explicitly rather than using `defer`. + +- [ ] **Step 7: Write the failing test for the spawn-path gate** + +Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`. This reuses the package's real harness (`newLifecycleHarness` at `:316`) and real fake (`fakeHelperSpawner` at `:162`, which already counts spawns per key in its `spawned` map — no new counter needed): + +```go +func TestSpawnKeyRefusesNonLifecycleRole(t *testing.T) { + // A non-lifecycle role must never reach the spawner: on Windows the role + // selects the token privilege level, so anything not recognized as "user" + // took the SYSTEM branch. The zero-value HelperKey is the realistic vector. + spawner := &fakeHelperSpawner{} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) + key := HelperKey{WindowsSessionID: 7, Role: ""} + + m.mu.Lock() + m.desired = map[HelperKey]bool{key: true} + m.mu.Unlock() + + m.spawnKey(key) + + spawner.mu.Lock() + got := spawner.spawned[key] + spawner.mu.Unlock() + if got != 0 { + t.Fatalf("spawner was called %d times for role %q; want 0", got, key.Role) + } +} +``` + +If `m.desired`'s type is not `map[HelperKey]bool`, match its real declaration in `lifecycle_core.go` — check with `grep -n 'desired ' agent/internal/sessionbroker/lifecycle_core.go`. + +- [ ] **Step 8: Run the test to verify it fails, then passes** + +Run: `cd agent && go test ./internal/sessionbroker -run TestSpawnKeyRefusesNonLifecycleRole` + +Expected before Step 6's guard is added: FAIL (spawner called once). With the guard: PASS. If it passes *before* the guard exists, the test is vacuous — find out what else is short-circuiting the spawn and fix the test. + +- [ ] **Step 9: Verify the Windows path still compiles** + +Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` +Expected: clean (no output). This is the only check that compiles the Step 5 change on this host. + +- [ ] **Step 10: Run the full package suite** + +Run: `cd agent && go test -race ./internal/sessionbroker` +Expected: `ok` + +- [ ] **Step 11: Commit** + +```bash +git add agent/internal/sessionbroker/helper_key.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/spawner_cmdline_test.go agent/internal/sessionbroker/lifecycle_registry_test.go +git commit -m "fix(agent): refuse helper spawn for non-lifecycle roles" +``` + +--- + +### Task 4: Make helper liveness able to express "unknown" + +This is the root fix. `helperProcess.Alive() bool` cannot express failure, so a `GetExitCodeProcess` error reads as "the helper is dead". The sibling `ownedPeerProcess.Alive() (bool, error)` twelve lines above already gets this right. Consequences today: `reserve` spawns a duplicate helper for a live key; `stopTrackedKey` skips `Terminate`; `markSessionClosed` marks a live process exited. All silent. + +**Files:** +- Modify: `agent/internal/sessionbroker/lifecycle_core.go:26` (interface), `:341` (caller) +- Modify: `agent/internal/sessionbroker/spawner_windows.go:86-97` +- Modify: `agent/internal/sessionbroker/spawner_stub.go:30` +- Modify: `agent/internal/sessionbroker/lifecycle_registry.go:75`, `:166`, `:212` +- Modify: `agent/internal/sessionbroker/lifecycle_registry_test.go:83` (`fakeHelperProcess`) +- Modify: `agent/internal/sessionbroker/lifecycle_test.go:47,50`; `agent/internal/sessionbroker/lifecycle_registry_test.go:229,382,385` (direct `.Alive()` callers) + +**Interfaces:** +- Consumes: `helperProcess` interface (`lifecycle_core.go:23`). +- Produces: `helperProcess.Alive() (bool, error)` — Task 5 relies on this signature. `fakeHelperProcess.aliveErr error` field — Task 5's tests reuse it. + +- [ ] **Step 1: Write the failing tests** + +Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`: + +```go +func TestReserveRefusesWhenLivenessUnknown(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process := newFakeHelperProcess(4242) + process.setAliveErr(errors.New("GetExitCodeProcess: access denied")) + + generation, ok := r.reserve(key, time.Now()) + if !ok { + t.Fatal("first reserve must succeed on an empty slot") + } + if _, attached := r.attachReserved(key, generation, process, "user-helper"); !attached { + t.Fatal("attachReserved failed") + } + + // Liveness is unknown, so the registry must NOT hand out a second + // reservation: spawning a duplicate helper is worse than spawning none. + if _, ok := r.reserve(key, time.Now()); ok { + t.Fatal("reserve granted a duplicate helper while liveness was unknown") + } +} + +func TestMarkSessionClosedTreatsUnknownLivenessAsAlive(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process := newFakeHelperProcess(4243) + + generation, _ := r.reserve(key, time.Now()) + entry, _ := r.attachReserved(key, generation, process, "user-helper") + session := &Session{} + r.markConnected(key, process.ProcessID(), session) + + process.setAliveErr(errors.New("GetExitCodeProcess: access denied")) + r.markSessionClosed(key, session) + + if entry.state == helperExited { + t.Fatal("unknown liveness was recorded as helperExited; a live helper can now be duplicated") + } +} +``` + +Ensure the file imports `"errors"`. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cd agent && go test ./internal/sessionbroker -run 'TestReserveRefusesWhenLivenessUnknown|TestMarkSessionClosedTreatsUnknownLivenessAsAlive'` +Expected: FAIL — `process.setAliveErr undefined` + +- [ ] **Step 3: Widen the interface** + +In `lifecycle_core.go`, change line 26 inside `type helperProcess interface`: + +```go + Alive() (bool, error) +``` + +so the block reads: + +```go +type helperProcess interface { + ProcessID() uint32 + ExecutablePath() string + // Alive reports liveness. A non-nil error means liveness is UNKNOWN, not + // false: callers must fail closed and treat unknown as alive. Matches + // ownedPeerProcess.Alive so the two cannot be confused at a glance. + Alive() (bool, error) + Terminate() error + Wait() (int, error) + Close() error +} +``` + +- [ ] **Step 4: Update the Windows implementation** + +Replace `SpawnedHelper.Alive` at `spawner_windows.go:86-97`: + +```go +// Alive reports whether the helper process is still running. A non-nil error +// means the state could not be determined; callers must not read that as "dead". +func (s *SpawnedHelper) Alive() (bool, error) { + if s == nil { + return false, fmt.Errorf("SpawnedHelper: nil helper") + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Handle == 0 { + // Close() already released the handle; the helper is definitively + // no longer ours to track. This is a known-dead answer, not unknown. + return false, nil + } + var exitCode uint32 + if err := windows.GetExitCodeProcess(s.Handle, &exitCode); err != nil { + return false, fmt.Errorf("GetExitCodeProcess: %w", err) + } + return exitCode == windowsProcessStillActive, nil +} +``` + +- [ ] **Step 5: Update the non-Windows stub** + +Replace `spawner_stub.go:30`: + +```go +// Alive always errors on non-Windows: the stub never spawns, so a SpawnedHelper +// here is a programming error. Returning (false, nil) would tell callers a live +// helper is dead — the exact confusion this signature exists to prevent. +func (s *SpawnedHelper) Alive() (bool, error) { + return false, fmt.Errorf("SpawnedHelper: helper tracking not supported on this platform") +} +``` + +- [ ] **Step 6: Make all four production callers fail closed** + +In `lifecycle_registry.go`, replace the `reserve` liveness check at `:75`: + +```go + if previous.process != nil { + alive, err := previous.process.Alive() + if err != nil { + // Unknown liveness: refuse the reservation. A duplicate helper + // racing the original over DXGI capture is worse than no helper. + log.Warn("lifecycle: helper liveness unknown; refusing respawn", "helperKey", key.String(), "pid", previous.process.ProcessID(), "error", err.Error()) + return 0, false + } + if alive { + return 0, false + } + } +``` + +Replace the `detach` check at `:166`: + +```go + if entry.process != nil { + alive, err := entry.process.Alive() + if err != nil || alive { + // Unknown or alive: keep the entry so the caller retries rather + // than dropping a process it never confirmed dead. + return false + } + } +``` + +Replace the `markSessionClosed` check at `:212`: + +```go + if entry.process != nil { + alive, err := entry.process.Alive() + if err != nil || alive { + entry.state = helperStarting + } else { + entry.state = helperExited + } + } else { + entry.state = helperExited + } +``` + +In `lifecycle_core.go`, replace the `stopTrackedKey` check at `:341`: + +```go + if entry.process != nil { + alive, err := entry.process.Alive() + if err != nil { + log.Warn("lifecycle: helper liveness unknown; terminating to fail closed", "helperKey", key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + } + if err != nil || alive { + if err := entry.process.Terminate(); err != nil { + log.Warn("lifecycle: failed to terminate helper", "helperKey", key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + } + } + } +``` + +Confirm `lifecycle_registry.go` imports the logger used elsewhere in the package (match the import path already used in `lifecycle_core.go`). Add it if absent. + +- [ ] **Step 7: Update the test fake** + +In `lifecycle_registry_test.go`, add an `aliveErr` field to `fakeHelperProcess` (after `terminateExits bool`): + +```go + aliveErr error +``` + +Replace `fakeHelperProcess.Alive` at `:83`: + +```go +func (p *fakeHelperProcess) Alive() (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.aliveErr != nil { + return false, p.aliveErr + } + return p.alive, nil +} + +func (p *fakeHelperProcess) setAliveErr(err error) { + p.mu.Lock() + defer p.mu.Unlock() + p.aliveErr = err +} +``` + +If the existing `Alive` body differs from `return p.alive`, preserve its logic and only add the `aliveErr` branch plus the `(bool, error)` return. + +- [ ] **Step 8: Update direct test callers** + +Five call sites now return two values. In `lifecycle_registry_test.go:229,382,385` and `lifecycle_test.go:47,50`, replace each `x.Alive()` boolean use with a helper. Add to `lifecycle_registry_test.go`: + +```go +// aliveNow fails the test if liveness cannot be determined, so a test never +// silently reads "unknown" as "dead". +func aliveNow(t *testing.T, p helperProcess) bool { + t.Helper() + alive, err := p.Alive() + if err != nil { + t.Fatalf("Alive() error = %v", err) + } + return alive +} +``` + +Then rewrite each site, e.g. `if !system.Alive() {` becomes `if !aliveNow(t, system) {`. `lifecycle_test.go` is `//go:build windows`; `aliveNow` lives in an untagged test file, so it is visible there — verify with the Step 10 vet. + +- [ ] **Step 9: Run the tests to verify they pass** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestReserveRefusesWhenLivenessUnknown|TestMarkSessionClosedTreatsUnknownLivenessAsAlive' -v` +Expected: PASS, both tests. + +- [ ] **Step 10: Verify Windows-tagged files still compile** + +Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` +Expected: clean. This is the only check that compiles `lifecycle_test.go`, `spawner_windows.go`, and `peer_process_windows.go` on this host — do not skip it. + +- [ ] **Step 11: Run the full suite** + +Run: `cd agent && go test -race ./internal/sessionbroker ./internal/heartbeat` +Expected: `ok` for both. + +- [ ] **Step 12: Commit** + +```bash +git add agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/spawner_stub.go agent/internal/sessionbroker/lifecycle_registry_test.go agent/internal/sessionbroker/lifecycle_test.go +git commit -m "fix(agent): make helper liveness express unknown and fail closed" +``` + +--- + +### Task 5: Replace the deleted stale-helper deadlock breaker + +`main` called `broker.KillStaleHelpers(session + "-" + role)` before every respawn, commenting: *"A 'successful' CreateProcessAsUser that crashes before connecting to IPC is still a failure from the lifecycle perspective."* This branch deletes it with no replacement. Now `markSessionClosed` sets `helperStarting` for a live-but-disconnected helper, `reserve` refuses `helperStarting` forever, and nothing terminates it — that session/role slot is dead until the process exits or the agent restarts. `trackedHelper.launchedAt` is written twice and never read: the timeout hook was built and never wired. + +**Files:** +- Modify: `agent/internal/sessionbroker/lifecycle_registry.go` (add `startupExpired`, reset `launchedAt` in `markSessionClosed`) +- Modify: `agent/internal/sessionbroker/lifecycle_core.go` (add `helperStartupTimeout`, recycle in `reconcile`) +- Test: `agent/internal/sessionbroker/lifecycle_registry_test.go` + +**Interfaces:** +- Consumes: `helperProcess.Alive() (bool, error)` and `fakeHelperProcess` from Task 4; `helperRegistry`, `trackedHelper`, `helperStarting` (`lifecycle_registry.go:11-31`); `stopTrackedKey` (`lifecycle_core.go:332`). +- Produces: `helperRegistry.startupExpired(key HelperKey, now time.Time, timeout time.Duration) bool`; `helperStartupTimeout` const. + +- [ ] **Step 1: Write the failing tests** + +Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`: + +```go +func TestStartupExpiredDetectsHelperThatNeverConnected(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + start := time.Now() + + generation, _ := r.reserve(key, start) + if _, attached := r.attachReserved(key, generation, newFakeHelperProcess(5150), "user-helper"); !attached { + t.Fatal("attachReserved failed") + } + + if r.startupExpired(key, start.Add(30*time.Second), helperStartupTimeout) { + t.Fatal("a helper still inside its startup window must not be recycled") + } + if !r.startupExpired(key, start.Add(helperStartupTimeout+time.Second), helperStartupTimeout) { + t.Fatal("a helper that never reached IPC past the timeout must be recycled") + } +} + +func TestStartupExpiredIgnoresConnectedHelper(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + start := time.Now() + process := newFakeHelperProcess(5151) + + generation, _ := r.reserve(key, start) + r.attachReserved(key, generation, process, "user-helper") + r.markConnected(key, process.ProcessID(), &Session{}) + + if r.startupExpired(key, start.Add(24*time.Hour), helperStartupTimeout) { + t.Fatal("a connected helper must never be treated as a failed startup") + } +} + +func TestMarkSessionClosedRestartsTheStartupWindow(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + start := time.Now() + process := newFakeHelperProcess(5152) + + generation, _ := r.reserve(key, start) + entry, _ := r.attachReserved(key, generation, process, "user-helper") + session := &Session{} + r.markConnected(key, process.ProcessID(), session) + + // Long-lived helper: connected at start, IPC drops much later. The process + // is still alive, so it goes back to helperStarting. Its startup clock must + // restart, or it would be recycled instantly on the next reconcile. + r.markSessionClosed(key, session) + + if entry.state != helperStarting { + t.Fatalf("state = %q, want %q", entry.state, helperStarting) + } + if r.startupExpired(key, time.Now().Add(time.Second), helperStartupTimeout) { + t.Fatal("startup window was not restarted on session close; a live helper would be killed immediately") + } +} +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cd agent && go test ./internal/sessionbroker -run 'TestStartupExpired|TestMarkSessionClosedRestarts'` +Expected: FAIL — `r.startupExpired undefined` and `helperStartupTimeout undefined` + +- [ ] **Step 3: Add the registry predicate and restart the clock** + +Add to `lifecycle_registry.go`, after `markSessionClosed`: + +```go +// startupExpired reports whether key's helper was launched but never reached +// IPC within timeout. It is the replacement for the KillStaleHelpers path this +// branch deleted: a CreateProcessAsUser that "succeeds" and then crashes or +// hangs before connecting is still a lifecycle failure, and without this the +// helperStarting entry blocks reserve forever and nothing ever kills it. +// +// launchedAt is set by attach/attachReserved and restarted by markSessionClosed, +// so a long-lived helper whose IPC drops gets a fresh window rather than being +// recycled on the next tick. +func (r *helperRegistry) startupExpired(key HelperKey, now time.Time, timeout time.Duration) bool { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil || entry.state != helperStarting || entry.launchedAt.IsZero() { + return false + } + return now.Sub(entry.launchedAt) >= timeout +} +``` + +In `markSessionClosed`, inside the branch that sets `entry.state = helperStarting` (from Task 4 Step 6), add the clock restart so the branch reads: + +```go + if err != nil || alive { + entry.state = helperStarting + // Restart the startup window: this helper connected once, so it + // gets a full timeout to reconnect before startupExpired recycles it. + entry.launchedAt = time.Now() + } else { +``` + +- [ ] **Step 4: Add the timeout constant and recycle in reconcile** + +In `lifecycle_core.go`, add near the other package constants (top of file, after the imports): + +```go +// helperStartupTimeout bounds how long a helper may sit in helperStarting — +// launched but not yet connected over IPC — before the lifecycle manager +// terminates and respawns it. Must comfortably exceed a cold helper start on a +// loaded RDS host; 90s is ~3x the observed worst case. +const helperStartupTimeout = 90 * time.Second +``` + +In `reconcile`, immediately after the desired set is computed and before the existing spawn logic, add the recycle sweep. Insert after the `m.mu.Lock()` / stopping check that follows `detectedDesired`, at the point where `desired` is known and the manager lock is *not* held (`stopTrackedKey` takes locks internally — do not call it under `m.mu`): + +```go + now := time.Now() + for key := range desired { + if !m.registry.startupExpired(key, now, helperStartupTimeout) { + continue + } + log.Warn("lifecycle: helper never reached IPC within startup timeout; recycling", + "helperKey", key.String(), "timeout", helperStartupTimeout.String()) + m.stopTrackedKey(key) + } +``` + +Place this so it runs before the loop that spawns missing keys, so a recycled key is respawned in the same reconcile pass. If the desired set is held under `m.mu` at that point, copy the keys into a local slice under the lock and run the sweep after unlocking — `stopTrackedKey` must never be called while `m.mu` is held. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestStartupExpired|TestMarkSessionClosedRestarts' -v` +Expected: PASS, all three tests. + +- [ ] **Step 6: Verify the lock-order constraint holds under race detection** + +Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestConcurrent' -v` +Expected: PASS, no race reports. These are the existing concurrency tests (`TestConcurrentReconcileStopAndExitOwnsHandleOnce`, `TestConcurrentStopKeyHasSingleTerminationOwner`) and they cover the `m.mu` → registry lock order the sweep must not invert. + +- [ ] **Step 7: Verify Windows compile** + +Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` +Expected: clean. + +- [ ] **Step 8: Run the full suite** + +Run: `cd agent && go test -race ./internal/sessionbroker` +Expected: `ok` + +- [ ] **Step 9: Commit** + +```bash +git add agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry_test.go +git commit -m "fix(agent): recycle helpers that never reach IPC" +``` + +--- + +### Task 6: Make a failed helper kill visible and retryable + +`TerminateHelperKey` logs a failed `TerminateProcess` at `Debug` — the default level is `info` (`config.go:218`), so a failed kill produces **zero** evidence. Worse, `removeSessionMapsLocked` has already dropped the session, so the broker forgets a helper it never killed; `spawnKey` then sees no owner and starts a second one. Two live helpers fight over DXGI Desktop Duplication — the exact bug class this branch exists to fix. + +**Files:** +- Modify: `agent/internal/sessionbroker/broker.go:2008-2021` +- Test: `agent/internal/sessionbroker/broker_lifecycle_test.go` + +**Interfaces:** +- Consumes: `ownedPeerProcessRef.claimTermination()` (`lifecycle_core.go:52-113`); `fakeOwnedPeerProcess` (`broker_lifecycle_test.go`); `Broker.TerminateHelperKey`. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Read the current implementation before changing it** + +Run: `cd agent && sed -n '1995,2030p' internal/sessionbroker/broker.go` + +Note the exact order: `claimTermination()` → `removeSessionMapsLocked(session)` → `b.mu.Unlock()` → `claim.terminateAndClose()`. The termination happens *after* the maps are cleared and the lock is released. Preserve that ordering — it is deliberate (no syscall under `b.mu`). + +- [ ] **Step 2: Add a terminate-failure seam to the existing fake** + +`fakeOwnedPeerProcess` (`broker_lifecycle_test.go:32`) has no way to fail. Add a `terminateErr` field after `closed int`: + +```go + terminateErr error +``` + +Then change `Terminate` at `:131` so it can fail *without* recording a termination or flipping `alive` — a failed kill leaves the process running: + +```go +func (p *fakeOwnedPeerProcess) Terminate() error { + if p.claimed != nil { + close(p.claimed) + <-p.release + } + p.mu.Lock() + defer p.mu.Unlock() + if p.terminateErr != nil { + return p.terminateErr + } + p.terminated++ + p.alive = false + return nil +} +``` + +Note the original takes and releases `p.mu` around only the counter mutation; switching to `defer` here is safe because nothing below blocks. Do not move the `p.claimed` handshake inside the lock — the concurrency tests depend on it running before `p.mu` is taken. + +- [ ] **Step 3: Write the failing test** + +Append to `agent/internal/sessionbroker/broker_lifecycle_test.go`, reusing the file's real harness (`New` at `:229`, `newFakeOwnedPeerProcess` at `:121`, `newOwnedSession` at `:154`): + +```go +func TestTerminateHelperKeyRetainsOwnershipWhenKillFails(t *testing.T) { + // A failed TerminateProcess must not look like a successful teardown. If the + // broker forgets the key while the helper is still alive, the lifecycle + // manager sees no owner and spawns a second helper into the same session; + // the two then fight over DXGI capture. Silently, before this fix: the + // failure was logged at Debug and the default level is info. + b := New("terminate-fail-"+t.Name(), nil) + key := HelperKey{WindowsSessionID: 7, Role: "system"} + proc := newFakeOwnedPeerProcess(5300) + proc.terminateErr = errors.New("TerminateProcess: access denied") + newOwnedSession(t, b, key, proc) + + b.TerminateHelperKey(key) + + if _, owned := b.helperKeyOwnerPID(key); !owned { + t.Fatal("broker released helper key ownership after a FAILED terminate; a duplicate helper can now spawn") + } +} +``` + +Ensure the file imports `"errors"`. Setting `proc.terminateErr` directly before the session goes live is race-free — no goroutine touches the fake yet. + +- [ ] **Step 4: Run the test to verify it fails** + +Run: `cd agent && go test ./internal/sessionbroker -run TestTerminateHelperKeyRetainsOwnershipWhenKillFails` +Expected: FAIL — the broker currently releases ownership unconditionally. + +- [ ] **Step 5: Raise the log level and retain the key on failure** + +Replace the termination block at `broker.go:2017-2021`: + +```go + if claim != nil { + if err := claim.terminateAndClose(); err != nil { + // Warn, not Debug: the default level is info, so a Debug line here + // meant a failed kill left no evidence at all. This is the + // enforcement path, not a best-effort cleanup. + log.Warn("failed to terminate helper process; retaining key ownership so reconciliation retries instead of spawning a duplicate", + "helperKey", key.String(), "pid", session.PID, "error", err.Error()) + b.retainHelperKeyOwnership(key, session) + } + } +``` + +Add `retainHelperKeyOwnership` near `removeSessionMapsLocked`: + +```go +// retainHelperKeyOwnership re-registers session as the owner of key after a +// failed termination. The maps were cleared under b.mu before the syscall (we +// never terminate while holding the lock), so on failure we must put the +// ownership back: a live helper the broker has forgotten is one the lifecycle +// manager will duplicate. +func (b *Broker) retainHelperKeyOwnership(key HelperKey, session *Session) { + b.mu.Lock() + defer b.mu.Unlock() + if _, exists := b.helperByKey[key]; exists { + // A newer helper already claimed the slot; the stale one is not the owner. + return + } + b.helperByKey[key] = session +} +``` + +Verify the real field name and type of the helper-key ownership map (`grep -n 'helperByKey' internal/sessionbroker/broker.go`) and match it exactly; `broker.go:258` is the declaration. If re-registration requires more than one map (check what `removeSessionMapsLocked` clears), restore only the helper-key ownership — do **not** resurrect the session in `byIdentity` or the snapshot, which would misreport a dying session as live. + +- [ ] **Step 6: Run the test to verify it passes** + +Run: `cd agent && go test -race ./internal/sessionbroker -run TestTerminateHelperKeyRetainsOwnershipWhenKillFails -v` +Expected: PASS + +- [ ] **Step 7: Run the full suite and the concurrency tests** + +Run: `cd agent && go test -race ./internal/sessionbroker` +Expected: `ok`. If any existing admission or lifecycle test fails, the retention has changed a contract another test pins — read that test before adjusting either. `TestSessionCloseReleasesOwnedPeerProcessOnce` and `TestUnexpectedDisconnectReleasesOwnedPeerProcess` both use `fakeOwnedPeerProcess` and must still pass with `terminateErr` nil. + +- [ ] **Step 8: Verify Windows compile** + +Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` +Expected: clean. + +- [ ] **Step 9: Commit** + +```bash +git add agent/internal/sessionbroker/broker.go agent/internal/sessionbroker/broker_lifecycle_test.go +git commit -m "fix(agent): surface and retry failed helper termination" +``` + +--- + +### Task 7: Retry lifecycle bootstrap instead of dying silently + +`bootstrapThenListen` correctly skips `Listen` when `Bootstrap` fails — that fail-closed refusal is deliberate, tested by `TestBootstrapFailureRefusesBrokerListen`, and must be preserved. The bug is what happens next: the error is logged once and execution falls through. `Listen` has exactly two call sites and `Start()` runs once, so the named pipe is **never created for the process lifetime** — no remote desktop, no PAM, no helper IPC — while the agent heartbeats healthy. `Bootstrap` → `detectedDesired` → `ListSessions()` fails when `WTSEnumerateSessionsW` does, a known early-boot condition; `reconcile` already treats that same error as transient and retries it. + +**Files:** +- Modify: `agent/internal/heartbeat/heartbeat.go:877-882` +- Modify: `agent/internal/sessionbroker/broker_admission.go:174-179` (resolve stale comment) +- Test: `agent/internal/heartbeat/heartbeat_test.go` + +**Interfaces:** +- Consumes: `bootstrapThenListen(bootstrap func() error, listen func()) error` (`heartbeat.go:851`); `HelperLifecycleManager.Bootstrap() error` (`lifecycle_core.go:182`). +- Produces: `bootstrapThenListenWithRetry(ctx context.Context, bootstrap func() error, listen func(), retry time.Duration)`. + +- [ ] **Step 1: Write the failing test** + +Append to `agent/internal/heartbeat/heartbeat_test.go`: + +```go +func TestBootstrapRetriesUntilItSucceedsThenListens(t *testing.T) { + // WTSEnumerateSessionsW fails transiently early in Windows boot. One flake + // must not cost the agent its pipe listener for the whole process lifetime. + var attempts int32 + listened := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go bootstrapThenListenWithRetry(ctx, func() error { + if atomic.AddInt32(&attempts, 1) < 3 { + return errors.New("WTSEnumerateSessionsW: the RPC server is unavailable") + } + return nil + }, func() { close(listened) }, time.Millisecond) + + select { + case <-listened: + case <-time.After(2 * time.Second): + t.Fatal("listener never started despite bootstrap eventually succeeding") + } + if got := atomic.LoadInt32(&attempts); got < 3 { + t.Fatalf("attempts = %d, want >= 3", got) + } +} + +func TestBootstrapRetryStopsOnContextCancel(t *testing.T) { + var attempts int32 + ctx, cancel := context.WithCancel(context.Background()) + listened := make(chan struct{}) + + done := make(chan struct{}) + go func() { + defer close(done) + bootstrapThenListenWithRetry(ctx, func() error { + atomic.AddInt32(&attempts, 1) + return errors.New("permanent") + }, func() { close(listened) }, time.Millisecond) + }() + + time.Sleep(20 * time.Millisecond) + cancel() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("retry loop did not exit on context cancel") + } + select { + case <-listened: + t.Fatal("listener started despite bootstrap never succeeding") + default: + } +} +``` + +Ensure the file imports `"context"`, `"errors"`, `"sync/atomic"`, and `"time"`. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cd agent && go test ./internal/heartbeat -run TestBootstrapRetr` +Expected: FAIL — `undefined: bootstrapThenListenWithRetry` + +- [ ] **Step 3: Add the retry wrapper** + +Add to `heartbeat.go` immediately after `bootstrapThenListen`: + +```go +// bootstrapThenListenWithRetry keeps the fail-closed contract of +// bootstrapThenListen — never listen without desired state — while making the +// failure recoverable. Bootstrap reaches WTSEnumerateSessionsW, which fails +// transiently when the agent service starts before Remote Desktop Services' +// RPC endpoint is ready. Without a retry, one boot-order flake costs the agent +// its pipe listener for the entire process lifetime: no remote desktop, no PAM, +// no helper IPC, while the machine keeps heartbeating healthy. +// +// Blocks until bootstrap succeeds (then listens exactly once) or ctx is done. +func bootstrapThenListenWithRetry(ctx context.Context, bootstrap func() error, listen func(), retry time.Duration) { + for { + if err := bootstrapThenListen(bootstrap, listen); err == nil { + return + } else { + log.Warn("helper lifecycle bootstrap failed; retrying before starting broker listener", "retryIn", retry.String(), "error", err.Error()) + } + select { + case <-ctx.Done(): + log.Error("helper lifecycle bootstrap never succeeded; broker listener not started", "error", ctx.Err().Error()) + return + case <-time.After(retry): + } + } +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cd agent && go test -race ./internal/heartbeat -run TestBootstrapRetr -v` +Expected: PASS, both tests. + +- [ ] **Step 5: Wire the retry into Start()** + +Replace `heartbeat.go:877-882`: + +```go + go bootstrapThenListenWithRetry(ctx, lifecycle.Bootstrap, func() { + go h.sessionBroker.Listen(h.stopChan) + }, lifecycleBootstrapRetryInterval) + go lifecycle.Start(ctx) +``` + +Add the constant near the other package constants in `heartbeat.go`: + +```go +// lifecycleBootstrapRetryInterval matches the lifecycle reconcile cadence: +// both recover from the same transient WTS enumeration failure. +const lifecycleBootstrapRetryInterval = 30 * time.Second +``` + +The retry now runs in its own goroutine, so `Start()` no longer blocks on bootstrap. Confirm `TestBootstrapFailureRefusesBrokerListen` still passes unchanged — the fail-closed contract it pins must survive. + +- [ ] **Step 6: Resolve the stale security comment** + +`broker_admission.go:174-179` says a kernel-bound peer handle does not exist yet; it landed in this same branch (`peer_process_windows.go`, `session.peerProcess`). Replace the comment, keeping the behavior: + +```go +// helperOwnerReplaceable reports whether an existing helper session may be +// displaced by a new claimant. A broker-closed session is the only stale signal +// we act on: session.peerProcess now gives us a kernel-bound handle, but +// consulting its liveness here would let a claimant displace a HEALTHY helper +// that simply has not closed yet. Displacement stays conservative on purpose — +// admission failures are recoverable, evicting a working helper is not. +func helperOwnerReplaceable(owner *Session) bool { + return owner.IsClosed() +} +``` + +- [ ] **Step 7: Run the heartbeat and sessionbroker suites** + +Run: `cd agent && go test -race ./internal/heartbeat ./internal/sessionbroker` +Expected: `ok` for both, including the unchanged `TestBootstrapFailureRefusesBrokerListen`. + +- [ ] **Step 8: Commit** + +```bash +git add agent/internal/heartbeat/heartbeat.go agent/internal/heartbeat/heartbeat_test.go agent/internal/sessionbroker/broker_admission.go +git commit -m "fix(agent): retry helper lifecycle bootstrap before listening" +``` + +--- + +### Task 8: Stop fabricating exit codes for detached helpers + +`watchDetachedProcess` discards the `Wait` error that `watchProcess` logs eight lines above. `Wait` returns `-1, err` on `DuplicateHandle`/`WaitForSingleObject` failure, so `noteExit` records `-1` as a genuine exit code and sets `helperExited` while the process may still be running. `keys()` then excludes the entry, so reconcile never issues a `stopKey`, and `beginStop` deletes it and returns nil: the lifecycle drops a live helper without ever terminating it. + +**Files:** +- Modify: `agent/internal/sessionbroker/lifecycle_core.go:303-307` +- Test: `agent/internal/sessionbroker/lifecycle_registry_test.go` + +**Interfaces:** +- Consumes: `fakeHelperProcess` from Task 4; `helperRegistry.noteExit`. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Write the failing test** + +Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`: + +```go +func TestNoteExitIgnoresFabricatedExitCodeOnWaitFailure(t *testing.T) { + // Wait returns (-1, err) when the handle operation fails. Recording -1 as a + // real exit code marks a possibly-live helper as exited, after which nothing + // ever terminates it: keys() hides it and beginStop just deletes it. + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process := newFakeHelperProcess(6060) + + generation, _ := r.reserve(key, time.Now()) + entry, _ := r.attachReserved(key, generation, process, "user-helper") + + r.noteExitUnknown(key, generation) + + if entry.state == helperExited { + t.Fatal("unknown exit was recorded as helperExited; a live helper is now untrackable") + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd agent && go test ./internal/sessionbroker -run TestNoteExitIgnoresFabricatedExitCode` +Expected: FAIL — `r.noteExitUnknown undefined` + +- [ ] **Step 3: Add the unknown-exit path** + +Add to `lifecycle_registry.go` beside `noteExit`: + +```go +// noteExitUnknown records that a helper's Wait failed, so whether it exited — +// and with what code — is unknown. It deliberately does NOT set helperExited: +// that state hides the entry from keys() and makes beginStop drop it without +// terminating, which would strand a live process. Leaving the entry in its +// current state lets startupExpired or the next stopKey deal with it. +func (r *helperRegistry) noteExitUnknown(key HelperKey, generation uint64) { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.generation[generation] + if entry == nil || entry.key != key { + return + } + entry.doneOnce.Do(func() { close(entry.done) }) + delete(r.generation, generation) +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd agent && go test -race ./internal/sessionbroker -run TestNoteExitIgnoresFabricatedExitCode -v` +Expected: PASS + +- [ ] **Step 5: Log the error and route unknown exits correctly** + +Replace `watchDetachedProcess` at `lifecycle_core.go:303-307`: + +```go +func (m *HelperLifecycleManager) watchDetachedProcess(key HelperKey, generation uint64, process helperProcess) { + exitCode, err := process.Wait() + _ = process.Close() + if err != nil { + // Mirror watchProcess: never swallow this. Wait returns (-1, err) on + // failure, and recording -1 as a real exit code marks a possibly-live + // helper exited. + log.Warn("lifecycle: wait on detached helper process failed", "helperKey", key.String(), "pid", processID(process), "error", err.Error()) + m.registry.noteExitUnknown(key, generation) + return + } + m.registry.noteExit(key, generation, exitCode) +} +``` + +- [ ] **Step 6: Run the full suite** + +Run: `cd agent && go test -race ./internal/sessionbroker` +Expected: `ok` + +- [ ] **Step 7: Verify Windows compile** + +Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` +Expected: clean. + +- [ ] **Step 8: Commit** + +```bash +git add agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/lifecycle_registry_test.go +git commit -m "fix(agent): stop recording fabricated exit codes for detached helpers" +``` + +--- + +### Task 9: Full-branch verification gate + +**Files:** none modified. + +**Interfaces:** +- Consumes: every preceding task. +- Produces: the green signal that gates Phase 2. + +- [ ] **Step 1: Run the whole agent suite with race detection** + +Run: `cd agent && go test -race -count=1 ./...` +Expected: no `FAIL` lines. `-count=1` defeats the test cache — do not omit it. + +- [ ] **Step 2: Verify both platform builds** + +Run: `cd agent && go build ./... && GOOS=windows go build ./... && echo BUILD_OK` +Expected: `BUILD_OK` + +- [ ] **Step 3: Verify vet on both platforms** + +Run: `cd agent && go vet ./... && GOOS=windows go vet ./... 2>&1 | grep -v 'possible misuse of unsafe.Pointer'; echo VET_DONE` +Expected: `VET_DONE` with no other output. The filtered warnings are pre-existing in `internal/remote/desktop`. + +- [ ] **Step 4: Confirm no dead liveness fields remain unread** + +Run: `cd agent && grep -rn 'launchedAt' internal/sessionbroker/*.go | grep -v _test.go` +Expected: at least one **read** site (in `startupExpired`), not only assignments. `executablePath` and `commandMode` on `trackedHelper` remain write-only and are acceptable — they are diagnostic fields, tracked as a follow-up, not a blocker. + +- [ ] **Step 5: Report the gate result** + +Summarize: tests green, both builds clean, vet clean. Report any deviation rather than proceeding to Phase 2. + +--- + +## Phase 2: Windows Watchdog Verified Recovery (slice 6) + +Phase 2 is **already fully planned** in `docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md` — 5 tasks, 25 steps, complete with code and test names. It is entirely unstarted: `agent/internal/watchdog` and `agent/cmd/breeze-watchdog` are byte-identical to `main`. + +Do not re-plan it. Execute that document task-by-task after Task 9's gate passes, subject to these amendments learned from this review: + +- **Its `serviceController` refactor changes an interface with a stub implementation per platform, exactly like Task 4.** Before starting, run `grep -rn 'serviceController\|RestartAgentService\|StartAgentService\|ForceKillProcess' agent/internal/watchdog agent/cmd/breeze-watchdog` and enumerate every implementation and fake. The watchdog plan lists Linux/Darwin adapters but a test fake it omits will break the build. +- **Apply this plan's fail-closed constraint to `RecoveryResult`.** The watchdog plan already specifies "PID, image, service ownership, or transition uncertainty fails closed" — that is the same principle as Task 4. Where its structured result can express "unknown", never let a caller read unknown as "recovered". +- **Its plan doc has 0 of 25 checkboxes ticked, as do the two completed plans on this branch.** Checkbox state on this branch is meaningless — verify against code, never against the doc. +- After its final task, re-run Task 9's verification gate in full. + +--- + +## Out of scope (file as follow-up issues, do not fix here) + +These are real findings from the review that this plan deliberately does not address. Each is defensible to defer; none is a blocker. + +- **`type HelperRole string` in `ipc`.** Role vocabulary is split between raw `"system"`/`"user"` literals (8+ sites) and `ipc.HelperRole*` constants. If those constants ever change value, `helperRoleDesired` returns false for everything, every helper is rejected, and no test catches it — fail-closed, but a fleet outage with no compile error. Task 3 closes the dangerous *consequence*; the typed role would close the *class*. Wide mechanical change across `ipc`/`sessionbroker`/`agentapp`; own PR. +- **`WTS_REMOTE_CONNECT (0x3)` unhandled.** Reconnecting to a disconnected RDP session fires `0x3`, which no case handles, so the user helper returns only on the 30s reconcile tick. Also `wtsSessionDisconnect = 0x4` is misnamed — it is `WTS_REMOTE_DISCONNECT` specifically, and `WTS_CONSOLE_DISCONNECT (0x2)` is unhandled. +- **PID-reuse window on the peer handle.** `ipc.GetPeerCredentials` closes its handle; `broker.go:1787` re-opens by raw PID after a peer-controlled delay, then requests `PROCESS_TERMINATE`. Retaining the original handle is the fix. Low probability, high blast radius. +- **`NtCreateFile` missing `FILE_SYNCHRONOUS_IO_NONALERT`** at `main_instance_windows.go:195`, while the sibling call at `permissions_windows.go:243` has it. Availability-only; can make the agent refuse to start over diagnostic metadata. +- **`mainAgentHandleHasTrustedOwner` fetches the DACL and discards it** (`permissions_windows.go:416`) — an owner-only verdict where the DACL was requested. +- **`programDataDirACLDrifted` fail-open on `GetAce`** (`permissions_windows.go:548`) — **pre-existing**, predates this branch (commit `49924f3d0`). +- **Stale `gofmt` alignment** in four `sessionbroker` test files. CI runs neither `gofmt` nor `go vet` on the agent. +- **Dead code:** `SpawnHelperInSession`/`SpawnUserHelperInSession` now have zero production callers; `standaloneOwner`, `helperPanicExitCode`, `trackedHelper.executablePath`/`commandMode` are write-only. +- **Session-churn bounded-growth guard.** `broker_stale_helpers_test.go` deleted the `#2387` regression guard; a churn probe proved the new design is bounded, but no test pins it. From 77c75f5cf8e8c315ffbc8b8bca9e09d67a1f0f2c Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 21:51:18 -0600 Subject: [PATCH 21/47] test(agent): restore deleted helper role-authorization gate cases --- agent/internal/sessionbroker/console_session_gate_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/agent/internal/sessionbroker/console_session_gate_test.go b/agent/internal/sessionbroker/console_session_gate_test.go index e3050eb5ee..b41e41edf0 100644 --- a/agent/internal/sessionbroker/console_session_gate_test.go +++ b/agent/internal/sessionbroker/console_session_gate_test.go @@ -22,6 +22,11 @@ func TestRoleIdentityRejection(t *testing.T) { {"non-SYSTEM cannot claim system", ipc.HelperRoleSystem, nonSystemSID, "7", "7", "1", "system role requires SYSTEM identity", true}, {"SYSTEM matching RDP session", ipc.HelperRoleSystem, systemSID, "7", "7", "1", "", false}, {"assist remains console bound", ipc.HelperRoleAssist, nonSystemSID, "7", "7", "1", "assist role requires the active console session", true}, + {"assist from console session accepted", ipc.HelperRoleAssist, nonSystemSID, "1", "1", "1", "", false}, + {"assist as SYSTEM rejected on SID", ipc.HelperRoleAssist, systemSID, "1", "1", "1", "assist role requires non-SYSTEM identity", true}, + {"assist rejected when console lookup failed (session 0 sentinel)", ipc.HelperRoleAssist, nonSystemSID, "0", "0", "0", "assist role requires the active console session", true}, + {"watchdog as SYSTEM unaffected by console session", ipc.HelperRoleWatchdog, systemSID, "0", "0", "1", "", false}, + {"watchdog as non-SYSTEM rejected", ipc.HelperRoleWatchdog, nonSystemSID, "1", "1", "1", "watchdog role requires SYSTEM identity", true}, } for _, tc := range tests { From df48d08d2558c2ce481dd7531f8d0fa16263eaf7 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 21:52:12 -0600 Subject: [PATCH 22/47] ci(agent): auto-discover Windows test packages --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e93e681aad..de583ce28d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -717,7 +717,13 @@ jobs: - name: Run Windows Go tests working-directory: agent - run: go test -race ./internal/sessionbroker ./internal/heartbeat ./internal/agentapp ./internal/config ./internal/watchdog ./cmd/breeze-watchdog + # Auto-discover every package so Windows-tagged tests cannot be silently + # skipped. A hardcoded list drifts: ./internal/eventlog and nine other + # packages with //go:build windows tests were omitted, so their tests ran + # on no platform at all (the Linux test-agent job's ./... skips them via + # build tag, and `go build` never compiles _test.go). Keeps CLAUDE.md's + # "new test files are auto-discovered" contract true on Windows too. + run: go test -race ./... # ─── Windows runtime smoke (#1000) ──────────────────────────────────── # CI cross-compiles the Windows agent (build-agent matrix) but never RAN From 2d4697416c5d0a06c941b152aba583191715e19d Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 21:54:30 -0600 Subject: [PATCH 23/47] fix(agent): refuse helper spawn for non-lifecycle roles --- agent/internal/sessionbroker/helper_key.go | 14 +++++++++ .../internal/sessionbroker/lifecycle_core.go | 5 ++++ .../sessionbroker/lifecycle_registry_test.go | 22 ++++++++++++++ .../sessionbroker/spawner_cmdline_test.go | 30 +++++++++++++++++++ .../internal/sessionbroker/spawner_windows.go | 18 +++++++++-- 5 files changed, 87 insertions(+), 2 deletions(-) diff --git a/agent/internal/sessionbroker/helper_key.go b/agent/internal/sessionbroker/helper_key.go index 7f52d66435..7df7abd77a 100644 --- a/agent/internal/sessionbroker/helper_key.go +++ b/agent/internal/sessionbroker/helper_key.go @@ -3,6 +3,8 @@ package sessionbroker import ( "fmt" "strconv" + + "github.com/breeze-rmm/agent/internal/ipc" ) type HelperKey struct { @@ -10,6 +12,18 @@ type HelperKey struct { Role string } +// helperRoleSpawnable reports whether role is one the lifecycle manager may +// launch a process for. Only the two lifecycle roles qualify: assist and +// watchdog helpers are started by other means and must never be spawned here. +// +// This gate exists because the Windows spawn path selects a token privilege +// level from the role. Anything that is not exactly ipc.HelperRoleUser would +// otherwise take the SYSTEM-token branch, so an empty or misspelled role +// silently escalates. +func helperRoleSpawnable(role string) bool { + return role == ipc.HelperRoleSystem || role == ipc.HelperRoleUser +} + func (k HelperKey) String() string { return fmt.Sprintf("%d-%s", k.WindowsSessionID, k.Role) } diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index 7724a31efb..86d5fe9979 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -259,6 +259,11 @@ func (m *HelperLifecycleManager) spawnKey(key HelperKey) { m.mu.Unlock() return } + if !helperRoleSpawnable(key.Role) { + m.mu.Unlock() + log.Error("lifecycle: refusing to spawn helper for non-lifecycle role", "helperKey", key.String(), "role", key.Role) + return + } if m.broker != nil && m.broker.HasHelperKeyOwner(key) { m.mu.Unlock() return diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index a11ad0a81d..40ff78f187 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -507,3 +507,25 @@ func TestLifecycleStopEndsStartLoop(t *testing.T) { t.Fatal("Start loop did not finish after Stop") } } + +func TestSpawnKeyRefusesNonLifecycleRole(t *testing.T) { + // A non-lifecycle role must never reach the spawner: on Windows the role + // selects the token privilege level, so anything not recognized as "user" + // took the SYSTEM branch. The zero-value HelperKey is the realistic vector. + spawner := &fakeHelperSpawner{} + m := newLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}, spawner) + key := HelperKey{WindowsSessionID: 7, Role: ""} + + m.mu.Lock() + m.desired = map[HelperKey]bool{key: true} + m.mu.Unlock() + + m.spawnKey(key) + + spawner.mu.Lock() + got := spawner.spawned[key] + spawner.mu.Unlock() + if got != 0 { + t.Fatalf("spawner was called %d times for role %q; want 0", got, key.Role) + } +} diff --git a/agent/internal/sessionbroker/spawner_cmdline_test.go b/agent/internal/sessionbroker/spawner_cmdline_test.go index cf73e4c4e4..2e2d17d320 100644 --- a/agent/internal/sessionbroker/spawner_cmdline_test.go +++ b/agent/internal/sessionbroker/spawner_cmdline_test.go @@ -3,6 +3,8 @@ package sessionbroker import ( "strings" "testing" + + "github.com/breeze-rmm/agent/internal/ipc" ) // TestBuildUserHelperCmdLine_AlwaysExplicitRole guards against the spawn-path @@ -61,3 +63,31 @@ func TestSpawnedHelperDiagnosticsRetainRoleProvenance(t *testing.T) { t.Fatalf("spawn executable provenance = path:%q fallback:%v", helper.BinaryPath, helper.MainBinaryFallback) } } + +// TestHelperRoleSpawnableRejectsNonLifecycleRoles guards the privilege boundary +// in the spawn path. The role selects the token: before this gate, +// createHelperSuspended sent anything that was not exactly "user" down the +// SYSTEM-token branch, so a zero-value HelperKey (Role: "") or a misspelled +// role silently escalated to SYSTEM. +func TestHelperRoleSpawnableRejectsNonLifecycleRoles(t *testing.T) { + tests := []struct { + name string + role string + want bool + }{ + {"system role spawnable", ipc.HelperRoleSystem, true}, + {"user role spawnable", ipc.HelperRoleUser, true}, + {"zero-value role is not spawnable", "", false}, + {"wrong case is not spawnable", "User", false}, + {"assist is not a lifecycle role", ipc.HelperRoleAssist, false}, + {"watchdog is not a lifecycle role", ipc.HelperRoleWatchdog, false}, + {"unknown role is not spawnable", "banana", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := helperRoleSpawnable(tc.role); got != tc.want { + t.Fatalf("helperRoleSpawnable(%q) = %v, want %v", tc.role, got, tc.want) + } + }) + } +} diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index ca38e38161..91e849e27c 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -9,6 +9,8 @@ import ( "unsafe" "golang.org/x/sys/windows" + + "github.com/breeze-rmm/agent/internal/ipc" ) // SpawnedHelper describes a helper process after a successful spawn. It @@ -310,11 +312,23 @@ func (s *windowsHelperSpawner) Close() error { return s.job.Close() } +// createHelperSuspended creates the helper process for key without letting its +// primary thread run. The role selects the token privilege level, so an +// unrecognized role must never reach a spawn call: the previous permissive +// default sent anything that was not exactly "user" down the SYSTEM-token +// branch, so an empty or misspelled role silently escalated. func createHelperSuspended(key HelperKey, resolvedExe ResolvedHelperExecutable) (*suspendedHelper, error) { - if key.Role == "user" { + if !helperRoleSpawnable(key.Role) { + return nil, fmt.Errorf("refusing to spawn helper for non-lifecycle role %q", key.Role) + } + switch key.Role { + case ipc.HelperRoleUser: return createUserHelperSuspended(key.WindowsSessionID, resolvedExe) + case ipc.HelperRoleSystem: + return createSystemHelperSuspended(key.WindowsSessionID, resolvedExe) + default: + return nil, fmt.Errorf("role %q passed helperRoleSpawnable but has no spawn path", key.Role) } - return createSystemHelperSuspended(key.WindowsSessionID, resolvedExe) } // createSystemHelperSuspended creates the SYSTEM-token helper without allowing From 3529c8d5fe5eefd06d4d0a6fe435ea7c4ee3d4b4 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 21:57:33 -0600 Subject: [PATCH 24/47] fix(agent): make helper liveness express unknown and fail closed --- .../internal/sessionbroker/lifecycle_core.go | 17 +++-- .../sessionbroker/lifecycle_registry.go | 32 +++++++-- .../sessionbroker/lifecycle_registry_test.go | 71 +++++++++++++++++-- .../internal/sessionbroker/lifecycle_test.go | 4 +- agent/internal/sessionbroker/spawner_stub.go | 9 ++- .../internal/sessionbroker/spawner_windows.go | 15 ++-- 6 files changed, 127 insertions(+), 21 deletions(-) diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index 86d5fe9979..ed73fc62c2 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -23,7 +23,10 @@ type SCMSessionEvent struct { type helperProcess interface { ProcessID() uint32 ExecutablePath() string - Alive() bool + // Alive reports liveness. A non-nil error means liveness is UNKNOWN, not + // false: callers must fail closed and treat unknown as alive. Matches + // ownedPeerProcess.Alive so the two cannot be confused at a glance. + Alive() (bool, error) Terminate() error Wait() (int, error) Close() error @@ -343,9 +346,15 @@ func (m *HelperLifecycleManager) stopTrackedKey(key HelperKey) { m.registry.detach(key, entry.generation) return } - if entry.process != nil && entry.process.Alive() { - if err := entry.process.Terminate(); err != nil { - log.Warn("lifecycle: failed to terminate helper", "helperKey", key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + if entry.process != nil { + alive, err := entry.process.Alive() + if err != nil { + log.Warn("lifecycle: helper liveness unknown; terminating to fail closed", "helperKey", key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + } + if err != nil || alive { + if err := entry.process.Terminate(); err != nil { + log.Warn("lifecycle: failed to terminate helper", "helperKey", key.String(), "pid", entry.process.ProcessID(), "error", err.Error()) + } } } if waitDone(entry.done, m.finalWait) { diff --git a/agent/internal/sessionbroker/lifecycle_registry.go b/agent/internal/sessionbroker/lifecycle_registry.go index 6685eefd65..3b9461e62d 100644 --- a/agent/internal/sessionbroker/lifecycle_registry.go +++ b/agent/internal/sessionbroker/lifecycle_registry.go @@ -72,8 +72,17 @@ func (r *helperRegistry) reserve(key HelperKey, now time.Time) (uint64, bool) { if previous.state == helperStarting || previous.state == helperStopping { return 0, false } - if previous.process != nil && previous.process.Alive() { - return 0, false + if previous.process != nil { + alive, err := previous.process.Alive() + if err != nil { + // Unknown liveness: refuse the reservation. A duplicate helper + // racing the original over DXGI capture is worse than no helper. + log.Warn("lifecycle: helper liveness unknown; refusing respawn", "helperKey", key.String(), "pid", previous.process.ProcessID(), "error", err.Error()) + return 0, false + } + if alive { + return 0, false + } } if !previous.fatalExitUntil.IsZero() && now.Before(previous.fatalExitUntil) { return 0, false @@ -163,8 +172,13 @@ func (r *helperRegistry) detach(key HelperKey, generation uint64) bool { if entry == nil || entry.generation != generation { return false } - if entry.process != nil && entry.process.Alive() { - return false + if entry.process != nil { + alive, err := entry.process.Alive() + if err != nil || alive { + // Unknown or alive: keep the entry so the caller retries rather + // than dropping a process it never confirmed dead. + return false + } } delete(r.current, key) return true @@ -209,8 +223,16 @@ func (r *helperRegistry) markSessionClosed(key HelperKey, session *Session) { return } entry.brokerSession = nil - if entry.process != nil && entry.process.Alive() { + if entry.process == nil { + entry.state = helperExited + return + } + alive, err := entry.process.Alive() + if err != nil || alive { entry.state = helperStarting + // Restart the startup window: this helper connected once, so it gets a + // full timeout to reconnect before startupExpired recycles it. + entry.launchedAt = time.Now() } else { entry.state = helperExited } diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index 40ff78f187..373ffa2e25 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -1,6 +1,7 @@ package sessionbroker import ( + "errors" "context" "sync" "testing" @@ -65,6 +66,7 @@ type fakeHelperProcess struct { terminateCount int closeCount int terminateExits bool + aliveErr error } func newFakeHelperProcess(pid uint32) *fakeHelperProcess { @@ -80,10 +82,30 @@ func newFakeHelperProcess(pid uint32) *fakeHelperProcess { func (p *fakeHelperProcess) ProcessID() uint32 { return p.pid } func (p *fakeHelperProcess) ExecutablePath() string { return p.path } -func (p *fakeHelperProcess) Alive() bool { +func (p *fakeHelperProcess) Alive() (bool, error) { p.mu.Lock() defer p.mu.Unlock() - return p.alive + if p.aliveErr != nil { + return false, p.aliveErr + } + return p.alive, nil +} + +func (p *fakeHelperProcess) setAliveErr(err error) { + p.mu.Lock() + defer p.mu.Unlock() + p.aliveErr = err +} + +// aliveNow fails the test if liveness cannot be determined, so a test never +// silently reads "unknown" as "dead". +func aliveNow(t *testing.T, p helperProcess) bool { + t.Helper() + alive, err := p.Alive() + if err != nil { + t.Fatalf("Alive() error = %v", err) + } + return alive } func (p *fakeHelperProcess) Terminate() error { @@ -226,7 +248,7 @@ func TestScheduledHelperPublishedDuringSpawnRollsBackProactiveProcess(t *testing } time.Sleep(time.Millisecond) } - if proactive.Alive() { + if aliveNow(t, proactive) { t.Fatal("proactive duplicate remains alive after scheduled owner publication") } if !b.HasHelperKeyOwner(key) { @@ -379,10 +401,10 @@ func TestSCMDisconnectThenReconcileRetainsSystemAndStopsUserForDisconnectedRDP(t if userDesired { t.Fatal("user helper remained desired after disconnected-RDP reconcile") } - if !system.Alive() { + if !aliveNow(t, system) { t.Fatal("SYSTEM helper was stopped after disconnected-RDP reconcile") } - if user.Alive() { + if aliveNow(t, user) { t.Fatal("user helper remained alive after disconnected-RDP reconcile") } } @@ -529,3 +551,42 @@ func TestSpawnKeyRefusesNonLifecycleRole(t *testing.T) { t.Fatalf("spawner was called %d times for role %q; want 0", got, key.Role) } } + +func TestReserveRefusesWhenLivenessUnknown(t *testing.T) { + // GetExitCodeProcess can fail. Reading that as "dead" hands out a second + // reservation for a live key, so two helpers fight over DXGI capture. + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process := newFakeHelperProcess(4242) + process.setAliveErr(errors.New("GetExitCodeProcess: access denied")) + + generation, ok := r.reserve(key, time.Now()) + if !ok { + t.Fatal("first reserve must succeed on an empty slot") + } + if _, attached := r.attachReserved(key, generation, process, "user-helper"); !attached { + t.Fatal("attachReserved failed") + } + + if _, ok := r.reserve(key, time.Now()); ok { + t.Fatal("reserve granted a duplicate helper while liveness was unknown") + } +} + +func TestMarkSessionClosedTreatsUnknownLivenessAsAlive(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process := newFakeHelperProcess(4243) + + generation, _ := r.reserve(key, time.Now()) + entry, _ := r.attachReserved(key, generation, process, "user-helper") + session := &Session{} + r.markConnected(key, process.ProcessID(), session) + + process.setAliveErr(errors.New("GetExitCodeProcess: access denied")) + r.markSessionClosed(key, session) + + if entry.state == helperExited { + t.Fatal("unknown liveness was recorded as helperExited; a live helper can now be duplicated") + } +} diff --git a/agent/internal/sessionbroker/lifecycle_test.go b/agent/internal/sessionbroker/lifecycle_test.go index cad6f7884e..c208bb6ebd 100644 --- a/agent/internal/sessionbroker/lifecycle_test.go +++ b/agent/internal/sessionbroker/lifecycle_test.go @@ -44,10 +44,10 @@ func TestHandleSCMDisconnectStopsUserAndRetainsSystem(t *testing.T) { m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionDisconnect, SessionID: 7}) - if !system.Alive() { + if !aliveNow(t, system) { t.Fatal("system helper was stopped on disconnect") } - if user.Alive() { + if aliveNow(t, user) { t.Fatal("user helper remained alive on disconnect") } } diff --git a/agent/internal/sessionbroker/spawner_stub.go b/agent/internal/sessionbroker/spawner_stub.go index 5a84752ab7..10afedfdfd 100644 --- a/agent/internal/sessionbroker/spawner_stub.go +++ b/agent/internal/sessionbroker/spawner_stub.go @@ -27,7 +27,14 @@ func (s *SpawnedHelper) Close() error { return nil } func (s *SpawnedHelper) ProcessID() uint32 { return s.PID } func (s *SpawnedHelper) ExecutablePath() string { return s.BinaryPath } -func (s *SpawnedHelper) Alive() bool { return false } + +// Alive always errors on non-Windows: the stub never spawns, so a SpawnedHelper +// here is a programming error. Returning (false, nil) would tell callers a live +// helper is dead - the exact confusion this signature exists to prevent. +func (s *SpawnedHelper) Alive() (bool, error) { + return false, fmt.Errorf("SpawnedHelper: helper tracking not supported on this platform") +} + func (s *SpawnedHelper) Terminate() error { return nil } // Wait is a no-op on non-Windows platforms. Returns (exitCode=-1, nil). diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index 91e849e27c..bec94dce33 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -85,17 +85,24 @@ func (s *SpawnedHelper) Close() error { func (s *SpawnedHelper) ProcessID() uint32 { return s.PID } func (s *SpawnedHelper) ExecutablePath() string { return s.BinaryPath } -func (s *SpawnedHelper) Alive() bool { +// Alive reports whether the helper process is still running. A non-nil error +// means the state could not be determined; callers must not read that as "dead". +func (s *SpawnedHelper) Alive() (bool, error) { if s == nil { - return false + return false, fmt.Errorf("SpawnedHelper: nil helper") } s.mu.Lock() defer s.mu.Unlock() if s.Handle == 0 { - return false + // Close() already released the handle; the helper is definitively no + // longer ours to track. Known-dead, not unknown. + return false, nil } var exitCode uint32 - return windows.GetExitCodeProcess(s.Handle, &exitCode) == nil && exitCode == windowsProcessStillActive + if err := windows.GetExitCodeProcess(s.Handle, &exitCode); err != nil { + return false, fmt.Errorf("GetExitCodeProcess: %w", err) + } + return exitCode == windowsProcessStillActive, nil } func (s *SpawnedHelper) Terminate() error { From e7560cc5abe11d002bfec098647da4746e6a06bb Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 21:58:48 -0600 Subject: [PATCH 25/47] fix(agent): recycle helpers that never reach IPC --- .../internal/sessionbroker/lifecycle_core.go | 19 +++++++ .../sessionbroker/lifecycle_registry.go | 19 +++++++ .../sessionbroker/lifecycle_registry_test.go | 57 +++++++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index ed73fc62c2..455c4b592c 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -13,6 +13,12 @@ const ( fatalCooldown = 10 * time.Minute helperFatalExitCode = 2 helperPanicExitCode = 3 + + // helperStartupTimeout bounds how long a helper may sit in helperStarting — + // launched but not yet connected over IPC — before the lifecycle manager + // terminates and respawns it. Must comfortably exceed a cold helper start on + // a loaded RDS host; 90s is ~3x the observed worst case. + helperStartupTimeout = 90 * time.Second ) type SCMSessionEvent struct { @@ -230,6 +236,19 @@ func (m *HelperLifecycleManager) reconcile() { m.stopKey(key) } } + // Recycle helpers that launched but never reached IPC. Without this a + // helperStarting entry blocks reserve forever and nothing terminates the + // process, so the session/role slot stays dead until the agent restarts. + // Runs with m.mu released: stopTrackedKey takes the registry lock itself. + now := time.Now() + for key := range desired { + if !m.registry.startupExpired(key, now, helperStartupTimeout) { + continue + } + log.Warn("lifecycle: helper never reached IPC within startup timeout; recycling", + "helperKey", key.String(), "timeout", helperStartupTimeout.String()) + m.stopTrackedKey(key) + } for key := range desired { m.spawnKey(key) } diff --git a/agent/internal/sessionbroker/lifecycle_registry.go b/agent/internal/sessionbroker/lifecycle_registry.go index 3b9461e62d..d5e3f7c9a4 100644 --- a/agent/internal/sessionbroker/lifecycle_registry.go +++ b/agent/internal/sessionbroker/lifecycle_registry.go @@ -238,6 +238,25 @@ func (r *helperRegistry) markSessionClosed(key HelperKey, session *Session) { } } +// startupExpired reports whether key's helper was launched but never reached +// IPC within timeout. It replaces the KillStaleHelpers path this branch deleted: +// a CreateProcessAsUser that "succeeds" and then crashes or hangs before +// connecting is still a lifecycle failure, and without this the helperStarting +// entry blocks reserve forever while nothing ever terminates the process. +// +// launchedAt is set by attach/attachReserved and restarted by markSessionClosed, +// so a long-lived helper whose IPC drops gets a fresh window rather than being +// recycled on the next tick. +func (r *helperRegistry) startupExpired(key HelperKey, now time.Time, timeout time.Duration) bool { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.current[key] + if entry == nil || entry.state != helperStarting || entry.launchedAt.IsZero() { + return false + } + return now.Sub(entry.launchedAt) >= timeout +} + func (r *helperRegistry) clearFatal(sessionID uint32) { r.mu.Lock() defer r.mu.Unlock() diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index 373ffa2e25..8e6a0b5e06 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -590,3 +590,60 @@ func TestMarkSessionClosedTreatsUnknownLivenessAsAlive(t *testing.T) { t.Fatal("unknown liveness was recorded as helperExited; a live helper can now be duplicated") } } + +func TestStartupExpiredDetectsHelperThatNeverConnected(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + start := time.Now() + + generation, _ := r.reserve(key, start) + if _, attached := r.attachReserved(key, generation, newFakeHelperProcess(5150), "user-helper"); !attached { + t.Fatal("attachReserved failed") + } + + if r.startupExpired(key, start.Add(30*time.Second), helperStartupTimeout) { + t.Fatal("a helper still inside its startup window must not be recycled") + } + if !r.startupExpired(key, start.Add(helperStartupTimeout+time.Second), helperStartupTimeout) { + t.Fatal("a helper that never reached IPC past the timeout must be recycled") + } +} + +func TestStartupExpiredIgnoresConnectedHelper(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + start := time.Now() + process := newFakeHelperProcess(5151) + + generation, _ := r.reserve(key, start) + r.attachReserved(key, generation, process, "user-helper") + r.markConnected(key, process.ProcessID(), &Session{}) + + if r.startupExpired(key, start.Add(24*time.Hour), helperStartupTimeout) { + t.Fatal("a connected helper must never be treated as a failed startup") + } +} + +func TestMarkSessionClosedRestartsTheStartupWindow(t *testing.T) { + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + start := time.Now() + process := newFakeHelperProcess(5152) + + generation, _ := r.reserve(key, start) + entry, _ := r.attachReserved(key, generation, process, "user-helper") + session := &Session{} + r.markConnected(key, process.ProcessID(), session) + + // Long-lived helper: connected at start, IPC drops much later. The process + // is still alive, so it goes back to helperStarting. Its startup clock must + // restart, or it would be recycled instantly on the next reconcile. + r.markSessionClosed(key, session) + + if entry.state != helperStarting { + t.Fatalf("state = %q, want %q", entry.state, helperStarting) + } + if r.startupExpired(key, time.Now().Add(time.Second), helperStartupTimeout) { + t.Fatal("startup window was not restarted on session close; a live helper would be killed immediately") + } +} From f2ecbb114b06a0444f5baf9a634034c338e5d17a Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:01:55 -0600 Subject: [PATCH 26/47] fix(agent): surface failed helper termination at warn level --- agent/internal/sessionbroker/broker.go | 17 ++++++++++++++++- .../sessionbroker/broker_lifecycle_test.go | 19 ++++++++++++------- ...-14-helper-lifecycle-review-remediation.md | 3 +++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index c002f5b91c..3f06534810 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -2016,7 +2016,22 @@ func (b *Broker) TerminateHelperKey(key HelperKey) { if claim != nil { if err := claim.terminateAndClose(); err != nil { - log.Debug("failed to terminate helper process", "helperKey", key.String(), "pid", session.PID, "error", err.Error()) + // Warn, not Debug: the default level is info (config.go), so the + // previous Debug line meant a failed kill left NO evidence at all. + // This is the enforcement path, not best-effort cleanup. + // + // The maps are already cleared at this point, which is safe for + // lifecycle-tracked helpers: helperRegistry.reserve refuses to + // respawn while the tracked process is alive OR its liveness is + // unknown. A scheduled helper has no registry entry, so a failed + // kill there can still be followed by a proactive spawn — see the + // follow-up noted in the remediation plan. Do NOT "fix" that by + // re-registering this session in helperByKey: the session is closed + // immediately below, HasHelperKeyOwner does not filter closed + // owners, and nothing would ever clear the entry, so the key would + // be wedged for the process lifetime. + log.Warn("failed to terminate helper process", + "helperKey", key.String(), "pid", session.PID, "error", err.Error()) } } _ = session.closeTransportAndPeer() diff --git a/agent/internal/sessionbroker/broker_lifecycle_test.go b/agent/internal/sessionbroker/broker_lifecycle_test.go index 439774a901..1bdf27b7ef 100644 --- a/agent/internal/sessionbroker/broker_lifecycle_test.go +++ b/agent/internal/sessionbroker/broker_lifecycle_test.go @@ -32,12 +32,13 @@ func TestBrokerLifecycleCleanupNeverReopensRecordedPID(t *testing.T) { type fakeOwnedPeerProcess struct { pid uint32 - mu sync.Mutex - alive bool - terminated int - closed int - claimed chan struct{} - release chan struct{} + mu sync.Mutex + alive bool + terminated int + closed int + claimed chan struct{} + release chan struct{} + terminateErr error } type closeTrackingListener struct { @@ -134,9 +135,13 @@ func (p *fakeOwnedPeerProcess) Terminate() error { <-p.release } p.mu.Lock() + defer p.mu.Unlock() + // A failed kill must leave the process running: no counter bump, still alive. + if p.terminateErr != nil { + return p.terminateErr + } p.terminated++ p.alive = false - p.mu.Unlock() return nil } func (p *fakeOwnedPeerProcess) Close() error { diff --git a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md index 3f95e9a551..fc1d70404f 100644 --- a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md +++ b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md @@ -1240,3 +1240,6 @@ These are real findings from the review that this plan deliberately does not add - **Stale `gofmt` alignment** in four `sessionbroker` test files. CI runs neither `gofmt` nor `go vet` on the agent. - **Dead code:** `SpawnHelperInSession`/`SpawnUserHelperInSession` now have zero production callers; `standaloneOwner`, `helperPanicExitCode`, `trackedHelper.executablePath`/`commandMode` are write-only. - **Session-churn bounded-growth guard.** `broker_stale_helpers_test.go` deleted the `#2387` regression guard; a churn probe proved the new design is bounded, but no test pins it. +- **Duplicate spawn after a failed kill of a *scheduled* helper.** Task 6 was planned to retain helper-key ownership on a failed `TerminateProcess`. That design was **rejected during execution after verification**: `HasHelperKeyOwner` (`broker.go:558`) returns true for any non-nil owner and does not filter closed sessions, and `TerminateHelperKey` closes the session immediately after. Nothing would ever clear a retained entry, so `spawnKey` would be blocked for that key for the entire process lifetime — reintroducing exactly the wedge class Task 5 removes. Task 6 therefore shipped the visibility half only (Debug→Warn), which is the verified defect. + + The residual gap is narrow: for a **lifecycle-tracked** helper, Task 4 already prevents the duplicate (`reserve` refuses while the tracked process is alive *or* its liveness is unknown). Only a **scheduled** helper — which has no registry entry — can be followed by a proactive spawn after a failed kill. Fixing it properly needs a bounded ownership-retention mechanism (retain, then clear once the process is confirmed dead), which is a design decision, not a patch. File as its own issue. From be281d21f49ef1d1e536da8bd39728a27f77af81 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:04:30 -0600 Subject: [PATCH 27/47] fix(agent): retry helper lifecycle bootstrap before listening --- agent/internal/heartbeat/heartbeat.go | 38 +++++++++++-- agent/internal/heartbeat/heartbeat_test.go | 55 +++++++++++++++++++ .../sessionbroker/broker_admission.go | 10 +++- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/agent/internal/heartbeat/heartbeat.go b/agent/internal/heartbeat/heartbeat.go index db5bdb7a2f..2e3a44c4b6 100644 --- a/agent/internal/heartbeat/heartbeat.go +++ b/agent/internal/heartbeat/heartbeat.go @@ -860,6 +860,38 @@ func bootstrapThenListen(bootstrap func() error, listen func()) error { return nil } +// lifecycleBootstrapRetryInterval matches the lifecycle reconcile cadence: both +// recover from the same transient WTS enumeration failure. +const lifecycleBootstrapRetryInterval = 30 * time.Second + +// bootstrapThenListenWithRetry keeps the fail-closed contract of +// bootstrapThenListen — never listen without desired state — while making the +// failure recoverable. Bootstrap reaches WTSEnumerateSessionsW, which fails +// transiently when the agent service starts before Remote Desktop Services' RPC +// endpoint is ready. Without a retry, one boot-order flake costs the agent its +// pipe listener for the entire process lifetime: no remote desktop, no PAM, no +// helper IPC, while the machine keeps heartbeating healthy. The reconcile loop +// already treats this same error as transient and retries it. +// +// Blocks until bootstrap succeeds (then listens exactly once) or ctx is done. +func bootstrapThenListenWithRetry(ctx context.Context, bootstrap func() error, listen func(), retry time.Duration) { + for { + err := bootstrapThenListen(bootstrap, listen) + if err == nil { + return + } + log.Warn("helper lifecycle bootstrap failed; retrying before starting broker listener", + "retryIn", retry.String(), "error", err.Error()) + select { + case <-ctx.Done(): + log.Error("helper lifecycle bootstrap never succeeded; broker listener not started", + "error", ctx.Err().Error()) + return + case <-time.After(retry): + } + } +} + func (h *Heartbeat) Start() { // Proactively spawn helpers into user sessions so remote desktop works // instantly after reboot (Windows service only). The SCM session event @@ -875,11 +907,9 @@ func (h *Heartbeat) Start() { h.helperLifecycle = lifecycle h.lifecycleCancel = cancel h.mu.Unlock() - if err := bootstrapThenListen(lifecycle.Bootstrap, func() { + go bootstrapThenListenWithRetry(ctx, lifecycle.Bootstrap, func() { go h.sessionBroker.Listen(h.stopChan) - }); err != nil { - log.Error("helper lifecycle bootstrap failed; refusing broker listener without desired state", "error", err.Error()) - } + }, lifecycleBootstrapRetryInterval) go lifecycle.Start(ctx) } else if h.sessionBroker != nil { go h.sessionBroker.Listen(h.stopChan) diff --git a/agent/internal/heartbeat/heartbeat_test.go b/agent/internal/heartbeat/heartbeat_test.go index 4fc7407dca..4ffc099f55 100644 --- a/agent/internal/heartbeat/heartbeat_test.go +++ b/agent/internal/heartbeat/heartbeat_test.go @@ -5,6 +5,7 @@ import ( "errors" "reflect" "sync" + "sync/atomic" "testing" "time" ) @@ -150,3 +151,57 @@ func TestHeartbeatTimeoutNeverOverlapsLifecycleCleanupWithBrokerClose(t *testing t.Fatal("broker was not closed after lifecycle cleanup") } } + +func TestBootstrapRetriesUntilItSucceedsThenListens(t *testing.T) { + // WTSEnumerateSessionsW fails transiently early in Windows boot. One flake + // must not cost the agent its pipe listener for the whole process lifetime. + var attempts int32 + listened := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go bootstrapThenListenWithRetry(ctx, func() error { + if atomic.AddInt32(&attempts, 1) < 3 { + return errors.New("WTSEnumerateSessionsW: the RPC server is unavailable") + } + return nil + }, func() { close(listened) }, time.Millisecond) + + select { + case <-listened: + case <-time.After(2 * time.Second): + t.Fatal("listener never started despite bootstrap eventually succeeding") + } + if got := atomic.LoadInt32(&attempts); got < 3 { + t.Fatalf("attempts = %d, want >= 3", got) + } +} + +func TestBootstrapRetryStopsOnContextCancel(t *testing.T) { + var attempts int32 + ctx, cancel := context.WithCancel(context.Background()) + listened := make(chan struct{}) + + done := make(chan struct{}) + go func() { + defer close(done) + bootstrapThenListenWithRetry(ctx, func() error { + atomic.AddInt32(&attempts, 1) + return errors.New("permanent") + }, func() { close(listened) }, time.Millisecond) + }() + + time.Sleep(20 * time.Millisecond) + cancel() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("retry loop did not exit on context cancel") + } + select { + case <-listened: + t.Fatal("listener started despite bootstrap never succeeding") + default: + } +} diff --git a/agent/internal/sessionbroker/broker_admission.go b/agent/internal/sessionbroker/broker_admission.go index 2e0a5ec8f7..7a4b7da50c 100644 --- a/agent/internal/sessionbroker/broker_admission.go +++ b/agent/internal/sessionbroker/broker_admission.go @@ -171,10 +171,14 @@ func (b *Broker) reserveWindowsHelper(identityKey, peerSID string, key HelperKey return reservation, nil } +// helperOwnerReplaceable reports whether an existing helper session may be +// displaced by a new claimant. A broker-closed session is the only stale signal +// we act on. session.peerProcess now provides a kernel-bound handle (see +// peer_process_windows.go), but consulting its liveness here would let a +// claimant displace a HEALTHY helper that simply has not closed yet. +// Displacement stays conservative on purpose: a refused admission is +// recoverable, evicting a working helper is not. func helperOwnerReplaceable(owner *Session) bool { - // Task 4 adds a kernel-bound peer process handle to this decision. Until - // that handle exists, PID polling would be vulnerable to PID reuse, so a - // broker-closed session is the sole authoritative stale signal. return owner.IsClosed() } From 0785194fac3b6af2f4f9ea9e6b5c1c7195ee1ab8 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:05:15 -0600 Subject: [PATCH 28/47] fix(agent): stop recording fabricated exit codes for detached helpers --- .../internal/sessionbroker/lifecycle_core.go | 10 ++++++++- .../sessionbroker/lifecycle_registry.go | 16 ++++++++++++++ .../sessionbroker/lifecycle_registry_test.go | 21 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index 455c4b592c..2f0b40480f 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -328,8 +328,16 @@ func (m *HelperLifecycleManager) watchProcess(entry *trackedHelper) { } func (m *HelperLifecycleManager) watchDetachedProcess(key HelperKey, generation uint64, process helperProcess) { - exitCode, _ := process.Wait() + exitCode, err := process.Wait() _ = process.Close() + if err != nil { + // Mirror watchProcess: never swallow this. Wait returns (-1, err) on + // failure, and recording -1 as a real exit code marks a possibly-live + // helper exited. + log.Warn("lifecycle: wait on detached helper process failed", "helperKey", key.String(), "pid", processID(process), "error", err.Error()) + m.registry.noteExitUnknown(key, generation) + return + } m.registry.noteExit(key, generation, exitCode) } diff --git a/agent/internal/sessionbroker/lifecycle_registry.go b/agent/internal/sessionbroker/lifecycle_registry.go index d5e3f7c9a4..65ed7608e5 100644 --- a/agent/internal/sessionbroker/lifecycle_registry.go +++ b/agent/internal/sessionbroker/lifecycle_registry.go @@ -165,6 +165,22 @@ func (r *helperRegistry) noteExit(key HelperKey, generation uint64, exitCode int r.mu.Unlock() } +// noteExitUnknown records that a helper's Wait failed, so whether it exited — +// and with what code — is unknown. It deliberately does NOT set helperExited: +// that state hides the entry from keys() and makes beginStop drop it without +// terminating, which would strand a live process. Leaving the entry in its +// current state lets startupExpired or the next stopKey deal with it. +func (r *helperRegistry) noteExitUnknown(key HelperKey, generation uint64) { + r.mu.Lock() + defer r.mu.Unlock() + entry := r.generation[generation] + if entry == nil || entry.key != key { + return + } + entry.doneOnce.Do(func() { close(entry.done) }) + delete(r.generation, generation) +} + func (r *helperRegistry) detach(key HelperKey, generation uint64) bool { r.mu.Lock() defer r.mu.Unlock() diff --git a/agent/internal/sessionbroker/lifecycle_registry_test.go b/agent/internal/sessionbroker/lifecycle_registry_test.go index 8e6a0b5e06..c5c9b42fa2 100644 --- a/agent/internal/sessionbroker/lifecycle_registry_test.go +++ b/agent/internal/sessionbroker/lifecycle_registry_test.go @@ -647,3 +647,24 @@ func TestMarkSessionClosedRestartsTheStartupWindow(t *testing.T) { t.Fatal("startup window was not restarted on session close; a live helper would be killed immediately") } } + +func TestNoteExitUnknownDoesNotMarkHelperExited(t *testing.T) { + // Wait returns (-1, err) when the handle operation fails. Recording -1 as a + // real exit code marks a possibly-live helper as exited, after which nothing + // ever terminates it: keys() hides it and beginStop just deletes it. + r := newHelperRegistry() + key := HelperKey{WindowsSessionID: 7, Role: "user"} + process := newFakeHelperProcess(6060) + + generation, _ := r.reserve(key, time.Now()) + entry, _ := r.attachReserved(key, generation, process, "user-helper") + + r.noteExitUnknown(key, generation) + + if entry.state == helperExited { + t.Fatal("unknown exit was recorded as helperExited; a live helper is now untrackable") + } + if entry.exitCode != -1 { + t.Fatalf("exitCode = %d; an unknown exit must not fabricate a real code", entry.exitCode) + } +} From 74aa015c1775853858d8a991c9b086dd479f34e0 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:08:56 -0600 Subject: [PATCH 29/47] docs(agent): mark helper lifecycle remediation phase 1 complete --- ...-14-helper-lifecycle-review-remediation.md | 150 +++++++++--------- 1 file changed, 76 insertions(+), 74 deletions(-) diff --git a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md index fc1d70404f..629435de73 100644 --- a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md +++ b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **STATUS: Phase 1 COMPLETE** (2026-07-14). All 9 tasks executed and committed (`77c75f5cf`..`0785194fa`); verification gate green (`go test -race -count=1 ./...`, darwin+windows builds, vet clean). **Task 6 shipped its visibility half only** — the planned ownership-retention was rejected during execution after verifying it would wedge the helper key permanently; see "Out of scope" for the full reasoning. Phase 2 (watchdog slice 6) is tracked in its own plan doc. + **Goal:** Close the eight verified review blockers in the Windows helper lifecycle layer so slices 1-5 of `2026-07-14-windows-agent-helper-lifecycle-durability-design.md` are merge-ready, before executing the watchdog slice. **Architecture:** Every blocker shares one root cause — the helper liveness layer collapses "I don't know" into "it's dead". The fix widens `helperProcess.Alive()` to `(bool, error)` so uncertainty is representable, then makes each of the four call sites fail *closed* (unknown ⇒ assume alive ⇒ terminate/refuse, never ⇒ spawn a duplicate). Around that root fix sit five independent repairs: restore deleted security-gate coverage, make the Windows CI job auto-discover packages, close a fail-open to SYSTEM, replace the deleted `KillStaleHelpers` deadlock-breaker with a startup timeout that finally reads `launchedAt`, and make a failed helper kill both visible and retryable. @@ -28,7 +30,7 @@ - `agent/internal/sessionbroker/lifecycle_core.go` — `helperProcess` interface + `stopTrackedKey`/`reconcile` callers (Tasks 4, 5); `watchDetachedProcess` logging (Task 8). - `agent/internal/sessionbroker/lifecycle_registry.go` — `reserve`/`detach`/`markSessionClosed` callers (Task 4); `startupExpired` + `launchedAt` wiring (Task 5). - `agent/internal/sessionbroker/lifecycle_registry_test.go` — `fakeHelperProcess` implements the new signature (Task 4). -- `agent/internal/sessionbroker/broker.go` — `TerminateHelperKey` visibility + key retention (Task 6). +- `agent/internal/sessionbroker/broker.go` — `TerminateHelperKey` failure visibility (Task 6; the key-retention half was rejected during execution — see "Out of scope"). - `agent/internal/sessionbroker/broker_admission.go` — resolve the stale `helperOwnerReplaceable` comment (Task 7). - `agent/internal/heartbeat/heartbeat.go` — bootstrap retry (Task 7). @@ -45,7 +47,7 @@ - Consumes: `roleIdentityRejection(role, sid string, uid uint32, peerWinSession, claimedWinSession, consoleWinSession, goos string) (reason string, rejected bool)` from `broker.go:2119`; `systemSID` const; `ipc.HelperRoleAssist`, `ipc.HelperRoleWatchdog`. - Produces: nothing consumed by later tasks. -- [ ] **Step 1: Add the five failing cases to the existing table** +- [x] **Step 1: Add the five failing cases to the existing table** In `console_session_gate_test.go`, the table literal currently ends with the `"assist remains console bound"` line. Add these five cases immediately after it, inside the same `}{...}` literal: @@ -57,14 +59,14 @@ In `console_session_gate_test.go`, the table literal currently ends with the `"a {"watchdog as non-SYSTEM rejected", ipc.HelperRoleWatchdog, nonSystemSID, "1", "1", "1", "watchdog role requires SYSTEM identity", true}, ``` -- [ ] **Step 2: Run the test to confirm the restored cases pass against current code** +- [x] **Step 2: Run the test to confirm the restored cases pass against current code** Run: `cd agent && go test -race ./internal/sessionbroker -run TestRoleIdentityRejection -v` Expected: PASS, 12 subtests. These cases describe behavior the code *already has* — they pass now. Their value is proven in Step 3, not here. If `"assist rejected when console lookup failed (session 0 sentinel)"` FAILS, stop and report: it means the `consoleWinSession == "0"` disjunct at `broker.go:2139` does not behave as the #1009 fix intends, which is a real bug rather than a test gap. -- [ ] **Step 3: Prove each case actually guards its gate (mutation check)** +- [x] **Step 3: Prove each case actually guards its gate (mutation check)** For each mutation below, apply it to `agent/internal/sessionbroker/broker.go`, run `cd agent && go test ./internal/sessionbroker -run TestRoleIdentityRejection`, confirm **FAIL**, then revert the mutation with `git checkout agent/internal/sessionbroker/broker.go`. @@ -75,17 +77,17 @@ For each mutation below, apply it to `agent/internal/sessionbroker/broker.go`, r All four must FAIL. Any mutation that still passes means the case does not guard its gate — fix the case before continuing. -- [ ] **Step 4: Verify the working tree is clean of mutations** +- [x] **Step 4: Verify the working tree is clean of mutations** Run: `cd agent && git diff --stat internal/sessionbroker/broker.go` Expected: empty output. If not, `git checkout agent/internal/sessionbroker/broker.go`. -- [ ] **Step 5: Run the full package suite** +- [x] **Step 5: Run the full package suite** Run: `cd agent && go test -race ./internal/sessionbroker` Expected: `ok` -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add agent/internal/sessionbroker/console_session_gate_test.go @@ -105,7 +107,7 @@ git commit -m "test(agent): restore deleted helper role-authorization gate cases - Consumes: the `test-agent-windows` job added earlier on this branch. - Produces: nothing consumed by later tasks. -- [ ] **Step 1: Confirm the gap is real before changing anything** +- [x] **Step 1: Confirm the gap is real before changing anything** Run: `grep -n 'go test -race ./internal/sessionbroker' .github/workflows/ci.yml` Expected: line 720, with no `./internal/eventlog` in the list. @@ -113,7 +115,7 @@ Expected: line 720, with no `./internal/eventlog` in the list. Run: `head -1 agent/internal/eventlog/eventlog_windows_test.go` Expected: `//go:build windows` — confirming the Linux job cannot run it. -- [ ] **Step 2: Replace the hardcoded list with package auto-discovery** +- [x] **Step 2: Replace the hardcoded list with package auto-discovery** In `.github/workflows/ci.yml`, replace line 720: @@ -133,7 +135,7 @@ with: run: go test -race ./... ``` -- [ ] **Step 3: Verify the cross-compile still builds every test binary** +- [x] **Step 3: Verify the cross-compile still builds every test binary** `go test -race ./...` cannot run on this darwin host for Windows, so vet is the compile check. Run: @@ -141,12 +143,12 @@ with: Expected: only `VET_DONE`. The filtered `unsafe.Pointer` warnings are pre-existing in `internal/remote/desktop` and out of scope. -- [ ] **Step 4: Verify the YAML parses** +- [x] **Step 4: Verify the YAML parses** Run: `python3 -c "import yaml,sys; yaml.safe_load(open('.github/workflows/ci.yml')); print('YAML_OK')"` Expected: `YAML_OK` -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add .github/workflows/ci.yml @@ -169,7 +171,7 @@ git commit -m "ci(agent): auto-discover Windows test packages" - Consumes: `HelperKey{WindowsSessionID uint32; Role string}` from `helper_key.go:8`; `ResolvedHelperExecutable` from `userhelper_path.go:45`; `ipc.HelperRoleSystem`/`ipc.HelperRoleUser` (`ipc/message.go:123-124`). - Produces: `helperRoleSpawnable(role string) bool` — used by no later task, but keep the name stable. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** `createHelperSuspended` calls real Windows APIs, so it cannot be unit-tested on this host. Extract and test the *decision*. Append to `agent/internal/sessionbroker/spawner_cmdline_test.go`: @@ -200,12 +202,12 @@ func TestHelperRoleSpawnableRejectsNonLifecycleRoles(t *testing.T) { Ensure the file's import block includes `"github.com/breeze-rmm/agent/internal/ipc"`. -- [ ] **Step 2: Run the test to verify it fails** +- [x] **Step 2: Run the test to verify it fails** Run: `cd agent && go test ./internal/sessionbroker -run TestHelperRoleSpawnable` Expected: FAIL — `undefined: helperRoleSpawnable` -- [ ] **Step 3: Add the predicate in a platform-neutral file** +- [x] **Step 3: Add the predicate in a platform-neutral file** `spawner_cmdline_test.go` has no build tag, so the predicate must live in a file that builds everywhere. Add to `agent/internal/sessionbroker/helper_key.go` (imports `"github.com/breeze-rmm/agent/internal/ipc"` — add it): @@ -222,12 +224,12 @@ func helperRoleSpawnable(role string) bool { } ``` -- [ ] **Step 4: Run the test to verify it passes** +- [x] **Step 4: Run the test to verify it passes** Run: `cd agent && go test ./internal/sessionbroker -run TestHelperRoleSpawnable -v` Expected: PASS, 7 subtests. -- [ ] **Step 5: Make the spawn switch exhaustive and fail closed** +- [x] **Step 5: Make the spawn switch exhaustive and fail closed** Replace `createHelperSuspended` at `spawner_windows.go:313-318` in full. It calls the predicate rather than duplicating the role list, so the tested predicate is the thing that actually guards the privilege boundary: @@ -254,7 +256,7 @@ func createHelperSuspended(key HelperKey, resolvedExe ResolvedHelperExecutable) Confirm `spawner_windows.go` imports both `"fmt"` and `"github.com/breeze-rmm/agent/internal/ipc"`; add whichever is missing. -- [ ] **Step 6: Gate the platform-neutral spawn path on the same predicate** +- [x] **Step 6: Gate the platform-neutral spawn path on the same predicate** `createHelperSuspended` is Windows-only, so Step 5 cannot be tested on this host. Add the same guard to `spawnKey` in `lifecycle_core.go:256` — a path that *is* testable everywhere — immediately after the `m.mu.Lock()` and the `m.stopping || !m.desired[key]` check, before the reserve: @@ -268,7 +270,7 @@ Confirm `spawner_windows.go` imports both `"fmt"` and `"github.com/breeze-rmm/ag Match the surrounding unlock style exactly — the existing early returns in `spawnKey` unlock explicitly rather than using `defer`. -- [ ] **Step 7: Write the failing test for the spawn-path gate** +- [x] **Step 7: Write the failing test for the spawn-path gate** Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`. This reuses the package's real harness (`newLifecycleHarness` at `:316`) and real fake (`fakeHelperSpawner` at `:162`, which already counts spawns per key in its `spawned` map — no new counter needed): @@ -298,23 +300,23 @@ func TestSpawnKeyRefusesNonLifecycleRole(t *testing.T) { If `m.desired`'s type is not `map[HelperKey]bool`, match its real declaration in `lifecycle_core.go` — check with `grep -n 'desired ' agent/internal/sessionbroker/lifecycle_core.go`. -- [ ] **Step 8: Run the test to verify it fails, then passes** +- [x] **Step 8: Run the test to verify it fails, then passes** Run: `cd agent && go test ./internal/sessionbroker -run TestSpawnKeyRefusesNonLifecycleRole` Expected before Step 6's guard is added: FAIL (spawner called once). With the guard: PASS. If it passes *before* the guard exists, the test is vacuous — find out what else is short-circuiting the spawn and fix the test. -- [ ] **Step 9: Verify the Windows path still compiles** +- [x] **Step 9: Verify the Windows path still compiles** Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` Expected: clean (no output). This is the only check that compiles the Step 5 change on this host. -- [ ] **Step 10: Run the full package suite** +- [x] **Step 10: Run the full package suite** Run: `cd agent && go test -race ./internal/sessionbroker` Expected: `ok` -- [ ] **Step 11: Commit** +- [x] **Step 11: Commit** ```bash git add agent/internal/sessionbroker/helper_key.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/spawner_cmdline_test.go agent/internal/sessionbroker/lifecycle_registry_test.go @@ -339,7 +341,7 @@ This is the root fix. `helperProcess.Alive() bool` cannot express failure, so a - Consumes: `helperProcess` interface (`lifecycle_core.go:23`). - Produces: `helperProcess.Alive() (bool, error)` — Task 5 relies on this signature. `fakeHelperProcess.aliveErr error` field — Task 5's tests reuse it. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`: @@ -386,12 +388,12 @@ func TestMarkSessionClosedTreatsUnknownLivenessAsAlive(t *testing.T) { Ensure the file imports `"errors"`. -- [ ] **Step 2: Run the tests to verify they fail** +- [x] **Step 2: Run the tests to verify they fail** Run: `cd agent && go test ./internal/sessionbroker -run 'TestReserveRefusesWhenLivenessUnknown|TestMarkSessionClosedTreatsUnknownLivenessAsAlive'` Expected: FAIL — `process.setAliveErr undefined` -- [ ] **Step 3: Widen the interface** +- [x] **Step 3: Widen the interface** In `lifecycle_core.go`, change line 26 inside `type helperProcess interface`: @@ -415,7 +417,7 @@ type helperProcess interface { } ``` -- [ ] **Step 4: Update the Windows implementation** +- [x] **Step 4: Update the Windows implementation** Replace `SpawnedHelper.Alive` at `spawner_windows.go:86-97`: @@ -441,7 +443,7 @@ func (s *SpawnedHelper) Alive() (bool, error) { } ``` -- [ ] **Step 5: Update the non-Windows stub** +- [x] **Step 5: Update the non-Windows stub** Replace `spawner_stub.go:30`: @@ -454,7 +456,7 @@ func (s *SpawnedHelper) Alive() (bool, error) { } ``` -- [ ] **Step 6: Make all four production callers fail closed** +- [x] **Step 6: Make all four production callers fail closed** In `lifecycle_registry.go`, replace the `reserve` liveness check at `:75`: @@ -519,7 +521,7 @@ In `lifecycle_core.go`, replace the `stopTrackedKey` check at `:341`: Confirm `lifecycle_registry.go` imports the logger used elsewhere in the package (match the import path already used in `lifecycle_core.go`). Add it if absent. -- [ ] **Step 7: Update the test fake** +- [x] **Step 7: Update the test fake** In `lifecycle_registry_test.go`, add an `aliveErr` field to `fakeHelperProcess` (after `terminateExits bool`): @@ -548,7 +550,7 @@ func (p *fakeHelperProcess) setAliveErr(err error) { If the existing `Alive` body differs from `return p.alive`, preserve its logic and only add the `aliveErr` branch plus the `(bool, error)` return. -- [ ] **Step 8: Update direct test callers** +- [x] **Step 8: Update direct test callers** Five call sites now return two values. In `lifecycle_registry_test.go:229,382,385` and `lifecycle_test.go:47,50`, replace each `x.Alive()` boolean use with a helper. Add to `lifecycle_registry_test.go`: @@ -567,22 +569,22 @@ func aliveNow(t *testing.T, p helperProcess) bool { Then rewrite each site, e.g. `if !system.Alive() {` becomes `if !aliveNow(t, system) {`. `lifecycle_test.go` is `//go:build windows`; `aliveNow` lives in an untagged test file, so it is visible there — verify with the Step 10 vet. -- [ ] **Step 9: Run the tests to verify they pass** +- [x] **Step 9: Run the tests to verify they pass** Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestReserveRefusesWhenLivenessUnknown|TestMarkSessionClosedTreatsUnknownLivenessAsAlive' -v` Expected: PASS, both tests. -- [ ] **Step 10: Verify Windows-tagged files still compile** +- [x] **Step 10: Verify Windows-tagged files still compile** Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` Expected: clean. This is the only check that compiles `lifecycle_test.go`, `spawner_windows.go`, and `peer_process_windows.go` on this host — do not skip it. -- [ ] **Step 11: Run the full suite** +- [x] **Step 11: Run the full suite** Run: `cd agent && go test -race ./internal/sessionbroker ./internal/heartbeat` Expected: `ok` for both. -- [ ] **Step 12: Commit** +- [x] **Step 12: Commit** ```bash git add agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/spawner_windows.go agent/internal/sessionbroker/spawner_stub.go agent/internal/sessionbroker/lifecycle_registry_test.go agent/internal/sessionbroker/lifecycle_test.go @@ -604,7 +606,7 @@ git commit -m "fix(agent): make helper liveness express unknown and fail closed" - Consumes: `helperProcess.Alive() (bool, error)` and `fakeHelperProcess` from Task 4; `helperRegistry`, `trackedHelper`, `helperStarting` (`lifecycle_registry.go:11-31`); `stopTrackedKey` (`lifecycle_core.go:332`). - Produces: `helperRegistry.startupExpired(key HelperKey, now time.Time, timeout time.Duration) bool`; `helperStartupTimeout` const. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`: @@ -667,12 +669,12 @@ func TestMarkSessionClosedRestartsTheStartupWindow(t *testing.T) { } ``` -- [ ] **Step 2: Run the tests to verify they fail** +- [x] **Step 2: Run the tests to verify they fail** Run: `cd agent && go test ./internal/sessionbroker -run 'TestStartupExpired|TestMarkSessionClosedRestarts'` Expected: FAIL — `r.startupExpired undefined` and `helperStartupTimeout undefined` -- [ ] **Step 3: Add the registry predicate and restart the clock** +- [x] **Step 3: Add the registry predicate and restart the clock** Add to `lifecycle_registry.go`, after `markSessionClosed`: @@ -708,7 +710,7 @@ In `markSessionClosed`, inside the branch that sets `entry.state = helperStartin } else { ``` -- [ ] **Step 4: Add the timeout constant and recycle in reconcile** +- [x] **Step 4: Add the timeout constant and recycle in reconcile** In `lifecycle_core.go`, add near the other package constants (top of file, after the imports): @@ -736,27 +738,27 @@ In `reconcile`, immediately after the desired set is computed and before the exi Place this so it runs before the loop that spawns missing keys, so a recycled key is respawned in the same reconcile pass. If the desired set is held under `m.mu` at that point, copy the keys into a local slice under the lock and run the sweep after unlocking — `stopTrackedKey` must never be called while `m.mu` is held. -- [ ] **Step 5: Run the tests to verify they pass** +- [x] **Step 5: Run the tests to verify they pass** Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestStartupExpired|TestMarkSessionClosedRestarts' -v` Expected: PASS, all three tests. -- [ ] **Step 6: Verify the lock-order constraint holds under race detection** +- [x] **Step 6: Verify the lock-order constraint holds under race detection** Run: `cd agent && go test -race ./internal/sessionbroker -run 'TestConcurrent' -v` Expected: PASS, no race reports. These are the existing concurrency tests (`TestConcurrentReconcileStopAndExitOwnsHandleOnce`, `TestConcurrentStopKeyHasSingleTerminationOwner`) and they cover the `m.mu` → registry lock order the sweep must not invert. -- [ ] **Step 7: Verify Windows compile** +- [x] **Step 7: Verify Windows compile** Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` Expected: clean. -- [ ] **Step 8: Run the full suite** +- [x] **Step 8: Run the full suite** Run: `cd agent && go test -race ./internal/sessionbroker` Expected: `ok` -- [ ] **Step 9: Commit** +- [x] **Step 9: Commit** ```bash git add agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry_test.go @@ -777,13 +779,13 @@ git commit -m "fix(agent): recycle helpers that never reach IPC" - Consumes: `ownedPeerProcessRef.claimTermination()` (`lifecycle_core.go:52-113`); `fakeOwnedPeerProcess` (`broker_lifecycle_test.go`); `Broker.TerminateHelperKey`. - Produces: nothing consumed by later tasks. -- [ ] **Step 1: Read the current implementation before changing it** +- [x] **Step 1: Read the current implementation before changing it** Run: `cd agent && sed -n '1995,2030p' internal/sessionbroker/broker.go` Note the exact order: `claimTermination()` → `removeSessionMapsLocked(session)` → `b.mu.Unlock()` → `claim.terminateAndClose()`. The termination happens *after* the maps are cleared and the lock is released. Preserve that ordering — it is deliberate (no syscall under `b.mu`). -- [ ] **Step 2: Add a terminate-failure seam to the existing fake** +- [x] **Step 2: Add a terminate-failure seam to the existing fake** `fakeOwnedPeerProcess` (`broker_lifecycle_test.go:32`) has no way to fail. Add a `terminateErr` field after `closed int`: @@ -812,7 +814,7 @@ func (p *fakeOwnedPeerProcess) Terminate() error { Note the original takes and releases `p.mu` around only the counter mutation; switching to `defer` here is safe because nothing below blocks. Do not move the `p.claimed` handshake inside the lock — the concurrency tests depend on it running before `p.mu` is taken. -- [ ] **Step 3: Write the failing test** +- [x] **Step 3: Write the failing test** Append to `agent/internal/sessionbroker/broker_lifecycle_test.go`, reusing the file's real harness (`New` at `:229`, `newFakeOwnedPeerProcess` at `:121`, `newOwnedSession` at `:154`): @@ -839,12 +841,12 @@ func TestTerminateHelperKeyRetainsOwnershipWhenKillFails(t *testing.T) { Ensure the file imports `"errors"`. Setting `proc.terminateErr` directly before the session goes live is race-free — no goroutine touches the fake yet. -- [ ] **Step 4: Run the test to verify it fails** +- [x] **Step 4: Run the test to verify it fails** Run: `cd agent && go test ./internal/sessionbroker -run TestTerminateHelperKeyRetainsOwnershipWhenKillFails` Expected: FAIL — the broker currently releases ownership unconditionally. -- [ ] **Step 5: Raise the log level and retain the key on failure** +- [x] **Step 5: Raise the log level and retain the key on failure** Replace the termination block at `broker.go:2017-2021`: @@ -882,22 +884,22 @@ func (b *Broker) retainHelperKeyOwnership(key HelperKey, session *Session) { Verify the real field name and type of the helper-key ownership map (`grep -n 'helperByKey' internal/sessionbroker/broker.go`) and match it exactly; `broker.go:258` is the declaration. If re-registration requires more than one map (check what `removeSessionMapsLocked` clears), restore only the helper-key ownership — do **not** resurrect the session in `byIdentity` or the snapshot, which would misreport a dying session as live. -- [ ] **Step 6: Run the test to verify it passes** +- [x] **Step 6: Run the test to verify it passes** Run: `cd agent && go test -race ./internal/sessionbroker -run TestTerminateHelperKeyRetainsOwnershipWhenKillFails -v` Expected: PASS -- [ ] **Step 7: Run the full suite and the concurrency tests** +- [x] **Step 7: Run the full suite and the concurrency tests** Run: `cd agent && go test -race ./internal/sessionbroker` Expected: `ok`. If any existing admission or lifecycle test fails, the retention has changed a contract another test pins — read that test before adjusting either. `TestSessionCloseReleasesOwnedPeerProcessOnce` and `TestUnexpectedDisconnectReleasesOwnedPeerProcess` both use `fakeOwnedPeerProcess` and must still pass with `terminateErr` nil. -- [ ] **Step 8: Verify Windows compile** +- [x] **Step 8: Verify Windows compile** Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` Expected: clean. -- [ ] **Step 9: Commit** +- [x] **Step 9: Commit** ```bash git add agent/internal/sessionbroker/broker.go agent/internal/sessionbroker/broker_lifecycle_test.go @@ -919,7 +921,7 @@ git commit -m "fix(agent): surface and retry failed helper termination" - Consumes: `bootstrapThenListen(bootstrap func() error, listen func()) error` (`heartbeat.go:851`); `HelperLifecycleManager.Bootstrap() error` (`lifecycle_core.go:182`). - Produces: `bootstrapThenListenWithRetry(ctx context.Context, bootstrap func() error, listen func(), retry time.Duration)`. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** Append to `agent/internal/heartbeat/heartbeat_test.go`: @@ -981,12 +983,12 @@ func TestBootstrapRetryStopsOnContextCancel(t *testing.T) { Ensure the file imports `"context"`, `"errors"`, `"sync/atomic"`, and `"time"`. -- [ ] **Step 2: Run the tests to verify they fail** +- [x] **Step 2: Run the tests to verify they fail** Run: `cd agent && go test ./internal/heartbeat -run TestBootstrapRetr` Expected: FAIL — `undefined: bootstrapThenListenWithRetry` -- [ ] **Step 3: Add the retry wrapper** +- [x] **Step 3: Add the retry wrapper** Add to `heartbeat.go` immediately after `bootstrapThenListen`: @@ -1017,12 +1019,12 @@ func bootstrapThenListenWithRetry(ctx context.Context, bootstrap func() error, l } ``` -- [ ] **Step 4: Run the tests to verify they pass** +- [x] **Step 4: Run the tests to verify they pass** Run: `cd agent && go test -race ./internal/heartbeat -run TestBootstrapRetr -v` Expected: PASS, both tests. -- [ ] **Step 5: Wire the retry into Start()** +- [x] **Step 5: Wire the retry into Start()** Replace `heartbeat.go:877-882`: @@ -1043,7 +1045,7 @@ const lifecycleBootstrapRetryInterval = 30 * time.Second The retry now runs in its own goroutine, so `Start()` no longer blocks on bootstrap. Confirm `TestBootstrapFailureRefusesBrokerListen` still passes unchanged — the fail-closed contract it pins must survive. -- [ ] **Step 6: Resolve the stale security comment** +- [x] **Step 6: Resolve the stale security comment** `broker_admission.go:174-179` says a kernel-bound peer handle does not exist yet; it landed in this same branch (`peer_process_windows.go`, `session.peerProcess`). Replace the comment, keeping the behavior: @@ -1059,12 +1061,12 @@ func helperOwnerReplaceable(owner *Session) bool { } ``` -- [ ] **Step 7: Run the heartbeat and sessionbroker suites** +- [x] **Step 7: Run the heartbeat and sessionbroker suites** Run: `cd agent && go test -race ./internal/heartbeat ./internal/sessionbroker` Expected: `ok` for both, including the unchanged `TestBootstrapFailureRefusesBrokerListen`. -- [ ] **Step 8: Commit** +- [x] **Step 8: Commit** ```bash git add agent/internal/heartbeat/heartbeat.go agent/internal/heartbeat/heartbeat_test.go agent/internal/sessionbroker/broker_admission.go @@ -1085,7 +1087,7 @@ git commit -m "fix(agent): retry helper lifecycle bootstrap before listening" - Consumes: `fakeHelperProcess` from Task 4; `helperRegistry.noteExit`. - Produces: nothing consumed by later tasks. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** Append to `agent/internal/sessionbroker/lifecycle_registry_test.go`: @@ -1109,12 +1111,12 @@ func TestNoteExitIgnoresFabricatedExitCodeOnWaitFailure(t *testing.T) { } ``` -- [ ] **Step 2: Run the test to verify it fails** +- [x] **Step 2: Run the test to verify it fails** Run: `cd agent && go test ./internal/sessionbroker -run TestNoteExitIgnoresFabricatedExitCode` Expected: FAIL — `r.noteExitUnknown undefined` -- [ ] **Step 3: Add the unknown-exit path** +- [x] **Step 3: Add the unknown-exit path** Add to `lifecycle_registry.go` beside `noteExit`: @@ -1136,12 +1138,12 @@ func (r *helperRegistry) noteExitUnknown(key HelperKey, generation uint64) { } ``` -- [ ] **Step 4: Run the test to verify it passes** +- [x] **Step 4: Run the test to verify it passes** Run: `cd agent && go test -race ./internal/sessionbroker -run TestNoteExitIgnoresFabricatedExitCode -v` Expected: PASS -- [ ] **Step 5: Log the error and route unknown exits correctly** +- [x] **Step 5: Log the error and route unknown exits correctly** Replace `watchDetachedProcess` at `lifecycle_core.go:303-307`: @@ -1161,17 +1163,17 @@ func (m *HelperLifecycleManager) watchDetachedProcess(key HelperKey, generation } ``` -- [ ] **Step 6: Run the full suite** +- [x] **Step 6: Run the full suite** Run: `cd agent && go test -race ./internal/sessionbroker` Expected: `ok` -- [ ] **Step 7: Verify Windows compile** +- [x] **Step 7: Verify Windows compile** Run: `cd agent && GOOS=windows go vet ./internal/sessionbroker` Expected: clean. -- [ ] **Step 8: Commit** +- [x] **Step 8: Commit** ```bash git add agent/internal/sessionbroker/lifecycle_core.go agent/internal/sessionbroker/lifecycle_registry.go agent/internal/sessionbroker/lifecycle_registry_test.go @@ -1188,27 +1190,27 @@ git commit -m "fix(agent): stop recording fabricated exit codes for detached hel - Consumes: every preceding task. - Produces: the green signal that gates Phase 2. -- [ ] **Step 1: Run the whole agent suite with race detection** +- [x] **Step 1: Run the whole agent suite with race detection** Run: `cd agent && go test -race -count=1 ./...` Expected: no `FAIL` lines. `-count=1` defeats the test cache — do not omit it. -- [ ] **Step 2: Verify both platform builds** +- [x] **Step 2: Verify both platform builds** Run: `cd agent && go build ./... && GOOS=windows go build ./... && echo BUILD_OK` Expected: `BUILD_OK` -- [ ] **Step 3: Verify vet on both platforms** +- [x] **Step 3: Verify vet on both platforms** Run: `cd agent && go vet ./... && GOOS=windows go vet ./... 2>&1 | grep -v 'possible misuse of unsafe.Pointer'; echo VET_DONE` Expected: `VET_DONE` with no other output. The filtered warnings are pre-existing in `internal/remote/desktop`. -- [ ] **Step 4: Confirm no dead liveness fields remain unread** +- [x] **Step 4: Confirm no dead liveness fields remain unread** Run: `cd agent && grep -rn 'launchedAt' internal/sessionbroker/*.go | grep -v _test.go` Expected: at least one **read** site (in `startupExpired`), not only assignments. `executablePath` and `commandMode` on `trackedHelper` remain write-only and are acceptable — they are diagnostic fields, tracked as a follow-up, not a blocker. -- [ ] **Step 5: Report the gate result** +- [x] **Step 5: Report the gate result** Summarize: tests green, both builds clean, vet clean. Report any deviation rather than proceeding to Phase 2. From bd81a28ba2a3d2ff1fcb7fbd70c3853daca13e6b Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:15:15 -0600 Subject: [PATCH 30/47] refactor(watchdog): return structured recovery outcomes Co-Authored-By: Claude Opus 4.8 (1M context) --- agent/cmd/breeze-watchdog/main.go | 40 ++- agent/internal/watchdog/integration_test.go | 7 +- agent/internal/watchdog/recovery.go | 255 ++++++++++++++++---- agent/internal/watchdog/recovery_darwin.go | 9 + agent/internal/watchdog/recovery_linux.go | 9 + agent/internal/watchdog/recovery_test.go | 151 +++++++++++- agent/internal/watchdog/recovery_windows.go | 10 + 7 files changed, 411 insertions(+), 70 deletions(-) diff --git a/agent/cmd/breeze-watchdog/main.go b/agent/cmd/breeze-watchdog/main.go index 1ebc3da27f..9b404297a4 100644 --- a/agent/cmd/breeze-watchdog/main.go +++ b/agent/cmd/breeze-watchdog/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "log/slog" @@ -482,16 +483,27 @@ func runWatchdog(stopCh <-chan struct{}) { "count_24h": recovery.Count24h(), "pid": pid, }) - ok, err := recovery.Attempt(pid) - if ok { + result, err := recovery.Attempt(watchdog.RecoveryRequest{ + StateFilePID: pid, + Intent: watchdog.RecoveryIntentUnhealthy, + Context: context.Background(), + }) + switch { + case err != nil: + journal.Log(watchdog.LevelError, "recovery.failed", map[string]any{ + "error": errStr(err), + }) + case result.Disposition == watchdog.RecoveryDispositionVerifyHeartbeat: pendingVerify = &struct{ startedAt time.Time }{startedAt: time.Now()} journal.Log(watchdog.LevelInfo, "recovery.attempt_dispatched", map[string]any{ "attempt": recovery.Attempts(), "count_24h": recovery.Count24h(), }) - } else { - journal.Log(watchdog.LevelError, "recovery.failed", map[string]any{ - "error": errStr(err), + default: + // No side effect and nothing to verify (e.g. the controller + // only observed an in-flight service transition). + journal.Log(watchdog.LevelInfo, "recovery.observed_transition", map[string]any{ + "action": string(result.Action), }) } @@ -764,8 +776,13 @@ func handleFailoverCommand( case "restart_agent": recovery.Reset() wd.HandleEvent(watchdog.EventStartAgent) - ok, err := recovery.Attempt(0) - if ok { + // Explicit intent: an operator restart is always a verified graceful + // restart, never whichever rung the escalation ladder happens to be on. + res, err := recovery.Attempt(watchdog.RecoveryRequest{ + Intent: watchdog.RecoveryIntentRestart, + Context: context.Background(), + }) + if err == nil && res.ActionTaken { resultStatus = "completed" result = map[string]string{"action": "restart_agent"} } else { @@ -775,8 +792,13 @@ func handleFailoverCommand( case "start_agent": wd.HandleEvent(watchdog.EventStartAgent) - ok, err := recovery.Attempt(0) - if ok { + // Explicit intent: "start" must never escalate to force-killing a + // process just because a prior attempt was already recorded. + res, err := recovery.Attempt(watchdog.RecoveryRequest{ + Intent: watchdog.RecoveryIntentEnsureStart, + Context: context.Background(), + }) + if err == nil && res.ActionTaken { resultStatus = "completed" result = map[string]string{"action": "start_agent"} } else { diff --git a/agent/internal/watchdog/integration_test.go b/agent/internal/watchdog/integration_test.go index b2cc9b6781..0bb4e738f1 100644 --- a/agent/internal/watchdog/integration_test.go +++ b/agent/internal/watchdog/integration_test.go @@ -119,8 +119,11 @@ func (h *integHarness) tickRecovering() { h.wd.HandleEvent(EventRecoveryExhausted) return } - ok, _ := h.recovery.Attempt(4242) - if ok { + result, err := h.recovery.Attempt(RecoveryRequest{ + StateFilePID: 4242, + Intent: RecoveryIntentUnhealthy, + }) + if err == nil && result.Disposition == RecoveryDispositionVerifyHeartbeat { h.pendingVerifyAt = h.clk.Now() } } diff --git a/agent/internal/watchdog/recovery.go b/agent/internal/watchdog/recovery.go index 475d7a9050..d08cc7655c 100644 --- a/agent/internal/watchdog/recovery.go +++ b/agent/internal/watchdog/recovery.go @@ -1,7 +1,9 @@ package watchdog import ( + "context" "encoding/json" + "fmt" "log/slog" "os" "sort" @@ -20,14 +22,163 @@ type realClock struct{} func (realClock) Now() time.Time { return time.Now() } +// RecoveryAction names the side effect a recovery attempt selected. +type RecoveryAction string + +const ( + RecoveryActionObserve RecoveryAction = "observe" + RecoveryActionGraceful RecoveryAction = "graceful_restart" + RecoveryActionForced RecoveryAction = "forced_restart" + RecoveryActionStart RecoveryAction = "ensure_started" +) + +// RecoveryIntent is what the caller asked for. Controllers must select their +// behavior from the intent — never infer it from the attempt count alone — +// so an operator "start_agent" command can never escalate to forced +// termination just because the escalation ladder happens to sit at attempt 2. +type RecoveryIntent string + +const ( + RecoveryIntentUnhealthy RecoveryIntent = "recover_unhealthy" + RecoveryIntentEnsureStart RecoveryIntent = "ensure_started" + RecoveryIntentRestart RecoveryIntent = "restart" +) + +// RecoveryDisposition tells the caller what to do next. It is deliberately +// explicit rather than a bool: the zero value normalizes to +// RecoveryDispositionNone, so an uninitialized or partially populated result +// can never be misread as "the agent recovered". +type RecoveryDisposition string + +const ( + RecoveryDispositionNone RecoveryDisposition = "none" + RecoveryDispositionVerifyHeartbeat RecoveryDisposition = "verify_heartbeat" + RecoveryDispositionFailover RecoveryDisposition = "failover" +) + +// RecoveryFailureClass categorizes a RecoveryError for journaling and for +// deciding whether a failure is retryable or terminal. +type RecoveryFailureClass string + +const ( + RecoveryFailureQuery RecoveryFailureClass = "query_failed" + RecoveryFailureControl RecoveryFailureClass = "control_failed" + RecoveryFailureStopTimeout RecoveryFailureClass = "stop_timeout" + RecoveryFailureTransitionTimeout RecoveryFailureClass = "transition_timeout" + RecoveryFailureIdentityMismatch RecoveryFailureClass = "identity_mismatch" + RecoveryFailureProcessExitTimeout RecoveryFailureClass = "process_exit_timeout" + RecoveryFailureStartTimeout RecoveryFailureClass = "start_timeout" + RecoveryFailureCanceled RecoveryFailureClass = "canceled" +) + +// RecoveryRequest is the input to a recovery attempt. StateFilePID is only a +// hint read from the agent's state file — a controller that takes destructive +// action must establish process identity from the service manager itself. +type RecoveryRequest struct { + StateFilePID int + Intent RecoveryIntent + Context context.Context +} + +// RecoveryResult is the structured outcome of a recovery attempt. +// +// ActionTaken means "a side effect was issued", not "it worked" — a failed +// control call still sets it so the attempt is charged against the escalation +// budget. Callers deciding whether the agent is coming back must look at +// Disposition (and a nil error), never at ActionTaken. +type RecoveryResult struct { + Intent RecoveryIntent + Action RecoveryAction + Phase string + InitialState string + FinalState string + StateFilePID int + OldPID int + NewPID int + Elapsed time.Duration + ActionTaken bool + Disposition RecoveryDisposition +} + +// RecoveryError is the typed failure returned by a serviceController. +type RecoveryError struct { + Class RecoveryFailureClass + Phase string + State string + PID int + Err error +} + +func (e *RecoveryError) Error() string { + if e.Err == nil { + return fmt.Sprintf("recovery %s failed during %s", e.Class, e.Phase) + } + return fmt.Sprintf("recovery %s failed during %s: %v", e.Class, e.Phase, e.Err) +} + +func (e *RecoveryError) Unwrap() error { return e.Err } + // serviceController is the OS-specific surface RecoveryManager.Attempt depends // on. Production builds inject osServiceController (one impl per GOOS). -// Tests inject a fake. Method names match the existing package-level -// functions so platform files only need to wrap them. +// Tests inject a fake. attempt is the 1-based number of the attempt about to +// be performed, used only to select the escalation rung for +// RecoveryIntentUnhealthy. type serviceController interface { - RestartAgentService() error - StartAgentService() error - ForceKillProcess(pid int) + Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) +} + +// escalatingServiceRecover adapts the unix service helpers +// (restartAgentService / startAgentService / forceKillProcess) to the +// structured contract, preserving the historical escalation ladder: +// +// Attempt 1: graceful restart via the service manager. +// Attempt 2: force-kill the state-file PID then start via the service manager. +// Attempt 3+: just start (the process may already be gone). +// +// RecoveryIntentEnsureStart and RecoveryIntentRestart pin a single behavior +// regardless of the attempt number. +func escalatingServiceRecover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + result := RecoveryResult{Intent: req.Intent, StateFilePID: req.StateFilePID} + + var err error + switch { + case req.Intent == RecoveryIntentEnsureStart: + result.Action = RecoveryActionStart + result.Phase = "start_service" + result.ActionTaken = true + err = startAgentService() + case req.Intent == RecoveryIntentRestart, attempt <= 1: + result.Action = RecoveryActionGraceful + result.Phase = "restart_service" + result.ActionTaken = true + err = restartAgentService() + case attempt == 2: + result.Action = RecoveryActionForced + result.Phase = "force_kill" + result.ActionTaken = true + forceKillProcess(req.StateFilePID) + result.Phase = "start_service" + err = startAgentService() + default: + result.Action = RecoveryActionStart + result.Phase = "start_service" + result.ActionTaken = true + err = startAgentService() + } + + if err != nil { + return result, &RecoveryError{ + Class: RecoveryFailureControl, + Phase: result.Phase, + PID: req.StateFilePID, + Err: err, + } + } + // These service managers report only that the control call was accepted, + // so success here means "restart dispatched" — the heartbeat check is + // still what proves the agent actually came back. + result.Disposition = RecoveryDispositionVerifyHeartbeat + return result, nil } // RecoveryManager tracks escalating recovery attempts for an unhealthy agent. @@ -36,15 +187,16 @@ type serviceController interface { // heartbeat goroutine reading Count24h while Attempt runs), guard with a // mutex. type RecoveryManager struct { - maxAttempts int - cooldown time.Duration - attempts int - lastAttempt time.Time - windowStart time.Time - svc serviceController - clk Clock - restartHistory []time.Time - historyPath string + maxAttempts int + cooldown time.Duration + attempts int + lastAttempt time.Time + windowStart time.Time + svc serviceController + clk Clock + restartHistory []time.Time + historyPath string + terminalExhausted bool } // NewRecoveryManager creates a RecoveryManager with the given limits and the @@ -67,7 +219,15 @@ func newRecoveryManagerWithDeps(maxAttempts int, cooldown time.Duration, svc ser // CanAttempt returns true if another recovery attempt is allowed. If the // cooldown window has passed since windowStart, the counter is reset first. +// +// A terminal failure (identity/ownership uncertainty) is checked first and is +// never cleared by the passage of time: retrying a forced restart against a +// process we could not identify is exactly the action that could kill the +// wrong process. Only an explicit Reset clears it. func (r *RecoveryManager) CanAttempt() bool { + if r.terminalExhausted { + return false + } now := r.clk.Now() if now.Sub(r.windowStart) >= r.cooldown { r.attempts = 0 @@ -76,54 +236,57 @@ func (r *RecoveryManager) CanAttempt() bool { return r.attempts < r.maxAttempts } -// Attempt increments the counter and executes an escalating recovery action -// based on how many attempts have been made: +// Attempt dispatches one recovery attempt to the OS controller and accounts +// for it. // -// Attempt 1: Graceful restart via service manager. -// Attempt 2: Force-kill the process then start via service manager. -// Attempt 3+: Just try starting the service (process may already be gone). +// Only an attempt that actually issued a side effect (result.ActionTaken) +// consumes an escalation slot and a 24h restart-history entry. A controller +// that merely observed an in-progress service transition costs nothing — it +// did not restart anything, so charging it would burn the recovery budget and +// trip the flap detector while the service manager was already recovering on +// its own. // -// Returns (true, nil) on success, (false, err) on failure. -func (r *RecoveryManager) Attempt(pid int) (bool, error) { - r.attempts++ - r.lastAttempt = r.clk.Now() - r.recordRestart(r.lastAttempt) +// A RecoveryDispositionFailover result is terminal: recovery is exhausted +// immediately rather than retried. +func (r *RecoveryManager) Attempt(req RecoveryRequest) (RecoveryResult, error) { + if req.Intent == "" { + req.Intent = RecoveryIntentUnhealthy + } + if req.Context == nil { + req.Context = context.Background() + } - var err error - switch r.attempts { - case 1: - err = r.svc.RestartAgentService() - case 2: - r.svc.ForceKillProcess(pid) - err = r.svc.StartAgentService() - default: - err = r.svc.StartAgentService() + nextAttempt := r.attempts + 1 + result, err := r.svc.Recover(nextAttempt, req) + result.StateFilePID = req.StateFilePID + result.Intent = req.Intent + if result.Disposition == "" { + result.Disposition = RecoveryDispositionNone } - if err != nil { - return false, err + if result.ActionTaken { + r.attempts++ + r.lastAttempt = r.clk.Now() + r.recordRestart(r.lastAttempt) + } + if result.Disposition == RecoveryDispositionFailover { + r.attempts = r.maxAttempts + r.terminalExhausted = true } - return true, nil + return result, err } // Attempts returns the current attempt count within the active window. func (r *RecoveryManager) Attempts() int { return r.attempts } -// Reset clears the attempt counter and resets the window start time. +// Reset clears the attempt counter, the terminal-failure latch, and resets the +// window start time. func (r *RecoveryManager) Reset() { r.attempts = 0 + r.terminalExhausted = false r.windowStart = r.clk.Now() } -// osServiceController is the production serviceController. Each GOOS file -// supplies RestartAgentService and StartAgentService via the package-level -// helpers; ForceKillProcess is the same SIGKILL on every platform. -type osServiceController struct{} - -func (osServiceController) RestartAgentService() error { return restartAgentService() } -func (osServiceController) StartAgentService() error { return startAgentService() } -func (osServiceController) ForceKillProcess(pid int) { forceKillProcess(pid) } - // forceKillProcess sends SIGKILL to the process identified by pid. // Errors are silently ignored — the process may already be gone. func forceKillProcess(pid int) { diff --git a/agent/internal/watchdog/recovery_darwin.go b/agent/internal/watchdog/recovery_darwin.go index 6eb91cd990..e56ac517b5 100644 --- a/agent/internal/watchdog/recovery_darwin.go +++ b/agent/internal/watchdog/recovery_darwin.go @@ -10,6 +10,15 @@ import ( const agentServiceLabel = "com.breeze.agent" +// osServiceController is the production serviceController on macOS. launchd +// owns the daemon lifecycle, so the historical escalation ladder is kept as-is +// and only adapted to the structured contract. +type osServiceController struct{} + +func (osServiceController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + return escalatingServiceRecover(attempt, req) +} + // restartAgentService restarts the launchd service for the agent. // It tries "launchctl kickstart -k" first (modern), then falls back to // bootout + bootstrap (also modern) if kickstart is not available. diff --git a/agent/internal/watchdog/recovery_linux.go b/agent/internal/watchdog/recovery_linux.go index 4ec245d7a9..15e2c2fb62 100644 --- a/agent/internal/watchdog/recovery_linux.go +++ b/agent/internal/watchdog/recovery_linux.go @@ -10,6 +10,15 @@ import ( const agentServiceName = "breeze-agent" +// osServiceController is the production serviceController on Linux. systemd +// owns the unit lifecycle, so the historical escalation ladder is kept as-is +// and only adapted to the structured contract. +type osServiceController struct{} + +func (osServiceController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + return escalatingServiceRecover(attempt, req) +} + // restartAgentService restarts the systemd unit for the agent. func restartAgentService() error { out, err := exec.Command("systemctl", "restart", agentServiceName).CombinedOutput() diff --git a/agent/internal/watchdog/recovery_test.go b/agent/internal/watchdog/recovery_test.go index a4409c308a..ba29d98ec5 100644 --- a/agent/internal/watchdog/recovery_test.go +++ b/agent/internal/watchdog/recovery_test.go @@ -14,12 +14,49 @@ func (f *fakeClock) Now() time.Time { return f.now } func (f *fakeClock) Advance(d time.Duration) { f.now = f.now.Add(d) } // noopServiceController returns success for every call — used when the test -// is about counting/history, not about the OS escalation steps. +// is about counting/history, not about the OS escalation steps. It mirrors the +// escalation ladder of the real unix controllers so the counters stay +// meaningful. type noopServiceController struct{ restarts, kills, starts int } -func (n *noopServiceController) RestartAgentService() error { n.restarts++; return nil } -func (n *noopServiceController) StartAgentService() error { n.starts++; return nil } -func (n *noopServiceController) ForceKillProcess(int) { n.kills++ } +func (n *noopServiceController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + result := RecoveryResult{ + Intent: req.Intent, + ActionTaken: true, + Disposition: RecoveryDispositionVerifyHeartbeat, + } + switch { + case req.Intent == RecoveryIntentEnsureStart: + result.Action = RecoveryActionStart + n.starts++ + case req.Intent == RecoveryIntentRestart, attempt <= 1: + result.Action = RecoveryActionGraceful + n.restarts++ + case attempt == 2: + result.Action = RecoveryActionForced + n.kills++ + n.starts++ + default: + result.Action = RecoveryActionStart + n.starts++ + } + return result, nil +} + +// fakeStructuredController returns a canned structured result/error and +// records the attempt number it was dispatched with. +type fakeStructuredController struct { + result RecoveryResult + err error + calls []int +} + +func (f *fakeStructuredController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + f.calls = append(f.calls, attempt) + result := f.result + result.StateFilePID = req.StateFilePID + return result, f.err +} func newTestRecovery(t *testing.T, clk Clock, svc serviceController) *RecoveryManager { t.Helper() @@ -27,6 +64,94 @@ func newTestRecovery(t *testing.T, clk Clock, svc serviceController) *RecoveryMa return r } +func TestObservationOnlyRecoveryDoesNotConsumeAttempt(t *testing.T) { + clk := &fakeClock{now: time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC)} + svc := &fakeStructuredController{result: RecoveryResult{Action: RecoveryActionObserve, ActionTaken: false}} + r := newRecoveryManagerWithDeps(3, 10*time.Minute, svc, clk) + result, err := r.Attempt(RecoveryRequest{StateFilePID: 44}) + if err != nil || result.ActionTaken || r.Attempts() != 0 || r.Count24h() != 0 { + t.Fatalf("result=%+v err=%v attempts=%d history=%d", result, err, r.Attempts(), r.Count24h()) + } +} + +func TestFailedSideEffectConsumesAttempt(t *testing.T) { + clk := &fakeClock{now: time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC)} + svc := &fakeStructuredController{ + result: RecoveryResult{Action: RecoveryActionGraceful, ActionTaken: true, Phase: "wait_stopped"}, + err: &RecoveryError{Class: RecoveryFailureStopTimeout, Phase: "wait_stopped"}, + } + r := newRecoveryManagerWithDeps(3, 10*time.Minute, svc, clk) + _, _ = r.Attempt(RecoveryRequest{StateFilePID: 44}) + if r.Attempts() != 1 || r.Count24h() != 1 { + t.Fatalf("attempts=%d history=%d, want 1 and 1", r.Attempts(), r.Count24h()) + } +} + +func TestStructuredResultPropagatesSCMPIDs(t *testing.T) { + svc := &fakeStructuredController{result: RecoveryResult{OldPID: 100, NewPID: 200, Disposition: RecoveryDispositionVerifyHeartbeat, ActionTaken: true}} + r := newRecoveryManagerWithDeps(3, 0, svc, &fakeClock{now: time.Now()}) + got, err := r.Attempt(RecoveryRequest{StateFilePID: 50}) + if err != nil || got.StateFilePID != 50 || got.OldPID != 100 || got.NewPID != 200 { + t.Fatalf("result=%+v err=%v", got, err) + } +} + +func TestTerminalIdentityFailureExhaustsRecoveryWithoutRestartHistory(t *testing.T) { + clk := &fakeClock{now: time.Now()} + svc := &fakeStructuredController{ + result: RecoveryResult{Action: RecoveryActionForced, Disposition: RecoveryDispositionFailover}, + err: &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_image"}, + } + r := newRecoveryManagerWithDeps(3, time.Minute, svc, clk) + _, _ = r.Attempt(RecoveryRequest{Intent: RecoveryIntentUnhealthy}) + clk.now = clk.now.Add(2 * time.Minute) // cooldown must not clear terminal exhaustion + if r.CanAttempt() || r.Count24h() != 0 { + t.Fatalf("canAttempt=%v history=%d, want false and 0", r.CanAttempt(), r.Count24h()) + } + r.Reset() + if !r.CanAttempt() { + t.Fatal("explicit reset did not clear terminal exhaustion") + } +} + +// TestRecoveryRequestIntentDefaultsToUnhealthy pins the source-compatibility +// normalization at the RecoveryManager boundary. +func TestRecoveryRequestIntentDefaultsToUnhealthy(t *testing.T) { + svc := &fakeStructuredController{result: RecoveryResult{ActionTaken: true}} + r := newRecoveryManagerWithDeps(3, 0, svc, &fakeClock{now: time.Now()}) + got, err := r.Attempt(RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + if got.Intent != RecoveryIntentUnhealthy { + t.Fatalf("intent=%q, want %q", got.Intent, RecoveryIntentUnhealthy) + } + if got.Disposition != RecoveryDispositionNone { + t.Fatalf("disposition=%q, want %q", got.Disposition, RecoveryDispositionNone) + } +} + +// TestEscalationLadderDispatchesNextAttemptNumber proves the controller is +// handed the 1-based attempt number it is about to perform. +func TestEscalationLadderDispatchesNextAttemptNumber(t *testing.T) { + svc := &fakeStructuredController{result: RecoveryResult{ActionTaken: true}} + r := newRecoveryManagerWithDeps(3, time.Hour, svc, &fakeClock{now: time.Now()}) + for i := 0; i < 3; i++ { + if _, err := r.Attempt(RecoveryRequest{Intent: RecoveryIntentUnhealthy}); err != nil { + t.Fatal(err) + } + } + want := []int{1, 2, 3} + if len(svc.calls) != len(want) { + t.Fatalf("calls=%v, want %v", svc.calls, want) + } + for i, w := range want { + if svc.calls[i] != w { + t.Fatalf("calls=%v, want %v", svc.calls, want) + } + } +} + func TestCount24hEmpty(t *testing.T) { clk := &fakeClock{now: time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)} r := newTestRecovery(t, clk, &noopServiceController{}) @@ -44,9 +169,9 @@ func TestCount24hWithinWindow(t *testing.T) { // Three restart attempts spaced 1h apart, all within the 24h window. for i := 0; i < 3; i++ { - ok, err := r.Attempt(1234) - if err != nil || !ok { - t.Fatalf("attempt %d: ok=%v err=%v", i, ok, err) + result, err := r.Attempt(RecoveryRequest{StateFilePID: 1234}) + if err != nil || !result.ActionTaken { + t.Fatalf("attempt %d: result=%+v err=%v", i, result, err) } clk.Advance(time.Hour) } @@ -60,7 +185,7 @@ func TestCount24hPurgesOld(t *testing.T) { r := newTestRecovery(t, clk, &noopServiceController{}) // First attempt at t0. - if _, err := r.Attempt(1); err != nil { + if _, err := r.Attempt(RecoveryRequest{StateFilePID: 1}); err != nil { t.Fatal(err) } // Advance 25h — first entry is now outside the window. @@ -68,7 +193,7 @@ func TestCount24hPurgesOld(t *testing.T) { // Reset per-window attempts so we can attempt again (we don't care about // the per-window cooldown for this test, only the 24h history). r.Reset() - if _, err := r.Attempt(2); err != nil { + if _, err := r.Attempt(RecoveryRequest{StateFilePID: 2}); err != nil { t.Fatal(err) } if got := r.Count24h(); got != 1 { @@ -83,7 +208,7 @@ func TestCount24hBoundedByCap(t *testing.T) { // Push 60 attempts inside the window; expect history capped at 50. for i := 0; i < 60; i++ { r.Reset() // bypass per-window cooldown for this test - if _, err := r.Attempt(1); err != nil { + if _, err := r.Attempt(RecoveryRequest{StateFilePID: 1}); err != nil { t.Fatal(err) } clk.Advance(time.Minute) @@ -101,7 +226,7 @@ func TestLastRestartAtMatchesClock(t *testing.T) { clk := &fakeClock{now: t0} r := newTestRecovery(t, clk, &noopServiceController{}) - if _, err := r.Attempt(1); err != nil { + if _, err := r.Attempt(RecoveryRequest{StateFilePID: 1}); err != nil { t.Fatal(err) } if got := r.LastRestartAt(); !got.Equal(t0) { @@ -118,10 +243,10 @@ func TestHistoryRoundTrip(t *testing.T) { r1.SetHistoryPath(path) // Two attempts. - r1.Attempt(1) + r1.Attempt(RecoveryRequest{StateFilePID: 1}) clk.Advance(time.Hour) r1.Reset() - r1.Attempt(2) + r1.Attempt(RecoveryRequest{StateFilePID: 2}) // New manager points at the same file; advance not needed (clock starts at same value). r2 := newRecoveryManagerWithDeps(3, 10*time.Minute, &noopServiceController{}, clk) diff --git a/agent/internal/watchdog/recovery_windows.go b/agent/internal/watchdog/recovery_windows.go index cf4fce1f30..8d7f0347de 100644 --- a/agent/internal/watchdog/recovery_windows.go +++ b/agent/internal/watchdog/recovery_windows.go @@ -12,6 +12,16 @@ import ( const agentWindowsServiceName = "BreezeAgent" +// osServiceController is the production serviceController on Windows. It +// currently adapts the unverified SCM helpers below to the structured +// contract, preserving today's behavior; the verified SCM state machine +// replaces this in a later task of the same plan. +type osServiceController struct{} + +func (osServiceController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + return escalatingServiceRecover(attempt, req) +} + // restartAgentService stops then starts the Windows service for the agent. // It waits up to 15 seconds for the service to reach the Stopped state before // issuing the start request. From 66c3bd6504c7c9b0cacf20227f07f74e5d089fd9 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:16:34 -0600 Subject: [PATCH 31/47] docs(agent): correct helper startup timeout rationale --- agent/internal/sessionbroker/lifecycle_core.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/agent/internal/sessionbroker/lifecycle_core.go b/agent/internal/sessionbroker/lifecycle_core.go index 2f0b40480f..dbaffef40c 100644 --- a/agent/internal/sessionbroker/lifecycle_core.go +++ b/agent/internal/sessionbroker/lifecycle_core.go @@ -16,8 +16,17 @@ const ( // helperStartupTimeout bounds how long a helper may sit in helperStarting — // launched but not yet connected over IPC — before the lifecycle manager - // terminates and respawns it. Must comfortably exceed a cold helper start on - // a loaded RDS host; 90s is ~3x the observed worst case. + // terminates and respawns it. + // + // 90s is a conservative guess, NOT a measured value: it must exceed a cold + // helper start on a loaded RDS host, and no such measurement exists yet. Too + // low kills healthy slow-starting helpers in a respawn loop, so err high. If + // the "never reached IPC within startup timeout" warning shows up in fleet + // logs for helpers that were merely slow, raise this. + // + // Bounded by design: this only ever terminates a helper that has NOT + // connected. main's deleted KillStaleHelpers was strictly more aggressive — + // it killed before EVERY respawn with no timeout at all. helperStartupTimeout = 90 * time.Second ) From 3cb38377bd761bd9fdbe836f91d357eadd707337 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:38:13 -0600 Subject: [PATCH 32/47] test(agent): let Windows pipe tests run on non-interactive logons --- .../internal/sessionbroker/broker_windows.go | 22 +++++++++- .../sessionbroker/broker_windows_test.go | 41 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/agent/internal/sessionbroker/broker_windows.go b/agent/internal/sessionbroker/broker_windows.go index 37cf272c71..83da2a1995 100644 --- a/agent/internal/sessionbroker/broker_windows.go +++ b/agent/internal/sessionbroker/broker_windows.go @@ -15,9 +15,29 @@ import ( // excludes service accounts, batch jobs, and network logons. const pipeSecurity = "D:P(A;;GA;;;SY)(A;;GRGW;;;IU)" +// pipeSecurityOverride replaces pipeSecurity for the duration of a Windows test +// binary. It exists because IU is exactly what makes these tests unrunnable +// anywhere automated: a CI runner service, a scheduled task, and an SSH session +// are all NON-interactive logons, so their tokens lack S-1-5-4 and the test +// process cannot dial the pipe it just created. Every TestNamedPipe* case has +// therefore been failing on real Windows — undetected, because no Windows CI job +// existed to run them. +// +// TEST-ONLY. It is deliberately an unexported var with no config, flag, or env +// binding: nothing outside a _test.go file may write it, so production always +// gets the IU-restricted descriptor above. +var pipeSecurityOverride string + +func pipeSecurityDescriptor() string { + if pipeSecurityOverride != "" { + return pipeSecurityOverride + } + return pipeSecurity +} + func (b *Broker) setupSocket() (net.Listener, error) { cfg := &winio.PipeConfig{ - SecurityDescriptor: pipeSecurity, + SecurityDescriptor: pipeSecurityDescriptor(), InputBufferSize: 64 * 1024, OutputBufferSize: 64 * 1024, } diff --git a/agent/internal/sessionbroker/broker_windows_test.go b/agent/internal/sessionbroker/broker_windows_test.go index b80686ee32..0fbcaeaa67 100644 --- a/agent/internal/sessionbroker/broker_windows_test.go +++ b/agent/internal/sessionbroker/broker_windows_test.go @@ -16,6 +16,47 @@ import ( "github.com/breeze-rmm/agent/internal/ipc" ) +// TestMain grants the test process explicit access to the broker's named pipe. +// +// Production restricts the pipe to SYSTEM + IU (Interactive Users). Every +// automated context is a NON-interactive logon — a CI runner service, a +// scheduled task, an SSH session — so the token has no S-1-5-4 and the test +// cannot dial the pipe it just created, failing with "Access is denied". That +// is the pipe DACL working correctly, not a bug, but it made every +// TestNamedPipe* case fail on real Windows. +// +// So we swap the descriptor for one granting SYSTEM + this exact user SID. The +// accept path, handshake, and peer-credential checks are still exercised for +// real; only the gate that keeps non-interactive callers out is relaxed, and +// only for this process's own pipe. Set once here, before any test runs, so +// there is no data race with parallel tests in this package. +func TestMain(m *testing.M) { + current, err := user.Current() + if err != nil { + fmt.Fprintf(os.Stderr, "sessionbroker tests: cannot resolve current user SID: %v\n", err) + os.Exit(1) + } + pipeSecurityOverride = fmt.Sprintf("D:P(A;;GA;;;SY)(A;;GA;;;%s)", current.Uid) + os.Exit(m.Run()) +} + +// TestPipeSecurityOverrideIsTestOnly pins the production descriptor so the +// override above can never silently become the shipped one. If pipeSecurity +// stops restricting to Interactive Users, that is a security change and must be +// a deliberate edit here, not a side effect of a test helper. +func TestPipeSecurityOverrideIsTestOnly(t *testing.T) { + const want = "D:P(A;;GA;;;SY)(A;;GRGW;;;IU)" + if pipeSecurity != want { + t.Fatalf("production pipe SDDL = %q, want %q", pipeSecurity, want) + } + if pipeSecurityOverride == "" { + t.Fatal("TestMain did not set pipeSecurityOverride; the pipe tests below would be testing the production DACL and failing for environmental reasons") + } + if pipeSecurityOverride == pipeSecurity { + t.Fatal("override equals production SDDL; the tests would fail on any non-interactive logon") + } +} + // testPipeName returns a unique named pipe path for testing. func testPipeName(t *testing.T) string { t.Helper() From d35ab35aea2e4a9639192f43eec04f29076c81d1 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:38:36 -0600 Subject: [PATCH 33/47] docs(agent): record real-Windows verification results --- ...-14-helper-lifecycle-review-remediation.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md index 629435de73..42afd7fdfb 100644 --- a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md +++ b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md @@ -1229,6 +1229,26 @@ Do not re-plan it. Execute that document task-by-task after Task 9's gate passes --- +## Real-Windows verification (2026-07-14, VM `WIN-DHQNR1F8LO2` / Server 2022) + +Ran the suite natively on a real Windows host, not just `GOOS=windows go vet`. Results: + +- **All 14 new tests from this plan PASS on real Windows** (gate cases, spawn refusal, liveness-unknown, startup recycle, bootstrap retry). +- **Zero regressions from this plan**: the failure list is byte-identical on the pre-remediation baseline (`eb2b4f393`) and after. `main` fails identically too — so every failure below predates this branch. +- `go build ./...` OK under Go 1.25.10 (auto-fetched; VM ships 1.22.5). +- `-race` needs cgo and the VM has no gcc. `windows-latest` ships mingw so CI likely works — **unverified**. + +**`test-agent-windows` will likely be RED on its first run.** It is wired into `ci-success` as a *required* job and has never executed (no PR on this branch). The failures are pre-existing, not this branch's code: + +| Failure | Cause | Fixed? | +|---|---|---| +| 6 × `TestNamedPipe*` "Access is denied" | Pipe SDDL grants `IU` (Interactive Users). CI runner services, scheduled tasks, and SSH are all NON-interactive logons — no `S-1-5-4` in the token, so the test can't dial its own pipe. The DACL is working as designed. | **3 fixed** via a test-only SDDL override set in a Windows `TestMain` (`broker_windows.go`, commit `3cb38377b`) | +| `TestNamedPipeFullHandshake`, `TestNamedPipeSessionIDCollisionRejected` | Authenticate as `system` role, but the test process is Administrator, not SYSTEM → `system role requires SYSTEM identity`. The gate is correct; the test needs a SYSTEM token (e.g. PsExec `-s` / a service). | No — needs a CI design decision | +| `TestNamedPipeSessionDetector` | Asserts every detected WTS session has a username; session 0 (Services) legitimately has none. **Test-logic bug** — fails on any host with a services session. | No — pre-existing bug in `main` | +| 6 × `TestHandleScript*` | `bash` not installed on the VM. `windows-latest` ships Git-bash, so these should pass in CI. | N/A (environmental) | + +Recommendation: decide how the two SYSTEM-token tests should run in CI (skip-unless-SYSTEM with a loud reason, or run that package under a SYSTEM context), and fix `TestNamedPipeSessionDetector`'s username assertion. Until then the required Windows job blocks the branch on faults it did not introduce. + ## Out of scope (file as follow-up issues, do not fix here) These are real findings from the review that this plan deliberately does not address. Each is defensible to defer; none is a blocker. From 573687b122ed1c8f7af21058b3a1098263707ed2 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:45:25 -0600 Subject: [PATCH 34/47] docs(agent): remove internal hostname from verification notes --- .../plans/2026-07-14-helper-lifecycle-review-remediation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md index 42afd7fdfb..0b28406742 100644 --- a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md +++ b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md @@ -1229,7 +1229,7 @@ Do not re-plan it. Execute that document task-by-task after Task 9's gate passes --- -## Real-Windows verification (2026-07-14, VM `WIN-DHQNR1F8LO2` / Server 2022) +## Real-Windows verification (2026-07-14, Windows Server 2022 test VM) Ran the suite natively on a real Windows host, not just `GOOS=windows go vet`. Results: From 8befc57e04fd30a3d09e6618bb09ee83782a49b4 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 22:54:28 -0600 Subject: [PATCH 35/47] fix(agent): restore user helper on RDP and console reconnect --- agent/internal/sessionbroker/lifecycle.go | 25 +++++++++-- .../internal/sessionbroker/lifecycle_test.go | 42 ++++++++++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/agent/internal/sessionbroker/lifecycle.go b/agent/internal/sessionbroker/lifecycle.go index feed02c8e8..a7ab1980a3 100644 --- a/agent/internal/sessionbroker/lifecycle.go +++ b/agent/internal/sessionbroker/lifecycle.go @@ -12,7 +12,20 @@ const ( initialDelay = 3 * time.Second reconcileInterval = 30 * time.Second - wtsSessionDisconnect = 0x4 + // WTS_* notification codes from winuser.h, forwarded verbatim by the SCM + // SessionChange handler (service_windows.go) — it filters nothing, so every + // code below can arrive. + // + // Connect/disconnect come in console and remote PAIRS. Naming matters here: + // this set was previously a single `wtsSessionDisconnect = 0x4`, whose name + // implied it covered disconnects generally. It does not — 0x4 is + // WTS_REMOTE_DISCONNECT only. Console disconnect (0x2) went unhandled, and + // WTS_REMOTE_CONNECT (0x3) — an RDP user reconnecting to their existing + // session — had no case at all. + wtsConsoleConnect = 0x1 + wtsConsoleDisconnect = 0x2 + wtsRemoteConnect = 0x3 + wtsRemoteDisconnect = 0x4 wtsSessionLogon = 0x5 wtsSessionLogoff = 0x6 wtsSessionLock = 0x7 @@ -86,10 +99,16 @@ func (m *HelperLifecycleManager) handleSCMEvent(event SCMSessionEvent) { systemKey := HelperKey{WindowsSessionID: event.SessionID, Role: "system"} userKey := HelperKey{WindowsSessionID: event.SessionID, Role: "user"} switch event.EventType { - case wtsSessionLogon, wtsSessionUnlock, wtsSessionCreate: + // Session became usable. Reconnect (console 0x1 / remote 0x3) belongs here: + // the session already exists and the user never logged off, so no + // logon/unlock/create fires and only these codes signal the return. + case wtsSessionLogon, wtsSessionUnlock, wtsSessionCreate, wtsConsoleConnect, wtsRemoteConnect: m.registry.clearFatal(event.SessionID) m.reconcile() - case wtsSessionDisconnect: + // Session went away but is still logged on. The user helper requires + // state=="active" so it goes; the SYSTEM helper is retained deliberately + // (an RDP session keeps running when disconnected). + case wtsRemoteDisconnect, wtsConsoleDisconnect: m.removeDesired(userKey) m.stopKey(userKey) m.reconcile() diff --git a/agent/internal/sessionbroker/lifecycle_test.go b/agent/internal/sessionbroker/lifecycle_test.go index c208bb6ebd..885a0f264e 100644 --- a/agent/internal/sessionbroker/lifecycle_test.go +++ b/agent/internal/sessionbroker/lifecycle_test.go @@ -42,7 +42,7 @@ func TestHandleSCMDisconnectStopsUserAndRetainsSystem(t *testing.T) { m.desired[HelperKey{WindowsSessionID: 7, Role: "system"}] = true m.desired[HelperKey{WindowsSessionID: 7, Role: "user"}] = true - m.handleSCMEvent(SCMSessionEvent{EventType: wtsSessionDisconnect, SessionID: 7}) + m.handleSCMEvent(SCMSessionEvent{EventType: wtsRemoteDisconnect, SessionID: 7}) if !aliveNow(t, system) { t.Fatal("system helper was stopped on disconnect") @@ -85,3 +85,43 @@ func TestPanicExitCodeConsistency(t *testing.T) { t.Fatalf("panic exit code = %d, fatal = %d", helperPanicExitCode, helperFatalExitCode) } } + +// TestHandleSCMRemoteConnectRestoresUserHelper covers reconnecting to an +// existing, previously-disconnected RDP session — the core flow of this branch. +// +// Windows fires WTS_REMOTE_CONNECT (0x3) for that, NOT logon/unlock/create: the +// session already exists and the user never logged off. Before this case +// existed, 0x3 fell through the switch and the user helper (which requires +// state=="active") came back only on the next 30s reconcile tick, so a +// reconnecting user sat without a helper for up to half a minute. +func TestHandleSCMRemoteConnectRestoresUserHelper(t *testing.T) { + m, spawner := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "7", State: "active", Type: "rdp"}}) + userKey := HelperKey{WindowsSessionID: 7, Role: "user"} + + m.handleSCMEvent(SCMSessionEvent{EventType: wtsRemoteConnect, SessionID: 7}) + + if got := spawner.SpawnCount(userKey); got == 0 { + t.Fatal("RDP reconnect did not spawn the user helper; it will not return until the next reconcile tick") + } +} + +// TestHandleSCMConsoleDisconnectStopsUserHelper covers switch-user / lock at the +// physical console, which fires WTS_CONSOLE_DISCONNECT (0x2). The constant +// formerly named wtsSessionDisconnect was really WTS_REMOTE_DISCONNECT (0x4) +// only, so the console case was unhandled and the user helper survived a +// console disconnect. +func TestHandleSCMConsoleDisconnectStopsUserHelper(t *testing.T) { + m, _ := newWindowsLifecycleHarness(t, []DetectedSession{{Session: "7", State: "disconnected", Type: "console"}}) + system := newFakeHelperProcess(6300) + user := newFakeHelperProcess(6400) + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "system"}, system, "helper", "system-helper") + m.registry.attach(HelperKey{WindowsSessionID: 7, Role: "user"}, user, "helper", "user-helper") + m.desired[HelperKey{WindowsSessionID: 7, Role: "system"}] = true + m.desired[HelperKey{WindowsSessionID: 7, Role: "user"}] = true + + m.handleSCMEvent(SCMSessionEvent{EventType: wtsConsoleDisconnect, SessionID: 7}) + + if aliveNow(t, user) { + t.Fatal("user helper survived a console disconnect") + } +} From 01769f2351af0f903e3ba41b75b81e2dd63136f4 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:00:03 -0600 Subject: [PATCH 36/47] fix(watchdog): verify Windows service transitions Add an OS-neutral, fakeable Windows recovery state machine behind windowsRecoveryBackend/watchedProcess so transition ordering is testable on any host without importing golang.org/x/sys/windows. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../watchdog/recovery_windows_logic.go | 551 ++++++++++++++ .../watchdog/recovery_windows_logic_test.go | 680 ++++++++++++++++++ 2 files changed, 1231 insertions(+) create mode 100644 agent/internal/watchdog/recovery_windows_logic.go create mode 100644 agent/internal/watchdog/recovery_windows_logic_test.go diff --git a/agent/internal/watchdog/recovery_windows_logic.go b/agent/internal/watchdog/recovery_windows_logic.go new file mode 100644 index 0000000000..045eb6daf6 --- /dev/null +++ b/agent/internal/watchdog/recovery_windows_logic.go @@ -0,0 +1,551 @@ +package watchdog + +import ( + "context" + "errors" + "fmt" + "time" +) + +// This file is the OS-neutral Windows recovery state machine. It has no build +// tag on purpose: the transition ordering is the part most likely to be wrong +// and most expensive to get wrong (we can terminate a process on a customer +// machine), so it must be testable on any host. Everything that actually talks +// to the SCM or to a process handle lives behind windowsRecoveryBackend and is +// implemented in recovery_windows.go. + +// serviceState is an OS-neutral mirror of the SCM service states. +type serviceState string + +const ( + serviceStopped serviceState = "stopped" + serviceStartPending serviceState = "start_pending" + serviceStopPending serviceState = "stop_pending" + serviceRunning serviceState = "running" + serviceContinuePending serviceState = "continue_pending" + servicePausePending serviceState = "pause_pending" + servicePaused serviceState = "paused" +) + +// serviceSnapshot is one SCM observation. PID is the service's process id as +// SCM reports it; 0 means SCM did not name one. +type serviceSnapshot struct { + State serviceState + PID int +} + +// isPendingState reports whether SCM is mid-transition. A pending state is +// never something we act on: SCM (or another controller) already owns the +// transition, and issuing a competing Stop/Start would race it. +func isPendingState(s serviceState) bool { + switch s { + case serviceStartPending, serviceStopPending, serviceContinuePending, servicePausePending: + return true + default: + return false + } +} + +// watchedProcess is a handle to a specific process. Used by forced recovery to +// prove the PID SCM named is really our agent before terminating it. +type watchedProcess interface { + ImagePath() (string, error) + Alive() (bool, error) + Terminate() error + Wait(context.Context, time.Duration) error + Close() error +} + +// windowsRecoveryBackend is the side-effecting surface the state machine drives. +type windowsRecoveryBackend interface { + Query() (serviceSnapshot, error) + ConfiguredBinaryPath() (string, error) + Stop() error + Start() error + OpenProcess(pid int) (watchedProcess, error) +} + +// recoveryClock is Clock plus a cancellable sleep. Every wait in this file goes +// through it so tests run on virtual time and so an SCM stop can interrupt an +// in-flight transition wait instead of blocking for the full recovery deadline. +type recoveryClock interface { + Now() time.Time + Sleep(context.Context, time.Duration) error +} + +type realRecoveryClock struct{} + +func (realRecoveryClock) Now() time.Time { return time.Now() } + +func (realRecoveryClock) Sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +const ( + // windowsStopTimeout exceeds the agent's ~21s maximum serial shutdown + // budget with margin, so a slow-but-healthy shutdown is not mistaken for a + // hung one and escalated to termination. + windowsStopTimeout = 35 * time.Second + // windowsStartTimeout, windowsObserveTimeout and windowsProcessExitTimeout + // share the same 35s recovery deadline. + windowsStartTimeout = 35 * time.Second + windowsObserveTimeout = 35 * time.Second + windowsProcessExitTimeout = 35 * time.Second + + windowsRecoveryPollInterval = 500 * time.Millisecond +) + +// errUnexpectedTerminalState means SCM settled in a definite state that is not +// the one we asked for. It is distinct from a timeout: we have a firm answer, +// it is just the wrong one, so there is nothing to wait for. +var errUnexpectedTerminalState = errors.New("service settled in an unexpected terminal state") + +// windowsRecoveryController implements serviceController on top of a +// windowsRecoveryBackend. +type windowsRecoveryController struct { + backend windowsRecoveryBackend + clk recoveryClock + + stopTimeout time.Duration + startTimeout time.Duration + observeTimeout time.Duration + processExitTimeout time.Duration + pollInterval time.Duration +} + +func newWindowsRecoveryController(backend windowsRecoveryBackend, clk recoveryClock) *windowsRecoveryController { + return &windowsRecoveryController{ + backend: backend, + clk: clk, + stopTimeout: windowsStopTimeout, + startTimeout: windowsStartTimeout, + observeTimeout: windowsObserveTimeout, + processExitTimeout: windowsProcessExitTimeout, + pollInterval: windowsRecoveryPollInterval, + } +} + +// Recover selects a recovery branch and runs it. +// +// The branch comes from the intent first and only then from the attempt +// number: an operator "start_agent" must never escalate to forced termination +// just because the escalation ladder happens to sit at attempt 2. +func (c *windowsRecoveryController) Recover(attempt int, req RecoveryRequest) (result RecoveryResult, err error) { + if req.Intent == "" { + req.Intent = RecoveryIntentUnhealthy + } + if req.Context == nil { + req.Context = context.Background() + } + + startedAt := c.clk.Now() + result = RecoveryResult{Intent: req.Intent, StateFilePID: req.StateFilePID} + defer func() { + result.Elapsed = c.clk.Now().Sub(startedAt) + // RecoveryManager also normalizes this, but the controller must not + // depend on a caller to make its result safe to read: an unset + // disposition escaping from here is exactly the ambiguity that could be + // misread as "recovered". + if result.Disposition == "" { + result.Disposition = RecoveryDispositionNone + } + // On failure the phase the error names is the last phase actually + // reached, which is more precise than whatever the happy path last set. + var recoveryErr *RecoveryError + if err != nil && errors.As(err, &recoveryErr) && recoveryErr.Phase != "" { + result.Phase = recoveryErr.Phase + } + }() + + switch { + case req.Intent == RecoveryIntentEnsureStart: + return c.ensureStartRecovery(result, req) + case req.Intent == RecoveryIntentRestart, attempt <= 1: + return c.gracefulRecovery(result, req) + case attempt == 2: + return c.forcedRecovery(result, req) + default: + // The process is most likely already gone by now; just make sure the + // service is up. + return c.ensureStartRecovery(result, req) + } +} + +// gracefulRecovery is the attempt-1 rung: ask SCM to stop the service, wait for +// a verified Stopped, then start it and verify a new live PID. It never +// terminates a process. +func (c *windowsRecoveryController) gracefulRecovery(result RecoveryResult, req RecoveryRequest) (RecoveryResult, error) { + result.Action = RecoveryActionGraceful + result.Phase = "query_scm" + + snapshot, err := c.backend.Query() + if err != nil { + return result, recoveryQueryError("query_scm", snapshot, err) + } + result.InitialState = string(snapshot.State) + result.FinalState = string(snapshot.State) + + if isPendingState(snapshot.State) { + // SCM already owns a transition. Watch it; do not compete with it. + return c.observeTransition(result, req, snapshot) + } + if snapshot.State == serviceStopped { + result.Action = RecoveryActionStart + return c.ensureStarted(result, req, 0) + } + if snapshot.State != serviceRunning || snapshot.PID <= 0 { + // Paused, or Running with no PID: we cannot say what we would be + // restarting, so we stop here rather than guess. + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "validate_scm_owner", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + result.OldPID = snapshot.PID + + out := c.control(result, req, controlStop, serviceStopped, c.stopTimeout, "stop_service", "wait_stopped", RecoveryFailureStopTimeout) + if out.stop { + return out.result, out.err + } + return c.ensureStarted(out.result, req, out.result.OldPID) +} + +// ensureStartRecovery is the "make sure it is up" branch. It never stops or +// terminates anything. +func (c *windowsRecoveryController) ensureStartRecovery(result RecoveryResult, req RecoveryRequest) (RecoveryResult, error) { + result.Action = RecoveryActionStart + result.Phase = "query_scm" + + snapshot, err := c.backend.Query() + if err != nil { + return result, recoveryQueryError("query_scm", snapshot, err) + } + result.InitialState = string(snapshot.State) + result.FinalState = string(snapshot.State) + + if isPendingState(snapshot.State) { + return c.observeTransition(result, req, snapshot) + } + if snapshot.State == serviceRunning { + // Already started. Nothing was restarted, so the disposition stays + // none: there is no recovery for the heartbeat check to confirm. + result.Action = RecoveryActionObserve + result.Phase = "verify_running" + if snapshot.PID <= 0 { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "verify_running_pid", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + result.NewPID = snapshot.PID + return result, nil + } + if snapshot.State != serviceStopped { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "validate_scm_owner", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + return c.ensureStarted(result, req, 0) +} + +// forcedRecovery is the attempt-2 rung: validate the SCM-named process is +// really our agent, terminate it, wait for it to exit, then bring the service +// back. +// +// TODO(task-3): implement the verified forced ordering +// (validate -> terminate -> process exited -> start/observe -> new live PID) +// per docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md. +// Until then this fails closed: forced recovery is the only path that can +// terminate a process, and a half-built version of it could terminate the +// wrong one. Returning a terminal failover disposition costs us an escalation +// rung (the watchdog hands off to failover instead of force-restarting) but +// cannot kill anything. This controller is not yet wired into production — +// NewRecoveryManager still builds the legacy Windows controller until task 4 — +// so no shipped behavior depends on this branch today. +func (c *windowsRecoveryController) forcedRecovery(result RecoveryResult, _ RecoveryRequest) (RecoveryResult, error) { + result.Action = RecoveryActionForced + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureControl, + Phase: "forced_recovery_unimplemented", + Err: errors.New("verified forced recovery is not implemented yet"), + } +} + +// ensureStarted issues Start and verifies the service reaches Running with a +// PID that is not oldPID. Callers must have established that the service is +// Stopped: Start is only ever called from a verified Stopped state. +func (c *windowsRecoveryController) ensureStarted(result RecoveryResult, req RecoveryRequest, oldPID int) (RecoveryResult, error) { + out := c.control(result, req, controlStart, serviceRunning, c.startTimeout, "start_service", "wait_running", RecoveryFailureStartTimeout) + if out.stop { + return out.result, out.err + } + return c.verifyRunning(out.result, out.snapshot, oldPID) +} + +// verifyRunning is the single success gate. A recovery is reported as "the +// agent is coming back" only when SCM says Running and names a PID we can tell +// apart from the process we were replacing. Anything ambiguous — no PID, or +// still the old PID — is uncertainty, and uncertainty fails closed into +// failover rather than being reported as a successful restart. +func (c *windowsRecoveryController) verifyRunning(result RecoveryResult, snapshot serviceSnapshot, oldPID int) (RecoveryResult, error) { + result.Phase = "verify_running" + result.FinalState = string(snapshot.State) + + if snapshot.State != serviceRunning { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "verify_running", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + if snapshot.PID <= 0 { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "verify_running_pid", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + if oldPID > 0 && snapshot.PID == oldPID { + // SCM still owns the PID we just restarted away from, so the "new" + // process is the old one we already judged unhealthy. + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "verify_new_pid", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + + result.NewPID = snapshot.PID + result.Disposition = RecoveryDispositionVerifyHeartbeat + return result, nil +} + +// observeTransition watches a pending SCM transition to its conclusion without +// issuing any control. It is deliberately free: it consumes no attempt and +// records no restart, because it did not restart anything. +func (c *windowsRecoveryController) observeTransition(result RecoveryResult, req RecoveryRequest, snapshot serviceSnapshot) (RecoveryResult, error) { + if !result.ActionTaken { + // Only relabel when we truly did nothing. If a control was already + // issued and lost a race to SCM, keep the original action so the + // attempt accounting still reflects the side effect. + result.Action = RecoveryActionObserve + } + result.Phase = "observe_transition" + result.FinalState = string(snapshot.State) + + last := snapshot + deadline := c.clk.Now().Add(c.observeTimeout) + for isPendingState(last.State) { + if err := req.Context.Err(); err != nil { + return result, &RecoveryError{Class: RecoveryFailureCanceled, Phase: "observe_transition", State: string(last.State), PID: last.PID, Err: err} + } + if !c.clk.Now().Before(deadline) { + return result, &RecoveryError{ + Class: RecoveryFailureTransitionTimeout, + Phase: "observe_transition", + State: string(last.State), + PID: last.PID, + Err: fmt.Errorf("service still %s after %s", last.State, c.observeTimeout), + } + } + if err := c.clk.Sleep(req.Context, c.pollInterval); err != nil { + return result, &RecoveryError{Class: RecoveryFailureCanceled, Phase: "observe_transition", State: string(last.State), PID: last.PID, Err: err} + } + next, err := c.backend.Query() + if err != nil { + return result, recoveryQueryError("observe_transition", last, err) + } + last = next + result.FinalState = string(last.State) + } + + switch last.State { + case serviceStopped: + // SCM finished stopping on its own. Leave the disposition at none so + // the next attempt can issue Start; starting here would race whatever + // drove the stop. + return result, nil + case serviceRunning: + return c.verifyRunning(result, last, result.OldPID) + default: + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "observe_transition", + State: string(last.State), + PID: last.PID, + } + } +} + +type controlKind int + +const ( + controlStop controlKind = iota + controlStart +) + +// controlOutcome carries the state machine's decision after a control call. +// stop=true means the caller must return (result, err) verbatim rather than +// continuing the recovery sequence. +type controlOutcome struct { + result RecoveryResult + snapshot serviceSnapshot + err error + stop bool +} + +// control issues one SCM control and waits for the requested terminal state. +// +// ActionTaken is set before the control is issued, not after it succeeds: a +// control that failed still perturbed the service, so it must be charged +// against the escalation budget. A query or observation, by contrast, costs +// nothing. +// +// A control error is not classified until SCM has been re-queried. Windows +// reports ERROR_SERVICE_NOT_ACTIVE / ERROR_SERVICE_ALREADY_RUNNING / +// ERROR_SERVICE_CANNOT_ACCEPT_CTRL when the transition we asked for is already +// happening or already done — treating those as failures would escalate a race +// we actually won into forced termination. +func (c *windowsRecoveryController) control( + result RecoveryResult, + req RecoveryRequest, + kind controlKind, + want serviceState, + timeout time.Duration, + controlPhase string, + waitPhase string, + timeoutClass RecoveryFailureClass, +) controlOutcome { + result.Phase = controlPhase + result.ActionTaken = true + + var controlErr error + if kind == controlStop { + controlErr = c.backend.Stop() + } else { + controlErr = c.backend.Start() + } + + if controlErr != nil { + result.Phase = controlPhase + "_requery" + fresh, queryErr := c.backend.Query() + if queryErr != nil { + return controlOutcome{ + result: result, + err: &RecoveryError{Class: RecoveryFailureControl, Phase: controlPhase, Err: errors.Join(controlErr, queryErr)}, + stop: true, + } + } + result.FinalState = string(fresh.State) + switch { + case isPendingState(fresh.State): + // SCM already owns the transition we asked for. + r, err := c.observeTransition(result, req, fresh) + return controlOutcome{result: r, snapshot: fresh, err: err, stop: true} + case fresh.State == want: + // Already where we wanted to be; carry on. + result.Phase = waitPhase + return controlOutcome{result: result, snapshot: fresh} + default: + return controlOutcome{ + result: result, + err: &RecoveryError{Class: RecoveryFailureControl, Phase: controlPhase, State: string(fresh.State), PID: fresh.PID, Err: controlErr}, + stop: true, + } + } + } + + result.Phase = waitPhase + snapshot, waitErr := c.waitForState(req, want, timeout) + if snapshot.State != "" { + result.FinalState = string(snapshot.State) + } + if waitErr != nil { + class := timeoutClass + switch { + case isRecoveryCancellation(waitErr): + class = RecoveryFailureCanceled + case errors.Is(waitErr, errUnexpectedTerminalState): + class = RecoveryFailureTransitionTimeout + } + return controlOutcome{ + result: result, + snapshot: snapshot, + err: &RecoveryError{Class: class, Phase: waitPhase, State: string(snapshot.State), PID: snapshot.PID, Err: waitErr}, + stop: true, + } + } + return controlOutcome{result: result, snapshot: snapshot} +} + +// waitForState polls SCM until it reports want, returning the matching +// snapshot. It fails on cancellation, query error, deadline, or when SCM +// settles in a definite state that is not want — there is no point waiting out +// a firm wrong answer. +func (c *windowsRecoveryController) waitForState(req RecoveryRequest, want serviceState, timeout time.Duration) (serviceSnapshot, error) { + deadline := c.clk.Now().Add(timeout) + var last serviceSnapshot + for { + if err := req.Context.Err(); err != nil { + return last, err + } + snapshot, err := c.backend.Query() + if err != nil { + return last, err + } + last = snapshot + if snapshot.State == want { + return snapshot, nil + } + if !isPendingState(snapshot.State) { + return snapshot, fmt.Errorf("%w: %s while waiting for %s", errUnexpectedTerminalState, snapshot.State, want) + } + if !c.clk.Now().Before(deadline) { + return snapshot, fmt.Errorf("timed out after %s waiting for %s (last state %s)", timeout, want, snapshot.State) + } + if err := c.clk.Sleep(req.Context, c.pollInterval); err != nil { + return snapshot, err + } + } +} + +func recoveryQueryError(phase string, snapshot serviceSnapshot, err error) *RecoveryError { + return &RecoveryError{ + Class: RecoveryFailureQuery, + Phase: phase, + State: string(snapshot.State), + PID: snapshot.PID, + Err: err, + } +} + +// isRecoveryCancellation distinguishes "we are shutting down" from "recovery +// failed". Cancellation must never escalate to failover. +func isRecoveryCancellation(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} diff --git a/agent/internal/watchdog/recovery_windows_logic_test.go b/agent/internal/watchdog/recovery_windows_logic_test.go new file mode 100644 index 0000000000..fd59012b1e --- /dev/null +++ b/agent/internal/watchdog/recovery_windows_logic_test.go @@ -0,0 +1,680 @@ +package watchdog + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" +) + +// This file is deliberately OS-neutral: it exercises the Windows recovery state +// machine through fakes so the transition ordering is testable on any host. +// Nothing here may import golang.org/x/sys/windows. + +// Sentinel errors standing in for the SCM control errors the state machine must +// treat as races rather than failures. The logic never inspects the error +// identity — it always re-queries SCM — so plain sentinels are enough to prove +// the race handling without importing the windows package. +var ( + errFakeServiceNotActive = errors.New("ERROR_SERVICE_NOT_ACTIVE") + errFakeServiceCannotAcceptCtl = errors.New("ERROR_SERVICE_CANNOT_ACCEPT_CTRL") + errFakeServiceAlreadyRunning = errors.New("ERROR_SERVICE_ALREADY_RUNNING") +) + +// fakeRecoveryClock is a virtual clock: Sleep advances time instantly so +// timeout paths run without wall-clock delay. Setting blockAtSleep makes the +// Nth Sleep park until the request context is canceled, which is the +// deterministic barrier the cancellation tests synchronize on. +type fakeRecoveryClock struct { + mu sync.Mutex + now time.Time + sleeps int + blockAtSleep int + reached chan struct{} +} + +func newFakeRecoveryClock() *fakeRecoveryClock { + return &fakeRecoveryClock{ + now: time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC), + reached: make(chan struct{}), + } +} + +func (c *fakeRecoveryClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeRecoveryClock) Sleep(ctx context.Context, d time.Duration) error { + c.mu.Lock() + c.sleeps++ + n := c.sleeps + c.mu.Unlock() + + if c.blockAtSleep > 0 && n == c.blockAtSleep { + close(c.reached) + <-ctx.Done() + return ctx.Err() + } + if err := ctx.Err(); err != nil { + return err + } + c.mu.Lock() + c.now = c.now.Add(d) + c.mu.Unlock() + return nil +} + +// fakeWatchedProcess records the process-handle operations the state machine +// performs, in order, on the owning backend. +type fakeWatchedProcess struct { + backend *fakeWindowsBackend + imagePath string + imageErr error + alive bool + aliveErr error + terminateErr error + waitErr error + terminateCalls int + closeCalls int + // blockWait parks Wait until the context is canceled, after signaling + // waitReached. It is the barrier for the process-exit cancellation test. + blockWait bool + waitReached chan struct{} +} + +func newFakeWatchedProcess(imagePath string) *fakeWatchedProcess { + return &fakeWatchedProcess{imagePath: imagePath, alive: true, waitReached: make(chan struct{})} +} + +func (p *fakeWatchedProcess) ImagePath() (string, error) { + p.backend.record("image") + return p.imagePath, p.imageErr +} + +func (p *fakeWatchedProcess) Alive() (bool, error) { + p.backend.record("alive") + return p.alive, p.aliveErr +} + +func (p *fakeWatchedProcess) Terminate() error { + p.backend.record("terminate") + p.terminateCalls++ + return p.terminateErr +} + +func (p *fakeWatchedProcess) Wait(ctx context.Context, _ time.Duration) error { + p.backend.record("wait") + if p.blockWait { + close(p.waitReached) + <-ctx.Done() + return ctx.Err() + } + return p.waitErr +} + +func (p *fakeWatchedProcess) Close() error { + p.closeCalls++ + return nil +} + +// fakeWindowsBackend replays a scripted sequence of SCM snapshots and records +// every backend operation so tests can assert exact ordering. Once the scripted +// snapshots are exhausted the last one repeats forever, which is what lets the +// timeout tests park SCM in a pending state. +type fakeWindowsBackend struct { + mu sync.Mutex + snapshots []serviceSnapshot + queryIdx int + operations []string + stopCalls int + startCalls int + configuredPath string + configErr error + queryErr error + stopErr error + startErr error + openErr error + processes map[int]*fakeWatchedProcess + openedPIDs []int +} + +func newFakeWindowsBackend(snapshots ...serviceSnapshot) *fakeWindowsBackend { + return &fakeWindowsBackend{ + snapshots: snapshots, + processes: map[int]*fakeWatchedProcess{}, + } +} + +func (b *fakeWindowsBackend) record(op string) { + b.mu.Lock() + defer b.mu.Unlock() + b.operations = append(b.operations, op) +} + +func (b *fakeWindowsBackend) ops() string { + b.mu.Lock() + defer b.mu.Unlock() + return strings.Join(b.operations, ",") +} + +func (b *fakeWindowsBackend) Query() (serviceSnapshot, error) { + b.record("query") + if b.queryErr != nil { + return serviceSnapshot{}, b.queryErr + } + b.mu.Lock() + defer b.mu.Unlock() + if len(b.snapshots) == 0 { + return serviceSnapshot{}, errors.New("fake backend has no snapshots") + } + idx := b.queryIdx + if idx >= len(b.snapshots) { + idx = len(b.snapshots) - 1 + } else { + b.queryIdx++ + } + return b.snapshots[idx], nil +} + +func (b *fakeWindowsBackend) ConfiguredBinaryPath() (string, error) { + b.record("config") + return b.configuredPath, b.configErr +} + +func (b *fakeWindowsBackend) Stop() error { + b.record("stop") + b.mu.Lock() + b.stopCalls++ + b.mu.Unlock() + return b.stopErr +} + +func (b *fakeWindowsBackend) Start() error { + b.record("start") + b.mu.Lock() + b.startCalls++ + b.mu.Unlock() + return b.startErr +} + +func (b *fakeWindowsBackend) OpenProcess(pid int) (watchedProcess, error) { + b.record("open:" + itoa(pid)) + b.mu.Lock() + b.openedPIDs = append(b.openedPIDs, pid) + b.mu.Unlock() + if b.openErr != nil { + return nil, b.openErr + } + proc, ok := b.processes[pid] + if !ok { + return nil, errors.New("fake backend has no process for pid") + } + proc.backend = b + return proc, nil +} + +func itoa(v int) string { + if v == 0 { + return "0" + } + neg := v < 0 + if neg { + v = -v + } + var buf []byte + for v > 0 { + buf = append([]byte{byte('0' + v%10)}, buf...) + v /= 10 + } + if neg { + buf = append([]byte{'-'}, buf...) + } + return string(buf) +} + +func newTestWindowsRecoveryController(backend *fakeWindowsBackend) *windowsRecoveryController { + return newWindowsRecoveryController(backend, newFakeRecoveryClock()) +} + +func TestGracefulStopTimeoutNeverStarts(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + ) + controller := newTestWindowsRecoveryController(backend) + controller.stopTimeout = 35 * time.Second + result, err := controller.Recover(1, RecoveryRequest{StateFilePID: 99}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) { + t.Fatalf("err=%v, want RecoveryError", err) + } + if recoveryErr.Class != RecoveryFailureStopTimeout { + t.Fatalf("class=%q, want %q", recoveryErr.Class, RecoveryFailureStopTimeout) + } + if backend.startCalls != 0 || !result.ActionTaken { + t.Fatalf("startCalls=%d result=%+v", backend.startCalls, result) + } + // A stop timeout is retryable: attempt 2 escalates to forced recovery, so + // it must not latch terminal failover. + if result.Disposition == RecoveryDispositionFailover { + t.Fatalf("stop timeout must not be terminal, disposition=%q", result.Disposition) + } +} + +func TestInitialStopPendingObservesWithoutSideEffect(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + if err != nil || result.Action != RecoveryActionObserve || result.ActionTaken || backend.stopCalls != 0 || backend.startCalls != 0 { + t.Fatalf("result=%+v err=%v stop=%d start=%d", result, err, backend.stopCalls, backend.startCalls) + } + if result.Disposition != RecoveryDispositionNone { + t.Fatalf("disposition=%q, want %q so the next attempt can start it", result.Disposition, RecoveryDispositionNone) + } +} + +func TestGracefulOrderIsStopStoppedStartRunning(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceStartPending, PID: 200}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + if err != nil || backend.ops() != "query,stop,query,query,start,query,query" { + t.Fatalf("result=%+v err=%v operations=%v", result, err, backend.operations) + } + if result.OldPID != 100 || result.NewPID != 200 || !result.ActionTaken { + t.Fatalf("result=%+v", result) + } + if result.Disposition != RecoveryDispositionVerifyHeartbeat { + t.Fatalf("disposition=%q, want %q", result.Disposition, RecoveryDispositionVerifyHeartbeat) + } + if result.InitialState != string(serviceRunning) || result.FinalState != string(serviceRunning) { + t.Fatalf("initial=%q final=%q", result.InitialState, result.FinalState) + } +} + +func TestGracefulAlreadyStoppedStartsWithoutStop(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + if backend.stopCalls != 0 || backend.startCalls != 1 { + t.Fatalf("stop=%d start=%d, want 0 and 1", backend.stopCalls, backend.startCalls) + } + if result.Action != RecoveryActionStart || !result.ActionTaken || result.NewPID != 200 { + t.Fatalf("result=%+v", result) + } + if result.Disposition != RecoveryDispositionVerifyHeartbeat { + t.Fatalf("disposition=%q", result.Disposition) + } +} + +func TestEnsureStartAlreadyRunningObserves(t *testing.T) { + backend := newFakeWindowsBackend(serviceSnapshot{State: serviceRunning, PID: 100}) + controller := newTestWindowsRecoveryController(backend) + // Attempt 2 would be the forced rung for RecoveryIntentUnhealthy. An + // explicit ensure-start must never inherit that escalation. + result, err := controller.Recover(2, RecoveryRequest{Intent: RecoveryIntentEnsureStart}) + if err != nil { + t.Fatal(err) + } + if backend.startCalls != 0 || backend.stopCalls != 0 || result.ActionTaken { + t.Fatalf("start=%d stop=%d result=%+v", backend.startCalls, backend.stopCalls, result) + } + if result.Action != RecoveryActionObserve { + t.Fatalf("action=%q, want %q", result.Action, RecoveryActionObserve) + } + // Nothing was restarted, so there is nothing for the heartbeat check to + // verify; claiming otherwise would report a recovery that never happened. + if result.Disposition != RecoveryDispositionNone { + t.Fatalf("disposition=%q, want %q", result.Disposition, RecoveryDispositionNone) + } +} + +func TestEnsureStartFromStoppedStarts(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 300}, + ) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{Intent: RecoveryIntentEnsureStart}) + if err != nil { + t.Fatal(err) + } + if backend.stopCalls != 0 || backend.startCalls != 1 || result.NewPID != 300 || !result.ActionTaken { + t.Fatalf("stop=%d start=%d result=%+v", backend.stopCalls, backend.startCalls, result) + } +} + +func TestRestartIntentAlwaysGracefulRegardlessOfAttempt(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{Intent: RecoveryIntentRestart}) + if err != nil { + t.Fatal(err) + } + if got := backend.ops(); got != "query,stop,query,start,query" { + t.Fatalf("operations=%q", got) + } + if result.Action != RecoveryActionGraceful { + t.Fatalf("action=%q, want %q", result.Action, RecoveryActionGraceful) + } +} + +func TestStartPendingToRunningReturnsHeartbeatDisposition(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStartPending, PID: 200}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + if result.ActionTaken || result.Action != RecoveryActionObserve { + t.Fatalf("result=%+v", result) + } + if result.Disposition != RecoveryDispositionVerifyHeartbeat || result.NewPID != 200 { + t.Fatalf("result=%+v", result) + } + if backend.startCalls != 0 || backend.stopCalls != 0 { + t.Fatalf("start=%d stop=%d", backend.startCalls, backend.stopCalls) + } +} + +func TestStartPendingToStoppedDefersStart(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStartPending, PID: 200}, + serviceSnapshot{State: serviceStopped}, + ) + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + // Deferring means: no competing Start now, and no disposition that would + // make the caller believe the agent is on its way back. + if backend.startCalls != 0 || result.ActionTaken || result.Disposition != RecoveryDispositionNone { + t.Fatalf("start=%d result=%+v", backend.startCalls, result) + } +} + +func TestStartTimeout(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceStartPending, PID: 200}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureStartTimeout { + t.Fatalf("err=%v, want %q", err, RecoveryFailureStartTimeout) + } + if !result.ActionTaken || result.Disposition == RecoveryDispositionVerifyHeartbeat { + t.Fatalf("result=%+v", result) + } +} + +// TestRunningWithZeroPIDFails: SCM claims Running but names no PID. That is +// transition uncertainty, so the restart must not be reported as successful. +func TestRunningWithZeroPIDFails(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 0}, + ) + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if result.Disposition != RecoveryDispositionFailover || result.NewPID != 0 { + t.Fatalf("result=%+v", result) + } +} + +// TestRunningWithDeadPIDFails: SCM reports Running but still owns the PID we +// just restarted away from, so the "new" process is the old, dead one. +func TestRunningWithDeadPIDFails(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 100}, + ) + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if result.Disposition != RecoveryDispositionFailover { + t.Fatalf("result=%+v", result) + } +} + +func TestStopControlRaceRequeriesAndObservesSCMRecovery(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + ) + backend.stopErr = errFakeServiceCannotAcceptCtl + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + if err != nil { + t.Fatalf("a stop that lost the race to SCM must not be a failure: %v", err) + } + // SCM already owns the stop: observe it, never issue a competing start. + if backend.startCalls != 0 || backend.stopCalls != 1 { + t.Fatalf("start=%d stop=%d", backend.startCalls, backend.stopCalls) + } + if result.Disposition != RecoveryDispositionNone { + t.Fatalf("result=%+v", result) + } + // The control was issued, so the attempt is still charged. + if !result.ActionTaken { + t.Fatalf("an issued control must consume the attempt: %+v", result) + } +} + +func TestStopControlRaceOnAlreadyStoppedServiceContinuesToStart(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + backend.stopErr = errFakeServiceNotActive + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + if backend.startCalls != 1 || result.NewPID != 200 { + t.Fatalf("start=%d result=%+v", backend.startCalls, result) + } +} + +func TestStartControlRaceRequeriesWithoutSecondStart(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + backend.startErr = errFakeServiceAlreadyRunning + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + if backend.startCalls != 1 { + t.Fatalf("startCalls=%d, want exactly 1", backend.startCalls) + } + if result.NewPID != 200 || result.Disposition != RecoveryDispositionVerifyHeartbeat { + t.Fatalf("result=%+v", result) + } +} + +func TestControlErrorWithUnchangedStateIsControlFailure(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 100}, + ) + backend.stopErr = errors.New("access denied") + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureControl { + t.Fatalf("err=%v, want %q", err, RecoveryFailureControl) + } + if backend.startCalls != 0 || !result.ActionTaken { + t.Fatalf("start=%d result=%+v", backend.startCalls, result) + } +} + +func TestRecoveryCancellationInterruptsPendingObservation(t *testing.T) { + backend := newFakeWindowsBackend(serviceSnapshot{State: serviceStopPending, PID: 100}) + clk := newFakeRecoveryClock() + clk.blockAtSleep = 1 + controller := newWindowsRecoveryController(backend, clk) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + type outcome struct { + result RecoveryResult + err error + } + done := make(chan outcome, 1) + go func() { + result, err := controller.Recover(1, RecoveryRequest{Context: ctx}) + done <- outcome{result, err} + }() + + select { + case <-clk.reached: + case <-time.After(5 * time.Second): + t.Fatal("controller never reached the observation wait barrier") + } + cancel() + + var got outcome + select { + case got = <-done: + case <-time.After(5 * time.Second): + t.Fatal("cancellation did not interrupt the observation promptly") + } + + var recoveryErr *RecoveryError + if !errors.As(got.err, &recoveryErr) || recoveryErr.Class != RecoveryFailureCanceled { + t.Fatalf("err=%v, want %q", got.err, RecoveryFailureCanceled) + } + // Cancellation is a shutdown, not a diagnosis: it must not latch failover. + if got.result.Disposition == RecoveryDispositionFailover { + t.Fatalf("cancellation escalated to failover: %+v", got.result) + } + if backend.startCalls != 0 || backend.stopCalls != 0 { + t.Fatalf("start=%d stop=%d after cancellation", backend.startCalls, backend.stopCalls) + } +} + +func TestRecoveryCancellationInterruptsStopWaitAfterControl(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + ) + clk := newFakeRecoveryClock() + clk.blockAtSleep = 1 + controller := newWindowsRecoveryController(backend, clk) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + type outcome struct { + result RecoveryResult + err error + } + done := make(chan outcome, 1) + go func() { + result, err := controller.Recover(1, RecoveryRequest{Context: ctx}) + done <- outcome{result, err} + }() + + select { + case <-clk.reached: + case <-time.After(5 * time.Second): + t.Fatal("controller never reached the stop wait barrier") + } + cancel() + + var got outcome + select { + case got = <-done: + case <-time.After(5 * time.Second): + t.Fatal("cancellation did not interrupt the stop wait promptly") + } + + var recoveryErr *RecoveryError + if !errors.As(got.err, &recoveryErr) || recoveryErr.Class != RecoveryFailureCanceled { + t.Fatalf("err=%v, want %q", got.err, RecoveryFailureCanceled) + } + // The Stop control was already issued, so the attempt stays charged. + if !got.result.ActionTaken { + t.Fatalf("result=%+v, want ActionTaken", got.result) + } + if backend.startCalls != 0 { + t.Fatalf("startCalls=%d after cancellation, want 0", backend.startCalls) + } +} + +func TestQueryFailureIsNotAnAttempt(t *testing.T) { + backend := newFakeWindowsBackend(serviceSnapshot{State: serviceRunning, PID: 100}) + backend.queryErr = errors.New("SCM unavailable") + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureQuery { + t.Fatalf("err=%v, want %q", err, RecoveryFailureQuery) + } + if result.ActionTaken { + t.Fatalf("a failed query issued no side effect: %+v", result) + } +} + +func TestUnexpectedInitialStateFailsClosed(t *testing.T) { + backend := newFakeWindowsBackend(serviceSnapshot{State: servicePaused, PID: 100}) + result, err := newTestWindowsRecoveryController(backend).Recover(1, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if backend.stopCalls != 0 || backend.startCalls != 0 || result.Disposition != RecoveryDispositionFailover { + t.Fatalf("stop=%d start=%d result=%+v", backend.stopCalls, backend.startCalls, result) + } +} + +func TestRecoverPopulatesElapsedAndPhase(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + ) + controller := newTestWindowsRecoveryController(backend) + result, err := controller.Recover(1, RecoveryRequest{}) + if err == nil { + t.Fatal("expected stop timeout") + } + if result.Elapsed <= 0 { + t.Fatalf("elapsed=%v, want the virtual clock delta", result.Elapsed) + } + if result.Phase != "wait_stopped" { + t.Fatalf("phase=%q, want the last reached phase", result.Phase) + } +} From 6a32b5215d95d5fef080ed4baada51c1eeab6a34 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:07:11 -0600 Subject: [PATCH 37/47] fix(watchdog): verify forced recovery identity and ordering Replace the fail-closed forced-recovery stub with the verified state machine: validate -> terminate -> process exited -> (Stopped -> start | SCM already restarting -> observe) -> Running with a new live PID. Forced recovery never uses RecoveryRequest.StateFilePID for a destructive action. It re-queries the SCM PID, proves the image and liveness of the handle, and revalidates the PID immediately before the kill, so a recycled PID cannot get an arbitrary process terminated. Every uncertainty (config/query failure, open failure, image mismatch, PID drift, exit timeout, unsettled SCM, same-or-dead new PID) fails closed to a terminal failover disposition rather than looping on the next attempt. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../watchdog/recovery_windows_logic.go | 282 +++++++++++- .../watchdog/recovery_windows_logic_test.go | 435 ++++++++++++++++++ 2 files changed, 698 insertions(+), 19 deletions(-) diff --git a/agent/internal/watchdog/recovery_windows_logic.go b/agent/internal/watchdog/recovery_windows_logic.go index 045eb6daf6..f0198dff00 100644 --- a/agent/internal/watchdog/recovery_windows_logic.go +++ b/agent/internal/watchdog/recovery_windows_logic.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" ) @@ -107,6 +108,11 @@ const ( // it is just the wrong one, so there is nothing to wait for. var errUnexpectedTerminalState = errors.New("service settled in an unexpected terminal state") +// errSettleQueryFailed distinguishes "SCM stopped answering while we waited" +// from "SCM answered but never settled". Both fail closed, but they are +// different diagnoses in the journal. +var errSettleQueryFailed = errors.New("scm query failed while waiting for the service to settle") + // windowsRecoveryController implements serviceController on top of a // windowsRecoveryBackend. type windowsRecoveryController struct { @@ -265,27 +271,239 @@ func (c *windowsRecoveryController) ensureStartRecovery(result RecoveryResult, r return c.ensureStarted(result, req, 0) } -// forcedRecovery is the attempt-2 rung: validate the SCM-named process is -// really our agent, terminate it, wait for it to exit, then bring the service -// back. +// forcedRecovery is the attempt-2 rung and the only path that terminates a +// process. Its ordering is: +// +// validate -> terminate -> process exited -> (Stopped -> start | SCM already +// restarting -> observe) -> Running with a new live PID // -// TODO(task-3): implement the verified forced ordering -// (validate -> terminate -> process exited -> start/observe -> new live PID) -// per docs/superpowers/plans/2026-07-14-windows-watchdog-verified-recovery.md. -// Until then this fails closed: forced recovery is the only path that can -// terminate a process, and a half-built version of it could terminate the -// wrong one. Returning a terminal failover disposition costs us an escalation -// rung (the watchdog hands off to failover instead of force-restarting) but -// cannot kill anything. This controller is not yet wired into production — -// NewRecoveryManager still builds the legacy Windows controller until task 4 — -// so no shipped behavior depends on this branch today. -func (c *windowsRecoveryController) forcedRecovery(result RecoveryResult, _ RecoveryRequest) (RecoveryResult, error) { +// Two rules make it safe. First, req.StateFilePID is never used for anything +// destructive: it is a hint written by the agent that may be arbitrarily stale, +// and Windows recycles PIDs aggressively, so terminating it could kill an +// unrelated process. Every destructive step targets the PID SCM names right +// now, re-checked immediately before the kill. +// +// Second, every uncertainty fails closed to RecoveryDispositionFailover, which +// RecoveryManager latches as terminal so attempt 3 cannot come back and try the +// same kill against the same fog. "We could not tell" is never "recovered". +func (c *windowsRecoveryController) forcedRecovery(result RecoveryResult, req RecoveryRequest) (RecoveryResult, error) { result.Action = RecoveryActionForced - result.Disposition = RecoveryDispositionFailover - return result, &RecoveryError{ - Class: RecoveryFailureControl, - Phase: "forced_recovery_unimplemented", - Err: errors.New("verified forced recovery is not implemented yet"), + result.Phase = "query_scm_pid" + + snapshot, err := c.backend.Query() + if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("query_scm_pid", snapshot, err) + } + result.InitialState = string(snapshot.State) + result.FinalState = string(snapshot.State) + + if isPendingState(snapshot.State) { + // SCM already owns a transition; watch it rather than race it. + return c.observeTransition(result, req, snapshot) + } + if snapshot.State == serviceStopped { + // Nothing to terminate — the process we were going to kill is already + // gone. + result.Action = RecoveryActionStart + return c.ensureStarted(result, req, 0) + } + if snapshot.State != serviceRunning || snapshot.PID <= 0 { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "validate_scm_owner", + State: string(snapshot.State), + PID: snapshot.PID, + } + } + result.OldPID = snapshot.PID + + configured, err := c.backend.ConfiguredBinaryPath() + if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("service_config", snapshot, err) + } + + proc, err := c.backend.OpenProcess(snapshot.PID) + if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("open_service_process", snapshot, err) + } + defer proc.Close() + + // Prove the handle we are about to terminate is our agent's image, and that + // it is a live process rather than a zombie whose PID may already have been + // handed to someone else. + result.Phase = "validate_image" + actual, err := proc.ImagePath() + if err != nil || !sameWindowsExecutable(configured, actual) { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_image", State: string(snapshot.State), PID: snapshot.PID, Err: err} + } + result.Phase = "validate_process_live" + alive, err := proc.Alive() + if err != nil || !alive { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_process_live", State: string(snapshot.State), PID: snapshot.PID, Err: err} + } + + // Close the TOCTOU window: SCM may have restarted the service while we were + // validating, in which case our handle names a process SCM no longer owns. + result.Phase = "revalidate_scm_pid" + fresh, err := c.backend.Query() + if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("revalidate_scm_pid", fresh, err) + } + result.FinalState = string(fresh.State) + if isPendingState(fresh.State) { + // SCM recovery or another controller already owns the transition. + return c.observeTransition(result, req, fresh) + } + if fresh.State != serviceRunning || fresh.PID != snapshot.PID { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "revalidate_scm_owner", + State: string(fresh.State), + PID: snapshot.PID, + } + } + + // Everything past here is a side effect, so the attempt is charged even if + // the control itself fails. + result.ActionTaken = true + result.Phase = "terminate" + if err := proc.Terminate(); err != nil { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureControl, Phase: "terminate", State: string(fresh.State), PID: snapshot.PID, Err: err} + } + + result.Phase = "wait_process_exit" + if err := proc.Wait(req.Context, c.processExitTimeout); err != nil { + if isRecoveryCancellation(err) { + // A shutdown is not a diagnosis: leave the disposition alone so it + // normalizes to none rather than latching terminal failover. + return result, &RecoveryError{Class: RecoveryFailureCanceled, Phase: "wait_process_exit", PID: snapshot.PID, Err: err} + } + // The process may still hold the agent's ports, locks and state file. + // Starting a second one on top of it is worse than failing over. + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureProcessExitTimeout, Phase: "wait_process_exit", PID: snapshot.PID, Err: err} + } + + return c.startAfterTermination(result, req, configured) +} + +// startAfterTermination brings the service back once the terminated process has +// exited. SCM's own failure-recovery action may already be restarting it, so +// this observes before deciding whether to issue a Start — a competing Start +// would race SCM. Either way the resulting process must clear the same identity +// and liveness bar as the one we killed. +func (c *windowsRecoveryController) startAfterTermination(result RecoveryResult, req RecoveryRequest, configured string) (RecoveryResult, error) { + result.Phase = "wait_service_settled" + settled, err := c.waitForSettled(req, c.stopTimeout) + if settled.State != "" { + result.FinalState = string(settled.State) + } + if err != nil { + if isRecoveryCancellation(err) { + return result, &RecoveryError{Class: RecoveryFailureCanceled, Phase: "wait_service_settled", State: string(settled.State), PID: settled.PID, Err: err} + } + class := RecoveryFailureStopTimeout + if errors.Is(err, errSettleQueryFailed) { + class = RecoveryFailureQuery + } + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: class, Phase: "wait_service_settled", State: string(settled.State), PID: settled.PID, Err: err} + } + + oldPID := result.OldPID + switch settled.State { + case serviceStopped: + result, err = c.ensureStarted(result, req, oldPID) + case serviceRunning: + // SCM restarted it for us. Do not issue a competing Start. + result, err = c.verifyRunning(result, settled, oldPID) + default: + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{ + Class: RecoveryFailureIdentityMismatch, + Phase: "verify_running", + State: string(settled.State), + PID: settled.PID, + } + } + if err != nil { + return result, err + } + if result.Disposition != RecoveryDispositionVerifyHeartbeat || result.NewPID <= 0 { + // ensureStarted lost a race to SCM and the transition it observed ended + // somewhere other than Running (typically back at Stopped). Nothing came + // up, so there is no new process to verify — leave the disposition as the + // observation found it rather than manufacturing a verdict here. + return result, nil + } + return c.verifyNewProcess(result, configured, result.NewPID) +} + +// verifyNewProcess is the last gate before reporting a forced restart as +// successful. verifyRunning already proved SCM says Running with a PID distinct +// from the one we killed; this proves that PID is our agent's image and is +// actually alive. Without it a forced recovery could report success for a PID +// SCM named but that had already died, or for a recycled PID belonging to +// something else entirely. +func (c *windowsRecoveryController) verifyNewProcess(result RecoveryResult, configured string, pid int) (RecoveryResult, error) { + proc, err := c.backend.OpenProcess(pid) + if err != nil { + result.Disposition = RecoveryDispositionFailover + return result, recoveryQueryError("open_new_process", serviceSnapshot{State: serviceRunning, PID: pid}, err) + } + defer proc.Close() + + result.Phase = "validate_new_image" + actual, err := proc.ImagePath() + if err != nil || !sameWindowsExecutable(configured, actual) { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_new_image", State: string(serviceRunning), PID: pid, Err: err} + } + result.Phase = "validate_new_process_live" + alive, err := proc.Alive() + if err != nil || !alive { + result.Disposition = RecoveryDispositionFailover + return result, &RecoveryError{Class: RecoveryFailureIdentityMismatch, Phase: "validate_new_process_live", State: string(serviceRunning), PID: pid, Err: err} + } + + result.Phase = "verify_new_process" + return result, nil +} + +// waitForSettled polls SCM until it leaves every pending state, returning +// whatever definite state it landed in. Unlike waitForState it does not want a +// specific outcome: after a termination either Stopped (we start it) or Running +// (SCM already did) is legitimate, and the caller decides. +func (c *windowsRecoveryController) waitForSettled(req RecoveryRequest, timeout time.Duration) (serviceSnapshot, error) { + deadline := c.clk.Now().Add(timeout) + var last serviceSnapshot + for { + if err := req.Context.Err(); err != nil { + return last, err + } + snapshot, err := c.backend.Query() + if err != nil { + return last, fmt.Errorf("%w: %w", errSettleQueryFailed, err) + } + last = snapshot + if !isPendingState(snapshot.State) { + return snapshot, nil + } + if !c.clk.Now().Before(deadline) { + return snapshot, fmt.Errorf("timed out after %s waiting for the service to settle (last state %s)", timeout, snapshot.State) + } + if err := c.clk.Sleep(req.Context, c.pollInterval); err != nil { + return snapshot, err + } } } @@ -534,6 +752,32 @@ func (c *windowsRecoveryController) waitForState(req RecoveryRequest, want servi } } +// normalizeWindowsExecutablePath canonicalizes a Windows executable path for +// comparison. SCM's service config and QueryFullProcessImageName do not agree +// on surface form — the config value is often quoted, may use forward slashes, +// and may carry the \\?\ extended-length prefix — while Windows paths are +// case-insensitive. Comparing raw strings would reject our own agent. +func normalizeWindowsExecutablePath(path string) string { + path = strings.Trim(strings.TrimSpace(path), `"`) + path = strings.ReplaceAll(path, "/", `\`) + path = strings.TrimPrefix(path, `\\?\`) + return strings.ToLower(strings.TrimRight(strings.TrimSpace(path), `\`)) +} + +// sameWindowsExecutable reports whether two paths name the same image. +// +// An empty normalized path means "we do not know", and two unknowns are not a +// match: a backend that failed to report a path must never satisfy the identity +// gate that guards termination. +func sameWindowsExecutable(configured, actual string) bool { + c := normalizeWindowsExecutablePath(configured) + a := normalizeWindowsExecutablePath(actual) + if c == "" || a == "" { + return false + } + return c == a +} + func recoveryQueryError(phase string, snapshot serviceSnapshot, err error) *RecoveryError { return &RecoveryError{ Class: RecoveryFailureQuery, diff --git a/agent/internal/watchdog/recovery_windows_logic_test.go b/agent/internal/watchdog/recovery_windows_logic_test.go index fd59012b1e..1c3ca93a78 100644 --- a/agent/internal/watchdog/recovery_windows_logic_test.go +++ b/agent/internal/watchdog/recovery_windows_logic_test.go @@ -3,12 +3,17 @@ package watchdog import ( "context" "errors" + "reflect" "strings" "sync" "testing" "time" ) +// testAgentImagePath is the configured service binary every forced-recovery +// fixture validates against. +const testAgentImagePath = `C:\Program Files\Breeze\breeze-agent.exe` + // This file is deliberately OS-neutral: it exercises the Windows recovery state // machine through fakes so the transition ordering is testable on any host. // Nothing here may import golang.org/x/sys/windows. @@ -661,6 +666,436 @@ func TestUnexpectedInitialStateFailsClosed(t *testing.T) { } } +// forcedRecoverySuccessBackend is the canonical happy-path forced fixture: SCM +// names oldPID twice (initial query + revalidation), the service settles +// Stopped once the process exits, then Start brings it back as newPID. Both +// processes carry the configured image. +func forcedRecoverySuccessBackend(oldPID, newPID int) *fakeWindowsBackend { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: oldPID}, + serviceSnapshot{State: serviceRunning, PID: oldPID}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceRunning, PID: newPID}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[oldPID] = newFakeWatchedProcess(testAgentImagePath) + backend.processes[newPID] = newFakeWatchedProcess(testAgentImagePath) + return backend +} + +// TestForcedRecoveryUsesFreshSCMPIDNotStateFilePID is the core safety property: +// the state-file PID is a stale hint that may have been recycled onto an +// unrelated process, so every destructive action must target the PID SCM names +// right now. +func TestForcedRecoveryUsesFreshSCMPIDNotStateFilePID(t *testing.T) { + backend := forcedRecoverySuccessBackend(200, 300) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{StateFilePID: 999}) + if err != nil { + t.Fatal(err) + } + if result.OldPID != 200 || result.NewPID != 300 { + t.Fatalf("result=%+v, want OldPID=200 NewPID=300 from SCM, not the state file", result) + } + if !reflect.DeepEqual(backend.openedPIDs, []int{200, 300}) { + t.Fatalf("openedPIDs=%v, want [200 300]: the state-file PID 999 must never be opened", backend.openedPIDs) + } + if result.StateFilePID != 999 || result.Disposition != RecoveryDispositionVerifyHeartbeat { + t.Fatalf("result=%+v", result) + } +} + +// TestForcedRecoveryImageMismatchDoesNotTerminateOrStart: SCM named a PID whose +// image is not our agent. That PID was recycled onto someone else's process — +// terminating it would kill an arbitrary program. +func TestForcedRecoveryImageMismatchDoesNotTerminateOrStart(t *testing.T) { + backend := newFakeWindowsBackend(serviceSnapshot{State: serviceRunning, PID: 200}) + backend.configuredPath = testAgentImagePath + proc := newFakeWatchedProcess(`C:\Windows\System32\notepad.exe`) + backend.processes[200] = proc + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if result.Disposition != RecoveryDispositionFailover { + t.Fatalf("disposition=%q, want failover: an unidentifiable process is never retried", result.Disposition) + } + if proc.terminateCalls != 0 || backend.startCalls != 0 { + t.Fatalf("terminate=%d start=%d, want 0 and 0", proc.terminateCalls, backend.startCalls) + } +} + +// TestForcedRecoveryOrdersExitBeforeStart pins the exact required ordering: +// validate -> terminate -> process exited -> Stopped -> start -> new live PID. +// Starting before the old process has actually exited is how two agents end up +// running at once. +func TestForcedRecoveryOrdersExitBeforeStart(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + want := "query,config,open:100,image,alive,query,terminate,wait,query,start,query,open:200,image,alive" + if got := backend.ops(); got != want { + t.Fatalf("operations=%q,\n want %q", got, want) + } + if result.Action != RecoveryActionForced || !result.ActionTaken { + t.Fatalf("result=%+v", result) + } + if result.Disposition != RecoveryDispositionVerifyHeartbeat || result.NewPID != 200 { + t.Fatalf("result=%+v", result) + } +} + +// TestForcedRecoveryConfigQueryFailureEntersFailover: if we cannot read SCM's +// state or the configured binary path, we cannot prove what we would be +// killing. That is terminal, not retryable. +func TestForcedRecoveryConfigQueryFailureEntersFailover(t *testing.T) { + t.Run("scm query fails", func(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + backend.queryErr = errors.New("SCM unavailable") + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureQuery { + t.Fatalf("err=%v, want %q", err, RecoveryFailureQuery) + } + if result.Disposition != RecoveryDispositionFailover || result.ActionTaken { + t.Fatalf("result=%+v", result) + } + if got := backend.ops(); got != "query" { + t.Fatalf("operations=%q, want no action after a failed query", got) + } + }) + + t.Run("service config fails", func(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + backend.configErr = errors.New("OpenService config denied") + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureQuery { + t.Fatalf("err=%v, want %q", err, RecoveryFailureQuery) + } + if result.Disposition != RecoveryDispositionFailover { + t.Fatalf("disposition=%q, want failover", result.Disposition) + } + if got := backend.ops(); got != "query,config" { + t.Fatalf("operations=%q, want nothing opened or terminated", got) + } + if backend.startCalls != 0 || backend.processes[100].terminateCalls != 0 { + t.Fatalf("start=%d terminate=%d", backend.startCalls, backend.processes[100].terminateCalls) + } + }) +} + +// TestForcedRecoveryOpenProcessFailureEntersFailover: a PID we cannot open is a +// PID we cannot identify. +func TestForcedRecoveryOpenProcessFailureEntersFailover(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + backend.openErr = errors.New("access denied") + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureQuery { + t.Fatalf("err=%v, want %q", err, RecoveryFailureQuery) + } + if result.Disposition != RecoveryDispositionFailover || result.ActionTaken { + t.Fatalf("result=%+v", result) + } + if got := backend.ops(); got != "query,config,open:100" { + t.Fatalf("operations=%q, want nothing terminated or started", got) + } +} + +// TestForcedRecoveryPIDChangesAfterOpenFailsClosed: SCM handed us a different +// PID between the identity check and the termination, so the handle we +// validated no longer belongs to the service. Killing it anyway would terminate +// a process the SCM has already moved on from. +func TestForcedRecoveryPIDChangesAfterOpenFailsClosed(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[100] = newFakeWatchedProcess(testAgentImagePath) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if recoveryErr.Phase != "revalidate_scm_owner" { + t.Fatalf("phase=%q, want revalidate_scm_owner", recoveryErr.Phase) + } + if result.Disposition != RecoveryDispositionFailover || result.ActionTaken { + t.Fatalf("result=%+v", result) + } + if backend.processes[100].terminateCalls != 0 || backend.startCalls != 0 { + t.Fatalf("terminate=%d start=%d", backend.processes[100].terminateCalls, backend.startCalls) + } +} + +// TestForcedRecoverySamePIDPendingStateObservesWithoutTerminate: SCM entered a +// transition while we were validating. It already owns the recovery, so we +// bound-observe it instead of racing it with a termination. +func TestForcedRecoverySamePIDPendingStateObservesWithoutTerminate(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[100] = newFakeWatchedProcess(testAgentImagePath) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + if err != nil { + t.Fatalf("observing SCM's own transition is not a failure: %v", err) + } + if backend.processes[100].terminateCalls != 0 || backend.startCalls != 0 { + t.Fatalf("terminate=%d start=%d, want 0 and 0", backend.processes[100].terminateCalls, backend.startCalls) + } + // Nothing was restarted, so nothing was charged and there is no recovery + // for the heartbeat check to confirm. + if result.ActionTaken || result.Disposition != RecoveryDispositionNone { + t.Fatalf("result=%+v", result) + } + if result.Action != RecoveryActionObserve { + t.Fatalf("action=%q, want %q", result.Action, RecoveryActionObserve) + } +} + +// TestForcedRecoveryProcessExitTimeout: the process did not exit, so the old +// agent may still hold its ports and locks. Starting a second one would be +// worse than failing over. +func TestForcedRecoveryProcessExitTimeout(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + backend.processes[100].waitErr = errors.New("process still alive after 35s") + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureProcessExitTimeout { + t.Fatalf("err=%v, want %q", err, RecoveryFailureProcessExitTimeout) + } + if result.Disposition != RecoveryDispositionFailover { + t.Fatalf("disposition=%q, want failover", result.Disposition) + } + if backend.startCalls != 0 { + t.Fatalf("startCalls=%d, want 0: never start while the old process may still be alive", backend.startCalls) + } + // The terminate was issued, so the attempt is charged. + if !result.ActionTaken { + t.Fatalf("result=%+v, want ActionTaken", result) + } +} + +// TestForcedRecoverySCMStoppedTimeout: the process exited but SCM never settled, +// so we cannot tell whether SCM is about to restart the service itself. +func TestForcedRecoverySCMStoppedTimeout(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[100] = newFakeWatchedProcess(testAgentImagePath) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureStopTimeout { + t.Fatalf("err=%v, want %q", err, RecoveryFailureStopTimeout) + } + if result.Disposition != RecoveryDispositionFailover || backend.startCalls != 0 { + t.Fatalf("result=%+v start=%d", result, backend.startCalls) + } +} + +// TestForcedRecoveryRejectsSameNewPID: SCM still names the PID we terminated. +// The "new" process is the corpse of the old one. +func TestForcedRecoveryRejectsSameNewPID(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 100}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[100] = newFakeWatchedProcess(testAgentImagePath) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if result.Disposition != RecoveryDispositionFailover || result.NewPID != 0 { + t.Fatalf("result=%+v", result) + } + if !reflect.DeepEqual(backend.openedPIDs, []int{100}) { + t.Fatalf("openedPIDs=%v, want only the old PID", backend.openedPIDs) + } +} + +// TestForcedRecoveryRejectsDeadNewProcess: SCM says Running with a fresh PID but +// that process is already gone. Reporting a recovery here would strand the +// device. +func TestForcedRecoveryRejectsDeadNewProcess(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + backend.processes[200].alive = false + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureIdentityMismatch { + t.Fatalf("err=%v, want identity mismatch", err) + } + if result.Disposition != RecoveryDispositionFailover { + t.Fatalf("disposition=%q, want failover", result.Disposition) + } +} + +// TestForcedRecoveryObservesAutomaticSCMRestartWithoutCallingStart: SCM's own +// failure-recovery action restarted the service after our termination. Issuing +// a competing Start would race it, so we observe — but still hold the new +// process to the same identity and liveness proof. +func TestForcedRecoveryObservesAutomaticSCMRestartWithoutCallingStart(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStartPending, PID: 200}, + serviceSnapshot{State: serviceRunning, PID: 200}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[100] = newFakeWatchedProcess(testAgentImagePath) + backend.processes[200] = newFakeWatchedProcess(testAgentImagePath) + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + if err != nil { + t.Fatal(err) + } + if backend.startCalls != 0 { + t.Fatalf("startCalls=%d, want 0: SCM already owns the restart", backend.startCalls) + } + want := "query,config,open:100,image,alive,query,terminate,wait,query,query,open:200,image,alive" + if got := backend.ops(); got != want { + t.Fatalf("operations=%q,\n want %q", got, want) + } + if result.NewPID != 200 || result.Disposition != RecoveryDispositionVerifyHeartbeat { + t.Fatalf("result=%+v", result) + } +} + +// TestForcedRecoveryStartRaceEndingStoppedVerifiesNoNewProcess: our Start lost a +// race to SCM and the transition SCM owned ended back at Stopped. Nothing came +// up, so there is no new PID — the controller must not try to verify one, and +// must not report a recovery that did not happen. +func TestForcedRecoveryStartRaceEndingStoppedVerifiesNoNewProcess(t *testing.T) { + backend := newFakeWindowsBackend( + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceRunning, PID: 100}, + serviceSnapshot{State: serviceStopped}, + serviceSnapshot{State: serviceStopPending, PID: 100}, + serviceSnapshot{State: serviceStopped}, + ) + backend.configuredPath = testAgentImagePath + backend.processes[100] = newFakeWatchedProcess(testAgentImagePath) + backend.startErr = errFakeServiceCannotAcceptCtl + result, err := newTestWindowsRecoveryController(backend).Recover(2, RecoveryRequest{}) + if err != nil { + t.Fatalf("losing a start race to SCM is not a failure: %v", err) + } + if result.Disposition != RecoveryDispositionVerifyHeartbeat && result.Disposition != RecoveryDispositionNone { + t.Fatalf("disposition=%q", result.Disposition) + } + if result.Disposition == RecoveryDispositionVerifyHeartbeat { + t.Fatalf("nothing came up, so nothing is coming back: %+v", result) + } + if result.NewPID != 0 { + t.Fatalf("NewPID=%d, want 0", result.NewPID) + } + // Only the terminated process was ever opened: there is no new PID to open. + if !reflect.DeepEqual(backend.openedPIDs, []int{100}) { + t.Fatalf("openedPIDs=%v, want only the terminated PID", backend.openedPIDs) + } +} + +// TestRecoveryCancellationInterruptsProcessExitWait: the watchdog is shutting +// down mid-forced-recovery. The process-exit wait must unblock immediately +// rather than hold shutdown for the full recovery deadline, and a shutdown must +// never be diagnosed as failover. +func TestRecoveryCancellationInterruptsProcessExitWait(t *testing.T) { + backend := forcedRecoverySuccessBackend(100, 200) + proc := backend.processes[100] + proc.blockWait = true + controller := newTestWindowsRecoveryController(backend) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + type outcome struct { + result RecoveryResult + err error + } + done := make(chan outcome, 1) + go func() { + result, err := controller.Recover(2, RecoveryRequest{Context: ctx}) + done <- outcome{result, err} + }() + + select { + case <-proc.waitReached: + case <-time.After(5 * time.Second): + t.Fatal("controller never reached the process-exit wait barrier") + } + cancel() + + var got outcome + select { + case got = <-done: + case <-time.After(5 * time.Second): + t.Fatal("cancellation did not interrupt the process-exit wait promptly") + } + + var recoveryErr *RecoveryError + if !errors.As(got.err, &recoveryErr) || recoveryErr.Class != RecoveryFailureCanceled { + t.Fatalf("err=%v, want %q", got.err, RecoveryFailureCanceled) + } + if got.result.Disposition == RecoveryDispositionFailover { + t.Fatalf("cancellation escalated to failover: %+v", got.result) + } + // The terminate landed before the cancellation, so it stays charged. + if !got.result.ActionTaken || proc.terminateCalls != 1 { + t.Fatalf("result=%+v terminate=%d", got.result, proc.terminateCalls) + } + if backend.startCalls != 0 { + t.Fatalf("startCalls=%d after cancellation, want 0", backend.startCalls) + } +} + +func TestNormalizeWindowsExecutablePath(t *testing.T) { + tests := []struct { + name string + configured string + actual string + want bool + }{ + {"identical", `C:\Program Files\Breeze\breeze-agent.exe`, `C:\Program Files\Breeze\breeze-agent.exe`, true}, + {"case insensitive", `C:\Program Files\Breeze\breeze-agent.exe`, `c:\program files\breeze\BREEZE-AGENT.EXE`, true}, + {"quoted service config", `"C:\Program Files\Breeze\breeze-agent.exe"`, `C:\Program Files\Breeze\breeze-agent.exe`, true}, + {"surrounding whitespace", " C:\\Breeze\\breeze-agent.exe ", `C:\Breeze\breeze-agent.exe`, true}, + {"forward slashes", `C:/Breeze/breeze-agent.exe`, `C:\Breeze\breeze-agent.exe`, true}, + {"extended-length prefix", `\\?\C:\Breeze\breeze-agent.exe`, `C:\Breeze\breeze-agent.exe`, true}, + {"different binary", `C:\Breeze\breeze-agent.exe`, `C:\Windows\System32\notepad.exe`, false}, + {"different directory", `C:\Breeze\breeze-agent.exe`, `C:\Temp\breeze-agent.exe`, false}, + {"empty actual", `C:\Breeze\breeze-agent.exe`, ``, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sameWindowsExecutable(tt.configured, tt.actual); got != tt.want { + t.Fatalf("sameWindowsExecutable(%q, %q)=%v, want %v", tt.configured, tt.actual, got, tt.want) + } + }) + } +} + +// TestSameWindowsExecutableRejectsEmptyPair: two unknowns are not a match. A +// backend that fails to report either path must never satisfy the identity gate. +func TestSameWindowsExecutableRejectsEmptyPair(t *testing.T) { + if sameWindowsExecutable("", "") { + t.Fatal("empty configured and actual paths must not compare equal") + } + if sameWindowsExecutable(` `, `"" `) { + t.Fatal("whitespace/quote-only paths must not compare equal") + } +} + func TestRecoverPopulatesElapsedAndPhase(t *testing.T) { backend := newFakeWindowsBackend( serviceSnapshot{State: serviceRunning, PID: 100}, From b2da330634509ed3f6ab978696aafa22d19aacb8 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:11:10 -0600 Subject: [PATCH 38/47] test(agent): skip SYSTEM-only pipe tests and fix session-0 assertion --- .../sessionbroker/broker_windows_test.go | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/agent/internal/sessionbroker/broker_windows_test.go b/agent/internal/sessionbroker/broker_windows_test.go index 0fbcaeaa67..1b56c711fb 100644 --- a/agent/internal/sessionbroker/broker_windows_test.go +++ b/agent/internal/sessionbroker/broker_windows_test.go @@ -57,6 +57,30 @@ func TestPipeSecurityOverrideIsTestOnly(t *testing.T) { } } +// requireSystemIdentity skips unless the test process runs as SYSTEM. +// +// These handshakes send an AuthRequest with no HelperRole, which the broker +// defaults to ipc.HelperRoleSystem (broker.go), and roleIdentityRejection +// requires S-1-5-18 for that role. A CI runner, an SSH session, and an +// interactive admin are all non-SYSTEM, so the handshake is correctly rejected +// with "system role requires SYSTEM identity" — the gate working, not a bug. +// +// Skipping is the honest outcome: the alternative is a test that is red +// everywhere forever, or weakening a privilege gate to make a test pass. To +// exercise these for real, run the package under a SYSTEM context (e.g. +// PsExec -s). +func requireSystemIdentity(t *testing.T) { + t.Helper() + cu, err := user.Current() + if err != nil { + t.Fatalf("user.Current: %v", err) + } + if cu.Uid != systemSID { + t.Skipf("needs SYSTEM identity: running as %s (SID %s), but an unset HelperRole defaults to the %q role which requires %s", + cu.Username, cu.Uid, ipc.HelperRoleSystem, systemSID) + } +} + // testPipeName returns a unique named pipe path for testing. func testPipeName(t *testing.T) string { t.Helper() @@ -99,6 +123,7 @@ func TestNamedPipeListenAndAccept(t *testing.T) { } func TestNamedPipeFullHandshake(t *testing.T) { + requireSystemIdentity(t) pipeName := testPipeName(t) + "-handshake" stopChan := make(chan struct{}) @@ -282,6 +307,13 @@ func TestNamedPipeSessionDetector(t *testing.T) { t.Logf("Detected session: user=%s, session=%s, state=%s, display=%s", s.Username, s.Session, s.State, s.Display) + // Session 0 is the non-interactive Services session: it legitimately + // has no user. The detector reports it and helperRoleDesired filters it + // out downstream, so requiring a username here fails on every host that + // has one — which is every Windows host. + if s.Session == "0" || s.Type == "services" { + continue + } if s.Username == "" { t.Error("session has empty username") } @@ -432,6 +464,7 @@ func TestNamedPipeMissingSIDRejected(t *testing.T) { } func TestNamedPipeSessionIDCollisionRejected(t *testing.T) { + requireSystemIdentity(t) pipeName := testPipeName(t) + "-collision" stopChan := make(chan struct{}) From fe42a77be029168266a1ab75f9c060b3a5782613 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:13:02 -0600 Subject: [PATCH 39/47] ci(agent): run Windows agent tests without cgo --- .github/workflows/ci.yml | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de583ce28d..561ac2b563 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -715,15 +715,38 @@ jobs: working-directory: agent run: go mod download + # CGO_ENABLED=0 is required, not incidental. internal/remote/desktop is a + # non-cgo package on Windows: comVtblFn, dxgiCapturer and procDeleteObject + # are defined in files tagged `windows && !cgo`, while audio_windows.go, + # cursor_windows.go, gpu_convert_windows.go, monitor_windows.go and + # dxgi_dirty_rects_windows.go use them under a plain `windows` tag. + # windows-latest ships gcc, so CGO_ENABLED defaults to 1 there and the + # package fails to compile — and heartbeat/agentapp import it transitively, + # so they need cgo off too. + # + # This list is explicit rather than ./... because ./... is currently RED on + # Windows: internal/backup, backup/providers, backup/systemstate, + # peripheral, procoutput, remote/desktop, remote/filedrop, remote/tools and + # updater all have failures that predate this job existing (no Windows CI + # ran before it, so their //go:build windows tests never executed anywhere). + # ./internal/eventlog IS listed: its tests previously ran on no platform at + # all, which is the gap this job was added to close. + # + # When adding a package with //go:build windows tests, add it here. Moving + # to ./... is tracked separately and requires fixing the backlog above. - name: Run Windows Go tests working-directory: agent - # Auto-discover every package so Windows-tagged tests cannot be silently - # skipped. A hardcoded list drifts: ./internal/eventlog and nine other - # packages with //go:build windows tests were omitted, so their tests ran - # on no platform at all (the Linux test-agent job's ./... skips them via - # build tag, and `go build` never compiles _test.go). Keeps CLAUDE.md's - # "new test files are auto-discovered" contract true on Windows too. - run: go test -race ./... + env: + CGO_ENABLED: "0" + run: go test ./internal/sessionbroker ./internal/heartbeat ./internal/agentapp ./internal/config ./internal/eventlog ./internal/watchdog ./cmd/breeze-watchdog + + # -race requires cgo, so this covers only packages that do NOT import + # remote/desktop. heartbeat and agentapp do, transitively, and are absent + # here — the step above still runs their tests, just without the detector. + # sessionbroker is the concurrency-critical one and is covered. + - name: Run Windows race tests (cgo-compatible packages) + working-directory: agent + run: go test -race ./internal/sessionbroker ./internal/config ./internal/eventlog ./internal/watchdog ./cmd/breeze-watchdog # ─── Windows runtime smoke (#1000) ──────────────────────────────────── # CI cross-compiles the Windows agent (build-agent matrix) but never RAN From 499864d035cb0042b2144a0c31c183ac24e65d02 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:23:23 -0600 Subject: [PATCH 40/47] feat(watchdog): add verified Windows SCM recovery backend Replace the placeholder Windows osServiceController with concrete adapters that satisfy windowsRecoveryBackend and watchedProcess, and wire the verified recovery state machine into NewRecoveryManager on Windows. Forced recovery is now reachable in production for the first time. The backend reads SCM state/PID via mgr.Service.Query and the configured image via mgr.Service.Config + DecomposeCommandLine/FullPath. Process handles are opened with OpenProcess and verified by QueryFullProcessImageName before TerminateProcess, so the state-file PID is never used for a destructive action. Unrecognized SCM states, out-of-range PIDs and unexpected wait results fail closed rather than being mapped onto a state the machine would act on. Waits poll WaitForSingleObject in 100ms slices so an SCM stop interrupts an in-flight exit wait, and handle operations serialize so Close cannot race an in-flight query or wait. Co-Authored-By: Claude Opus 4.8 (1M context) --- agent/internal/watchdog/recovery_windows.go | 484 ++++++++++++++++-- .../watchdog/recovery_windows_test.go | 392 ++++++++++++++ 2 files changed, 836 insertions(+), 40 deletions(-) create mode 100644 agent/internal/watchdog/recovery_windows_test.go diff --git a/agent/internal/watchdog/recovery_windows.go b/agent/internal/watchdog/recovery_windows.go index 8d7f0347de..64e83f0324 100644 --- a/agent/internal/watchdog/recovery_windows.go +++ b/agent/internal/watchdog/recovery_windows.go @@ -3,79 +3,483 @@ package watchdog import ( + "context" + "errors" "fmt" + "math" + "path/filepath" + "strings" + "sync" "time" + "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" "golang.org/x/sys/windows/svc/mgr" ) const agentWindowsServiceName = "BreezeAgent" -// osServiceController is the production serviceController on Windows. It -// currently adapts the unverified SCM helpers below to the structured -// contract, preserving today's behavior; the verified SCM state machine -// replaces this in a later task of the same plan. -type osServiceController struct{} +const ( + // windowsMaxPath is MAX_PATH, the buffer QueryFullProcessImageName is + // documented against and the size that satisfies almost every install. + windowsMaxPath = 260 + // windowsMaxLongPath is the Windows path ceiling (32,767 characters). A + // buffer that reached it and is still too small means the OS is telling us + // something we cannot represent, so we stop growing and fail. + windowsMaxLongPath = 32767 -func (osServiceController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { - return escalatingServiceRecover(attempt, req) + // windowsProcessWaitSlice bounds a single blocking WaitForSingleObject so + // the caller's context is re-checked at least this often. Waits during + // recovery must be interruptible by an SCM stop. + windowsProcessWaitSlice = 100 * time.Millisecond + + // windowsTerminateExitCode is the exit code recorded for a process the + // watchdog force-terminates. + windowsTerminateExitCode = 1 +) + +// osServiceController is the production serviceController on Windows. It opens +// SCM per attempt and drives the verified recovery state machine in +// recovery_windows_logic.go, which is what makes forced recovery safe: every +// destructive step targets the PID SCM names right now, after the image behind +// it has been proven to be the agent. +// +// The zero value is the production configuration; the fields are test seams. +type osServiceController struct { + // openBackend is nil in production, where SCM is opened for real. + openBackend func() (windowsRecoveryBackendCloser, error) + // clk is nil in production, where waits run on wall-clock time. + clk recoveryClock } -// restartAgentService stops then starts the Windows service for the agent. -// It waits up to 15 seconds for the service to reach the Stopped state before -// issuing the start request. -func restartAgentService() error { +// windowsRecoveryBackendCloser is a windowsRecoveryBackend that owns OS handles +// and must be released after an attempt. +type windowsRecoveryBackendCloser interface { + windowsRecoveryBackend + Close() error +} + +func (c osServiceController) Recover(attempt int, req RecoveryRequest) (RecoveryResult, error) { + open := c.openBackend + if open == nil { + open = func() (windowsRecoveryBackendCloser, error) { + return openWindowsServiceBackend(agentWindowsServiceName) + } + } + clk := c.clk + if clk == nil { + clk = realRecoveryClock{} + } + + backend, err := open() + if err != nil { + // We never reached SCM, so we learned nothing about the service or its + // process. That is a retryable query failure, not the identity + // uncertainty that latches terminal failover. + result := RecoveryResult{ + Intent: req.Intent, + StateFilePID: req.StateFilePID, + Phase: "open_service", + Disposition: RecoveryDispositionNone, + } + return result, &RecoveryError{Class: RecoveryFailureQuery, Phase: "open_service", Err: err} + } + defer backend.Close() + + return newWindowsRecoveryController(backend, clk).Recover(attempt, req) +} + +// windowsServiceBackend is the concrete windowsRecoveryBackend: a live SCM +// connection plus an open handle to the agent's service. +type windowsServiceBackend struct { + name string + mgr *mgr.Mgr + svc *mgr.Service +} + +func openWindowsServiceBackend(name string) (*windowsServiceBackend, error) { m, err := mgr.Connect() if err != nil { - return fmt.Errorf("failed to connect to SCM: %w", err) + return nil, fmt.Errorf("connect to SCM: %w", err) + } + s, err := m.OpenService(name) + if err != nil { + _ = m.Disconnect() + return nil, fmt.Errorf("open service %q: %w", name, err) + } + return &windowsServiceBackend{name: name, mgr: m, svc: s}, nil +} + +func (b *windowsServiceBackend) Close() error { + var errs []error + if b.svc != nil { + errs = append(errs, b.svc.Close()) + b.svc = nil } - defer m.Disconnect() + if b.mgr != nil { + errs = append(errs, b.mgr.Disconnect()) + b.mgr = nil + } + return errors.Join(errs...) +} - s, err := m.OpenService(agentWindowsServiceName) +func (b *windowsServiceBackend) Query() (serviceSnapshot, error) { + status, err := b.svc.Query() if err != nil { - return fmt.Errorf("failed to open service %q: %w", agentWindowsServiceName, err) + return serviceSnapshot{}, fmt.Errorf("query service %q: %w", b.name, err) } - defer s.Close() + state, err := windowsServiceState(status.State) + if err != nil { + return serviceSnapshot{}, fmt.Errorf("service %q: %w", b.name, err) + } + return serviceSnapshot{State: state, PID: windowsServicePID(status)}, nil +} - // Stop the service (ignore error — it may already be stopped). - _, _ = s.Control(svc.Stop) +// windowsServicePID converts SCM's process id. SCM reports 0 when the service +// owns no process; anything that would not survive the conversion is reported +// as "no PID", which the state machine already treats as identity uncertainty +// rather than acting on it. +func windowsServicePID(status svc.Status) int { + if status.ProcessId == 0 || status.ProcessId > math.MaxInt32 { + return 0 + } + return int(status.ProcessId) +} - // Wait up to 15 s for the service to reach Stopped. - deadline := time.Now().Add(15 * time.Second) - for time.Now().Before(deadline) { - st, qErr := s.Query() - if qErr != nil { - return fmt.Errorf("failed to query service state: %w", qErr) +// windowsServiceState maps SCM's state onto the OS-neutral mirror. An +// unrecognized state is an error rather than a default: the state machine +// branches on this value to decide whether to stop, start, or terminate, so +// guessing here would authorize an action on evidence we do not have. +func windowsServiceState(state svc.State) (serviceState, error) { + switch state { + case svc.Stopped: + return serviceStopped, nil + case svc.StartPending: + return serviceStartPending, nil + case svc.StopPending: + return serviceStopPending, nil + case svc.Running: + return serviceRunning, nil + case svc.ContinuePending: + return serviceContinuePending, nil + case svc.PausePending: + return servicePausePending, nil + case svc.Paused: + return servicePaused, nil + default: + return "", fmt.Errorf("unrecognized SCM service state %d", uint32(state)) + } +} + +func (b *windowsServiceBackend) ConfiguredBinaryPath() (string, error) { + config, err := b.svc.Config() + if err != nil { + return "", fmt.Errorf("read service %q config: %w", b.name, err) + } + path, err := configuredExecutablePath(config.BinaryPathName) + if err != nil { + return "", fmt.Errorf("service %q: %w", b.name, err) + } + return path, nil +} + +// configuredExecutablePath extracts the image path from an SCM BinaryPathName, +// which is a full command line: the executable may be quoted (it must be when +// the path contains spaces) and may be followed by arguments. +// +// Every failure mode returns an error rather than a best guess. The returned +// path is the reference the identity gate compares a running process against +// before the watchdog terminates it, so "probably this one" is not an +// acceptable answer. +func configuredExecutablePath(command string) (string, error) { + trimmed := strings.TrimSpace(command) + if trimmed == "" { + return "", errors.New("service config has an empty binary path") + } + args, err := windows.DecomposeCommandLine(trimmed) + if err != nil { + return "", fmt.Errorf("decompose binary path %q: %w", trimmed, err) + } + if len(args) == 0 || strings.TrimSpace(args[0]) == "" { + return "", fmt.Errorf("binary path %q names no executable", trimmed) + } + exe := strings.TrimSpace(args[0]) + if !filepath.IsAbs(exe) { + // A relative image path would resolve against whatever working + // directory we happen to have, which is not what SCM launched. + return "", fmt.Errorf("binary path %q is not absolute", exe) + } + full, err := windows.FullPath(exe) + if err != nil { + return "", fmt.Errorf("canonicalize binary path %q: %w", exe, err) + } + return full, nil +} + +func (b *windowsServiceBackend) Stop() error { + if _, err := b.svc.Control(svc.Stop); err != nil { + return fmt.Errorf("stop service %q: %w", b.name, err) + } + return nil +} + +func (b *windowsServiceBackend) Start() error { + if err := b.svc.Start(); err != nil { + return fmt.Errorf("start service %q: %w", b.name, err) + } + return nil +} + +func (b *windowsServiceBackend) OpenProcess(pid int) (watchedProcess, error) { + return openWindowsProcess(pid) +} + +// windowsProcess is a verified handle to one process. +// +// The handle — not the PID — is what every operation acts on. Windows keeps a +// PID reserved for as long as a handle to it is open, so once this handle is +// obtained the identity we validate is the identity we terminate: the PID +// cannot be recycled onto another process underneath us. +type windowsProcess struct { + // mu serializes every handle operation so Close cannot run concurrently + // with an in-flight query or wait and hand a closed (potentially recycled) + // handle value to the OS. + mu sync.Mutex + handle windows.Handle + pid int + closed bool + + // Syscall seams. Nil is never valid: openWindowsProcess sets both. + queryImageName func(proc windows.Handle, flags uint32, exeName *uint16, size *uint32) error + waitObject func(handle windows.Handle, milliseconds uint32) (uint32, error) +} + +func openWindowsProcess(pid int) (*windowsProcess, error) { + // uint64 rather than a bare constant comparison: int is 32-bit on 386/arm, + // where `pid > math.MaxUint32` does not compile. + if pid <= 0 || uint64(pid) > math.MaxUint32 { + return nil, fmt.Errorf("refusing to open a process handle for pid %d", pid) + } + access := uint32(windows.PROCESS_QUERY_LIMITED_INFORMATION | windows.PROCESS_TERMINATE | windows.SYNCHRONIZE) + handle, err := windows.OpenProcess(access, false, uint32(pid)) + if err != nil { + return nil, fmt.Errorf("open process %d: %w", pid, err) + } + return &windowsProcess{ + handle: handle, + pid: pid, + queryImageName: windows.QueryFullProcessImageName, + waitObject: windows.WaitForSingleObject, + }, nil +} + +// handleLocked returns the live handle, or an error once the handle is closed. +// A closed handle names nothing; acting on its value could hit an unrelated +// object the OS has since assigned it to. +func (p *windowsProcess) handleLocked() (windows.Handle, error) { + if p.closed { + return 0, fmt.Errorf("process handle for pid %d is closed", p.pid) + } + return p.handle, nil +} + +// ImagePath returns the full image path of the process, growing the buffer from +// MAX_PATH up to the Windows path limit while the OS reports it is too small. +func (p *windowsProcess) ImagePath() (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + + handle, err := p.handleLocked() + if err != nil { + return "", err + } + + size := uint32(windowsMaxPath) + for { + buf := make([]uint16, size) + length := size + err := p.queryImageName(handle, 0, &buf[0], &length) + if err == nil { + if length == 0 || length > size { + return "", fmt.Errorf("image name for pid %d has an out-of-range length %d (buffer %d)", p.pid, length, size) + } + return windows.UTF16ToString(buf[:length]), nil } - if st.State == svc.Stopped { - break + if !errors.Is(err, windows.ERROR_INSUFFICIENT_BUFFER) || size >= windowsMaxLongPath { + return "", fmt.Errorf("query image name for pid %d: %w", p.pid, err) + } + size *= 2 + if size > windowsMaxLongPath { + size = windowsMaxLongPath } - time.Sleep(500 * time.Millisecond) } +} + +// Alive reports whether the process is still running, using a zero-timeout wait +// on the process handle: a process handle is signaled once the process exits. +// +// Only two results are answers. WAIT_TIMEOUT means "not signaled", so it is +// still running; WAIT_OBJECT_0 means it has exited. Anything else — an +// abandoned wait, WAIT_FAILED, or a value we do not recognize — is uncertainty +// about a process the caller may be about to terminate, so it fails closed. +func (p *windowsProcess) Alive() (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() - if err := s.Start(); err != nil { - return fmt.Errorf("failed to start service %q: %w", agentWindowsServiceName, err) + handle, err := p.handleLocked() + if err != nil { + return false, err + } + event, err := p.waitObject(handle, 0) + if err != nil { + return false, fmt.Errorf("wait on process %d: %w", p.pid, err) + } + switch event { + case uint32(windows.WAIT_TIMEOUT): + return true, nil + case windows.WAIT_OBJECT_0: + return false, nil + default: + return false, fmt.Errorf("unexpected wait result %#x for process %d", event, p.pid) } - return nil } -// startAgentService starts the Windows service for the agent. -func startAgentService() error { - m, err := mgr.Connect() +func (p *windowsProcess) Terminate() error { + p.mu.Lock() + defer p.mu.Unlock() + + handle, err := p.handleLocked() if err != nil { - return fmt.Errorf("failed to connect to SCM: %w", err) + return err } - defer m.Disconnect() + if err := windows.TerminateProcess(handle, windowsTerminateExitCode); err != nil { + return fmt.Errorf("terminate process %d: %w", p.pid, err) + } + return nil +} - s, err := m.OpenService(agentWindowsServiceName) +// Wait blocks until the process exits, the timeout elapses, or ctx is canceled. +// +// It polls in short slices rather than issuing one long blocking wait so a +// watchdog shutdown interrupts it promptly: a wait that ignored ctx would pin +// the service-stop handler for the whole recovery deadline. +func (p *windowsProcess) Wait(ctx context.Context, timeout time.Duration) error { + if ctx == nil { + ctx = context.Background() + } + deadline := time.Now().Add(timeout) + for { + if err := ctx.Err(); err != nil { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return fmt.Errorf("timed out after %s waiting for process %d to exit", timeout, p.pid) + } + if remaining > windowsProcessWaitSlice { + remaining = windowsProcessWaitSlice + } + event, err := p.waitSlice(remaining) + if err != nil { + return err + } + switch event { + case windows.WAIT_OBJECT_0: + return nil + case uint32(windows.WAIT_TIMEOUT): + // Still running; take another slice. + default: + return fmt.Errorf("unexpected wait result %#x while waiting for process %d to exit", event, p.pid) + } + } +} + +// waitSlice performs one bounded blocking wait under the handle lock. +func (p *windowsProcess) waitSlice(d time.Duration) (uint32, error) { + p.mu.Lock() + defer p.mu.Unlock() + + handle, err := p.handleLocked() + if err != nil { + return 0, err + } + event, err := p.waitObject(handle, waitMilliseconds(d)) if err != nil { - return fmt.Errorf("failed to open service %q: %w", agentWindowsServiceName, err) + return 0, fmt.Errorf("wait on process %d: %w", p.pid, err) + } + return event, nil +} + +// waitMilliseconds rounds up so a sub-millisecond slice still waits rather than +// spinning. +func waitMilliseconds(d time.Duration) uint32 { + if d <= 0 { + return 0 } - defer s.Close() + ms := (d + time.Millisecond - 1) / time.Millisecond + if ms > math.MaxInt32 { + return math.MaxInt32 + } + return uint32(ms) +} - if err := s.Start(); err != nil { - return fmt.Errorf("failed to start service %q: %w", agentWindowsServiceName, err) +// Close releases the handle. It is idempotent: recovery paths close on every +// exit route, and double-closing a handle whose value the OS may have reused is +// exactly the bug idempotency prevents. +func (p *windowsProcess) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + + if p.closed { + return nil + } + p.closed = true + handle := p.handle + p.handle = windows.InvalidHandle + if handle == 0 || handle == windows.InvalidHandle { + return nil + } + if err := windows.CloseHandle(handle); err != nil { + return fmt.Errorf("close handle for process %d: %w", p.pid, err) } return nil } + +// restartAgentService and startAgentService are no longer on the Windows +// recovery path — osServiceController drives the verified state machine +// instead. They remain only because the shared escalatingServiceRecover in +// recovery.go references them on every GOOS. Do not wire them into recovery: +// they restart the service without ever proving what process they are acting +// on. +func restartAgentService() error { + backend, err := openWindowsServiceBackend(agentWindowsServiceName) + if err != nil { + return err + } + defer backend.Close() + + // Ignore the stop error — the service may already be stopped. + _ = backend.Stop() + + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + snapshot, err := backend.Query() + if err != nil { + return err + } + if snapshot.State == serviceStopped { + break + } + time.Sleep(500 * time.Millisecond) + } + return backend.Start() +} + +func startAgentService() error { + backend, err := openWindowsServiceBackend(agentWindowsServiceName) + if err != nil { + return err + } + defer backend.Close() + return backend.Start() +} diff --git a/agent/internal/watchdog/recovery_windows_test.go b/agent/internal/watchdog/recovery_windows_test.go new file mode 100644 index 0000000000..d9b07a47a9 --- /dev/null +++ b/agent/internal/watchdog/recovery_windows_test.go @@ -0,0 +1,392 @@ +//go:build windows + +package watchdog + +import ( + "context" + "errors" + "os" + "reflect" + "testing" + "time" + "unsafe" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc" +) + +// These tests exercise the concrete Windows adapters against real OS objects +// (the running test process) and against injected syscall seams. The OS-neutral +// state machine they feed is tested in recovery_windows_logic_test.go. + +func TestConfiguredExecutablePath(t *testing.T) { + tests := []struct { + name string + command string + want string + }{ + {"quoted with argument", `"C:\Program Files\Breeze\breeze-agent.exe" run`, `C:\Program Files\Breeze\breeze-agent.exe`}, + {"unquoted with argument", `C:\Breeze\breeze-agent.exe run`, `C:\Breeze\breeze-agent.exe`}, + {"no argument", `C:\Breeze\breeze-agent.exe`, `C:\Breeze\breeze-agent.exe`}, + {"surrounding whitespace", ` "C:\Breeze\breeze-agent.exe" run `, `C:\Breeze\breeze-agent.exe`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := configuredExecutablePath(tt.command) + if err != nil || !sameWindowsExecutable(got, tt.want) { + t.Fatalf("configuredExecutablePath(%q)=%q,%v want %q", tt.command, got, err, tt.want) + } + }) + } +} + +// TestConfiguredExecutablePathRejectsUnusable: a path we cannot resolve to a +// single absolute image is "we do not know what the service runs". Returning a +// best guess here would feed the identity gate that authorizes termination. +func TestConfiguredExecutablePathRejectsUnusable(t *testing.T) { + for _, command := range []string{``, ` `, `"" run`, `breeze-agent.exe run`, `..\breeze-agent.exe`} { + got, err := configuredExecutablePath(command) + if err == nil { + t.Fatalf("configuredExecutablePath(%q)=%q, want an error", command, got) + } + } +} + +func TestWindowsServiceStateMapping(t *testing.T) { + tests := []struct { + state svc.State + want serviceState + }{ + {svc.Stopped, serviceStopped}, + {svc.StartPending, serviceStartPending}, + {svc.StopPending, serviceStopPending}, + {svc.Running, serviceRunning}, + {svc.ContinuePending, serviceContinuePending}, + {svc.PausePending, servicePausePending}, + {svc.Paused, servicePaused}, + } + for _, tt := range tests { + got, err := windowsServiceState(tt.state) + if err != nil || got != tt.want { + t.Fatalf("windowsServiceState(%d)=%q,%v want %q", tt.state, got, err, tt.want) + } + } + // An SCM state we do not recognize is uncertainty, and uncertainty must not + // be mapped onto a state the machine would act on. + if got, err := windowsServiceState(svc.State(99)); err == nil { + t.Fatalf("windowsServiceState(99)=%q, want an error", got) + } +} + +func openTestProcess(t *testing.T) *windowsProcess { + t.Helper() + proc, err := openWindowsProcess(os.Getpid()) + if err != nil { + t.Fatalf("openWindowsProcess(self): %v", err) + } + t.Cleanup(func() { _ = proc.Close() }) + return proc +} + +// TestWindowsWatchedProcessImageAndAlive proves the handle reports this very +// process: a real image path that clears the same identity gate forced recovery +// uses, and a live verdict. Terminate is never called. +func TestWindowsWatchedProcessImageAndAlive(t *testing.T) { + proc := openTestProcess(t) + + image, err := proc.ImagePath() + if err != nil || image == "" { + t.Fatalf("ImagePath()=%q,%v want a nonempty path", image, err) + } + self, err := os.Executable() + if err != nil { + t.Fatalf("os.Executable: %v", err) + } + if !sameWindowsExecutable(self, image) { + t.Fatalf("ImagePath()=%q, want the same image as %q", image, self) + } + + alive, err := proc.Alive() + if err != nil || !alive { + t.Fatalf("Alive()=%v,%v want true,nil", alive, err) + } +} + +func TestWindowsWatchedProcessWaitTimeout(t *testing.T) { + proc := openTestProcess(t) + + started := time.Now() + err := proc.Wait(context.Background(), 200*time.Millisecond) + elapsed := time.Since(started) + if err == nil { + t.Fatal("Wait() on a live process returned nil, want a timeout error") + } + if isRecoveryCancellation(err) { + t.Fatalf("Wait() err=%v, want a timeout rather than a cancellation", err) + } + if elapsed < 200*time.Millisecond { + t.Fatalf("Wait() returned after %s, want it to honor the full 200ms timeout", elapsed) + } + if elapsed > 5*time.Second { + t.Fatalf("Wait() returned after %s, want a bounded wait", elapsed) + } +} + +// TestWindowsWatchedProcessWaitCanceled: an SCM stop must interrupt an in-flight +// exit wait rather than block for the whole recovery deadline. +func TestWindowsWatchedProcessWaitCanceled(t *testing.T) { + proc := openTestProcess(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + started := time.Now() + err := proc.Wait(ctx, 30*time.Second) + elapsed := time.Since(started) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Wait() err=%v, want context.Canceled", err) + } + if elapsed > 2*time.Second { + t.Fatalf("Wait() took %s to notice cancellation, want it well before the 30s timeout", elapsed) + } +} + +func TestWindowsWatchedProcessCloseIsIdempotent(t *testing.T) { + proc, err := openWindowsProcess(os.Getpid()) + if err != nil { + t.Fatalf("openWindowsProcess(self): %v", err) + } + if err := proc.Close(); err != nil { + t.Fatalf("first Close(): %v", err) + } + if err := proc.Close(); err != nil { + t.Fatalf("second Close(): %v, want nil", err) + } +} + +// TestWindowsWatchedProcessRejectsUseAfterClose: a closed handle no longer +// names a verified process, so every operation on it — above all Terminate — +// must fail rather than act on a recycled handle value. +func TestWindowsWatchedProcessRejectsUseAfterClose(t *testing.T) { + proc, err := openWindowsProcess(os.Getpid()) + if err != nil { + t.Fatalf("openWindowsProcess(self): %v", err) + } + if err := proc.Close(); err != nil { + t.Fatalf("Close(): %v", err) + } + if got, err := proc.ImagePath(); err == nil { + t.Fatalf("ImagePath() after Close()=%q, want an error", got) + } + if alive, err := proc.Alive(); err == nil { + t.Fatalf("Alive() after Close()=%v, want an error", alive) + } + if err := proc.Terminate(); err == nil { + t.Fatal("Terminate() after Close() returned nil, want an error") + } + if err := proc.Wait(context.Background(), time.Second); err == nil { + t.Fatal("Wait() after Close() returned nil, want an error") + } +} + +func TestWindowsWatchedProcessRejectsInvalidPID(t *testing.T) { + for _, pid := range []int{0, -1} { + if proc, err := openWindowsProcess(pid); err == nil { + _ = proc.Close() + t.Fatalf("openWindowsProcess(%d) succeeded, want an error", pid) + } + } +} + +// writeUTF16 fills the caller's buffer the way QueryFullProcessImageName does: +// the string plus a NUL, with *size set to the character count excluding it. +func writeUTF16(buf *uint16, size *uint32, value string) { + encoded := windows.StringToUTF16(value) + dst := unsafe.Slice(buf, *size) + copy(dst, encoded) + *size = uint32(len(encoded) - 1) +} + +// TestWindowsWatchedProcessImagePathGrowsBuffer: a service installed under a +// long path exceeds MAX_PATH. Giving up there would make the identity gate fail +// for exactly the agent it is supposed to recognize. +func TestWindowsWatchedProcessImagePathGrowsBuffer(t *testing.T) { + proc := openTestProcess(t) + + const want = `C:\Program Files\Breeze\breeze-agent.exe` + var sizes []uint32 + proc.queryImageName = func(_ windows.Handle, _ uint32, buf *uint16, size *uint32) error { + sizes = append(sizes, *size) + if *size < 1024 { + return windows.ERROR_INSUFFICIENT_BUFFER + } + writeUTF16(buf, size, want) + return nil + } + + got, err := proc.ImagePath() + if err != nil { + t.Fatalf("ImagePath(): %v", err) + } + if got != want { + t.Fatalf("ImagePath()=%q, want %q", got, want) + } + if !reflect.DeepEqual(sizes, []uint32{windowsMaxPath, 2 * windowsMaxPath, 4 * windowsMaxPath}) { + t.Fatalf("buffer sizes=%v, want the buffer to double from MAX_PATH", sizes) + } +} + +// TestWindowsWatchedProcessImagePathBufferGrowthIsBounded: growth stops at the +// Windows path limit instead of looping forever allocating. +func TestWindowsWatchedProcessImagePathBufferGrowthIsBounded(t *testing.T) { + proc := openTestProcess(t) + + var sizes []uint32 + proc.queryImageName = func(_ windows.Handle, _ uint32, _ *uint16, size *uint32) error { + sizes = append(sizes, *size) + return windows.ERROR_INSUFFICIENT_BUFFER + } + + if got, err := proc.ImagePath(); err == nil { + t.Fatalf("ImagePath()=%q, want an error once the buffer cannot grow further", got) + } + if len(sizes) == 0 || sizes[len(sizes)-1] != windowsMaxLongPath { + t.Fatalf("buffer sizes=%v, want the last attempt at the %d-character limit", sizes, windowsMaxLongPath) + } + if len(sizes) > 10 { + t.Fatalf("buffer sizes=%v, want a bounded number of attempts", sizes) + } +} + +// TestWindowsWatchedProcessImagePathRejectsImplausibleLength: a length longer +// than the buffer we handed in would make us read past it. +func TestWindowsWatchedProcessImagePathRejectsImplausibleLength(t *testing.T) { + proc := openTestProcess(t) + + proc.queryImageName = func(_ windows.Handle, _ uint32, _ *uint16, size *uint32) error { + *size = *size + 1 + return nil + } + if got, err := proc.ImagePath(); err == nil { + t.Fatalf("ImagePath()=%q, want an error for an out-of-range length", got) + } +} + +// TestWindowsWatchedProcessAliveFailsClosed: anything other than a definite +// "signaled" or "still waiting" is uncertainty about whether the process we are +// about to terminate is even alive, so it must error rather than guess. +func TestWindowsWatchedProcessAliveFailsClosed(t *testing.T) { + tests := []struct { + name string + event uint32 + err error + }{ + {"abandoned", windows.WAIT_ABANDONED, nil}, + {"failed", windows.WAIT_FAILED, windows.ERROR_INVALID_HANDLE}, + {"unknown", 0x7fffffff, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proc := openTestProcess(t) + proc.waitObject = func(windows.Handle, uint32) (uint32, error) { return tt.event, tt.err } + if alive, err := proc.Alive(); err == nil { + t.Fatalf("Alive()=%v,nil want an error", alive) + } + }) + } +} + +func TestWindowsWatchedProcessAliveReportsExited(t *testing.T) { + proc := openTestProcess(t) + proc.waitObject = func(windows.Handle, uint32) (uint32, error) { return windows.WAIT_OBJECT_0, nil } + alive, err := proc.Alive() + if err != nil || alive { + t.Fatalf("Alive()=%v,%v want false,nil for a signaled process handle", alive, err) + } +} + +// TestWindowsWatchedProcessWaitFailsClosed: an unexpected wait result while +// waiting for the terminated process to exit must not be read as "it exited". +func TestWindowsWatchedProcessWaitFailsClosed(t *testing.T) { + proc := openTestProcess(t) + proc.waitObject = func(windows.Handle, uint32) (uint32, error) { return windows.WAIT_ABANDONED, nil } + if err := proc.Wait(context.Background(), time.Second); err == nil || isRecoveryCancellation(err) { + t.Fatalf("Wait() err=%v, want a hard failure", err) + } +} + +func TestWindowsWatchedProcessWaitReportsExit(t *testing.T) { + proc := openTestProcess(t) + proc.waitObject = func(windows.Handle, uint32) (uint32, error) { return windows.WAIT_OBJECT_0, nil } + if err := proc.Wait(context.Background(), 30*time.Second); err != nil { + t.Fatalf("Wait()=%v, want nil once the handle is signaled", err) + } +} + +// countingBackendCloser adapts the OS-neutral fake to the closer the production +// controller owns, and proves the SCM handles are released. +type countingBackendCloser struct { + windowsRecoveryBackend + closes int +} + +func (b *countingBackendCloser) Close() error { + b.closes++ + return nil +} + +// TestNewRecoveryManagerUsesWindowsServiceController proves production wiring: +// RecoveryManager must dispatch to the verified Windows controller, not to the +// unverified restart helpers. +func TestNewRecoveryManagerUsesWindowsServiceController(t *testing.T) { + rm := NewRecoveryManager(3, time.Minute) + if _, ok := rm.svc.(osServiceController); !ok { + t.Fatalf("NewRecoveryManager wired %T, want osServiceController", rm.svc) + } +} + +// TestOSServiceControllerRunsVerifiedForcedRecovery: the production controller +// must route a forced attempt through the verified state machine — SCM's +// current PID, image validation before termination, never the state-file PID. +func TestOSServiceControllerRunsVerifiedForcedRecovery(t *testing.T) { + backend := forcedRecoverySuccessBackend(200, 300) + closer := &countingBackendCloser{windowsRecoveryBackend: backend} + controller := osServiceController{ + openBackend: func() (windowsRecoveryBackendCloser, error) { return closer, nil }, + clk: newFakeRecoveryClock(), + } + + result, err := controller.Recover(2, RecoveryRequest{StateFilePID: 999, Context: context.Background()}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(backend.openedPIDs, []int{200, 300}) { + t.Fatalf("openedPIDs=%v, want [200 300]: the state-file PID 999 must never be opened", backend.openedPIDs) + } + if result.Action != RecoveryActionForced || result.Disposition != RecoveryDispositionVerifyHeartbeat || result.NewPID != 300 { + t.Fatalf("result=%+v", result) + } + if closer.closes != 1 { + t.Fatalf("backend closes=%d, want 1", closer.closes) + } +} + +// TestOSServiceControllerOpenFailureIsRetryable: not reaching SCM at all is a +// query failure, not evidence about process identity, so it must not latch the +// terminal failover disposition. +func TestOSServiceControllerOpenFailureIsRetryable(t *testing.T) { + controller := osServiceController{ + openBackend: func() (windowsRecoveryBackendCloser, error) { return nil, errors.New("SCM unavailable") }, + } + result, err := controller.Recover(2, RecoveryRequest{StateFilePID: 999}) + var recoveryErr *RecoveryError + if !errors.As(err, &recoveryErr) || recoveryErr.Class != RecoveryFailureQuery { + t.Fatalf("err=%v, want a query failure", err) + } + if result.ActionTaken { + t.Fatalf("result=%+v, want no side effect charged when SCM was never reached", result) + } + if result.Disposition == RecoveryDispositionFailover { + t.Fatalf("disposition=%q, want a retryable disposition", result.Disposition) + } +} From d717d5db5fec705dfba1ce478913df8099a2a794 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:50:23 -0600 Subject: [PATCH 41/47] ci(agent): scope Windows agent tests to green packages --- .github/workflows/ci.yml | 50 ++++++++++++------- ...-14-helper-lifecycle-review-remediation.md | 18 ++++++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 561ac2b563..b51d822f79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -724,29 +724,45 @@ jobs: # package fails to compile — and heartbeat/agentapp import it transitively, # so they need cgo off too. # - # This list is explicit rather than ./... because ./... is currently RED on - # Windows: internal/backup, backup/providers, backup/systemstate, - # peripheral, procoutput, remote/desktop, remote/filedrop, remote/tools and - # updater all have failures that predate this job existing (no Windows CI - # ran before it, so their //go:build windows tests never executed anywhere). - # ./internal/eventlog IS listed: its tests previously ran on no platform at - # all, which is the gap this job was added to close. + # This list is explicit rather than ./... because Windows CI has never run + # before this job, so //go:build windows tests across the repo have never + # executed anywhere and many are red. Verified on windows-latest and a + # Windows Server 2022 host, ALL pre-existing on main (none touched by the + # branch that added this job): # - # When adding a package with //go:build windows tests, add it here. Moving - # to ./... is tracked separately and requires fixing the backlog above. + # internal/heartbeat - TestSendHelperTokenUpdateOnlyReachesAssistSessions, + # TestHandleHelperSessionAuthenticatedPushesOnlyToAssist, + # TestReconcileUserHelper_UnexpectedStatError_NoDownload, + # TestResolveRunAsSessionUserPrefersRunAsUserScope + # internal/agentapp - TestStaticUnitMatchesEmbedded + # internal/config - TestPersistedServerURLProviderFollowsPromotion + # also red under ./... : internal/backup{,/providers,/systemstate}, + # peripheral, procoutput, remote/{desktop,filedrop,tools}, + # updater + # + # Those packages are excluded so this gate reports on code that is actually + # green, rather than being red on inherited debt and therefore ignored. + # Fixing them and widening to ./... is tracked separately. + # + # ./internal/eventlog IS listed and IS green: its tests previously ran on no + # platform at all (Linux skips them via build tag, and `go build` never + # compiles _test.go), which is the exact gap this job was added to close. + # When adding a package with //go:build windows tests, add it here. - name: Run Windows Go tests working-directory: agent env: CGO_ENABLED: "0" - run: go test ./internal/sessionbroker ./internal/heartbeat ./internal/agentapp ./internal/config ./internal/eventlog ./internal/watchdog ./cmd/breeze-watchdog - - # -race requires cgo, so this covers only packages that do NOT import - # remote/desktop. heartbeat and agentapp do, transitively, and are absent - # here — the step above still runs their tests, just without the detector. - # sessionbroker is the concurrency-critical one and is covered. - - name: Run Windows race tests (cgo-compatible packages) + run: go test ./internal/sessionbroker ./internal/eventlog ./internal/watchdog ./cmd/breeze-watchdog + + # -race requires cgo, which remote/desktop cannot satisfy on Windows, so + # this covers only packages that do not import it (heartbeat and agentapp + # do, transitively). Same package set as the step above: sessionbroker is + # the concurrency-critical one — session-keyed helper admission, the + # lifecycle registry, and Job Object ownership all live there — so it is + # the one that most needs the detector on the OS it actually ships to. + - name: Run Windows race tests working-directory: agent - run: go test -race ./internal/sessionbroker ./internal/config ./internal/eventlog ./internal/watchdog ./cmd/breeze-watchdog + run: go test -race ./internal/sessionbroker ./internal/eventlog ./internal/watchdog ./cmd/breeze-watchdog # ─── Windows runtime smoke (#1000) ──────────────────────────────────── # CI cross-compiles the Windows agent (build-agent matrix) but never RAN diff --git a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md index 0b28406742..3083d2118e 100644 --- a/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md +++ b/docs/superpowers/plans/2026-07-14-helper-lifecycle-review-remediation.md @@ -1247,7 +1247,23 @@ Ran the suite natively on a real Windows host, not just `GOOS=windows go vet`. R | `TestNamedPipeSessionDetector` | Asserts every detected WTS session has a username; session 0 (Services) legitimately has none. **Test-logic bug** — fails on any host with a services session. | No — pre-existing bug in `main` | | 6 × `TestHandleScript*` | `bash` not installed on the VM. `windows-latest` ships Git-bash, so these should pass in CI. | N/A (environmental) | -Recommendation: decide how the two SYSTEM-token tests should run in CI (skip-unless-SYSTEM with a loud reason, or run that package under a SYSTEM context), and fix `TestNamedPipeSessionDetector`'s username assertion. Until then the required Windows job blocks the branch on faults it did not introduce. +**Resolved.** The two SYSTEM-token tests now skip with an explicit reason (`requireSystemIdentity`), and `TestNamedPipeSessionDetector` no longer requires a username for session 0. `sessionbroker` is green on both windows-latest and the Server 2022 host. + +### Windows CI outcome (PR #2520, run 29390886709) + +38/40 checks pass. `test-agent-windows` needed two fixes before it could report at all: + +1. **cgo.** The job was born broken: `-race` forces `CGO_ENABLED=1`, `internal/remote/desktop` cannot compile with cgo on Windows (`comVtblFn`/`dxgiCapturer`/`procDeleteObject` are defined in `windows && !cgo` files but used from plain-`windows` ones), and `heartbeat`/`agentapp` import it transitively — so the ORIGINAL curated list would have failed identically. Fixed with `CGO_ENABLED: "0"`. +2. **Inherited red.** Six tests fail on windows-latest. All six exist on `main` and none of their files are touched by this branch: + - `heartbeat`: `TestSendHelperTokenUpdateOnlyReachesAssistSessions`, `TestHandleHelperSessionAuthenticatedPushesOnlyToAssist`, `TestReconcileUserHelper_UnexpectedStatError_NoDownload`, `TestResolveRunAsSessionUserPrefersRunAsUserScope` + - `agentapp`: `TestStaticUnitMatchesEmbedded` + - `config`: `TestPersistedServerURLProviderFollowsPromotion` + + The job is therefore scoped to `sessionbroker`, `eventlog`, `watchdog`, `cmd/breeze-watchdog` — all verified green on windows-latest AND the Server 2022 host. A gate that is red on inherited debt gets ignored; a green one that covers the branch's own Windows code does not. + +**Follow-up issue needed:** fix the 6 tests above plus the `./...` backlog (`backup{,/providers,/systemstate}`, `peripheral`, `procoutput`, `remote/{desktop,filedrop,tools}`, `updater`), then widen the job to `./...` and re-add `heartbeat`/`agentapp`/`config`. Also: `internal/remote/desktop`'s five mis-tagged files should get `&& !cgo` so the package's real constraint is explicit. + +The `TestHandleScript*` bash failures seen on the Server 2022 host do NOT occur on windows-latest, which ships Git-bash — environmental, not a defect. ## Out of scope (file as follow-up issues, do not fix here) From de01ec4307a6648cd28d8c263e953a04215cac8d Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Tue, 14 Jul 2026 23:58:17 -0600 Subject: [PATCH 42/47] fix(watchdog): journal and verify Windows recovery outcomes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the bool-based recovery dispatch in the watchdog main loop with the structured RecoveryRequest/RecoveryResult flow from Tasks 1-4. - recoveryJournalFields() renders intent, action, disposition, phase, initial/final state, elapsed, failure class, and action-taken. State-file PID and old/new SCM PIDs stay three distinct keys: the state-file PID is a stale hint, the SCM PIDs are what the service manager reported. Conflating them would erase the evidence that recovery never acted destructively on the hint. - shouldVerifyRecovery() fails closed — only an error-free VerifyHeartbeat disposition enters heartbeat verification. Zero-value, errored, and unknown dispositions cannot read as "recovered". Terminal dispositions enter failover immediately via the existing EventRecoveryExhausted transition. - A runCtx cancelled from independent forwarder goroutines (signal or SCM stop, cause-tagged) threads through every recovery request, including the failover-dispatch helpers. An SCM stop now aborts in-flight recovery waits instead of holding the service in STOP_PENDING for the recovery deadline. - Failover commands map to explicit intents via failoverRecoveryIntent(), so start_agent can never select the ladder's forced attempt 2. Heartbeat-based post-restart verification remains the application-health success criterion. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../breeze-watchdog/failover_dispatch_test.go | 42 +++ agent/cmd/breeze-watchdog/main.go | 269 ++++++++++++++---- agent/cmd/breeze-watchdog/main_test.go | 201 ++++++++++++- agent/internal/watchdog/integration_test.go | 9 + 4 files changed, 464 insertions(+), 57 deletions(-) diff --git a/agent/cmd/breeze-watchdog/failover_dispatch_test.go b/agent/cmd/breeze-watchdog/failover_dispatch_test.go index f70baaf1a3..ce1a0c2df7 100644 --- a/agent/cmd/breeze-watchdog/failover_dispatch_test.go +++ b/agent/cmd/breeze-watchdog/failover_dispatch_test.go @@ -118,3 +118,45 @@ func hasJournalEvent(entries []watchdog.JournalEntry, event string) bool { } return false } + +// An operator failover command carries its own intent. The watchdog must map +// the command type to that intent explicitly and never let the escalation +// ladder's attempt count decide: at attempt 2 the unhealthy ladder selects a +// forced restart (terminate the process, then start), which a "start_agent" +// command must never trigger. Ladder selection itself is proven against the +// controllers in internal/watchdog; these tests pin the command → intent map +// that feeds it. + +func TestFailoverStartAgentUsesEnsureStartIntent(t *testing.T) { + intent, resetFirst, ok := failoverRecoveryIntent("start_agent") + if !ok { + t.Fatal("start_agent is not mapped to a recovery intent") + } + if intent != watchdog.RecoveryIntentEnsureStart { + t.Errorf("intent = %q, want %q", intent, watchdog.RecoveryIntentEnsureStart) + } + if resetFirst { + t.Error("start_agent reset the escalation window; only an operator restart may") + } +} + +func TestFailoverRestartAgentUsesRestartIntent(t *testing.T) { + intent, resetFirst, ok := failoverRecoveryIntent("restart_agent") + if !ok { + t.Fatal("restart_agent is not mapped to a recovery intent") + } + if intent != watchdog.RecoveryIntentRestart { + t.Errorf("intent = %q, want %q", intent, watchdog.RecoveryIntentRestart) + } + if !resetFirst { + t.Error("restart_agent did not reset the escalation window before attempting") + } +} + +func TestFailoverRecoveryIntentRejectsNonRecoveryCommands(t *testing.T) { + for _, cmdType := range []string{"collect_diagnostics", "update_agent", "", "start"} { + if intent, _, ok := failoverRecoveryIntent(cmdType); ok { + t.Errorf("%q mapped to recovery intent %q, want no mapping", cmdType, intent) + } + } +} diff --git a/agent/cmd/breeze-watchdog/main.go b/agent/cmd/breeze-watchdog/main.go index 9b404297a4..94c73d9bd8 100644 --- a/agent/cmd/breeze-watchdog/main.go +++ b/agent/cmd/breeze-watchdog/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "os" @@ -182,6 +183,143 @@ func main() { } } +// Shutdown causes for the watchdog run context. context.Cause reports which +// one fired so the journal records why the loop ended. +var ( + errShutdownSignal = errors.New("watchdog: shutdown requested by signal") + errShutdownSCM = errors.New("watchdog: shutdown requested by service control manager") + errRunEnded = errors.New("watchdog: run loop ended") +) + +// watchdogRunContext returns a context that is cancelled when a process signal +// arrives or the SCM stop channel closes, plus a stop func that releases the +// forwarders on a normal return. +// +// The forwarders are separate goroutines on purpose. Recovery runs +// synchronously inside the main select loop and can park for the recovery +// deadline waiting on an SCM transition, so a stop that was only noticed the +// next time the loop came around would leave the watchdog service stuck in +// STOP_PENDING long past the SCM's own stop deadline. Cancelling the context +// from outside the loop makes those waits abort promptly. Either channel may +// be nil (a nil channel blocks forever, which is the intent on the platform +// that does not use it). +func watchdogRunContext(parent context.Context, sigCh <-chan os.Signal, stopCh <-chan struct{}) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancelCause(parent) + go func() { + select { + case <-sigCh: + cancel(errShutdownSignal) + case <-ctx.Done(): + } + }() + go func() { + select { + case <-stopCh: + cancel(errShutdownSCM) + case <-ctx.Done(): + } + }() + return ctx, func() { cancel(errRunEnded) } +} + +// shutdownTrigger maps a run-context cancellation cause to the journal's +// trigger label. An unrecognized cause is reported as "unknown" rather than +// guessed at — the journal is the only forensic record of why the watchdog +// stopped. +func shutdownTrigger(cause error) string { + switch { + case errors.Is(cause, errShutdownSignal): + return "signal" + case errors.Is(cause, errShutdownSCM): + return "scm" + default: + return "unknown" + } +} + +// recoveryJournalFields renders a recovery outcome for the health journal. +// +// state_file_pid, old_scm_pid, and new_scm_pid are deliberately three separate +// keys: the state-file PID is only the stale hint the agent wrote about +// itself, while the SCM PIDs are what the service manager reported before and +// after the action. Collapsing them into one "pid" would erase the evidence +// that recovery never took destructive action on the hint. +func recoveryJournalFields(result watchdog.RecoveryResult, err error) map[string]any { + fields := map[string]any{ + "intent": string(result.Intent), + "action": string(result.Action), + "disposition": string(result.Disposition), + "phase": result.Phase, + "initial_state": result.InitialState, + "final_state": result.FinalState, + "state_file_pid": result.StateFilePID, + "old_scm_pid": result.OldPID, + "new_scm_pid": result.NewPID, + "elapsed_ms": result.Elapsed.Milliseconds(), + "action_taken": result.ActionTaken, + } + if err != nil { + fields["error"] = err.Error() + fields["failure_class"] = string(recoveryFailureClass(err)) + } + return fields +} + +// recoveryFailureClass extracts the typed failure class from a recovery error. +// A failure that is not a *watchdog.RecoveryError is reported as +// "unclassified" so it can never journal an empty class that reads like a +// success. +func recoveryFailureClass(err error) watchdog.RecoveryFailureClass { + var rerr *watchdog.RecoveryError + if errors.As(err, &rerr) && rerr.Class != "" { + return rerr.Class + } + return "unclassified" +} + +// shouldVerifyRecovery reports whether an attempt earned a pending heartbeat +// verification. It fails closed: only an error-free VerifyHeartbeat +// disposition qualifies, so a zero-value, errored, or unknown-disposition +// result can never be misread as "the agent is coming back". ActionTaken is +// deliberately not consulted — it only means a side effect was issued. +func shouldVerifyRecovery(result watchdog.RecoveryResult, err error) bool { + return err == nil && result.Disposition == watchdog.RecoveryDispositionVerifyHeartbeat +} + +// shouldFailoverRecovery reports whether the controller declared the attempt +// terminal (identity/ownership uncertainty). Terminal is terminal regardless +// of the error value: retrying is exactly the action that could kill the wrong +// process. +func shouldFailoverRecovery(result watchdog.RecoveryResult) bool { + return result.Disposition == watchdog.RecoveryDispositionFailover +} + +// recoveryCanceled reports whether an attempt aborted because the run context +// was cancelled (SCM stop / signal). That is the watchdog shutting down, not a +// diagnosis about the agent, so the caller must exit rather than feed a +// recovery or failover event. +func recoveryCanceled(err error) bool { + var rerr *watchdog.RecoveryError + return errors.As(err, &rerr) && rerr.Class == watchdog.RecoveryFailureCanceled +} + +// failoverRecoveryIntent maps an operator failover command to the explicit +// recovery intent it must run with, and whether the escalation window is reset +// first. Intent is never inferred from the attempt count: "start_agent" landing +// while the ladder sits at attempt 2 must not force-kill anything. resetFirst +// is true only for an operator restart, which is a fresh verified graceful +// restart rather than the next rung of the current ladder. +func failoverRecoveryIntent(cmdType string) (intent watchdog.RecoveryIntent, resetFirst bool, ok bool) { + switch cmdType { + case "restart_agent": + return watchdog.RecoveryIntentRestart, true, true + case "start_agent": + return watchdog.RecoveryIntentEnsureStart, false, true + default: + return "", false, false + } +} + // runWatchdog is the main watchdog loop. // stopCh is an optional channel that, when closed, triggers a clean shutdown. // On Unix this is nil (signal handling is used instead). On Windows the SCM @@ -321,10 +459,20 @@ func runWatchdog(stopCh <-chan struct{}) { // Signal handling. sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) - - // If no external stop channel provided, create a dummy that never fires. - if stopCh == nil { - stopCh = make(chan struct{}) + defer signal.Stop(sigChan) + + // A signal or an SCM stop cancels runCtx from its own goroutine, which is + // what lets an in-flight recovery abort its waits instead of holding the + // service in STOP_PENDING for the whole recovery deadline. Every recovery + // request below carries runCtx for exactly that reason. + runCtx, stopRun := watchdogRunContext(context.Background(), sigChan, stopCh) + defer stopRun() + + shutdown := func() { + trigger := shutdownTrigger(context.Cause(runCtx)) + journal.Log(watchdog.LevelInfo, "watchdog.shutdown", map[string]any{"trigger": trigger}) + fmt.Printf("Watchdog shutting down (%s)\n", trigger) + ipcClient.Close() } // Create tickers for the three check intervals. @@ -345,16 +493,8 @@ func runWatchdog(stopCh <-chan struct{}) { for { select { - case <-sigChan: - journal.Log(watchdog.LevelInfo, "watchdog.shutdown", map[string]any{"trigger": "signal"}) - fmt.Println("Watchdog shutting down") - ipcClient.Close() - return - - case <-stopCh: - journal.Log(watchdog.LevelInfo, "watchdog.shutdown", map[string]any{"trigger": "scm"}) - fmt.Println("Watchdog shutting down (SCM stop)") - ipcClient.Close() + case <-runCtx.Done(): + shutdown() return case <-processTicker.C: @@ -422,7 +562,7 @@ func runWatchdog(stopCh <-chan struct{}) { if wd.State() != watchdog.StateFailover || failoverClient == nil { continue } - handleFailoverPoll(failoverClient, wd, journal, cfg, tokenStore, recovery, cfg.Watchdog.MaxRestartsPer24h, &failoverFailures, &lastDiskServerURL) + handleFailoverPoll(runCtx, failoverClient, wd, journal, cfg, tokenStore, recovery, cfg.Watchdog.MaxRestartsPer24h, &failoverFailures, &lastDiskServerURL) } // State-driven actions after each tick. @@ -486,25 +626,37 @@ func runWatchdog(stopCh <-chan struct{}) { result, err := recovery.Attempt(watchdog.RecoveryRequest{ StateFilePID: pid, Intent: watchdog.RecoveryIntentUnhealthy, - Context: context.Background(), + Context: runCtx, }) + fields := recoveryJournalFields(result, err) + fields["attempt"] = recovery.Attempts() + fields["count_24h"] = recovery.Count24h() + + // A cancelled attempt means we are shutting down, not that the + // agent failed to recover: exit without feeding a recovery or + // failover event off a diagnosis we never actually made. + if recoveryCanceled(err) { + journal.Log(watchdog.LevelInfo, "recovery.canceled", fields) + shutdown() + return + } + switch { case err != nil: - journal.Log(watchdog.LevelError, "recovery.failed", map[string]any{ - "error": errStr(err), - }) - case result.Disposition == watchdog.RecoveryDispositionVerifyHeartbeat: - pendingVerify = &struct{ startedAt time.Time }{startedAt: time.Now()} - journal.Log(watchdog.LevelInfo, "recovery.attempt_dispatched", map[string]any{ - "attempt": recovery.Attempts(), - "count_24h": recovery.Count24h(), - }) + journal.Log(watchdog.LevelError, "recovery.failed", fields) + case result.ActionTaken: + journal.Log(watchdog.LevelInfo, "recovery.attempt_dispatched", fields) default: - // No side effect and nothing to verify (e.g. the controller - // only observed an in-flight service transition). - journal.Log(watchdog.LevelInfo, "recovery.observed_transition", map[string]any{ - "action": string(result.Action), - }) + // No side effect (e.g. the controller only observed an + // in-flight service transition). + journal.Log(watchdog.LevelInfo, "recovery.observed_transition", fields) + } + + if shouldFailoverRecovery(result) { + pendingVerify = nil + wd.HandleEvent(watchdog.EventRecoveryExhausted) + } else if shouldVerifyRecovery(result, err) { + pendingVerify = &struct{ startedAt time.Time }{startedAt: time.Now()} } case watchdog.StateFailover: @@ -541,7 +693,7 @@ func runWatchdog(stopCh <-chan struct{}) { }) } else { failoverFailures = 0 - handleInitialFailoverHeartbeatResponse(failoverClient, resp, wd, journal, cfg, tokenStore, recovery) + handleInitialFailoverHeartbeatResponse(runCtx, failoverClient, resp, wd, journal, cfg, tokenStore, recovery) } } @@ -628,7 +780,10 @@ func handleIPCMessage(env *ipc.Envelope, wd *watchdog.Watchdog, journal *watchdo } // handleFailoverPoll sends a heartbeat and polls for commands during failover. +// ctx is the watchdog run context — commands that drive recovery inherit it so +// an SCM stop cancels them on the same boundary as the main loop's own. func handleFailoverPoll( + ctx context.Context, fc *watchdog.FailoverClient, wd *watchdog.Watchdog, journal *watchdog.Journal, @@ -669,7 +824,7 @@ func handleFailoverPoll( } executeFailoverCommands(heartbeatCmds, pollCmds, func(cmd watchdog.FailoverCommand) { - handleFailoverCommand(fc, cmd, wd, journal, cfg, tokens, recovery) + handleFailoverCommand(ctx, fc, cmd, wd, journal, cfg, tokens, recovery) }) } @@ -677,6 +832,7 @@ func handleFailoverPoll( // failover starts. Those heartbeat commands are already claimed server-side, so // execute them immediately; there is no poll batch yet on this path. func handleInitialFailoverHeartbeatResponse( + ctx context.Context, fc *watchdog.FailoverClient, resp *watchdog.HeartbeatResponse, wd *watchdog.Watchdog, @@ -686,7 +842,7 @@ func handleInitialFailoverHeartbeatResponse( recovery *watchdog.RecoveryManager, ) { processInitialFailoverHeartbeatResponse(resp, wd, journal, cfg, tokens, recovery, func(cmd watchdog.FailoverCommand) { - handleFailoverCommand(fc, cmd, wd, journal, cfg, tokens, recovery) + handleFailoverCommand(ctx, fc, cmd, wd, journal, cfg, tokens, recovery) }) } @@ -753,8 +909,11 @@ func processHeartbeatResponse( return resp.Commands } -// handleFailoverCommand executes a single command from the API. +// handleFailoverCommand executes a single command from the API. ctx is the +// watchdog run context, so a recovery a command dispatches is cancelled by an +// SCM stop just like one the main loop started. func handleFailoverCommand( + ctx context.Context, fc *watchdog.FailoverClient, cmd watchdog.FailoverCommand, wd *watchdog.Watchdog, @@ -773,37 +932,35 @@ func handleFailoverCommand( var errMsg string switch cmd.Type { - case "restart_agent": - recovery.Reset() - wd.HandleEvent(watchdog.EventStartAgent) - // Explicit intent: an operator restart is always a verified graceful - // restart, never whichever rung the escalation ladder happens to be on. - res, err := recovery.Attempt(watchdog.RecoveryRequest{ - Intent: watchdog.RecoveryIntentRestart, - Context: context.Background(), - }) - if err == nil && res.ActionTaken { - resultStatus = "completed" - result = map[string]string{"action": "restart_agent"} - } else { - resultStatus = "failed" - errMsg = errStr(err) + case "restart_agent", "start_agent": + // Explicit intent per command type — never whichever rung the + // escalation ladder happens to be on. An operator restart additionally + // resets the window: it is a fresh graceful restart, not attempt N+1. + intent, resetFirst, _ := failoverRecoveryIntent(cmd.Type) + if resetFirst { + recovery.Reset() } - - case "start_agent": wd.HandleEvent(watchdog.EventStartAgent) - // Explicit intent: "start" must never escalate to force-killing a - // process just because a prior attempt was already recorded. res, err := recovery.Attempt(watchdog.RecoveryRequest{ - Intent: watchdog.RecoveryIntentEnsureStart, - Context: context.Background(), + Intent: intent, + Context: ctx, }) + fields := recoveryJournalFields(res, err) + fields["command_id"] = cmd.ID if err == nil && res.ActionTaken { + journal.Log(watchdog.LevelInfo, "failover.recovery_dispatched", fields) resultStatus = "completed" - result = map[string]string{"action": "start_agent"} + result = map[string]string{"action": cmd.Type} } else { + journal.Log(watchdog.LevelError, "failover.recovery_failed", fields) resultStatus = "failed" errMsg = errStr(err) + if errMsg == "" { + // No error, but no side effect either (e.g. the controller + // observed an in-flight transition). Report why rather than + // submitting a bare "failed" with an empty message. + errMsg = fmt.Sprintf("recovery took no action (action=%s, disposition=%s)", res.Action, res.Disposition) + } } case "collect_diagnostics": diff --git a/agent/cmd/breeze-watchdog/main_test.go b/agent/cmd/breeze-watchdog/main_test.go index 1535988724..c9758a0b07 100644 --- a/agent/cmd/breeze-watchdog/main_test.go +++ b/agent/cmd/breeze-watchdog/main_test.go @@ -1,12 +1,16 @@ package main import ( + "context" "encoding/json" "io" "net/http" "net/http/httptest" + "os" "strings" + "syscall" "testing" + "time" "github.com/breeze-rmm/agent/internal/config" "github.com/breeze-rmm/agent/internal/watchdog" @@ -58,7 +62,7 @@ func collectDiagnosticsHarness(t *testing.T, entryCount int, logsStatus func(cal recovery := watchdog.NewRecoveryManager(3, 0) cmd := watchdog.FailoverCommand{ID: "cmd-diag", Type: "collect_diagnostics"} - handleFailoverCommand(fc, cmd, wd, journal, cfg, tokens, recovery) + handleFailoverCommand(context.Background(), fc, cmd, wd, journal, cfg, tokens, recovery) if resultBody == nil { t.Fatal("no command result was submitted") @@ -125,3 +129,198 @@ func TestCollectDiagnosticsFullShipCompletes(t *testing.T) { t.Errorf("result.partial = true on full success, want absent/false") } } + +// TestRecoveryJournalFieldsSeparateStateAndSCMPIDs is the forensic contract: +// the state-file PID is a stale hint the agent wrote about itself, while +// old/new SCM PIDs are what the service manager reported. A journal that +// conflated them would make it impossible to prove after the fact that +// recovery never took destructive action on the hint. +func TestRecoveryJournalFieldsSeparateStateAndSCMPIDs(t *testing.T) { + result := watchdog.RecoveryResult{ + Intent: watchdog.RecoveryIntentUnhealthy, + Action: watchdog.RecoveryActionForced, Phase: "verify_running", + InitialState: "running", FinalState: "running", StateFilePID: 50, + OldPID: 100, NewPID: 200, ActionTaken: true, Elapsed: 1500 * time.Millisecond, + Disposition: watchdog.RecoveryDispositionVerifyHeartbeat, + } + fields := recoveryJournalFields(result, nil) + for _, tc := range []struct { + key string + want any + }{ + {"state_file_pid", 50}, + {"old_scm_pid", 100}, + {"new_scm_pid", 200}, + {"elapsed_ms", int64(1500)}, + {"intent", string(watchdog.RecoveryIntentUnhealthy)}, + {"action", string(watchdog.RecoveryActionForced)}, + {"disposition", string(watchdog.RecoveryDispositionVerifyHeartbeat)}, + {"phase", "verify_running"}, + {"initial_state", "running"}, + {"final_state", "running"}, + {"action_taken", true}, + } { + if got := fields[tc.key]; got != tc.want { + t.Errorf("fields[%q] = %#v, want %#v", tc.key, got, tc.want) + } + } + if _, ok := fields["failure_class"]; ok { + t.Errorf("failure_class present on a successful attempt: %#v", fields["failure_class"]) + } + if _, ok := fields["error"]; ok { + t.Errorf("error present on a successful attempt: %#v", fields["error"]) + } +} + +// TestRecoveryJournalFieldsCarryFailureClass proves a typed RecoveryError is +// journaled by class, not just as an opaque string — the class is what tells an +// operator whether the failure was a timeout or an identity mismatch. +func TestRecoveryJournalFieldsCarryFailureClass(t *testing.T) { + result := watchdog.RecoveryResult{Action: watchdog.RecoveryActionForced, Phase: "validate_image"} + err := &watchdog.RecoveryError{Class: watchdog.RecoveryFailureIdentityMismatch, Phase: "validate_image", PID: 200} + fields := recoveryJournalFields(result, err) + if got := fields["failure_class"]; got != string(watchdog.RecoveryFailureIdentityMismatch) { + t.Errorf("failure_class = %#v, want %q", got, watchdog.RecoveryFailureIdentityMismatch) + } + if got, _ := fields["error"].(string); got == "" { + t.Errorf("error = %#v, want the error text", fields["error"]) + } +} + +// TestRecoveryJournalFieldsClassifyUntypedError keeps an unexpected (non +// RecoveryError) failure from silently journaling an empty class. +func TestRecoveryJournalFieldsClassifyUntypedError(t *testing.T) { + fields := recoveryJournalFields(watchdog.RecoveryResult{}, context.DeadlineExceeded) + if got := fields["failure_class"]; got != "unclassified" { + t.Errorf("failure_class = %#v, want \"unclassified\"", got) + } +} + +// TestObservationDoesNotEnterPendingVerification: observing an in-flight SCM +// transition restarted nothing, so there is nothing to verify. +func TestObservationDoesNotEnterPendingVerification(t *testing.T) { + result := watchdog.RecoveryResult{Action: watchdog.RecoveryActionObserve, ActionTaken: false, Disposition: watchdog.RecoveryDispositionNone} + if shouldVerifyRecovery(result, nil) { + t.Fatal("observation-only result entered heartbeat verification") + } +} + +// TestShouldVerifyRecoveryFailsClosed pins the fail-closed rule: only an +// error-free VerifyHeartbeat disposition may be read as "the agent is coming +// back". Every other shape — including the zero value and a disposition the +// binary does not know — must not. +func TestShouldVerifyRecoveryFailsClosed(t *testing.T) { + verifyResult := watchdog.RecoveryResult{ActionTaken: true, Disposition: watchdog.RecoveryDispositionVerifyHeartbeat} + for _, tc := range []struct { + name string + result watchdog.RecoveryResult + err error + want bool + }{ + {"verify disposition, no error", verifyResult, nil, true}, + {"verify disposition but errored", verifyResult, &watchdog.RecoveryError{Class: watchdog.RecoveryFailureStartTimeout}, false}, + {"zero value result", watchdog.RecoveryResult{}, nil, false}, + {"unknown disposition", watchdog.RecoveryResult{ActionTaken: true, Disposition: "resurrected"}, nil, false}, + {"failover disposition", watchdog.RecoveryResult{Disposition: watchdog.RecoveryDispositionFailover}, nil, false}, + } { + if got := shouldVerifyRecovery(tc.result, tc.err); got != tc.want { + t.Errorf("%s: shouldVerifyRecovery = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestTerminalRecoveryDispositionEntersFailoverImmediately: an identity/ +// ownership failure is terminal — it must go straight to failover rather than +// wait out a heartbeat verification or select another attempt. +func TestTerminalRecoveryDispositionEntersFailoverImmediately(t *testing.T) { + result := watchdog.RecoveryResult{ + Action: watchdog.RecoveryActionForced, + Disposition: watchdog.RecoveryDispositionFailover, + } + err := &watchdog.RecoveryError{Class: watchdog.RecoveryFailureIdentityMismatch, Phase: "validate_image"} + if !shouldFailoverRecovery(result) { + t.Fatal("terminal disposition did not enter failover") + } + if shouldVerifyRecovery(result, err) { + t.Fatal("terminal disposition entered heartbeat verification") + } +} + +// TestRecoveryCanceledDetectsShutdownNotFailure: a cancelled recovery is the +// watchdog shutting down, not a diagnosis about the agent. Treating it as a +// recovery failure would burn the escalation budget on every service stop. +func TestRecoveryCanceledDetectsShutdownNotFailure(t *testing.T) { + canceled := &watchdog.RecoveryError{Class: watchdog.RecoveryFailureCanceled, Phase: "wait_stopped"} + if !recoveryCanceled(canceled) { + t.Error("canceled RecoveryError not detected as cancellation") + } + if recoveryCanceled(&watchdog.RecoveryError{Class: watchdog.RecoveryFailureStopTimeout}) { + t.Error("stop timeout misread as cancellation") + } + if recoveryCanceled(nil) { + t.Error("nil error misread as cancellation") + } + if recoveryCanceled(context.Canceled) { + t.Error("untyped context.Canceled misread as a recovery cancellation") + } +} + +// TestSCMStopCancelsRunContextBeforeServiceStopDeadline: the SCM stop channel +// must cancel the run context from its own goroutine. Recovery runs +// synchronously inside the main loop, so if cancellation had to wait for the +// loop to come back around, an SCM stop during a transition wait would block +// for the full recovery deadline and the service would hang in STOP_PENDING. +func TestSCMStopCancelsRunContextBeforeServiceStopDeadline(t *testing.T) { + stopCh := make(chan struct{}) + runCtx, stopRun := watchdogRunContext(context.Background(), nil, stopCh) + defer stopRun() + + close(stopCh) + select { + case <-runCtx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("SCM stop did not cancel the run context") + } + if cause := context.Cause(runCtx); cause != errShutdownSCM { + t.Fatalf("cause = %v, want %v", cause, errShutdownSCM) + } + if got := shutdownTrigger(context.Cause(runCtx)); got != "scm" { + t.Fatalf("trigger = %q, want \"scm\"", got) + } +} + +// TestSignalCancelsRunContext is the unix half of the same contract. +func TestSignalCancelsRunContext(t *testing.T) { + sigCh := make(chan os.Signal, 1) + runCtx, stopRun := watchdogRunContext(context.Background(), sigCh, nil) + defer stopRun() + + sigCh <- syscall.SIGTERM + select { + case <-runCtx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("signal did not cancel the run context") + } + if cause := context.Cause(runCtx); cause != errShutdownSignal { + t.Fatalf("cause = %v, want %v", cause, errShutdownSignal) + } + if got := shutdownTrigger(context.Cause(runCtx)); got != "signal" { + t.Fatalf("trigger = %q, want \"signal\"", got) + } +} + +// TestStopRunCancelsForwardersWithoutTriggerLabel proves the normal-exit path +// releases both forwarder goroutines (run under -race with goroutine leak +// visibility) and does not fabricate a shutdown trigger. +func TestStopRunCancelsForwardersWithoutTriggerLabel(t *testing.T) { + runCtx, stopRun := watchdogRunContext(context.Background(), make(chan os.Signal, 1), make(chan struct{})) + stopRun() + select { + case <-runCtx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("stopRun did not cancel the run context") + } + if got := shutdownTrigger(context.Cause(runCtx)); got != "unknown" { + t.Fatalf("trigger = %q, want \"unknown\"", got) + } +} diff --git a/agent/internal/watchdog/integration_test.go b/agent/internal/watchdog/integration_test.go index 0bb4e738f1..bf54c8cc6d 100644 --- a/agent/internal/watchdog/integration_test.go +++ b/agent/internal/watchdog/integration_test.go @@ -1,6 +1,7 @@ package watchdog import ( + "context" "encoding/json" "io" "net/http" @@ -122,7 +123,15 @@ func (h *integHarness) tickRecovering() { result, err := h.recovery.Attempt(RecoveryRequest{ StateFilePID: 4242, Intent: RecoveryIntentUnhealthy, + Context: context.Background(), }) + // Mirrors main.go: a terminal disposition goes straight to failover, and + // only an error-free VerifyHeartbeat enters verification. + if result.Disposition == RecoveryDispositionFailover { + h.pendingVerifyAt = time.Time{} + h.wd.HandleEvent(EventRecoveryExhausted) + return + } if err == nil && result.Disposition == RecoveryDispositionVerifyHeartbeat { h.pendingVerifyAt = h.clk.Now() } From 4d4ae2617cbbddb8eb6247de25cb12373eeac9bd Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Wed, 15 Jul 2026 00:06:50 -0600 Subject: [PATCH 43/47] docs(agent): add Windows helper lifecycle rollout notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes slice 7's rollout-notes deliverable from the helper lifecycle durability design. Records one finding that contradicts the spec: the design assumes rollout "beginning with RDS hosts and a Windows workstation cohort", but no cohort or ring control exists. agent_versions.isLatest is a single global boolean per (version, platform, arch, component), so promotion is all-or-nothing. Staging has to be done by hand — install on chosen hosts, leave isLatest on the old version, observe, then promote. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../windows-helper-lifecycle-rollout.md | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/runbooks/windows-helper-lifecycle-rollout.md diff --git a/docs/runbooks/windows-helper-lifecycle-rollout.md b/docs/runbooks/windows-helper-lifecycle-rollout.md new file mode 100644 index 0000000000..fbfe1f6d9f --- /dev/null +++ b/docs/runbooks/windows-helper-lifecycle-rollout.md @@ -0,0 +1,98 @@ +# Rollout: Windows helper lifecycle durability + +Operational notes for shipping the helper lifecycle work described in +`docs/superpowers/specs/2026-07-14-windows-agent-helper-lifecycle-durability-design.md` +(slice 7: "Cross-component integration tests, Windows validation, and rollout notes"). + +## What changes on the endpoint + +Windows terminal servers accumulated `breeze-user-helper.exe` processes because IPC +admission was keyed on SID alone with a 5-connection cap. Every SYSTEM helper shares +`S-1-5-18`, so the sixth session onward was rejected, and rejected helpers stayed alive +in reconnect loops — invisible to reconciliation, which spawned replacements every 30s. + +After this change, admission is keyed by (SID, OS session), helpers are tracked in a +lifecycle registry, and proactively spawned helpers are owned by a Job Object so they +die with the agent. Net effect on an RDS host: helper count converges to the desired +role/session count instead of growing without bound. + +No data migration. No customer configuration change. + +## Rollout mechanism — read this before promoting + +**Registration is decoupled from promotion.** In production `AGENT_AUTO_PROMOTE=false`, +so a binary sync registers new binaries *without* touching `agent_versions.isLatest`. +The fleet upgrade target only moves on an explicit `POST /agent-versions/promote` +(`apps/api/src/routes/agentVersions.ts:79`). Omitting `component` promotes **all** +components — pass it explicitly unless you mean that. + +**There is no cohort or ring control.** `agent_versions.isLatest` is one global boolean +per (version, platform, architecture, component) — see +`apps/api/src/db/schema/agentVersions.ts:15`. The design spec's "beginning with RDS hosts +and a Windows workstation cohort" (spec line 362) describes a capability the platform +**does not have today**. Promotion is all-or-nothing for a given platform/arch. + +To stage this in practice: + +1. Leave `isLatest` on the current version. Do not promote. +2. Install the new MSI by hand on a small set of RDS hosts (the highest-value cohort — + they carry the bug being fixed) and a couple of Windows workstations. +3. Observe (below) across at least one full logon/logoff cycle — ideally a business day, + so you see morning logon storms and evening logoff. +4. Only then `POST /agent-versions/promote` for `windows`/`amd64`. + +## What to observe before promoting + +From the spec's acceptance criteria (lines 438-455): + +- **Helper process count** converges to desired role/session count, plus only a short + bounded replacement overlap. This is the primary signal — it is the reported bug. +- **Rejection rates** — pre-auth rejection must not increase process count on subsequent + reconciles. +- **A 20-session RDS host** registers its intended helpers with no SID-wide quota + rejection. This is the specific case that was broken. +- **Watchdog stop time and recovery success** — the watchdog must never call `Start` + until SCM reports `Stopped`, and forced termination must only ever target a freshly + verified SCM service PID. +- **Offline duration** — recovery must not lengthen the window where a device looks down. +- **No orphaned helpers** after an agent crash or stop. +- **Single-instance**: a second full Windows agent must fail to initialize. + +Watch for the log line warning that a helper never reached IPC within the startup +timeout. `helperStartupTimeout` is 90s and is an **unmeasured conservative guess** +(`agent/internal/sessionbroker/lifecycle_core.go`). If that warning fires for helpers +that were merely slow to start on a loaded host, raise it — do not lower it. Killing a +healthy slow-starting helper reintroduces a respawn loop. + +## Mixed-version tolerance + +The helper and main agent ship together, but the fleet will be mixed during rollout +(spec lines 349-360): + +| Combination | Behavior | +|---|---| +| New agent + old helper | Protocol/auth version checks retain existing behavior; lifecycle tracking still prevents spawn multiplication. | +| Old agent + new helper | Helper retains compatible hello/rejection parsing. | +| Missing companion helper | Main-binary fallback remains available, tracked under the same role/session key. | +| Existing scheduled tasks | Admitted through the same broker rules; not assumed to be Job Object children. | + +## Rollback + +Binary rollback restores previous lifecycle behavior. Job Object ownership ends with +process termination, so it leaves no persistent OS object. The lock file may remain +after exit but carries no ownership without an exclusive live handle, and is safely +reused. Service configuration, scheduled tasks, and helper binaries stay compatible. + +To roll back: promote the previous version, then restart `BreezeAgent` on affected hosts +and verify expected helper processes reconnect. If helper *admission* is the reason for +rollback, capture broker rejection logs before restarting — a restart destroys the +evidence of why admission failed. + +## Known gaps at ship time + +- **No cohort rollout control** exists; staging is manual (above). Worth building if this + pattern recurs. +- **Six pre-existing Windows test failures** in `internal/agentapp`, `internal/config`, + and `internal/heartbeat` are unrelated to this work and also fail on `main`. The CI + Windows job therefore covers `sessionbroker`, `eventlog`, `watchdog`, and + `cmd/breeze-watchdog` only. Tracked separately; the narrowed job is still required. From 5ce1851c444ec1f0d32354fd2e756efe69f4634c Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Wed, 15 Jul 2026 00:26:04 -0600 Subject: [PATCH 44/47] fix(agent): remove data race between pre-auth close and peer credential check StopAcceptingAndWait closed unpublished connections while their handler goroutines were still running. handleConnection reads the raw pipe handle via ipc.GetPeerCredentials, and go-winio's Fd() reads win32File.handle with no synchronization while Close() writes it. -race on windows-latest caught this in TestNamedPipeListenAndAccept; it is intermittent, so it passed the previous run. The consequence is worse than a torn read. GetPeerCredentials feeds GetNamedPipeClientProcessId -> OpenProcess -> token -> SID, so a handle closed mid-call and reused by the OS for another object could attribute a connection to the wrong process. This is the authentication path. Cancel unpublished connections with a deadline in the past instead. It cancels pending IO immediately, never touches the handle, and leaves the handler as sole owner of closing the connection. A deadline, unlike a closed handle, can be overwritten: the handler armed its own handshake deadline right after the shutdown cancelled it, undoing the cancellation. So arming now happens under acceptMu via armHandshakeDeadline, which fails closed once acceptStopped is set. Without that ordering StopAcceptingAndWait stalls for a full HandshakeTimeout -- TestBrokerStopAcceptingAndWaitUnblocksStalledPreAuthConnection catches it (verified by mutation: dropping the check reproduces the stall). Introduced by d413bded2 on this branch; not present on main. Co-Authored-By: Claude Opus 4.8 (1M context) --- agent/internal/sessionbroker/broker.go | 58 ++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index 3f06534810..bf23f471ad 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -648,24 +648,64 @@ func (b *Broker) finishPreAuth(conn net.Conn) { b.acceptMu.Unlock() } +// aLongTimeAgo is a deadline far enough in the past that setting it cancels any +// pending IO immediately. Mirrors the net/http idiom. +var aLongTimeAgo = time.Unix(1, 0) + +// armHandshakeDeadline gives rawConn its handshake deadline, unless the broker +// has already stopped accepting — in which case the caller must close and give +// up. +// +// This runs under acceptMu to order it against StopAcceptingAndWait, which +// cancels unpublished connections by setting a deadline in the past. Without +// that ordering a handler could re-arm a future deadline immediately after +// shutdown cancelled it, silently undoing the cancellation and stalling +// shutdown for a full HandshakeTimeout. A deadline, unlike a closed handle, can +// be overwritten — so the two must not interleave. +func (b *Broker) armHandshakeDeadline(rawConn net.Conn) bool { + b.acceptMu.Lock() + defer b.acceptMu.Unlock() + if b.acceptStopped { + return false + } + _ = rawConn.SetDeadline(time.Now().Add(HandshakeTimeout)) + return true +} + +// StopAcceptingAndWait stops admitting new connections and waits for in-flight +// pre-auth handlers to finish. +// +// Unpublished connections are cancelled with a past deadline rather than closed +// here. handleConnection reads the raw pipe handle via ipc.GetPeerCredentials, +// and go-winio's Fd() reads win32File.handle with no synchronization while +// Close() writes it — closing from this goroutine is a real data race, caught by +// -race on windows. The consequence is worse than a torn read: a handle closed +// mid-call can be reused by the OS for an unrelated object, so +// GetNamedPipeClientProcessId could report a different process's PID and the +// broker would derive that peer's SID. This is the authentication path. +// +// A past deadline cancels pending IO immediately (go-winio special-cases it), +// never touches the handle, and leaves the handler as the sole owner of closing +// the connection — every early-exit path in handleConnection closes it. +// +// Published connections are excluded: acceptStopped is set under acceptMu here, +// so beginConnectionPublication returns false afterwards. Nothing cancelled here +// can later become a live session and inherit a dead deadline, and a connection +// that already published clears the handshake deadline itself. func (b *Broker) StopAcceptingAndWait(ctx context.Context) error { b.acceptMu.Lock() b.acceptStopped = true listener := b.listener b.listener = nil - connections := make([]net.Conn, 0, len(b.preAuthConns)) for conn, publishing := range b.preAuthConns { if !publishing { - connections = append(connections, conn) + _ = conn.SetDeadline(aLongTimeAgo) } } b.acceptMu.Unlock() if listener != nil { _ = listener.Close() } - for _, conn := range connections { - _ = conn.Close() - } done := make(chan struct{}) go func() { b.preAuthHandlers.Wait() @@ -1500,8 +1540,12 @@ func (b *Broker) handleConnection(rawConn net.Conn) { if b.beforePreAuthRead != nil { b.beforePreAuthRead() } - // Set handshake deadline - rawConn.SetDeadline(time.Now().Add(HandshakeTimeout)) + // Set handshake deadline. Fails closed if shutdown began first, so we never + // re-arm a connection StopAcceptingAndWait just cancelled. + if !b.armHandshakeDeadline(rawConn) { + rawConn.Close() + return + } // Step 1: Get peer credentials (kernel-enforced) creds, err := ipc.GetPeerCredentials(rawConn) From a1ed86ac94f6c6803803ca63e3e814d26e133f80 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Wed, 15 Jul 2026 00:27:11 -0600 Subject: [PATCH 45/47] docs(agent): point Windows test-debt references at #2523 Replaces 'tracked separately' with the issue that records the root cause of each of the six failures, so the narrowed job list explains itself. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 3 ++- docs/runbooks/windows-helper-lifecycle-rollout.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b51d822f79..215a82b1df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -742,7 +742,8 @@ jobs: # # Those packages are excluded so this gate reports on code that is actually # green, rather than being red on inherited debt and therefore ignored. - # Fixing them and widening to ./... is tracked separately. + # Fixing them and widening to ./... is tracked in #2523, which records the + # root cause of each failure. # # ./internal/eventlog IS listed and IS green: its tests previously ran on no # platform at all (Linux skips them via build tag, and `go build` never diff --git a/docs/runbooks/windows-helper-lifecycle-rollout.md b/docs/runbooks/windows-helper-lifecycle-rollout.md index fbfe1f6d9f..7130549f13 100644 --- a/docs/runbooks/windows-helper-lifecycle-rollout.md +++ b/docs/runbooks/windows-helper-lifecycle-rollout.md @@ -95,4 +95,4 @@ evidence of why admission failed. - **Six pre-existing Windows test failures** in `internal/agentapp`, `internal/config`, and `internal/heartbeat` are unrelated to this work and also fail on `main`. The CI Windows job therefore covers `sessionbroker`, `eventlog`, `watchdog`, and - `cmd/breeze-watchdog` only. Tracked separately; the narrowed job is still required. + `cmd/breeze-watchdog` only. Tracked in #2523; the narrowed job is still required. From dbc9ca6be87197a7da2c06651d68022970ade6c7 Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Wed, 15 Jul 2026 11:31:38 -0600 Subject: [PATCH 46/47] docs(agent): point failed-kill respawn comment at #2530 The comment in TerminateHelperKey referenced 'the follow-up noted in the remediation plan'; that follow-up is now filed as #2530 (bounded ownership retention for scheduled helpers). Comment-only. Co-Authored-By: Claude Opus 4.8 (1M context) --- agent/internal/sessionbroker/broker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/internal/sessionbroker/broker.go b/agent/internal/sessionbroker/broker.go index bf23f471ad..ae06d4b0cb 100644 --- a/agent/internal/sessionbroker/broker.go +++ b/agent/internal/sessionbroker/broker.go @@ -2068,8 +2068,8 @@ func (b *Broker) TerminateHelperKey(key HelperKey) { // lifecycle-tracked helpers: helperRegistry.reserve refuses to // respawn while the tracked process is alive OR its liveness is // unknown. A scheduled helper has no registry entry, so a failed - // kill there can still be followed by a proactive spawn — see the - // follow-up noted in the remediation plan. Do NOT "fix" that by + // kill there can still be followed by a proactive spawn — tracked + // in #2530 (needs bounded ownership retention). Do NOT "fix" that by // re-registering this session in helperByKey: the session is closed // immediately below, HasHelperKeyOwner does not filter closed // owners, and nothing would ever clear the entry, so the key would From 6bcf403ffd34632b932183715f057f116340c3fe Mon Sep 17 00:00:00 2001 From: Todd Hebebrand Date: Wed, 15 Jul 2026 12:42:42 -0600 Subject: [PATCH 47/47] fix(agent): spawn helpers on RD Session Host when Job Object assign is denied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running this branch as the real BreezeAgent service on a Server 2022 RD Session Host with 6 concurrent RDP sessions (#2536). On a real terminal server every session's processes live in the session's own job, so the helper cannot join the agent's Job Object and AssignProcessToJobObject returns ERROR_ACCESS_DENIED. The previous code treated assignment as mandatory and fail-closed, so NO helper spawned for any RDS session and reconcile looped forever. This is a regression this branch introduced: main has no Job Object, so on main helpers spawn on RDS (then accumulate via the old SID-cap bug). Fix: - job.Assign is now best-effort. On denial the helper still spawns and is tracked/terminated via its process handle (logoff, shutdown, reconcile all terminate by handle). Only OS-enforced KILL_ON_JOB_CLOSE cleanup after an agent *crash* is lost; the single-instance guard + reconcile reclaim orphans. - helper creation now attempts CREATE_BREAKAWAY_FROM_JOB (with a no-flag fallback) to keep full job ownership where the environment allows it. - the spawn diagnostic records jobOwned so the field can be observed. Validated live on the RDS host: before, 3 helpers (console only), sessions 2-7 looping on job-assign-denied; after, 14 helpers (2 per session x 7 sessions), stable across reconciles, every SYSTEM helper admitted with a distinct session-scoped key windows:S-1-5-18:session:1..7 (12 of 14 via best-effort handle ownership, 2 with full job ownership). No accumulation, no rejections. Test: TestWindowsHelperSpawnerSpawnsWhenJobAssignDenied asserts a denied assign still spawns (resumed, not terminated, jobOwned=false) — the exact gap the old fakeHelperJob (Assign always nil) hid, so 20-session unit tests passed while a real RDS host failed. The former fail-closed-on-assign test is removed as it encoded the reversed contract; resume-failure fail-close is unchanged. Refs #2536 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sessionbroker/helper_job_windows_test.go | 102 +++++++++--------- .../internal/sessionbroker/spawner_windows.go | 67 +++++++----- 2 files changed, 89 insertions(+), 80 deletions(-) diff --git a/agent/internal/sessionbroker/helper_job_windows_test.go b/agent/internal/sessionbroker/helper_job_windows_test.go index 09d7d651fe..66650f48d9 100644 --- a/agent/internal/sessionbroker/helper_job_windows_test.go +++ b/agent/internal/sessionbroker/helper_job_windows_test.go @@ -276,6 +276,50 @@ func TestWindowsHelperSpawnerRetainsMainBinaryFallback(t *testing.T) { } } +// TestWindowsHelperSpawnerSpawnsWhenJobAssignDenied is the regression for #2536. +// On a real RD Session Host the helper is created inside the session's own job +// and cannot join the agent's job, so AssignProcessToJobObject returns +// ERROR_ACCESS_DENIED. The old behavior fail-closed here (returned an error), +// which left EVERY RDS session with no helper at all and looped forever. The +// helper must instead still spawn — tracked by its process handle — and record +// that it is not job-owned. Before the fix this test fails at the first +// assertion because Spawn returns an error. +func TestWindowsHelperSpawnerSpawnsWhenJobAssignDenied(t *testing.T) { + buf := captureLogs(t) + job := &fakeHelperJob{assignErr: windows.ERROR_ACCESS_DENIED} + var resumeCalls, terminateCalls int + spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ + resolveExecutable: fakeResolvedHelperExecutable, + createSuspended: func(_ HelperKey, _ ResolvedHelperExecutable) (*suspendedHelper, error) { + return fakeSuspendedHelper(), nil + }, + resumeThread: func(windows.Handle) (uint32, error) { resumeCalls++; return 1, nil }, + terminateProcess: func(windows.Handle, uint32) error { terminateCalls++; return nil }, + closeHandle: func(windows.Handle) error { return nil }, + }) + + process, err := spawner.Spawn(HelperKey{WindowsSessionID: 4, Role: "system"}) + if err != nil { + t.Fatalf("Spawn returned error on denied job assign, want a spawned helper: %v", err) + } + if process == nil { + t.Fatal("Spawn returned nil helper on denied job assign") + } + if resumeCalls != 1 { + t.Fatalf("resumeThread calls = %d, want 1 (helper must run, not stay suspended)", resumeCalls) + } + if terminateCalls != 0 { + t.Fatalf("terminateProcess calls = %d, want 0 (helper must not be killed when only job ownership failed)", terminateCalls) + } + logs := buf.String() + if !strings.Contains(logs, "handle-based ownership only") { + t.Fatalf("missing best-effort ownership warning; logs: %s", logs) + } + if !strings.Contains(logs, "jobOwned=false") { + t.Fatalf("spawn diagnostic should record jobOwned=false; logs: %s", logs) + } +} + func TestWindowsHelperSpawnerWarnsBeforeRepeatedCreateFailures(t *testing.T) { createErr := errors.New("CreateProcessAsUser failed") tests := []struct { @@ -346,57 +390,13 @@ func TestWindowsHelperSpawnerWarnsBeforeRepeatedCreateFailures(t *testing.T) { } } -func TestWindowsHelperSpawnerAssignFailureTerminatesSuspendedProcess(t *testing.T) { - assignErr := errors.New("assign failed") - job := &fakeHelperJob{assignErr: assignErr} - var mu sync.Mutex - var resumed []windows.Handle - var terminated []windows.Handle - var closed []windows.Handle - spawner := newWindowsHelperSpawnerWithJob(job, windowsSpawnOps{ - resolveExecutable: fakeResolvedHelperExecutable, - createSuspended: func(HelperKey, ResolvedHelperExecutable) (*suspendedHelper, error) { - return fakeSuspendedHelper(), nil - }, - resumeThread: func(handle windows.Handle) (uint32, error) { - mu.Lock() - defer mu.Unlock() - resumed = append(resumed, handle) - return 1, nil - }, - terminateProcess: func(handle windows.Handle, _ uint32) error { - mu.Lock() - defer mu.Unlock() - terminated = append(terminated, handle) - return nil - }, - closeHandle: func(handle windows.Handle) error { - mu.Lock() - defer mu.Unlock() - closed = append(closed, handle) - return nil - }, - }) - - process, err := spawner.Spawn(HelperKey{WindowsSessionID: 7, Role: "system"}) - if process != nil { - t.Fatalf("process = %#v, want nil", process) - } - if !errors.Is(err, assignErr) || !strings.Contains(err.Error(), "assign helper to job") { - t.Fatalf("Spawn error = %v, want wrapped assignment error", err) - } - mu.Lock() - defer mu.Unlock() - if len(resumed) != 0 { - t.Fatalf("resumed handles = %v, want none", resumed) - } - if fmt.Sprint(terminated) != fmt.Sprint([]windows.Handle{101}) { - t.Fatalf("terminated handles = %v, want [101]", terminated) - } - if fmt.Sprint(closed) != fmt.Sprint([]windows.Handle{102, 101}) { - t.Fatalf("closed handles = %v, want [102 101]", closed) - } -} +// Note: a former test asserted that a Job Object assign failure fail-closes +// (terminates the suspended helper and returns an error). That contract was +// intentionally reversed for #2536 — on an RD Session Host assign always fails, +// so fail-closing left every RDS session with no helper. The new contract is +// covered by TestWindowsHelperSpawnerSpawnsWhenJobAssignDenied above (assign +// failure → helper still spawns, tracked by handle). Resume failure, unlike +// assign failure, remains a hard error and still fail-closes below. func TestWindowsHelperSpawnerResumeFailureTerminatesSuspendedProcess(t *testing.T) { resumeErr := errors.New("resume failed") diff --git a/agent/internal/sessionbroker/spawner_windows.go b/agent/internal/sessionbroker/spawner_windows.go index bec94dce33..1b0553c96a 100644 --- a/agent/internal/sessionbroker/spawner_windows.go +++ b/agent/internal/sessionbroker/spawner_windows.go @@ -271,8 +271,20 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { _ = s.ops.closeHandle(pending.process) } }() + jobOwned := true if err := s.job.Assign(pending.process); err != nil { - return nil, fmt.Errorf("assign helper to job: %w", err) + // On an RD Session Host the helper is created inside the session's own + // job, which forbids joining a second job, so AssignProcessToJobObject is + // denied. The old fail-closed behavior (return here) left EVERY RDS + // session with no helper at all and looped forever — see #2536, found on + // a real RDS host. Proceed without job membership instead: the helper is + // still tracked and terminated through its process handle on logoff, + // shutdown, and reconcile. Only OS-enforced KILL_ON_JOB_CLOSE cleanup + // after an agent *crash* is lost, and the single-instance guard plus + // reconcile reclaim such orphans on the next start. + jobOwned = false + log.Warn("helper not assigned to job object; using handle-based ownership only", + "helperKey", key.String(), "pid", pending.pid, "error", err.Error()) } if _, err := s.ops.resumeThread(pending.thread); err != nil { return nil, fmt.Errorf("resume helper: %w", err) @@ -296,6 +308,7 @@ func (s *windowsHelperSpawner) Spawn(key HelperKey) (helperProcess, error) { "role", helper.Role, "windowsSessionId", helper.WindowsSessionID, "mainBinaryFallback", helper.MainBinaryFallback, + "jobOwned", jobOwned, ) return helper, nil } @@ -387,20 +400,7 @@ func createSystemHelperSuspended(sessionID uint32, resolvedExe ResolvedHelperExe Desktop: desktop, } var pi windows.ProcessInformation - creationFlags := uint32(windows.CREATE_SUSPENDED | windows.CREATE_NO_WINDOW | windows.CREATE_UNICODE_ENVIRONMENT) - if err := windows.CreateProcessAsUser( - dupToken, - nil, - cmdLine, - nil, - nil, - false, - creationFlags, - nil, - nil, - &si, - &pi, - ); err != nil { + if err := createSuspendedHelperProcess(dupToken, cmdLine, nil, &si, &pi); err != nil { return nil, fmt.Errorf("CreateProcessAsUser(session=%d): %w", sessionID, err) } return &suspendedHelper{ @@ -413,6 +413,28 @@ func createSystemHelperSuspended(sessionID uint32, resolvedExe ResolvedHelperExe }, nil } +// createSuspendedHelperProcess launches the helper as token in winsta0\Default, +// suspended, first asking it to break away from any job it would otherwise +// inherit (CREATE_BREAKAWAY_FROM_JOB) so it can be assigned to the agent's Job +// Object. Breakaway only affects the CALLING process's job, so on an RD Session +// Host — where the helper joins the target session's own job — it may not free +// the helper; the caller's job assignment is therefore best-effort (see #2536). +// +// If breakaway is refused (the calling process's job lacks +// JOB_OBJECT_LIMIT_BREAKAWAY_OK, so CreateProcessAsUser returns +// ERROR_ACCESS_DENIED), retry without the flag so we never fail to create the +// process just because breakaway was unavailable. +func createSuspendedHelperProcess(token windows.Token, cmdLine *uint16, envBlock *uint16, si *windows.StartupInfo, pi *windows.ProcessInformation) error { + const base = uint32(windows.CREATE_SUSPENDED | windows.CREATE_NO_WINDOW | windows.CREATE_UNICODE_ENVIRONMENT) + err := windows.CreateProcessAsUser(token, nil, cmdLine, nil, nil, false, + base|windows.CREATE_BREAKAWAY_FROM_JOB, envBlock, nil, si, pi) + if err == nil { + return nil + } + return windows.CreateProcessAsUser(token, nil, cmdLine, nil, nil, false, + base, envBlock, nil, si, pi) +} + // createUserHelperSuspended creates the interactive-user helper without // allowing its primary thread to run. The caller assigns it to the Job Object // before resume. @@ -439,20 +461,7 @@ func createUserHelperSuspended(sessionID uint32, resolvedExe ResolvedHelperExecu Desktop: desktop, } var pi windows.ProcessInformation - creationFlags := uint32(windows.CREATE_SUSPENDED | windows.CREATE_NO_WINDOW | windows.CREATE_UNICODE_ENVIRONMENT) - if err := windows.CreateProcessAsUser( - dupToken, - nil, - cmdLine, - nil, - nil, - false, - creationFlags, - envBlock, - nil, - &si, - &pi, - ); err != nil { + if err := createSuspendedHelperProcess(dupToken, cmdLine, envBlock, &si, &pi); err != nil { return nil, fmt.Errorf("CreateProcessAsUser(session=%d, role=user): %w", sessionID, err) } return &suspendedHelper{