diff --git a/crates/fakecloud-cloudformation/src/service.rs b/crates/fakecloud-cloudformation/src/service.rs index e6b7c14d9..f7e10ee18 100644 --- a/crates/fakecloud-cloudformation/src/service.rs +++ b/crates/fakecloud-cloudformation/src/service.rs @@ -613,6 +613,95 @@ pub struct CloudFormationService { /// call would trigger. Empty by default (memory mode, or no services /// wired); the server populates it via `with_snapshot_hooks`. pub(crate) snapshot_hooks: BTreeMap<&'static str, SnapshotHook>, + /// Serializes StackSets auto-deployment (see + /// `reconcile_auto_deployments`): one reconciliation runs at a time, and a + /// trigger that arrives while one is in flight sets `pending` so it is + /// served by a further pass instead of being dropped. Both flags live + /// under one lock, so a trigger can never land between the running pass + /// reading `pending` and releasing `running`. + pub(crate) auto_deployment_gate: Arc>, + /// Stack sets (by admin account and id) that already have a task waiting + /// for them to go idle, so a burst of organization changes against a busy + /// stack set cannot pile up one poller per change. + pub(crate) auto_deployment_retries: Arc>>, +} + +#[derive(Default)] +pub(crate) struct AutoDeploymentGate { + pub(crate) running: bool, + pub(crate) pending: bool, +} + +/// Holds the right to run a reconciliation, releasing it on drop so a +/// cancelled request (a client that hung up mid-mutation) cannot leave +/// auto-deployment permanently marked as running. +pub(crate) struct AutoDeploymentClaim { + service: CloudFormationService, + released: bool, +} + +impl AutoDeploymentClaim { + /// Claim the right to reconcile, or record that another pass is owed and + /// return `None` because one is already running. The claim covers every + /// trigger up to this moment, so it takes the pending flag with it. + pub(crate) fn take(service: &CloudFormationService) -> Option { + let mut guard = service.auto_deployment_gate.lock(); + if guard.running { + guard.pending = true; + return None; + } + guard.running = true; + guard.pending = false; + drop(guard); + Some(Self { + service: service.clone(), + released: false, + }) + } + + /// Decide, under one lock, whether the pass that just finished owes + /// another: either the claim keeps the run (a trigger landed while it was + /// working) or it releases it. Taking both decisions together is what + /// stops a trigger slipping between "nothing pending" and "not running", + /// which would strand the reconciliation it asked for. The flag is left + /// set for the next lap to consume, so a claim dropped before that lap + /// runs (a cancelled request) still leaves the trigger recorded. + pub(crate) fn another_pass_owed(&mut self) -> bool { + let mut guard = self.service.auto_deployment_gate.lock(); + if guard.pending { + return true; + } + guard.running = false; + drop(guard); + self.released = true; + false + } +} + +impl Drop for AutoDeploymentClaim { + fn drop(&mut self) { + // Released: the reconciliation ran to the end and found nothing more + // owed, so there is nothing to carry on. + if self.released { + return; + } + self.service.auto_deployment_gate.lock().running = false; + // A pass that panicked would panic again: carrying it on would spin + // on the same failure forever. Let it surface as the panic it is. + if std::thread::panicking() { + return; + } + // Otherwise the claim is being dropped mid-run: the request that was + // reconciling was cancelled (a client that hung up). Releasing the + // gate is not enough — the pass it was serving, and any trigger that + // arrived during it, would go unserved with nobody left to notice — + // so the work carries on in a task of its own, which no request can + // cancel. + if let Ok(handle) = tokio::runtime::Handle::try_current() { + let service = self.service.clone(); + handle.spawn(async move { service.reconcile_auto_deployments().await }); + } + } } /// Everything the async CreateStack provisioning task needs to provision @@ -1019,6 +1108,8 @@ impl CloudFormationService { snapshot_lock: Arc::new(AsyncMutex::new(())), s3_store: Arc::new(fakecloud_persistence::s3::MemoryS3Store::new()), snapshot_hooks: BTreeMap::new(), + auto_deployment_gate: Arc::new(parking_lot::Mutex::new(AutoDeploymentGate::default())), + auto_deployment_retries: Arc::new(parking_lot::Mutex::new(BTreeSet::new())), } } diff --git a/crates/fakecloud-cloudformation/src/stack_sets.rs b/crates/fakecloud-cloudformation/src/stack_sets.rs index fa1d72445..9d724bbac 100644 --- a/crates/fakecloud-cloudformation/src/stack_sets.rs +++ b/crates/fakecloud-cloudformation/src/stack_sets.rs @@ -26,7 +26,7 @@ use fakecloud_core::multi_account::MultiAccountState; use fakecloud_core::service::{AwsRequest, AwsResponse, AwsServiceError}; use crate::extras::{looks_like_url, xml_response, xml_response_no_result}; -use crate::service::CloudFormationService; +use crate::service::{AutoDeploymentClaim, CloudFormationService}; use crate::state::CloudFormationState; /// Service principal Organizations uses for StackSets trusted access and @@ -72,6 +72,24 @@ pub struct StackSet { pub permission_model: String, #[serde(default)] pub auto_deployment: Option, + /// The OUs a service-managed stack set is deployed to, each with the + /// regions it is deployed to there. Kept on the stack set rather than + /// derived from its instances, so an OU that momentarily has no accounts + /// stays a target and an account created in it later is still deployed + /// to. This is what `ListStackSetAutoDeploymentTargets` reports. + #[serde(default)] + pub auto_deployment_targets: BTreeMap>, + /// `(organizational unit, account, region)` triples that must not be + /// deployed to: ones an `AccountFilterType` left out when the instances + /// were created, and ones whose instance was deleted by hand. Without + /// this, reconciling against the organization would undo an explicit + /// `DeleteStackInstances` on the next membership change. + /// + /// Keyed by the OU the decision was made under, so it lasts exactly as + /// long as the account stays there: moving to another target OU (or out + /// of the targets altogether and back) deploys to it again, as AWS does. + #[serde(default)] + pub auto_deployment_excluded: BTreeSet<(String, String, String)>, #[serde(default)] pub managed_execution_active: bool, #[serde(default)] @@ -238,11 +256,42 @@ pub fn restore_stack_sets(accounts: &mut MultiAccountState) .get_mut(&account) .and_then(|s| s.stack_sets.get_mut(&set_id)) { + backfill_auto_deployment_targets(set); settle_interrupted_operations(set); } } } +/// A stack set persisted before auto-deployment targets were kept on the set +/// itself: recover them from where it is deployed, which is what the previous +/// build derived them from. A stack set from that build with no instances +/// left has nothing to recover them from — the previous build had already +/// stopped following its OUs in that state — so it restores with none, and +/// the next CreateStackInstances sets them again. +fn backfill_auto_deployment_targets(set: &mut StackSet) { + if set.permission_model != "SERVICE_MANAGED" || !set.auto_deployment_targets.is_empty() { + return; + } + let targets: Vec<(String, String)> = set + .instances + .iter() + // A refused import never deployed anything, so its OU was never a + // target — the same rule ImportStacksToStackSet applies. + .filter(|i| i.detailed_status != "FAILED_IMPORT") + .filter_map(|i| { + i.organizational_unit_id + .as_ref() + .map(|ou| (ou.clone(), i.region.clone())) + }) + .collect(); + for (ou, region) in targets { + set.auto_deployment_targets + .entry(ou) + .or_default() + .insert(region); + } +} + fn settle_interrupted_operations(set: &mut StackSet) { for op in &mut set.operations { if !matches!(op.status.as_str(), "RUNNING" | "STOPPING") { @@ -330,6 +379,8 @@ fn migrate_legacy_stack_sets(accounts: &mut MultiAccountState>>, + key: (String, String), +} + +impl Drop for RetryGuard { + fn drop(&mut self) { + self.retries.lock().remove(&self.key); + } +} + +/// Whether a stack set follows the organization: service-managed, active, +/// with auto-deployment on and somewhere to deploy to. +fn auto_deploys(set: &StackSet) -> bool { + set.status == "ACTIVE" + && set.permission_model == "SERVICE_MANAGED" + && set.auto_deployment.as_ref().is_some_and(|a| a.enabled) + && !set.auto_deployment_targets.is_empty() +} + +/// How a stack set's planned auto-deployment ended. +#[derive(Debug, PartialEq)] +enum PlanOutcome { + /// Nothing left to do for this stack set in this pass. + Done, + /// The stack set changed under the plan; re-derive it. + Stale, + /// The plan could not be finished now — another operation owns the stack + /// set, it is still deploying, or its task died. Re-plan once the stack + /// set is idle. + Retry, +} + +/// Preferences an auto-deployment operation runs with. Every account it +/// touches got there on its own, so one account's stack failing must not +/// cancel the accounts queued behind it the way an operator-issued operation +/// with the default zero tolerance would. +/// +/// As in AWS, a tolerance that is never exceeded also means the operation +/// settles as `SUCCEEDED` even when some of its targets failed; the failures +/// are on the per-target results, which is where `ListStackSetOperationResults` +/// and each instance's status report them. +fn auto_deployment_preferences() -> OperationPreferences { + OperationPreferences { + failure_tolerance_percentage: Some(100), + ..OperationPreferences::default() + } +} + +/// One operation auto-deployment has decided to run on a stack set. +struct PlannedDeployment { + action_name: &'static str, + action: TargetAction, + retain_stacks: Option, + targets: Vec, +} + +/// How far up the OU tree a lookup walks before giving up on a cycle. Shared +/// by `accounts_under` and `account_coverage`, which have to agree: one +/// decides what a deployment covers, the other what auto-deployment tears +/// down, so a walk that reached different ancestors would delete instances it +/// had just created. +const MAX_OU_DEPTH: usize = 16; + /// What an operation does to each target. #[derive(Debug, Clone)] enum TargetAction { /// Create the instance (or re-deploy it, when it already exists) with /// these overrides. - Create { - overrides: BTreeMap, - }, + Create { overrides: BTreeMap }, /// Re-deploy the instance's stack. `None` keeps the instance's overrides. Update { overrides: Option>, }, Delete { retain_stacks: bool, + /// A DeleteStackInstances the operator asked for, as opposed to one + /// auto-deployment ran because the account left the OU. The former is + /// a decision to remember (`auto_deployment_excluded`); the latter is + /// just the organization changing. + user_requested: bool, }, } @@ -1316,7 +1436,7 @@ fn accounts_under( break; } match org.ous.get(&parent) { - Some(p) if depth < 16 => { + Some(p) if depth < MAX_OU_DEPTH => { parent = p.parent_id.clone(); depth += 1; } @@ -1541,6 +1661,8 @@ impl CloudFormationService { execution_role_name, permission_model, auto_deployment, + auto_deployment_targets: BTreeMap::new(), + auto_deployment_excluded: BTreeSet::new(), managed_execution_active, instances: Vec::new(), operations: Vec::new(), @@ -1896,6 +2018,11 @@ impl CloudFormationService { service_managed, snapshot.auto_deployment.as_ref(), )?; + if !service_managed { + // A self-managed stack set has no OU targets to follow. + updated.auto_deployment_targets.clear(); + updated.auto_deployment_excluded.clear(); + } if let Some(active) = parse_bool(params, "ManagedExecution.Active")? { updated.managed_execution_active = active; } @@ -1966,6 +2093,30 @@ impl CloudFormationService { .ok_or_else(|| stack_set_not_found(&name))?; Self::check_not_stale(current, &snapshot)?; Self::check_can_start_operation(current, &op_id)?; + // Auto-deployment bookkeeping is written without recording an + // operation (an instance re-attributed to the OU it moved into, a + // stale exclusion dropped), so `check_not_stale` cannot see it. + // Carry the live values over the snapshot this update was built + // from, unless the update itself cleared them. + if updated.permission_model == "SERVICE_MANAGED" { + updated + .auto_deployment_targets + .clone_from(¤t.auto_deployment_targets); + updated + .auto_deployment_excluded + .clone_from(¤t.auto_deployment_excluded); + for instance in &mut updated.instances { + if let Some(live) = current + .instances + .iter() + .find(|i| i.account == instance.account && i.region == instance.region) + { + instance + .organizational_unit_id + .clone_from(&live.organizational_unit_id); + } + } + } accounts .get_or_create(&admin) .stack_sets @@ -2496,6 +2647,14 @@ impl CloudFormationService { None, )?; let set_id = snapshot.stack_set_id.clone(); + self.note_created_targets(&snapshot, &admin, deployment_targets.as_ref(), ®ions); + self.note_create_exclusions( + &snapshot, + &admin, + deployment_targets.as_ref(), + &targets, + ®ions, + ); self.launch_operation( req, &admin, @@ -2641,7 +2800,10 @@ impl CloudFormationService { &op_id, &spec, targets, - TargetAction::Delete { retain_stacks }, + TargetAction::Delete { + retain_stacks, + user_requested: true, + }, ) .await; Ok(xml_response( @@ -2964,7 +3126,7 @@ impl CloudFormationService { .filter(|(_, status, _)| status != "DELETE_COMPLETE"); match action { - TargetAction::Delete { retain_stacks } => { + TargetAction::Delete { retain_stacks, .. } => { let Some((stack_id, _, _)) = live_stack else { return (Outcome::Succeeded, None, None); }; @@ -3147,9 +3309,36 @@ impl CloudFormationService { .iter() .position(|i| i.account == target.account && i.region == target.region); match (action, position) { - (TargetAction::Delete { .. }, Some(idx)) => { + (TargetAction::Delete { user_requested, .. }, Some(idx)) => { if *outcome == Outcome::Succeeded { - set.instances.remove(idx); + // An instance the operator removed stays removed: record + // it so auto-deployment does not put it back while the + // account is still in the target OU, and stop following + // the OU in that region once it holds nothing. Both are + // recorded here, on the instance that actually went away, + // so a delete that failed changes neither. + let removed = set.instances.remove(idx); + if *user_requested && set.permission_model == "SERVICE_MANAGED" { + if let Some(ou) = &removed.organizational_unit_id { + set.auto_deployment_excluded.insert(( + ou.clone(), + removed.account.clone(), + removed.region.clone(), + )); + let still_deployed = set.instances.iter().any(|i| { + i.organizational_unit_id.as_deref() == Some(ou.as_str()) + && i.region == removed.region + }); + if !still_deployed { + if let Some(regions) = set.auto_deployment_targets.get_mut(ou) { + regions.remove(&removed.region); + if regions.is_empty() { + set.auto_deployment_targets.remove(ou); + } + } + } + } + } } else if !matches!(outcome, Outcome::Cancelled(_)) { let instance = &mut set.instances[idx]; apply_to_instance(instance, outcome); @@ -3499,6 +3688,22 @@ impl CloudFormationService { } } + // What the call asked for, by OU and region, whatever each stack's + // outcome turns out to be. + let requested_pairs: BTreeSet<(String, String)> = located + .iter() + .map(|(_, account, region)| (account.clone(), region.clone())) + .collect(); + let mut requested_by_ou: BTreeMap> = BTreeMap::new(); + for (_, account, region) in &located { + if let Some(ou) = ou_of_account.get(account) { + requested_by_ou + .entry(ou.clone()) + .or_default() + .insert(region.clone()); + } + } + let mut accounts = self.state.write(); let set_id = accounts .get(&admin) @@ -3637,6 +3842,63 @@ impl CloudFormationService { .stack_sets .get_mut(&set_id) .ok_or_else(|| stack_set_not_found(&name))?; + // Adopted instances are deployments like any other: record the OUs + // and regions they landed in, or auto-deployment would see them as + // outside the stack set's targets and tear them down. Only stacks + // that were really adopted count — one refused as FAILED_IMPORT is + // not deployed anywhere — and the accounts of the OU the operator did + // not adopt a stack from are left out, exactly as an + // `AccountFilterType` leaves them out of CreateStackInstances. + if set.permission_model == "SERVICE_MANAGED" { + let adopted: Vec<&StackInstance> = new_instances + .iter() + .filter(|i| i.detailed_status == "SUCCEEDED") + .collect(); + let mut regions_by_ou: BTreeMap> = BTreeMap::new(); + for instance in &adopted { + if let Some(ou) = &instance.organizational_unit_id { + regions_by_ou + .entry(ou.clone()) + .or_default() + .insert(instance.region.clone()); + } + } + for (ou, regions) in ®ions_by_ou { + set.auto_deployment_targets + .entry(ou.clone()) + .or_default() + .extend(regions.iter().cloned()); + } + // The accounts of a targeted OU the operator named no stack from + // are the ones left out, exactly as an `AccountFilterType` leaves + // accounts out of CreateStackInstances. Derived from what was + // asked for rather than from what was adopted: a stack refused + // (already managed elsewhere, or a template mismatch) was still + // asked for, and whether any one succeeded says nothing about the + // accounts that were never mentioned. + let live: BTreeSet<(String, String)> = set + .instances + .iter() + .map(|i| (i.account.clone(), i.region.clone())) + .collect(); + for (ou, regions) in &requested_by_ou { + for (account, account_ou) in &ou_of_account { + if account_ou != ou { + continue; + } + for region in regions { + let key = (account.clone(), region.clone()); + if !requested_pairs.contains(&key) && !live.contains(&key) { + set.auto_deployment_excluded.insert(( + ou.clone(), + account.clone(), + region.clone(), + )); + } + } + } + } + } set.instances.extend(new_instances); set.operations.push(op); Ok(xml_response( @@ -3654,17 +3916,11 @@ impl CloudFormationService { let name = required(params, "StackSetName")?; let admin = self.stack_set_admin_account(req, params)?; let set = self.read_stack_set(&admin, &name, Scope::of(params))?; - let mut by_ou: BTreeMap> = BTreeMap::new(); - if set.permission_model == "SERVICE_MANAGED" { - for instance in &set.instances { - if let Some(ou) = &instance.organizational_unit_id { - by_ou - .entry(ou.clone()) - .or_default() - .insert(instance.region.clone()); - } - } - } + let by_ou = if set.permission_model == "SERVICE_MANAGED" { + set.auto_deployment_targets.clone() + } else { + BTreeMap::new() + }; let entries: Vec<(String, BTreeSet)> = by_ou.into_iter().collect(); let (page, next) = paginate(entries, params)?; let inner = format!( @@ -3688,111 +3944,767 @@ impl CloudFormationService { )) } - // ── Drift ── + // ── Auto-deployment ── + + /// How long a deployment deferred behind another operation keeps waiting + /// for the stack set to go idle before giving up on this round. Outlasts + /// `STACK_WAIT_LIMIT`, so an operation waiting on a stack that provisions + /// in the background is waited out rather than abandoned. + const AUTO_DEPLOYMENT_MAX_WAIT: std::time::Duration = + std::time::Duration::from_secs(STACK_WAIT_LIMIT.as_secs() + 60); + /// How soon it first re-checks while waiting, and the longest it lets + /// that interval grow to. + const AUTO_DEPLOYMENT_POLL: std::time::Duration = std::time::Duration::from_millis(250); + const AUTO_DEPLOYMENT_MAX_POLL: std::time::Duration = std::time::Duration::from_secs(5); + /// How long the Organizations call that triggered a deployment waits for + /// it before handing the rest over to the background. Deployments are + /// near-instant unless a template provisions something real, so in + /// practice the caller sees the whole thing done; this only stops a slow + /// template holding an API request open past a client's read timeout. + const AUTO_DEPLOYMENT_INLINE_BUDGET: std::time::Duration = std::time::Duration::from_secs(20); + /// How many times one pass re-plans a stack set whose state moved under it + /// (a concurrent UpdateStackSet) before leaving it to the next pass. + const AUTO_DEPLOYMENT_REPLANS: usize = 8; + + /// Reconcile every service-managed stack set that has `AutoDeployment` + /// enabled against the organization as it stands now. + /// + /// Organizations calls this after a mutation (an account created, invited, + /// moved between OUs, removed or closed), as does a CloudFormation stack + /// that provisions organization resources itself. An account that has + /// joined an OU a stack set is deployed to gains that stack set's + /// instances; one that has left loses them, keeping its stacks when + /// `RetainStacksOnAccountRemoval` is set. That is what makes + /// `AutoDeployment` mean anything: without it a stack set only ever covers + /// the accounts that were in the OU at CreateStackInstances time. + /// + /// Reconciling against current state rather than a membership diff keeps + /// this idempotent and independent of which call triggered it: every pass + /// re-derives what is missing, so a pass that could not run is simply + /// repeated. The cost is that a change and its exact reversal inside one + /// pass (an account that leaves a target OU and re-joins it before the + /// pass reads the organization) is indistinguishable from nothing having + /// happened, because the end state is the same. + /// + /// Instances the operator deliberately left out — filtered out by + /// `AccountFilterType`, or removed with DeleteStackInstances — are + /// remembered on the stack set (`auto_deployment_excluded`) so reconciling + /// never undoes that decision. + /// + /// Deployment runs inline, so the caller that triggered it sees it done, + /// but never waits on a stack set that is busy: that one is re-planned by + /// a background pass once the operation in its way finishes. + pub async fn reconcile_auto_deployments(&self) { + // One reconciliation at a time: a stack deployed by this one can + // itself change the organization and trigger another, and concurrent + // organization mutations would otherwise plan against each other's + // half-applied state. A trigger that arrives while one is running is + // recorded under the same lock that releases the run, so it is always + // served by a further pass rather than lost. The claim is released on + // drop, so a cancelled request cannot wedge auto-deployment. + let Some(mut claim) = AutoDeploymentClaim::take(self) else { + return; + }; + // One budget for the whole call: the caller is an API request, and + // what matters to it is how long *it* is held, not how long any one + // stack set takes. Past it every remaining deployment carries on in + // the background. + let deadline = tokio::time::Instant::now() + Self::AUTO_DEPLOYMENT_INLINE_BUDGET; + loop { + // Consume the trigger this pass serves. One that lands while it + // runs sets the flag again and is served by another lap. + self.auto_deployment_gate.lock().pending = false; + self.reconcile_auto_deployments_once(deadline).await; + if !claim.another_pass_owed() { + return; + } + } + } - fn detect_stack_set_drift( - &self, - req: &AwsRequest, - params: &BTreeMap, - ) -> Result { - let name = required(params, "StackSetName")?; - let admin = self.stack_set_admin_account(req, params)?; - let preferences = parse_preferences(params)?; - let op_id = params - .get("OperationId") + async fn reconcile_auto_deployments_once(&self, deadline: tokio::time::Instant) { + let org = match self.deps.organizations.read().as_ref() { + Some(org) => org.clone(), + None => return, + }; + let candidates: Vec<(String, String)> = { + let accounts = self.state.read(); + accounts + .iter() + .flat_map(|(admin, state)| { + let admin = admin.to_string(); + state + .stack_sets + .values() + .filter(|set| auto_deploys(set)) + .map(move |set| (admin.clone(), set.stack_set_id.clone())) + .collect::>() + }) + .collect() + }; + for (admin, set_id) in candidates { + self.reconcile_stack_set_auto_deployment(&org, &admin, &set_id, deadline) + .await; + } + } + + /// The stack set as it stands, with any background-provisioning stack it + /// is waiting on folded in first. + fn refreshed_stack_set(&self, admin: &str, set_id: &str) -> Option { + let mut accounts = self.state.write(); + Self::refresh_stack_set(&mut accounts, admin, set_id); + accounts + .get(admin) + .and_then(|s| s.stack_sets.get(set_id)) .cloned() - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + } - let set = { - let mut accounts = self.state.write(); - let set_id = accounts - .get(&admin) - .and_then(|s| active_key(s, &name, Scope::of(params))) - .ok_or_else(|| stack_set_not_found(&name))?; - Self::refresh_stack_set(&mut accounts, &admin, &set_id); - let set = accounts - .get(&admin) - .and_then(|s| s.stack_sets.get(&set_id)) - .cloned() - .ok_or_else(|| stack_set_not_found(&name))?; - Self::check_can_start_drift(&set, &op_id)?; - set + fn with_stack_set( + &self, + admin: &str, + set_id: &str, + f: impl FnOnce(&mut StackSet) -> T, + ) -> Option { + let mut accounts = self.state.write(); + accounts + .get_mut(admin) + .and_then(|s| s.stack_sets.get_mut(set_id)) + .map(f) + } + + /// Record the OUs and regions a CreateStackInstances call deploys to, so + /// auto-deployment keeps following them even after every instance in one + /// of them is gone. + fn note_created_targets( + &self, + set: &StackSet, + admin: &str, + deployment_targets: Option<&DeploymentTargets>, + regions: &[String], + ) { + if set.permission_model != "SERVICE_MANAGED" { + return; + } + let Some(dt) = deployment_targets.filter(|d| !d.organizational_unit_ids.is_empty()) else { + return; }; + self.with_stack_set(admin, &set.stack_set_id, |set| { + for ou in &dt.organizational_unit_ids { + set.auto_deployment_targets + .entry(ou.clone()) + .or_default() + .extend(regions.iter().cloned()); + } + }); + } - // Check every instance's stack against the live backing resources. - let now = Utc::now(); - let mut op = Self::new_operation(&set, &op_id, "DETECT_DRIFT", preferences, None, None); - let mut instance_drift: Vec<(String, String, String)> = Vec::new(); - for instance in &set.instances { - let stack = instance.stack_id.as_ref().and_then(|id| { - self.state.read().get(&instance.account).and_then(|s| { - s.stacks - .values() - .find(|st| &st.stack_id == id && st.status != "DELETE_COMPLETE") - .cloned() - }) - }); - let Some(stack) = stack else { - instance_drift.push(( - instance.account.clone(), - instance.region.clone(), - "UNKNOWN".to_string(), - )); - op.results.push(OperationResult { - account: instance.account.clone(), - region: instance.region.clone(), - status: "FAILED".to_string(), - status_reason: Some("Stack instance does not have a stack".to_string()), - organizational_unit_id: instance.organizational_unit_id.clone(), - account_gate_status: None, - account_gate_reason: None, - }); - continue; - }; - let mut drifted = false; - for resource in &stack.resources { - let status = match self.resource_exists(&instance.account, resource) { - Some(true) => "IN_SYNC", - Some(false) => { - drifted = true; - "DELETED" + /// Record the accounts a CreateStackInstances call deliberately leaves + /// out of its target OUs, so auto-deployment does not add them later, and + /// clear the exclusion on the instances it does deploy. + fn note_create_exclusions( + &self, + set: &StackSet, + admin: &str, + deployment_targets: Option<&DeploymentTargets>, + targets: &[Target], + regions: &[String], + ) { + if set.permission_model != "SERVICE_MANAGED" { + return; + } + let Some(dt) = deployment_targets.filter(|d| !d.organizational_unit_ids.is_empty()) else { + return; + }; + let deployed: BTreeSet<(String, String)> = targets + .iter() + .map(|t| (t.account.clone(), t.region.clone())) + .collect(); + // An instance that already exists was not left out by this call. + let live: BTreeSet<(String, String)> = set + .instances + .iter() + .map(|i| (i.account.clone(), i.region.clone())) + .collect(); + let mut excluded: BTreeSet<(String, String, String)> = BTreeSet::new(); + { + let orgs = self.deps.organizations.read(); + // Without the organization there is nothing to work out who was + // left out, but what this call deployed is still known, and the + // exclusions on those are cleared below either way. + if let Some(org) = orgs.as_ref() { + for ou in &dt.organizational_unit_ids { + for (account, _) in accounts_under(org, ou) { + for region in regions { + let key = (account.clone(), region.clone()); + if !deployed.contains(&key) && !live.contains(&key) { + excluded.insert((ou.clone(), account.clone(), region.clone())); + } + } } - None => "NOT_CHECKED", - }; - op.resource_drifts.push(InstanceResourceDrift { - account: instance.account.clone(), - region: instance.region.clone(), - stack_id: stack.stack_id.clone(), - logical_id: resource.logical_id.clone(), - physical_id: resource.physical_id.clone(), - resource_type: resource.resource_type.clone(), - status: status.to_string(), - timestamp: now, - }); + } } - let status = if drifted { "DRIFTED" } else { "IN_SYNC" }; - instance_drift.push(( - instance.account.clone(), - instance.region.clone(), - status.to_string(), - )); - op.results.push(OperationResult { - account: instance.account.clone(), - region: instance.region.clone(), - status: "SUCCEEDED".to_string(), - status_reason: None, - organizational_unit_id: instance.organizational_unit_id.clone(), - account_gate_status: None, - account_gate_reason: None, + } + self.with_stack_set(admin, &set.stack_set_id, |set| { + set.auto_deployment_excluded.retain(|(_, account, region)| { + !deployed.contains(&(account.clone(), region.clone())) }); + set.auto_deployment_excluded.extend(excluded); + }); + } + + /// Where one account stands relative to a stack set's target OUs: every + /// target OU that contains it, nearest first, paired with that OU's + /// regions. + fn account_coverage( + org: &fakecloud_organizations::OrganizationState, + targets: &BTreeMap>, + account: &str, + ) -> Vec<(String, BTreeSet)> { + let mut matched: Vec<(String, BTreeSet)> = Vec::new(); + let Some(member) = org.accounts.get(account) else { + return matched; + }; + let mut parent = member.parent_id.clone(); + // Walk from the account's own parent up to the root, so the first + // target OU found is the most specific one containing it. + for _ in 0..=MAX_OU_DEPTH { + if let Some(regions) = targets.get(&parent) { + matched.push((parent.clone(), regions.clone())); + } + match org.ous.get(&parent) { + Some(ou) => parent = ou.parent_id.clone(), + None => break, + } } - let drifted = instance_drift - .iter() - .filter(|(_, _, s)| s == "DRIFTED") - .count(); - let in_sync = instance_drift - .iter() - .filter(|(_, _, s)| s == "IN_SYNC") + matched + } + + async fn reconcile_stack_set_auto_deployment( + &self, + org: &fakecloud_organizations::OrganizationState, + admin: &str, + set_id: &str, + deadline: tokio::time::Instant, + ) { + for _ in 0..Self::AUTO_DEPLOYMENT_REPLANS { + let Some(set) = self.refreshed_stack_set(admin, set_id) else { + return; + }; + // Re-checked each lap, not just when the candidates were picked: + // an UpdateStackSet is exactly what sends this loop round again, + // and it may be the one that turned auto-deployment off. + if !auto_deploys(&set) { + return; + } + let (plan, bookkeeping_changed) = self.plan_auto_deployment(org, admin, &set); + if bookkeeping_changed { + // Re-attribution and dropped exclusions are state changes of + // their own: there may be no operation to persist behind + // (an empty plan, or one that could not start), and without + // this they would be lost on a restart. + self.save_snapshot().await; + } + match self + .run_auto_deployment_plan(admin, &set, plan, deadline) + .await + { + PlanOutcome::Done => return, + // The stack set moved under the plan (a concurrent + // UpdateStackSet): re-derive it from what is there now. + PlanOutcome::Stale => continue, + // Let whatever is in the way finish, then plan again from + // scratch rather than replaying targets that may no longer be + // the right ones. + PlanOutcome::Retry => { + self.schedule_auto_deployment_retry(admin, set_id); + return; + } + } + } + // The stack set kept changing under every attempt; hand what is left + // to a later pass rather than dropping it. + self.schedule_auto_deployment_retry(admin, set_id); + } + + /// What one stack set needs to match the organization: the instances to + /// remove, then the ones to create. + fn plan_auto_deployment( + &self, + org: &fakecloud_organizations::OrganizationState, + admin: &str, + set: &StackSet, + ) -> (Vec, bool) { + let targets = &set.auto_deployment_targets; + // Where every member account stands relative to the target OUs. The + // management account is never a service-managed target. + let mut coverage: BTreeMap)>> = BTreeMap::new(); + for account in org.accounts.values() { + if account.id == org.management_account_id { + continue; + } + let matched = Self::account_coverage(org, targets, &account.id); + if !matched.is_empty() { + coverage.insert(account.id.clone(), matched); + } + } + // The OU an instance in this region belongs to: the nearest target OU + // containing the account that is actually deployed to that region. + let ou_for = |account: &str, region: &str| -> Option { + coverage + .get(account)? + .iter() + .find_map(|(ou, regions)| regions.contains(region).then(|| ou.clone())) + }; + let covers = |account: &str, region: &str| -> bool { + coverage + .get(account) + .is_some_and(|matched| matched.iter().any(|(_, r)| r.contains(region))) + }; + // An instance whose account has left the target OUs, or whose region + // they no longer cover, is no longer the stack set's. An excluded + // instance is not deployed but is not torn down either: an exclusion + // says "do not deploy here", not "remove what is there". + let removals: Vec = set + .instances + .iter() + .filter(|i| i.organizational_unit_id.is_some()) + // A refused import records the operator's own pre-existing stack + // against the stack set without adopting it. Deleting that stack + // would destroy something the stack set never deployed. + .filter(|i| i.detailed_status != "FAILED_IMPORT") + .filter(|i| !covers(&i.account, &i.region)) + .map(|i| Target { + account: i.account.clone(), + region: i.region.clone(), + ou: i.organizational_unit_id.clone(), + suspended: false, + }) + .collect(); + // An instance that exists but never got a stack because its target + // was cancelled — the failure tolerance of an operator-issued + // operation gave out before it ran — is not "already deployed": a + // later pass tries it again. One that was tried and failed (a denying + // account gate, a template the account rejects) is left alone, or + // every later organization change would re-run a deploy that is known + // to fail. One still PENDING belongs to an operation that has not + // finished with it yet. + let deployed: BTreeSet<(String, String)> = set + .instances + .iter() + .filter(|i| { + i.stack_id.is_some() + || matches!(i.detailed_status.as_str(), "PENDING" | "FAILED") + || i.status == "INOPERABLE" + || org + .accounts + .get(&i.account) + .is_some_and(|a| a.status != "ACTIVE") + }) + .map(|i| (i.account.clone(), i.region.clone())) + .collect(); + let mut creates: Vec = Vec::new(); + for (account, matched) in &coverage { + let suspended = org + .accounts + .get(account) + .is_some_and(|a| a.status != "ACTIVE"); + let regions: BTreeSet<&String> = matched.iter().flat_map(|(_, r)| r.iter()).collect(); + for region in regions { + if deployed.contains(&(account.clone(), region.clone())) { + continue; + } + let ou = ou_for(account, region); + let excluded = ou.as_ref().is_some_and(|ou| { + set.auto_deployment_excluded.contains(&( + ou.clone(), + account.clone(), + region.clone(), + )) + }); + if excluded { + continue; + } + creates.push(Target { + account: account.clone(), + region: region.clone(), + ou, + suspended, + }); + } + } + // Stale bookkeeping: an instance that has moved into another target + // OU is re-attributed to it, and an exclusion stops applying once the + // target OUs no longer cover it, so re-joining deploys again. + let reattributed: Vec<(String, String, String)> = set + .instances + .iter() + .filter(|i| i.organizational_unit_id.is_some()) + .filter_map(|i| { + let ou = ou_for(&i.account, &i.region)?; + // Keep the OU an instance was deployed through while that OU + // still covers it, so reconciling never rewrites an + // attribution CreateStackInstances chose. + let current = i.organizational_unit_id.as_deref()?; + let still_valid = coverage + .get(&i.account) + .is_some_and(|m| m.iter().any(|(o, r)| o == current && r.contains(&i.region))); + (!still_valid).then(|| (i.account.clone(), i.region.clone(), ou)) + }) + .collect(); + // An exclusion lapses as soon as the account stops resolving to the + // OU it was made under: it left the targets, or moved into another + // one, which is a membership change that deploys to it again. + let stale_exclusions: Vec<(String, String, String)> = set + .auto_deployment_excluded + .iter() + .filter(|(ou, account, region)| ou_for(account, region).as_deref() != Some(ou.as_str())) + .cloned() + .collect(); + let bookkeeping_changed = !reattributed.is_empty() || !stale_exclusions.is_empty(); + if bookkeeping_changed { + self.with_stack_set(admin, &set.stack_set_id, |set| { + for (account, region, ou) in &reattributed { + if let Some(instance) = set + .instances + .iter_mut() + .find(|i| i.account == *account && i.region == *region) + { + instance.organizational_unit_id = Some(ou.clone()); + } + } + for key in &stale_exclusions { + set.auto_deployment_excluded.remove(key); + } + }); + } + // Removals first, so an account that moved to an OU deployed in other + // regions does not briefly hold both sets of stacks. + let mut plan: Vec = Vec::new(); + if !removals.is_empty() { + let retain = set + .auto_deployment + .as_ref() + .is_some_and(|a| a.retain_stacks_on_account_removal); + plan.push(PlannedDeployment { + action_name: "DELETE", + action: TargetAction::Delete { + retain_stacks: retain, + user_requested: false, + }, + retain_stacks: Some(retain), + targets: removals, + }); + } + // A re-deployed instance keeps the overrides it was created with; a + // brand-new one has none. One operation per distinct override set, + // since overrides are a property of the operation. + let mut by_overrides: BTreeMap, Vec> = BTreeMap::new(); + for target in creates { + let overrides = set + .instances + .iter() + .find(|i| i.account == target.account && i.region == target.region) + .map(|i| i.parameter_overrides.clone()) + .unwrap_or_default(); + by_overrides.entry(overrides).or_default().push(target); + } + for (overrides, targets) in by_overrides { + plan.push(PlannedDeployment { + action_name: "CREATE", + action: TargetAction::Create { overrides }, + retain_stacks: None, + targets, + }); + } + (plan, bookkeeping_changed) + } + + /// Run a stack set's planned operations, in order, against `set`. + async fn run_auto_deployment_plan( + &self, + admin: &str, + set: &StackSet, + plan: Vec, + deadline: tokio::time::Instant, + ) -> PlanOutcome { + let mut snapshot = set.clone(); + for step in plan { + let regions: Vec = step + .targets + .iter() + .map(|t| t.region.clone()) + .collect::>() + .into_iter() + .collect(); + let record = DeploymentTargets { + organizational_unit_ids: step + .targets + .iter() + .filter_map(|t| t.ou.clone()) + .collect::>() + .into_iter() + .collect(), + ..DeploymentTargets::default() + }; + let targets = order_targets(step.targets, ®ions, &auto_deployment_preferences()); + let op_id = uuid::Uuid::new_v4().to_string(); + let spec = match self.start_instance_operation( + admin, + &snapshot, + &targets, + &op_id, + step.action_name, + auto_deployment_preferences(), + Some(record), + step.retain_stacks, + ) { + Ok(spec) => spec, + Err(err) if err.code() == "OperationInProgressException" => { + return PlanOutcome::Retry; + } + Err(err) if err.code() == "StaleRequestException" => return PlanOutcome::Stale, + // Contention and staleness are handled above; what is left + // is the stack set having gone (deleted, or no longer active), + // which no retry recovers. Abandoning the rest of the plan is + // the point — there is nothing to deploy to any more. + Err(err) => { + tracing::warn!( + stack_set = snapshot.stack_set_id, + error = %err.message(), + "stack set auto-deployment could not start an operation" + ); + return PlanOutcome::Done; + } + }; + // The operation runs in a task of its own and this only waits for + // it, the way `launch_operation` does for an API-driven one: the + // caller that triggered the deployment (an Organizations request) + // can be cancelled by a client that hangs up, and an operation + // abandoned half way would stay RUNNING and block the stack set. + let svc = self.clone(); + let (request_id, admin_owned, set_id_owned, op) = ( + uuid::Uuid::new_v4().to_string(), + admin.to_string(), + snapshot.stack_set_id.clone(), + op_id.clone(), + ); + let action = step.action; + let finished = tokio::spawn(async move { + svc.run_operation( + &request_id, + &admin_owned, + &set_id_owned, + &op, + &spec, + targets, + action, + ) + .await; + svc.save_snapshot().await; + }); + match tokio::time::timeout_at(deadline, finished).await { + Ok(Ok(())) => {} + Ok(Err(_)) => { + // The operation's task died. Settle it the way a restart + // settles one it interrupted, or it stays RUNNING and + // every later operation on this stack set is refused. + tracing::warn!( + stack_set = snapshot.stack_set_id, + "stack set auto-deployment operation did not finish" + ); + self.with_stack_set(admin, &snapshot.stack_set_id, |set| { + settle_interrupted_operations(set) + }); + self.save_snapshot().await; + // The rest of the plan never ran: re-plan it rather than + // leaving the accounts it covered undeployed. + return PlanOutcome::Retry; + } + // Still deploying. The operation owns the stack set until it + // is done, so leaving it to run is the same situation as + // finding the stack set busy: a waiter picks the rest of the + // work up once it goes idle, and the caller is let go. + Err(_) => { + tracing::info!( + stack_set = snapshot.stack_set_id, + "stack set auto-deployment still running; leaving it to the background" + ); + return PlanOutcome::Retry; + } + } + // The next step plans against what this one left behind. + match self.refreshed_stack_set(admin, &snapshot.stack_set_id) { + Some(current) => snapshot = current, + None => return PlanOutcome::Done, + } + } + PlanOutcome::Done + } + + /// Wait out the operation holding a stack set, then reconcile again. + /// + /// Detached: the caller that triggered the deployment (an Organizations + /// mutation, or a stack that provisioned an organization resource — which + /// may be the very operation in the way) must not be held up by it, and + /// on a busy stack set waiting inline would deadlock that second case. + fn schedule_auto_deployment_retry(&self, admin: &str, set_id: &str) { + let key = (admin.to_string(), set_id.to_string()); + // One waiter per stack set: a second change arriving while the first + // is still waiting is served by the reconciliation that waiter runs. + if !self.auto_deployment_retries.lock().insert(key.clone()) { + return; + } + let svc = self.clone(); + let (admin, set_id) = key; + // Built before the spawn: a task dropped before its first poll would + // otherwise leave the marker set and suppress every later retry. + let release = RetryGuard { + retries: self.auto_deployment_retries.clone(), + key: (admin.clone(), set_id.clone()), + }; + tokio::spawn(async move { + let release = release; + let deadline = tokio::time::Instant::now() + Self::AUTO_DEPLOYMENT_MAX_WAIT; + // Starts responsive, then backs off: a stack set stuck behind a + // long deployment must not take the global CloudFormation lock + // four times a second for an hour. + let mut poll = Self::AUTO_DEPLOYMENT_POLL; + loop { + // Read through `refreshed_stack_set`: an operation whose last + // stack has finished provisioning only settles when the stack + // set is refreshed, and a raw read would see it as forever + // running. + let busy = svc.refreshed_stack_set(&admin, &set_id).is_some_and(|set| { + set.operations + .iter() + .any(|o| matches!(o.status.as_str(), "RUNNING" | "STOPPING")) + }); + if !busy { + // Stand down first: the reconciliation this runs may find + // the stack set busy again (an operator's operation slipped + // in) and need to leave a waiter of its own, which it + // could not do while this one still held the marker. + drop(release); + svc.reconcile_auto_deployments().await; + return; + } + if tokio::time::Instant::now() >= deadline { + tracing::warn!( + stack_set = set_id, + "stack set auto-deployment gave up: another operation held the stack set" + ); + return; + } + tokio::time::sleep(poll).await; + poll = (poll * 2).min(Self::AUTO_DEPLOYMENT_MAX_POLL); + } + }); + } + + // ── Drift ── + + fn detect_stack_set_drift( + &self, + req: &AwsRequest, + params: &BTreeMap, + ) -> Result { + let name = required(params, "StackSetName")?; + let admin = self.stack_set_admin_account(req, params)?; + let preferences = parse_preferences(params)?; + let op_id = params + .get("OperationId") + .cloned() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + + let set = { + let mut accounts = self.state.write(); + let set_id = accounts + .get(&admin) + .and_then(|s| active_key(s, &name, Scope::of(params))) + .ok_or_else(|| stack_set_not_found(&name))?; + Self::refresh_stack_set(&mut accounts, &admin, &set_id); + let set = accounts + .get(&admin) + .and_then(|s| s.stack_sets.get(&set_id)) + .cloned() + .ok_or_else(|| stack_set_not_found(&name))?; + Self::check_can_start_drift(&set, &op_id)?; + set + }; + + // Check every instance's stack against the live backing resources. + let now = Utc::now(); + let mut op = Self::new_operation(&set, &op_id, "DETECT_DRIFT", preferences, None, None); + let mut instance_drift: Vec<(String, String, String)> = Vec::new(); + for instance in &set.instances { + let stack = instance.stack_id.as_ref().and_then(|id| { + self.state.read().get(&instance.account).and_then(|s| { + s.stacks + .values() + .find(|st| &st.stack_id == id && st.status != "DELETE_COMPLETE") + .cloned() + }) + }); + let Some(stack) = stack else { + instance_drift.push(( + instance.account.clone(), + instance.region.clone(), + "UNKNOWN".to_string(), + )); + op.results.push(OperationResult { + account: instance.account.clone(), + region: instance.region.clone(), + status: "FAILED".to_string(), + status_reason: Some("Stack instance does not have a stack".to_string()), + organizational_unit_id: instance.organizational_unit_id.clone(), + account_gate_status: None, + account_gate_reason: None, + }); + continue; + }; + let mut drifted = false; + for resource in &stack.resources { + let status = match self.resource_exists(&instance.account, resource) { + Some(true) => "IN_SYNC", + Some(false) => { + drifted = true; + "DELETED" + } + None => "NOT_CHECKED", + }; + op.resource_drifts.push(InstanceResourceDrift { + account: instance.account.clone(), + region: instance.region.clone(), + stack_id: stack.stack_id.clone(), + logical_id: resource.logical_id.clone(), + physical_id: resource.physical_id.clone(), + resource_type: resource.resource_type.clone(), + status: status.to_string(), + timestamp: now, + }); + } + let status = if drifted { "DRIFTED" } else { "IN_SYNC" }; + instance_drift.push(( + instance.account.clone(), + instance.region.clone(), + status.to_string(), + )); + op.results.push(OperationResult { + account: instance.account.clone(), + region: instance.region.clone(), + status: "SUCCEEDED".to_string(), + status_reason: None, + organizational_unit_id: instance.organizational_unit_id.clone(), + account_gate_status: None, + account_gate_reason: None, + }); + } + let drifted = instance_drift + .iter() + .filter(|(_, _, s)| s == "DRIFTED") + .count(); + let in_sync = instance_drift + .iter() + .filter(|(_, _, s)| s == "IN_SYNC") .count(); let failed = instance_drift .iter() @@ -4747,6 +5659,1547 @@ mod tests { (parent.id, child.id) } + const ACCT_D: &str = "333333333333"; + + /// A service-managed stack set with AutoDeployment enabled, deployed to + /// `workloads` in us-east-1. Returns the OU ids from `seed_org`. + async fn auto_deployed_set( + svc: &CloudFormationService, + name: &str, + retain_on_removal: bool, + ) -> (String, String) { + let (workloads, prod) = seed_org(svc); + ok(svc, "ActivateOrganizationsAccess", &[]).await; + ok( + svc, + "CreateStackSet", + &[ + ("StackSetName", name), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ( + "AutoDeployment.RetainStacksOnAccountRemoval", + if retain_on_removal { "true" } else { "false" }, + ), + ], + ) + .await; + ok( + svc, + "CreateStackInstances", + &[ + ("StackSetName", name), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + (workloads, prod) + } + + /// Put `account` in the organization under `parent`. + fn join_ou(svc: &CloudFormationService, account: &str, parent: &str) { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().expect("organization"); + let root = org.root_id.clone(); + org.enroll_account_if_missing(account); + org.move_account(account, &root, parent).unwrap(); + } + + fn instance_of<'a>(set: &'a StackSet, account: &str) -> Option<&'a StackInstance> { + set.instances.iter().find(|i| i.account == account) + } + + #[tokio::test] + async fn auto_deployment_covers_an_account_added_to_a_target_ou() { + let svc = service(); + let (workloads, prod) = auto_deployed_set(&svc, "org", false).await; + assert_eq!(stored_set(&svc, "org").instances.len(), 2); + + // A new account joins a nested OU below the target after the stack + // set was deployed. AWS deploys to it; so do we. + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + + let set = stored_set(&svc, "org"); + let instance = instance_of(&set, ACCT_D).expect("auto-deployed instance"); + assert_eq!(instance.status, "CURRENT", "{instance:?}"); + assert_eq!( + instance.organizational_unit_id.as_deref(), + Some(workloads.as_str()) + ); + assert_eq!(instance.region, "us-east-1"); + assert_eq!(queue_count(&svc, ACCT_D), 1); + + // Recorded as a CREATE operation against the target OU, as AWS does. + let op = set + .operations + .iter() + .find(|o| o.action == "CREATE" && o.results.iter().any(|r| r.account == ACCT_D)) + .expect("auto-deployment operation"); + assert_eq!(op.status, "SUCCEEDED"); + assert_eq!( + op.deployment_targets + .as_ref() + .map(|t| t.organizational_unit_ids.clone()), + Some(vec![workloads.clone()]) + ); + + // Reconciling again with nothing to do records no further operation. + let before = stored_set(&svc, "org").operations.len(); + svc.reconcile_auto_deployments().await; + assert_eq!(stored_set(&svc, "org").operations.len(), before); + } + + #[tokio::test] + async fn auto_deployment_removes_an_account_that_leaves_the_target_ou() { + let svc = service(); + auto_deployed_set(&svc, "org", false).await; + assert_eq!(queue_count(&svc, ACCT_C), 1); + + // ACCT_C moves out of the target OU tree, back to the root. + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().expect("organization"); + let root = org.root_id.clone(); + let parent = org.parent_of(ACCT_C).expect("parent").0; + org.move_account(ACCT_C, &parent, &root).unwrap(); + } + svc.reconcile_auto_deployments().await; + + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_C).is_none(), "{set:?}"); + assert!(instance_of(&set, ACCT_B).is_some()); + // RetainStacksOnAccountRemoval=false: the stack goes with the account. + assert_eq!(queue_count(&svc, ACCT_C), 0); + assert_eq!( + set.operations + .iter() + .find(|o| o.action == "DELETE") + .map(|o| o.retain_stacks), + Some(Some(false)) + ); + } + + #[tokio::test] + async fn auto_deployment_retains_stacks_when_configured() { + let svc = service(); + auto_deployed_set(&svc, "org", true).await; + assert_eq!(queue_count(&svc, ACCT_C), 1); + + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().expect("organization"); + let root = org.root_id.clone(); + let parent = org.parent_of(ACCT_C).expect("parent").0; + org.move_account(ACCT_C, &parent, &root).unwrap(); + } + svc.reconcile_auto_deployments().await; + + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_C).is_none(), "{set:?}"); + // The stack stays behind in the removed account. + assert_eq!(queue_count(&svc, ACCT_C), 1); + } + + #[tokio::test] + async fn auto_deployment_does_not_undo_a_deleted_instance() { + let svc = service(); + let (workloads, prod) = auto_deployed_set(&svc, "org", false).await; + // The operator drops one account's instance by hand (everything in + // the target OU except ACCT_B) while the account stays in that OU. + ok( + &svc, + "DeleteStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("DeploymentTargets.Accounts.member.1", ACCT_B), + ("DeploymentTargets.AccountFilterType", "DIFFERENCE"), + ("Regions.member.1", "us-east-1"), + ("RetainStacks", "false"), + ], + ) + .await; + assert_eq!(queue_count(&svc, ACCT_C), 0); + + // An unrelated organization change must not bring it back. + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_C).is_none(), "{set:?}"); + assert_eq!(queue_count(&svc, ACCT_C), 0); + // ...while the account that did join is deployed to. + assert!(instance_of(&set, ACCT_D).is_some(), "{set:?}"); + + // Leaving and re-joining the OU deploys again: the exclusion only + // covers the account while it is still there. + let (root, parent) = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + let parent = org.parent_of(ACCT_C).unwrap().0; + org.move_account(ACCT_C, &parent, &root).unwrap(); + (root, parent) + }; + svc.reconcile_auto_deployments().await; + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + org.move_account(ACCT_C, &root, &parent).unwrap(); + } + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_C).is_some(), "{set:?}"); + assert_eq!(queue_count(&svc, ACCT_C), 1); + } + + #[tokio::test] + async fn auto_deployment_honors_the_account_filter() { + let svc = service(); + let (workloads, _prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + // Deploy to the OU, but only to ACCT_B. + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("DeploymentTargets.Accounts.member.1", ACCT_B), + ("DeploymentTargets.AccountFilterType", "INTERSECTION"), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + let set = stored_set(&svc, "org"); + assert_eq!(set.instances.len(), 1); + assert_eq!(set.instances[0].account, ACCT_B); + + // Reconciling must not deploy to the account the filter left out. + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert_eq!(set.instances.len(), 1, "{set:?}"); + assert_eq!(queue_count(&svc, ACCT_C), 0); + } + + #[tokio::test] + async fn auto_deployment_reattributes_an_account_moved_between_target_ous() { + let svc = service(); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + // Two sibling OUs, each a target in the same region. + let other = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &other); + for ou in [&workloads, &other] { + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", ou), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + } + assert_eq!( + instance_of(&stored_set(&svc, "org"), ACCT_D) + .unwrap() + .organizational_unit_id + .as_deref(), + Some(other.as_str()) + ); + + // Moving between two target OUs keeps the stack but re-attributes it. + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + org.move_account(ACCT_D, &other, &prod).unwrap(); + } + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let instance = instance_of(&set, ACCT_D).expect("kept"); + assert_eq!( + instance.organizational_unit_id.as_deref(), + Some(workloads.as_str()) + ); + assert_eq!(queue_count(&svc, ACCT_D), 1); + } + + #[tokio::test] + async fn auto_deployment_keeps_following_an_ou_that_ran_empty() { + let svc = service(); + let (_workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + // `prod` holds exactly one account. + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", &prod), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + assert_eq!(stored_set(&svc, "org").instances.len(), 1); + + // It leaves, so the stack set has no instances at all left. + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.move_account(ACCT_C, &prod, &root).unwrap(); + } + svc.reconcile_auto_deployments().await; + assert!(stored_set(&svc, "org").instances.is_empty()); + + // The OU is still the stack set's target, so an account created in it + // afterwards is deployed to. + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let instance = instance_of(&set, ACCT_D).expect("deployed into the empty OU"); + assert_eq!( + instance.organizational_unit_id.as_deref(), + Some(prod.as_str()) + ); + assert_eq!(queue_count(&svc, ACCT_D), 1); + let targets = ok( + &svc, + "ListStackSetAutoDeploymentTargets", + &[("StackSetName", "org")], + ) + .await; + assert_eq!(tag(&targets, "OrganizationalUnitId"), prod); + } + + #[tokio::test] + async fn auto_deployment_moves_stacks_to_the_new_ous_regions() { + let svc = service(); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + // Two targets, deployed to different regions. + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", &sandbox), + ("Regions.member.1", "eu-west-1"), + ], + ) + .await; + + // Moving between the two targets moves the stacks with it: the + // instance in the OU it left must not be orphaned. + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + org.move_account(ACCT_D, &sandbox, &prod).unwrap(); + } + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let mine: Vec<&StackInstance> = set + .instances + .iter() + .filter(|i| i.account == ACCT_D) + .collect(); + assert_eq!(mine.len(), 1, "{set:?}"); + assert_eq!(mine[0].region, "us-east-1"); + assert_eq!( + mine[0].organizational_unit_id.as_deref(), + Some(workloads.as_str()) + ); + assert_eq!(queue_count(&svc, ACCT_D), 1); + } + + #[tokio::test] + async fn auto_deployment_keeps_the_ou_an_instance_was_deployed_through() { + let svc = service(); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + // ACCT_C sits in `prod`, nested under `workloads`; both are targets, + // each in its own region. + for (ou, region) in [(&workloads, "us-east-1"), (&prod, "eu-west-1")] { + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", ou), + ("Regions.member.1", region), + ], + ) + .await; + } + let before: Vec<(String, Option)> = stored_set(&svc, "org") + .instances + .iter() + .filter(|i| i.account == ACCT_C) + .map(|i| (i.region.clone(), i.organizational_unit_id.clone())) + .collect(); + assert_eq!(before.len(), 2, "{before:?}"); + + // Reconciling must not re-attribute either instance to the other OU. + svc.reconcile_auto_deployments().await; + let after: Vec<(String, Option)> = stored_set(&svc, "org") + .instances + .iter() + .filter(|i| i.account == ACCT_C) + .map(|i| (i.region.clone(), i.organizational_unit_id.clone())) + .collect(); + assert_eq!(before, after); + } + + #[tokio::test] + async fn a_failed_delete_does_not_exclude_the_account() { + let svc = service(); + let (workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + // Termination protection makes the instance's stack undeletable, so + // the delete fails and the instance stays INOPERABLE. + { + let mut accounts = svc.state.write(); + for stack in accounts.get_or_create(ACCT_B).stacks.values_mut() { + stack.enable_termination_protection = true; + } + } + ok( + &svc, + "DeleteStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("DeploymentTargets.Accounts.member.1", ACCT_C), + ("DeploymentTargets.AccountFilterType", "DIFFERENCE"), + ("Regions.member.1", "us-east-1"), + ("RetainStacks", "false"), + ], + ) + .await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_B).is_some(), "{set:?}"); + assert!( + set.auto_deployment_excluded.is_empty(), + "a delete that failed must not exclude the account: {:?}", + set.auto_deployment_excluded + ); + } + + #[tokio::test] + async fn auto_deployment_attributes_each_region_to_the_ou_deployed_there() { + let svc = service(); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + // Nested targets, each deployed to its own region. + for (ou, region) in [(&workloads, "us-east-1"), (&prod, "eu-west-1")] { + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", ou), + ("Regions.member.1", region), + ], + ) + .await; + } + // A new account in the nested OU is covered by both, and each of its + // instances belongs to the OU that is deployed to that region. + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let mut mine: Vec<(&str, &str)> = set + .instances + .iter() + .filter(|i| i.account == ACCT_D) + .map(|i| { + ( + i.region.as_str(), + i.organizational_unit_id.as_deref().unwrap_or(""), + ) + }) + .collect(); + mine.sort(); + assert_eq!( + mine, + [ + ("eu-west-1", prod.as_str()), + ("us-east-1", workloads.as_str()) + ], + "{set:?}" + ); + } + + #[tokio::test] + async fn a_deleted_instance_is_excluded_only_in_its_own_region() { + let svc = service(); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("Regions.member.1", "us-east-1"), + ("Regions.member.2", "eu-west-1"), + ], + ) + .await; + // Drop ACCT_C's us-east-1 instance only. + ok( + &svc, + "DeleteStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("DeploymentTargets.Accounts.member.1", ACCT_B), + ("DeploymentTargets.AccountFilterType", "DIFFERENCE"), + ("Regions.member.1", "us-east-1"), + ("RetainStacks", "false"), + ], + ) + .await; + + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let regions = |account: &str| -> Vec { + let mut out: Vec = set + .instances + .iter() + .filter(|i| i.account == account) + .map(|i| i.region.clone()) + .collect(); + out.sort(); + out + }; + // The removed one stays removed; the other region is untouched. + assert_eq!(regions(ACCT_C), ["eu-west-1"], "{set:?}"); + // The OU still deploys both regions to a new account. + assert_eq!(regions(ACCT_D), ["eu-west-1", "us-east-1"], "{set:?}"); + } + + #[tokio::test] + async fn a_failed_delete_keeps_the_ou_a_target() { + let svc = service(); + let (workloads, prod) = auto_deployed_set(&svc, "org", false).await; + { + let mut accounts = svc.state.write(); + for account in [ACCT_B, ACCT_C] { + for stack in accounts.get_or_create(account).stacks.values_mut() { + stack.enable_termination_protection = true; + } + } + } + // A whole-OU delete that cannot delete any stack. + ok( + &svc, + "DeleteStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("Regions.member.1", "us-east-1"), + ("RetainStacks", "false"), + ], + ) + .await; + let set = stored_set(&svc, "org"); + assert_eq!(set.instances.len(), 2, "{set:?}"); + assert_eq!( + set.auto_deployment_targets + .get(&workloads) + .map(BTreeSet::len), + Some(1), + "a delete that failed must not stop the OU being a target: {:?}", + set.auto_deployment_targets + ); + + // Reconciling leaves the still-deployed instances alone. + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_B).is_some(), "{set:?}"); + assert!(instance_of(&set, ACCT_D).is_some(), "{set:?}"); + } + + #[tokio::test] + async fn an_abandoned_reconciliation_releases_its_claim() { + let svc = service(); + let gate = svc.auto_deployment_gate.clone(); + let claim = AutoDeploymentClaim::take(&svc).expect("first claim"); + // A second trigger only records that a pass is owed. + assert!(AutoDeploymentClaim::take(&svc).is_none()); + assert!(gate.lock().pending); + // Finishing a pass with a trigger owed keeps the claim, and leaves the + // flag for the next lap to consume: a claim dropped before that lap + // runs still leaves the trigger recorded for whoever claims next. + let mut claim = claim; + assert!(claim.another_pass_owed()); + assert!(gate.lock().pending); + assert!(gate.lock().running); + // The lap that serves it consumes the flag, as the reconcile loop does. + gate.lock().pending = false; + // With nothing owed it releases, in the same lock that read the flag. + assert!(!claim.another_pass_owed()); + assert!(!gate.lock().running); + // Dropping the claim (a cancelled request) must not wedge the gate. + drop(claim); + assert!(!gate.lock().running); + assert!(AutoDeploymentClaim::take(&svc).is_some()); + } + + #[tokio::test] + async fn auto_deployment_retries_an_instance_that_never_deployed() { + let svc = service(); + let (workloads, prod) = auto_deployed_set(&svc, "org", false).await; + join_ou(&svc, ACCT_D, &prod); + // An earlier attempt left a record behind without ever creating the + // stack — a target cancelled when a sibling account's deploy failed. + svc.with_stack_set(ADMIN, &stored_set(&svc, "org").stack_set_id, |set| { + set.instances.push(StackInstance { + account: ACCT_D.to_string(), + region: "us-east-1".to_string(), + stack_id: None, + status: "OUTDATED".to_string(), + detailed_status: "CANCELLED".to_string(), + status_reason: Some("Cancelled since failure tolerance has exceeded".to_string()), + parameter_overrides: BTreeMap::new(), + organizational_unit_id: Some(workloads.clone()), + drift_status: "NOT_CHECKED".to_string(), + last_drift_check_timestamp: None, + last_operation_id: None, + }); + }); + + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let instance = instance_of(&set, ACCT_D).expect("instance"); + assert_eq!(instance.status, "CURRENT", "{instance:?}"); + assert!(instance.stack_id.is_some(), "{instance:?}"); + assert_eq!(queue_count(&svc, ACCT_D), 1); + } + + #[tokio::test] + async fn auto_deployment_does_not_let_one_account_cancel_the_others() { + let svc = service(); + let (_workloads, prod) = auto_deployed_set(&svc, "org", false).await; + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let op = set + .operations + .iter() + .find(|o| o.action == "CREATE" && o.results.iter().any(|r| r.account == ACCT_D)) + .expect("auto-deployment operation"); + // Accounts joined the OU independently, so a failure in one must not + // cancel the rest of the deployment. + assert_eq!(op.preferences.failure_tolerance_percentage, Some(100)); + assert_eq!(op.preferences.failure_tolerance_count, None); + } + + #[tokio::test] + async fn auto_deployment_leaves_imported_instances_alone() { + let svc = service(); + let (workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + // A stack in an account under a *different* OU, adopted into the set. + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + let xml = call_as( + &svc, + ACCT_D, + "CreateStack", + &[("StackName", "legacy"), ("TemplateBody", QUEUE_TEMPLATE)], + ) + .await + .unwrap(); + let stack_id = tag(&xml, "StackId"); + let xml = ok( + &svc, + "ImportStacksToStackSet", + &[ + ("StackSetName", "org"), + ("StackIds.member.1", &stack_id), + ("OrganizationalUnitIds.member.1", &sandbox), + ], + ) + .await; + let op_id = tag(&xml, "OperationId"); + let op = ok( + &svc, + "DescribeStackSetOperation", + &[("StackSetName", "org"), ("OperationId", &op_id)], + ) + .await; + assert_eq!(tag(&op, "Status"), "SUCCEEDED", "{op}"); + assert_eq!(queue_count(&svc, ACCT_D), 1); + + // Reconciling must not tear the adopted stack down: importing into an + // OU makes it one of the stack set's targets. + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_D).is_some(), "{set:?}"); + assert_eq!(queue_count(&svc, ACCT_D), 1); + assert!(set.auto_deployment_targets.contains_key(&sandbox)); + assert!(set.auto_deployment_targets.contains_key(&workloads)); + } + + #[tokio::test] + async fn a_retried_instance_keeps_its_parameter_overrides() { + let svc = service(); + let (workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + // ACCT_B's instance was created with an override and then lost its + // stack before it ever deployed. + let set_id = stored_set(&svc, "org").stack_set_id; + svc.with_stack_set(ADMIN, &set_id, |set| { + let instance = set + .instances + .iter_mut() + .find(|i| i.account == ACCT_B) + .unwrap(); + instance.stack_id = None; + instance.detailed_status = "CANCELLED".to_string(); + instance.parameter_overrides = + BTreeMap::from([("Env".to_string(), "prod".to_string())]); + instance.organizational_unit_id = Some(workloads.clone()); + }); + + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let instance = instance_of(&set, ACCT_B).expect("instance"); + assert!(instance.stack_id.is_some(), "{instance:?}"); + assert_eq!( + instance.parameter_overrides.get("Env").map(String::as_str), + Some("prod"), + "{instance:?}" + ); + let stack = stack_of(&svc, ACCT_B, instance.stack_id.as_deref().unwrap()); + assert_eq!( + stack.parameters.get("Env").map(String::as_str), + Some("prod") + ); + } + + #[tokio::test] + async fn a_second_create_does_not_exclude_a_live_instance() { + let svc = service(); + let (workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + // A narrower second call adds a region for ACCT_B only. ACCT_C's + // existing us-east-1 instance was not "left out" by it. + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("DeploymentTargets.Accounts.member.1", ACCT_B), + ("DeploymentTargets.AccountFilterType", "INTERSECTION"), + ("Regions.member.1", "us-east-1"), + ("Regions.member.2", "eu-west-1"), + ], + ) + .await; + let set = stored_set(&svc, "org"); + assert!( + !set.auto_deployment_excluded.contains(&( + workloads.clone(), + ACCT_C.to_string(), + "us-east-1".to_string() + )), + "{:?}", + set.auto_deployment_excluded + ); + // ACCT_C was left out of eu-west-1 though, so it is not deployed there. + assert!(set.auto_deployment_excluded.contains(&( + workloads.clone(), + ACCT_C.to_string(), + "eu-west-1".to_string() + ))); + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let mut regions: Vec<&str> = set + .instances + .iter() + .filter(|i| i.account == ACCT_C) + .map(|i| i.region.as_str()) + .collect(); + regions.sort(); + assert_eq!(regions, ["us-east-1"], "{set:?}"); + } + + /// A cancelled trigger (a client that hung up mid-mutation) must not + /// leave the stack set with an operation nothing will ever finish. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_cancelled_reconciliation_does_not_strand_an_operation() { + let svc = service(); + let (_workloads, prod) = auto_deployed_set(&svc, "org", false).await; + join_ou(&svc, ACCT_D, &prod); + let running = { + let svc = svc.clone(); + tokio::spawn(async move { svc.reconcile_auto_deployments().await }) + }; + // Drop the caller while it is deploying. + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + running.abort(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let set = stored_set(&svc, "org"); + let stuck = set + .operations + .iter() + .any(|o| matches!(o.status.as_str(), "RUNNING" | "STOPPING")); + if !stuck { + break; + } + assert!( + std::time::Instant::now() < deadline, + "operation left RUNNING after the trigger was cancelled: {:?}", + set.operations + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + // The work still lands: the cancelled pass was carried on, and a + // later trigger reconciles whatever it did not reach. Polled, since + // a trigger that arrives while the carried-on pass still holds the + // gate is served by that pass rather than inline. + svc.reconcile_auto_deployments().await; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while instance_of(&stored_set(&svc, "org"), ACCT_D).is_none_or(|i| i.stack_id.is_none()) { + assert!( + std::time::Instant::now() < deadline, + "the account that joined never got its instance" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + #[tokio::test] + async fn auto_deployment_does_not_retry_a_target_that_failed() { + let svc = service(); + let (workloads, prod) = auto_deployed_set(&svc, "org", false).await; + join_ou(&svc, ACCT_D, &prod); + let set_id = stored_set(&svc, "org").stack_set_id; + // A deploy that was tried and failed — a denying account gate, say. + svc.with_stack_set(ADMIN, &set_id, |set| { + set.instances.push(StackInstance { + account: ACCT_D.to_string(), + region: "us-east-1".to_string(), + stack_id: None, + status: "OUTDATED".to_string(), + detailed_status: "FAILED".to_string(), + status_reason: Some("Account gate check failed".to_string()), + parameter_overrides: BTreeMap::new(), + organizational_unit_id: Some(workloads.clone()), + drift_status: "NOT_CHECKED".to_string(), + last_drift_check_timestamp: None, + last_operation_id: None, + }); + }); + let before = stored_set(&svc, "org").operations.len(); + + // Unrelated organization changes must not re-run it every time. + for _ in 0..3 { + svc.reconcile_auto_deployments().await; + } + let set = stored_set(&svc, "org"); + assert_eq!(set.operations.len(), before, "{:?}", set.operations); + assert_eq!(instance_of(&set, ACCT_D).unwrap().detailed_status, "FAILED"); + } + + #[tokio::test] + async fn updating_a_stack_set_keeps_auto_deployment_bookkeeping() { + let svc = service(); + let (workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + let set_id = stored_set(&svc, "org").stack_set_id; + // Bookkeeping a reconciliation wrote without recording an operation. + svc.with_stack_set(ADMIN, &set_id, |set| { + set.auto_deployment_excluded.insert(( + workloads.clone(), + ACCT_D.to_string(), + "us-east-1".to_string(), + )); + }); + + ok( + &svc, + "UpdateStackSet", + &[ + ("StackSetName", "org"), + ("UsePreviousTemplate", "true"), + ("Description", "second revision"), + ], + ) + .await; + let set = stored_set(&svc, "org"); + assert!( + set.auto_deployment_excluded.contains(&( + workloads.clone(), + ACCT_D.to_string(), + "us-east-1".to_string() + )), + "{:?}", + set.auto_deployment_excluded + ); + assert!(set.auto_deployment_targets.contains_key(&workloads)); + } + + #[tokio::test] + async fn importing_one_stack_does_not_deploy_the_rest_of_its_ou() { + let svc = service(); + let (_workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + // A second account in the same OU, with no stack of its own. + const ACCT_E: &str = "444444444444"; + join_ou(&svc, ACCT_E, &sandbox); + let xml = call_as( + &svc, + ACCT_D, + "CreateStack", + &[("StackName", "legacy"), ("TemplateBody", QUEUE_TEMPLATE)], + ) + .await + .unwrap(); + let stack_id = tag(&xml, "StackId"); + ok( + &svc, + "ImportStacksToStackSet", + &[ + ("StackSetName", "org"), + ("StackIds.member.1", &stack_id), + ("OrganizationalUnitIds.member.1", &sandbox), + ], + ) + .await; + + // Adopting one account's stack must not deploy the template into the + // OU's other accounts. + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_D).is_some(), "{set:?}"); + assert!(instance_of(&set, ACCT_E).is_none(), "{set:?}"); + assert_eq!(queue_count(&svc, ACCT_E), 0); + } + + #[tokio::test] + async fn a_refused_import_does_not_make_its_ou_a_target() { + let svc = service(); + auto_deployed_set(&svc, "org", false).await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + // A stack whose template does not match the stack set's: adopted as + // FAILED_IMPORT, so nothing is actually deployed in that OU. + let xml = call_as( + &svc, + ACCT_D, + "CreateStack", + &[("StackName", "other"), ("TemplateBody", TOPIC_TEMPLATE)], + ) + .await + .unwrap(); + let stack_id = tag(&xml, "StackId"); + ok( + &svc, + "ImportStacksToStackSet", + &[ + ("StackSetName", "org"), + ("StackIds.member.1", &stack_id), + ("OrganizationalUnitIds.member.1", &sandbox), + ], + ) + .await; + let set = stored_set(&svc, "org"); + assert_eq!( + instance_of(&set, ACCT_D).map(|i| i.detailed_status.as_str()), + Some("FAILED_IMPORT"), + "{set:?}" + ); + assert!( + !set.auto_deployment_targets.contains_key(&sandbox), + "{:?}", + set.auto_deployment_targets + ); + } + + /// A reconciliation whose caller is cancelled part way still finishes: + /// the work moves to a task of its own rather than being forgotten. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_cancelled_pass_is_carried_on() { + let svc = service(); + let (_workloads, prod) = auto_deployed_set(&svc, "org", false).await; + join_ou(&svc, ACCT_D, &prod); + // A pass is under way — nobody else can claim the gate — and the + // request running it is cancelled. + let claim = AutoDeploymentClaim::take(&svc).expect("claim"); + assert!(AutoDeploymentClaim::take(&svc).is_none()); + drop(claim); + + // An instance is recorded before its stack is provisioned, as in + // AWS, so wait for the stack itself rather than for the record. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while instance_of(&stored_set(&svc, "org"), ACCT_D).is_none_or(|i| i.stack_id.is_none()) { + assert!( + std::time::Instant::now() < deadline, + "the interrupted reconciliation never ran" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + assert_eq!(queue_count(&svc, ACCT_D), 1); + } + + #[tokio::test] + async fn auto_deployment_never_deletes_a_refused_imports_stack() { + let svc = service(); + let (_workloads, prod) = auto_deployed_set(&svc, "org", false).await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + // An import refused for a template mismatch records the operator's + // own stack against the stack set without adopting it. + let xml = call_as( + &svc, + ACCT_D, + "CreateStack", + &[("StackName", "mine"), ("TemplateBody", TOPIC_TEMPLATE)], + ) + .await + .unwrap(); + let stack_id = tag(&xml, "StackId"); + ok( + &svc, + "ImportStacksToStackSet", + &[ + ("StackSetName", "org"), + ("StackIds.member.1", &stack_id), + ("OrganizationalUnitIds.member.1", &sandbox), + ], + ) + .await; + let topics = || { + svc.deps + .sns + .read() + .get(ACCT_D) + .map_or(0, |s| s.topics.len()) + }; + assert_eq!(topics(), 1); + + // Reconciling must not delete a stack the stack set never deployed. + join_ou(&svc, "444444444444", &prod); + svc.reconcile_auto_deployments().await; + assert_eq!(topics(), 1, "the operator's own stack was deleted"); + assert_eq!(stack_of(&svc, ACCT_D, &stack_id).status, "CREATE_COMPLETE"); + } + + #[test] + fn restoring_a_stack_set_ignores_refused_imports() { + let mut accounts = MultiAccountState::::new(ADMIN, "us-east-1", ""); + let set = StackSet { + stack_set_id: "org:1".to_string(), + name: "org".to_string(), + arn: String::new(), + status: "ACTIVE".to_string(), + description: None, + template_body: String::new(), + parameters: BTreeMap::new(), + capabilities: Vec::new(), + tags: Vec::new(), + administration_role_arn: None, + execution_role_name: None, + permission_model: "SERVICE_MANAGED".to_string(), + auto_deployment: Some(AutoDeployment { + enabled: true, + retain_stacks_on_account_removal: false, + }), + auto_deployment_targets: BTreeMap::new(), + auto_deployment_excluded: BTreeSet::new(), + managed_execution_active: false, + instances: vec![ + StackInstance { + account: ACCT_B.to_string(), + region: "us-east-1".to_string(), + stack_id: Some("stack-b".to_string()), + status: "CURRENT".to_string(), + detailed_status: "SUCCEEDED".to_string(), + status_reason: None, + parameter_overrides: BTreeMap::new(), + organizational_unit_id: Some("ou-kept".to_string()), + drift_status: "NOT_CHECKED".to_string(), + last_drift_check_timestamp: None, + last_operation_id: None, + }, + StackInstance { + account: ACCT_C.to_string(), + region: "us-east-1".to_string(), + stack_id: Some("stack-c".to_string()), + status: "OUTDATED".to_string(), + detailed_status: "FAILED_IMPORT".to_string(), + status_reason: None, + parameter_overrides: BTreeMap::new(), + organizational_unit_id: Some("ou-refused".to_string()), + drift_status: "NOT_CHECKED".to_string(), + last_drift_check_timestamp: None, + last_operation_id: None, + }, + ], + operations: Vec::new(), + drift: None, + created_at: Utc::now(), + }; + accounts + .get_or_create(ADMIN) + .stack_sets + .insert(set.stack_set_id.clone(), set); + + restore_stack_sets(&mut accounts); + let set = &accounts.get(ADMIN).unwrap().stack_sets["org:1"]; + assert!(set.auto_deployment_targets.contains_key("ou-kept")); + assert!( + !set.auto_deployment_targets.contains_key("ou-refused"), + "{:?}", + set.auto_deployment_targets + ); + } + + #[tokio::test] + async fn bookkeeping_only_reconciliation_is_persisted() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("cloudformation.json"); + let svc = service().with_snapshot_store(std::sync::Arc::new( + fakecloud_persistence::DiskSnapshotStore::new(path.clone()), + )); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "org"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "true"), + ], + ) + .await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + // Two targets covering the same region, so moving between them is a + // re-attribution and nothing else: no operation to persist behind. + for ou in [&workloads, &sandbox] { + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", ou), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + } + std::fs::remove_file(&path).ok(); + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + org.move_account(ACCT_D, &sandbox, &prod).unwrap(); + } + + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert_eq!( + instance_of(&set, ACCT_D) + .unwrap() + .organizational_unit_id + .as_deref(), + Some(workloads.as_str()) + ); + // The re-attribution has to reach disk: nothing else will write it. + let snapshot = std::fs::read_to_string(&path).expect("snapshot written"); + assert!(snapshot.contains(&workloads), "{snapshot}"); + assert!(snapshot.contains(ACCT_D), "{snapshot}"); + } + + #[tokio::test] + async fn a_busy_stack_set_gets_one_waiter_not_one_per_change() { + let svc = service(); + auto_deployed_set(&svc, "org", false).await; + let set_id = stored_set(&svc, "org").stack_set_id; + // A burst of organization changes against a stack set that is busy + // must not pile up a poller each. + for _ in 0..5 { + svc.schedule_auto_deployment_retry(ADMIN, &set_id); + } + assert_eq!( + svc.auto_deployment_retries.lock().len(), + 1, + "{:?}", + svc.auto_deployment_retries.lock() + ); + } + + #[tokio::test] + async fn a_deleted_instance_deploys_again_in_another_target_ou() { + let svc = service(); + let (workloads, _prod) = auto_deployed_set(&svc, "org", false).await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "org"), + ("DeploymentTargets.OrganizationalUnitIds.member.1", &sandbox), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + // The operator removes ACCT_C's instance while it sits in `workloads`. + ok( + &svc, + "DeleteStackInstances", + &[ + ("StackSetName", "org"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("DeploymentTargets.Accounts.member.1", ACCT_B), + ("DeploymentTargets.AccountFilterType", "DIFFERENCE"), + ("Regions.member.1", "us-east-1"), + ("RetainStacks", "false"), + ], + ) + .await; + svc.reconcile_auto_deployments().await; + assert!(instance_of(&stored_set(&svc, "org"), ACCT_C).is_none()); + + // Moving it into a different target OU is a membership change like + // any other: the decision was about the OU it left. + { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let parent = org.parent_of(ACCT_C).unwrap().0; + org.move_account(ACCT_C, &parent, &sandbox).unwrap(); + } + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + let instance = instance_of(&set, ACCT_C).expect("redeployed in the new OU"); + assert_eq!( + instance.organizational_unit_id.as_deref(), + Some(sandbox.as_str()) + ); + assert_eq!(queue_count(&svc, ACCT_C), 1); + } + + #[tokio::test] + async fn a_waiter_can_leave_another_waiter_behind_it() { + let svc = service(); + auto_deployed_set(&svc, "org", false).await; + let set_id = stored_set(&svc, "org").stack_set_id; + // While one waiter is marked, a second is suppressed... + svc.schedule_auto_deployment_retry(ADMIN, &set_id); + assert_eq!(svc.auto_deployment_retries.lock().len(), 1); + // ...but once it stands down, the reconciliation it runs can leave a + // fresh one, so a stack set that is busy again is not forgotten. + svc.auto_deployment_retries.lock().clear(); + svc.schedule_auto_deployment_retry(ADMIN, &set_id); + assert_eq!(svc.auto_deployment_retries.lock().len(), 1); + } + + #[tokio::test] + async fn turning_auto_deployment_off_stops_a_pass_in_its_tracks() { + let svc = service(); + let (_workloads, prod) = auto_deployed_set(&svc, "org", false).await; + join_ou(&svc, ACCT_D, &prod); + // The operator turns auto-deployment off between the scan that picked + // the stack set and the pass that plans it. + ok( + &svc, + "UpdateStackSet", + &[ + ("StackSetName", "org"), + ("UsePreviousTemplate", "true"), + ("AutoDeployment.Enabled", "false"), + ], + ) + .await; + + svc.reconcile_auto_deployments().await; + let set = stored_set(&svc, "org"); + assert!(instance_of(&set, ACCT_D).is_none(), "{set:?}"); + assert_eq!(queue_count(&svc, ACCT_D), 0); + } + + #[tokio::test] + async fn a_wholly_refused_import_still_leaves_the_ous_accounts_out() { + let svc = service(); + auto_deployed_set(&svc, "org", false).await; + let sandbox = { + let mut guard = svc.deps.organizations.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + org.create_ou(&root, "sandbox").unwrap().id + }; + join_ou(&svc, ACCT_D, &sandbox); + const ACCT_E: &str = "444444444444"; + join_ou(&svc, ACCT_E, &sandbox); + // The only stack named is refused, so nothing is adopted. + let xml = call_as( + &svc, + ACCT_D, + "CreateStack", + &[("StackName", "other"), ("TemplateBody", TOPIC_TEMPLATE)], + ) + .await + .unwrap(); + let stack_id = tag(&xml, "StackId"); + ok( + &svc, + "ImportStacksToStackSet", + &[ + ("StackSetName", "org"), + ("StackIds.member.1", &stack_id), + ("OrganizationalUnitIds.member.1", &sandbox), + ], + ) + .await; + let set = stored_set(&svc, "org"); + // The account whose stack was refused was asked for, so it is not + // recorded as deliberately left out... + assert!( + !set.auto_deployment_excluded.contains(&( + sandbox.clone(), + ACCT_D.to_string(), + "us-east-1".to_string() + )), + "{:?}", + set.auto_deployment_excluded + ); + // ...while the account nobody mentioned is. + assert!( + set.auto_deployment_excluded.contains(&( + sandbox.clone(), + ACCT_E.to_string(), + "us-east-1".to_string() + )), + "{:?}", + set.auto_deployment_excluded + ); + } + + #[tokio::test] + async fn auto_deployment_leaves_other_stack_sets_alone() { + let svc = service(); + let (workloads, prod) = seed_org(&svc); + ok(&svc, "ActivateOrganizationsAccess", &[]).await; + // Same OU target, but auto-deployment is off. + ok( + &svc, + "CreateStackSet", + &[ + ("StackSetName", "manual"), + ("TemplateBody", QUEUE_TEMPLATE), + ("PermissionModel", "SERVICE_MANAGED"), + ("AutoDeployment.Enabled", "false"), + ], + ) + .await; + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "manual"), + ( + "DeploymentTargets.OrganizationalUnitIds.member.1", + &workloads, + ), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + // A self-managed set deployed to named accounts is never touched either. + create_set(&svc, "self", TOPIC_TEMPLATE).await; + ok( + &svc, + "CreateStackInstances", + &[ + ("StackSetName", "self"), + ("Accounts.member.1", ACCT_B), + ("Regions.member.1", "us-east-1"), + ], + ) + .await; + + join_ou(&svc, ACCT_D, &prod); + svc.reconcile_auto_deployments().await; + + let manual = stored_set(&svc, "manual"); + assert!(instance_of(&manual, ACCT_D).is_none(), "{manual:?}"); + assert_eq!(manual.operations.len(), 1); + let self_managed = stored_set(&svc, "self"); + assert_eq!(self_managed.instances.len(), 1); + assert_eq!(self_managed.operations.len(), 1); + } + #[tokio::test] async fn service_managed_stack_sets_deploy_to_organizational_units() { let svc = service(); diff --git a/crates/fakecloud-e2e/tests/cloudformation_stack_sets_auto_deployment.rs b/crates/fakecloud-e2e/tests/cloudformation_stack_sets_auto_deployment.rs new file mode 100644 index 000000000..7f4f11bbc --- /dev/null +++ b/crates/fakecloud-e2e/tests/cloudformation_stack_sets_auto_deployment.rs @@ -0,0 +1,395 @@ +//! End-to-end coverage for StackSets auto-deployment: a service-managed stack +//! set with `AutoDeployment.Enabled` follows the organization. An account that +//! joins a target OU after the stack instances were created gets the stacks +//! too, and an account that leaves loses them. + +mod helpers; + +use aws_credential_types::Credentials; +use aws_sdk_cloudformation::error::ProvideErrorMetadata; +use aws_sdk_cloudformation::types::{ + AutoDeployment, Capability, DeploymentTargets, PermissionModels, StackSetOperationStatus, +}; +use helpers::TestServer; + +/// The account the default test credentials resolve to; it owns the org. +const MANAGEMENT: &str = "123456789012"; +const CHILD_ONE: &str = "111111111111"; +/// Joins the organization only after the stack instances exist. +const CHILD_TWO: &str = "222222222222"; + +const STACKSETS_PRINCIPAL: &str = "member.org.stacksets.cloudformation.amazonaws.com"; +const ROLE_NAME: &str = "readonly-role"; + +const TEMPLATE: &str = r#"{ + "Resources": { + "ReadOnlyExecutionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "readonly-role", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::123456789012:root" }, + "Action": "sts:AssumeRole" + }] + }, + "Path": "/" + } + } + } +}"#; + +async fn start() -> TestServer { + TestServer::start_with_env(&[ + // Organizations + IAM + StackSets are pure control plane here. + ("FAKECLOUD_CONTAINER_CLI", "false"), + ]) + .await +} + +/// An IAM client whose credentials belong to `account_id`. +async fn iam_for(server: &TestServer, account_id: &str) -> aws_sdk_iam::Client { + let (akid, secret) = server.create_admin(account_id, "root").await; + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url(server.endpoint()) + .region(aws_config::Region::new("us-east-1")) + .credentials_provider(Credentials::new(akid, secret, None, None, "member")) + .load() + .await; + aws_sdk_iam::Client::new(&config) +} + +async fn has_role(iam: &aws_sdk_iam::Client) -> bool { + match iam.get_role().role_name(ROLE_NAME).send().await { + Ok(_) => true, + Err(e) => { + assert_eq!( + e.code(), + Some("NoSuchEntity"), + "unexpected GetRole error: {e:?}" + ); + false + } + } +} + +async fn operation_status( + cfn: &aws_sdk_cloudformation::Client, + stack_set: &str, + operation_id: &str, +) -> StackSetOperationStatus { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + loop { + let status = cfn + .describe_stack_set_operation() + .stack_set_name(stack_set) + .operation_id(operation_id) + .send() + .await + .unwrap() + .stack_set_operation() + .and_then(|op| op.status()) + .cloned() + .expect("operation status"); + if !matches!( + status, + StackSetOperationStatus::Running + | StackSetOperationStatus::Queued + | StackSetOperationStatus::Stopping + ) { + return status; + } + assert!( + std::time::Instant::now() < deadline, + "operation {operation_id} never finished" + ); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } +} + +async fn instance_accounts(cfn: &aws_sdk_cloudformation::Client, stack_set: &str) -> Vec { + let mut accounts: Vec = cfn + .list_stack_instances() + .stack_set_name(stack_set) + .send() + .await + .unwrap() + .summaries() + .iter() + .filter_map(|s| s.account().map(str::to_string)) + .collect(); + accounts.sort(); + accounts +} + +#[tokio::test] +async fn auto_deployment_follows_accounts_in_and_out_of_the_target_ou() { + let server = start().await; + let orgs = server.organizations_client().await; + let cfn = server.cloudformation_client().await; + + orgs.create_organization() + .feature_set(aws_sdk_organizations::types::OrganizationFeatureSet::All) + .send() + .await + .unwrap(); + let root = orgs.list_roots().send().await.unwrap().roots()[0] + .id() + .unwrap() + .to_string(); + orgs.enable_aws_service_access() + .service_principal(STACKSETS_PRINCIPAL) + .send() + .await + .unwrap(); + + // One member account exists before the stack set is deployed. + let child_one = iam_for(&server, CHILD_ONE).await; + + cfn.create_stack_set() + .stack_set_name("auto") + .template_body(TEMPLATE) + .permission_model(PermissionModels::ServiceManaged) + .capabilities(Capability::CapabilityNamedIam) + .auto_deployment( + AutoDeployment::builder() + .enabled(true) + .retain_stacks_on_account_removal(false) + .build(), + ) + .send() + .await + .unwrap(); + let create = cfn + .create_stack_instances() + .stack_set_name("auto") + .deployment_targets( + DeploymentTargets::builder() + .organizational_unit_ids(&root) + .build(), + ) + .regions("us-east-1") + .send() + .await + .unwrap(); + assert_eq!( + operation_status(&cfn, "auto", create.operation_id().unwrap()).await, + StackSetOperationStatus::Succeeded + ); + assert!(has_role(&child_one).await, "member at deploy time"); + + // The management account is never a service-managed target. + assert_eq!( + orgs.describe_organization() + .send() + .await + .unwrap() + .organization() + .and_then(|o| o.master_account_id()), + Some(MANAGEMENT) + ); + let management = server.iam_client().await; + assert!(!has_role(&management).await, "management account"); + assert_eq!(instance_accounts(&cfn, "auto").await, [CHILD_ONE]); + + // The reported bug: an account that joins the target OU afterwards was + // left without the stack set's stacks. + let child_two = iam_for(&server, CHILD_TWO).await; + assert!(has_role(&child_two).await, "account added after deployment"); + assert_eq!( + instance_accounts(&cfn, "auto").await, + [CHILD_ONE, CHILD_TWO] + ); + + // Auto-deployment is recorded as an ordinary CREATE operation on the + // stack set, against the OU that gained the account. + let ops = cfn + .list_stack_set_operations() + .stack_set_name("auto") + .send() + .await + .unwrap(); + assert_eq!(ops.summaries().len(), 2, "{ops:?}"); + let targets = cfn + .list_stack_set_auto_deployment_targets() + .stack_set_name("auto") + .send() + .await + .unwrap(); + assert_eq!(targets.summaries().len(), 1); + assert_eq!( + targets.summaries()[0].organizational_unit_id(), + Some(&root[..]) + ); + + // Leaving the organization takes the stacks with it + // (RetainStacksOnAccountRemoval=false). + orgs.remove_account_from_organization() + .account_id(CHILD_TWO) + .send() + .await + .unwrap(); + assert_eq!(instance_accounts(&cfn, "auto").await, [CHILD_ONE]); + assert!( + !has_role(&child_two).await, + "stack removed with the account" + ); + assert!(has_role(&child_one).await, "untouched member"); +} + +#[tokio::test] +async fn auto_deployment_can_retain_stacks_when_an_account_is_removed() { + let server = start().await; + let orgs = server.organizations_client().await; + let cfn = server.cloudformation_client().await; + + orgs.create_organization() + .feature_set(aws_sdk_organizations::types::OrganizationFeatureSet::All) + .send() + .await + .unwrap(); + let root = orgs.list_roots().send().await.unwrap().roots()[0] + .id() + .unwrap() + .to_string(); + orgs.enable_aws_service_access() + .service_principal(STACKSETS_PRINCIPAL) + .send() + .await + .unwrap(); + let child_one = iam_for(&server, CHILD_ONE).await; + + cfn.create_stack_set() + .stack_set_name("retain") + .template_body(TEMPLATE) + .permission_model(PermissionModels::ServiceManaged) + .capabilities(Capability::CapabilityNamedIam) + .auto_deployment( + AutoDeployment::builder() + .enabled(true) + .retain_stacks_on_account_removal(true) + .build(), + ) + .send() + .await + .unwrap(); + let create = cfn + .create_stack_instances() + .stack_set_name("retain") + .deployment_targets( + DeploymentTargets::builder() + .organizational_unit_ids(&root) + .build(), + ) + .regions("us-east-1") + .send() + .await + .unwrap(); + assert_eq!( + operation_status(&cfn, "retain", create.operation_id().unwrap()).await, + StackSetOperationStatus::Succeeded + ); + assert!(has_role(&child_one).await); + + orgs.remove_account_from_organization() + .account_id(CHILD_ONE) + .send() + .await + .unwrap(); + assert!(instance_accounts(&cfn, "retain").await.is_empty()); + // The instance is gone from the stack set, but its stack stays behind. + assert!(has_role(&child_one).await, "retained stack"); + + // The OU is still the stack set's target even with nothing deployed in + // it, so the next account to join is deployed to. + let child_two = iam_for(&server, CHILD_TWO).await; + assert_eq!(instance_accounts(&cfn, "retain").await, [CHILD_TWO]); + assert!(has_role(&child_two).await); +} + +/// An `AWS::Organizations::Account` resource puts an account into the OU from +/// inside a CloudFormation stack, which never goes through the Organizations +/// API. Auto-deployment has to see that too. +#[tokio::test] +async fn auto_deployment_covers_an_account_created_by_a_cloudformation_stack() { + let server = start().await; + let orgs = server.organizations_client().await; + let cfn = server.cloudformation_client().await; + + orgs.create_organization() + .feature_set(aws_sdk_organizations::types::OrganizationFeatureSet::All) + .send() + .await + .unwrap(); + let root = orgs.list_roots().send().await.unwrap().roots()[0] + .id() + .unwrap() + .to_string(); + orgs.enable_aws_service_access() + .service_principal(STACKSETS_PRINCIPAL) + .send() + .await + .unwrap(); + iam_for(&server, CHILD_ONE).await; + + cfn.create_stack_set() + .stack_set_name("auto") + .template_body(TEMPLATE) + .permission_model(PermissionModels::ServiceManaged) + .capabilities(Capability::CapabilityNamedIam) + .auto_deployment(AutoDeployment::builder().enabled(true).build()) + .send() + .await + .unwrap(); + let create = cfn + .create_stack_instances() + .stack_set_name("auto") + .deployment_targets( + DeploymentTargets::builder() + .organizational_unit_ids(&root) + .build(), + ) + .regions("us-east-1") + .send() + .await + .unwrap(); + assert_eq!( + operation_status(&cfn, "auto", create.operation_id().unwrap()).await, + StackSetOperationStatus::Succeeded + ); + + let account_template = format!( + r#"{{"Resources":{{"Member":{{"Type":"AWS::Organizations::Account","Properties":{{"AccountName":"spawned","Email":"spawned@example.com","ParentIds":["{root}"]}}}}}}}}"# + ); + cfn.create_stack() + .stack_name("member-account") + .template_body(account_template) + .send() + .await + .unwrap(); + let account_id = helpers::wait_until(std::time::Duration::from_secs(30), || async { + let accounts = orgs.list_accounts().send().await.unwrap(); + accounts + .accounts() + .iter() + .find(|a| a.name() == Some("spawned")) + .and_then(|a| a.id().map(str::to_string)) + }) + .await + .expect("account provisioned by the stack"); + + let deployed = helpers::wait_until(std::time::Duration::from_secs(30), || async { + instance_accounts(&cfn, "auto") + .await + .contains(&account_id) + .then_some(()) + }) + .await; + assert!( + deployed.is_some(), + "stack-created account never got the stack set: {:?}", + instance_accounts(&cfn, "auto").await + ); +} diff --git a/crates/fakecloud-organizations/src/lib.rs b/crates/fakecloud-organizations/src/lib.rs index 7418cdcc9..f8b940072 100644 --- a/crates/fakecloud-organizations/src/lib.rs +++ b/crates/fakecloud-organizations/src/lib.rs @@ -3,7 +3,7 @@ pub mod resolver; pub(crate) mod service; pub(crate) mod state; -pub use service::OrganizationsService; +pub use service::{OrgChangeHook, OrgChangeHooks, OrganizationsService}; pub use state::{ MemberAccount, OrganizationState, OrganizationalUnit, OrganizationsSnapshot, Policy, ResponsibilityTransfer, SharedOrganizationsState, FEATURE_SET_ALL, diff --git a/crates/fakecloud-organizations/src/service/accounts.rs b/crates/fakecloud-organizations/src/service/accounts.rs index 74e6d74cf..e90a399e4 100644 --- a/crates/fakecloud-organizations/src/service/accounts.rs +++ b/crates/fakecloud-organizations/src/service/accounts.rs @@ -134,6 +134,7 @@ impl OrganizationsService { let state = self.state.clone(); let store = self.snapshot_store.clone(); let lock = self.snapshot_lock.clone(); + let hooks = self.change_hooks.clone(); let delay = { let mut rng = rand::thread_rng(); let span = CREATE_ACCOUNT_MAX_DELAY.saturating_sub(CREATE_ACCOUNT_MIN_DELAY); @@ -158,6 +159,9 @@ impl OrganizationsService { }; if completed { super::save_organizations_snapshot(&state, store, &lock).await; + // The account only joins the organization here, so this is + // where StackSets auto-deployment gets to see it. + hooks.fire_if_membership_changed(&state).await; } }); } diff --git a/crates/fakecloud-organizations/src/service/mod.rs b/crates/fakecloud-organizations/src/service/mod.rs index ce0a5b0af..5774b3f3b 100644 --- a/crates/fakecloud-organizations/src/service/mod.rs +++ b/crates/fakecloud-organizations/src/service/mod.rs @@ -1,3 +1,5 @@ +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -100,10 +102,142 @@ pub static ORGANIZATIONS_ACTIONS: &[&str] = &[ "ListOutboundResponsibilityTransfers", ]; +/// Called after a mutation that may have changed which accounts the +/// organization contains, or where they sit in the OU tree. Observers +/// re-read the organization themselves and reconcile against it, so the hook +/// carries no payload and is safe to fire more often than strictly needed. +pub type OrgChangeHook = Arc Pin + Send>> + Send + Sync>; + +/// The set of observers notified of organization changes. +/// +/// CloudFormation registers one to run StackSets auto-deployment: an account +/// joining (or leaving) an OU a service-managed stack set targets has to gain +/// (or lose) that stack set's instances. The registry is a shared handle so +/// the server can build Organizations first and install the CloudFormation +/// observer once that service exists. +#[derive(Clone, Default)] +pub struct OrgChangeHooks { + hooks: Arc>>, + /// Fingerprint of the organization the observers were last told about, so + /// a mutation that changed nothing they care about costs nothing. + seen: Arc>>, +} + +/// Everything an observer reacts to: which accounts exist, where they sit and +/// whether they are active, and the shape of the OU tree they sit in. +fn membership_fingerprint(org: &OrganizationState) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + org.org_id.hash(&mut hasher); + org.management_account_id.hash(&mut hasher); + for account in org.accounts.values() { + account.id.hash(&mut hasher); + account.parent_id.hash(&mut hasher); + account.status.hash(&mut hasher); + } + for ou in org.ous.values() { + ou.id.hash(&mut hasher); + ou.parent_id.hash(&mut hasher); + } + hasher.finish() +} + +impl OrgChangeHooks { + pub fn new() -> Self { + Self::default() + } + + pub fn register(&self, hook: OrgChangeHook) { + self.hooks.write().push(hook); + } + + /// Fire only when the organization's membership actually differs from + /// what the observers were last told. Most mutations (a tag, a policy, a + /// handshake that is still open) leave it untouched, and a stack set + /// reconciliation is far too heavy to run on each of those. + pub async fn fire_if_membership_changed(&self, state: &SharedOrganizationsState) { + // Read the organization under the same lock that records it. Reading + // first would let two concurrent mutations record a fingerprint for a + // state the organization has already left, after which the change + // that takes it back there looks like no change at all and is never + // announced. + let (previous, claimed) = { + let mut seen = self.seen.lock(); + let fingerprint = state.read().as_ref().map(membership_fingerprint); + if *seen == fingerprint { + return; + } + let previous = *seen; + // Claim it up front, so a mutation racing this one does not + // announce the same state twice. + *seen = fingerprint; + (previous, fingerprint) + }; + let claim = FingerprintClaim { + seen: self.seen.clone(), + previous, + claimed, + announced: false, + }; + self.fire().await; + claim.announced(); + } + + /// Run every registered observer to completion. Awaited by the mutation + /// that triggered it, so a caller that has just moved an account sees the + /// resulting deployment already done when the call returns. A registry + /// with no observers costs one lock. + pub async fn fire(&self) { + let hooks: Vec = self.hooks.read().clone(); + for hook in hooks { + hook().await; + } + } +} + +/// Holds a fingerprint claimed for announcement. If the caller is cancelled +/// before the observers have run, it puts back what was there so the change +/// is announced again rather than being remembered as already handled. +struct FingerprintClaim { + seen: Arc>>, + previous: Option, + claimed: Option, + announced: bool, +} + +impl FingerprintClaim { + fn announced(mut self) { + self.announced = true; + } +} + +impl Drop for FingerprintClaim { + fn drop(&mut self) { + if self.announced { + return; + } + let mut seen = self.seen.lock(); + // Only roll back what is still ours: a later change has its own + // claim and must keep it. + if *seen == self.claimed { + *seen = self.previous; + } + } +} + +impl std::fmt::Debug for OrgChangeHooks { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OrgChangeHooks") + .field("observers", &self.hooks.read().len()) + .finish() + } +} + pub struct OrganizationsService { state: SharedOrganizationsState, pub(crate) snapshot_store: Option>, pub(crate) snapshot_lock: Arc>, + pub(crate) change_hooks: OrgChangeHooks, } mod accounts; @@ -124,6 +258,7 @@ impl OrganizationsService { state, snapshot_store: None, snapshot_lock: Arc::new(AsyncMutex::new(())), + change_hooks: OrgChangeHooks::new(), } } @@ -132,6 +267,17 @@ impl OrganizationsService { self } + /// Share a change-hook registry with the server, which installs observers + /// into it after the services that react to organization changes exist. + pub fn with_change_hooks(mut self, hooks: OrgChangeHooks) -> Self { + self.change_hooks = hooks; + self + } + + pub fn change_hooks(&self) -> OrgChangeHooks { + self.change_hooks.clone() + } + pub fn shared() -> (Arc, SharedOrganizationsState) { let state: SharedOrganizationsState = Arc::new(parking_lot::RwLock::new(None)); (Arc::new(Self::new(state.clone())), state) @@ -351,6 +497,15 @@ impl AwsService for OrganizationsService { }; if mutates && matches!(result.as_ref(), Ok(resp) if resp.status.is_success()) { self.save_snapshot().await; + // Any successful mutation can have moved an account between OUs, + // added one to the organization or taken one out. Observers + // reconcile against the organization rather than against a diff, + // so this cannot miss a placement change (StackSets + // auto-deployment depends on seeing all of them), and the ones + // that changed nothing they care about are filtered out here. + self.change_hooks + .fire_if_membership_changed(&self.state) + .await; } result } diff --git a/crates/fakecloud-organizations/src/service/tests.rs b/crates/fakecloud-organizations/src/service/tests.rs index faadd55cd..00f06f307 100644 --- a/crates/fakecloud-organizations/src/service/tests.rs +++ b/crates/fakecloud-organizations/src/service/tests.rs @@ -1918,3 +1918,120 @@ async fn invite_responsibility_transfer_rejects_bad_type() { ); assert_eq!(err.code(), "InvalidInputException"); } + +#[tokio::test] +async fn a_mutation_that_changes_no_membership_notifies_nobody() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let state: SharedOrganizationsState = Arc::new(parking_lot::RwLock::new(None)); + let hooks = OrgChangeHooks::new(); + let fired = Arc::new(AtomicUsize::new(0)); + { + let fired = fired.clone(); + hooks.register(Arc::new(move || { + let fired = fired.clone(); + Box::pin(async move { + fired.fetch_add(1, Ordering::SeqCst); + }) + })); + } + // No organization at all: nothing to tell anyone about. + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), 0); + + *state.write() = Some(OrganizationState::bootstrap("000000000000")); + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), 1); + + // A tag is not a membership change. + state + .write() + .as_mut() + .unwrap() + .set_resource_tags("000000000000", &[("Env".to_string(), "dev".to_string())]); + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), 1); + + // An account joining is. + state + .write() + .as_mut() + .unwrap() + .enroll_account_if_missing("111111111111"); + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), 2); + + // So is moving it, and so is an OU appearing for it to move into. + let (root, ou) = { + let mut guard = state.write(); + let org = guard.as_mut().unwrap(); + let root = org.root_id.clone(); + let ou = org.create_ou(&root, "workloads").unwrap().id; + (root, ou) + }; + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), 3); + state + .write() + .as_mut() + .unwrap() + .move_account("111111111111", &root, &ou) + .unwrap(); + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), 4); +} + +#[tokio::test] +async fn a_membership_change_is_announced_even_after_a_reversal_races_it() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let state: SharedOrganizationsState = Arc::new(parking_lot::RwLock::new(Some( + OrganizationState::bootstrap("000000000000"), + ))); + let hooks = OrgChangeHooks::new(); + let fired = Arc::new(AtomicUsize::new(0)); + { + let fired = fired.clone(); + hooks.register(Arc::new(move || { + let fired = fired.clone(); + Box::pin(async move { + fired.fetch_add(1, Ordering::SeqCst); + }) + })); + } + let (root, ou) = { + let mut guard = state.write(); + let org = guard.as_mut().unwrap(); + org.enroll_account_if_missing("111111111111"); + let root = org.root_id.clone(); + let ou = org.create_ou(&root, "workloads").unwrap().id; + (root, ou) + }; + hooks.fire_if_membership_changed(&state).await; + let base = fired.load(Ordering::SeqCst); + + // Move out and straight back: the organization ends where it started, + // so what was recorded must describe that state, not the one in + // between — otherwise the next real move looks like no change. + { + let mut guard = state.write(); + let org = guard.as_mut().unwrap(); + org.move_account("111111111111", &root, &ou).unwrap(); + } + hooks.fire_if_membership_changed(&state).await; + { + let mut guard = state.write(); + let org = guard.as_mut().unwrap(); + org.move_account("111111111111", &ou, &root).unwrap(); + } + hooks.fire_if_membership_changed(&state).await; + let after_round_trip = fired.load(Ordering::SeqCst); + assert_eq!(after_round_trip, base + 2); + + // The same move again is a real change and has to be announced. + { + let mut guard = state.write(); + let org = guard.as_mut().unwrap(); + org.move_account("111111111111", &root, &ou).unwrap(); + } + hooks.fire_if_membership_changed(&state).await; + assert_eq!(fired.load(Ordering::SeqCst), after_round_trip + 1); +} diff --git a/crates/fakecloud-server/src/main.rs b/crates/fakecloud-server/src/main.rs index db840af1f..e4585f763 100644 --- a/crates/fakecloud-server/src/main.rs +++ b/crates/fakecloud-server/src/main.rs @@ -2379,19 +2379,46 @@ async fn main() { } else { None }; - let mut organizations_inner = OrganizationsService::new(organizations_state.clone()); + // Observers of organization membership changes. Installed further down, + // once the services that react to them exist (CloudFormation StackSets + // auto-deployment); Organizations fires whatever is registered by then. + let org_change_hooks = fakecloud_organizations::OrgChangeHooks::new(); + let mut organizations_inner = OrganizationsService::new(organizations_state.clone()) + .with_change_hooks(org_change_hooks.clone()); if let Some(store) = organizations_snapshot_store.clone() { organizations_inner = organizations_inner.with_snapshot_store(store); } - if let Some(h) = organizations_inner.snapshot_hook() { - cfn_snapshot_hooks.insert("organizations", h); + // The CloudFormation provisioner mutates Organizations state directly + // (an `AWS::Organizations::Account` resource, say) instead of going + // through the service, so this hook is its only notification: persist the + // change, then let StackSets auto-deployment react to the membership the + // stack just changed. + { + let persist = organizations_inner.snapshot_hook(); + let changed = org_change_hooks.clone(); + let orgs_state = organizations_state.clone(); + cfn_snapshot_hooks.insert( + "organizations", + Arc::new(move || { + let persist = persist.clone(); + let changed = changed.clone(); + let orgs_state = orgs_state.clone(); + Box::pin(async move { + if let Some(persist) = &persist { + persist().await; + } + // Every stack operation runs this hook, so it has to be + // free unless the stack really did change membership. + changed.fire_if_membership_changed(&orgs_state).await; + }) + }), + ); } - // Re-arm CreateAccount completion ticks for requests restored as IN_PROGRESS. - organizations_inner.rearm_in_progress_account_creations(); // Hook shared with the create-admin admin endpoint, which auto-enrolls an // account into the org directly and must persist that through to disk. let organizations_persist_hook = organizations_inner.snapshot_hook(); - registry.register(Arc::new(organizations_inner)); + let organizations_service = Arc::new(organizations_inner); + registry.register(organizations_service.clone()); // EC2 (ec2Query protocol). Instances are backed by the optional container // runtime (Docker/Podman); persistence is wired in later batches. We keep a // clone of the shared state so the introspection router can expose @@ -6861,6 +6888,20 @@ async fn main() { // provisioners one resource at a time via this service. let cloudformation_arc = Arc::new(cloudformation_service); registry.register(cloudformation_arc.clone()); + // StackSets auto-deployment: an account added to (or removed from) an OU a + // service-managed stack set targets gains (or loses) its stack instances. + { + let cfn = cloudformation_arc.clone(); + org_change_hooks.register(Arc::new(move || { + let cfn = cfn.clone(); + Box::pin(async move { cfn.reconcile_auto_deployments().await }) + })); + } + // Re-arm CreateAccount completion ticks for requests restored as + // IN_PROGRESS. Done here, after the observers are registered: the tick + // fires a second or two later and enrolls the account, which is a + // membership change auto-deployment has to see. + organizations_service.rearm_in_progress_account_creations(); // Cloud Control API (cloudcontrolapi): uniform CRUD+L over every CFN // resource type, delegating to the CloudFormation provisioner bridge. @@ -11601,11 +11642,17 @@ async fn main() { let iam = iam_state.clone(); let orgs = organizations_state.clone(); let persist = organizations_persist_hook.clone(); + let changed = org_change_hooks.clone(); move |axum::Json(body): axum::Json| { let iam = iam.clone(); let orgs = orgs.clone(); let persist = persist.clone(); + let changed = changed.clone(); async move { + let was_member = orgs + .read() + .as_ref() + .is_some_and(|org| org.accounts.contains_key(&body.account_id)); let resp = reset::create_admin_in_account( &iam, &orgs, @@ -11617,6 +11664,11 @@ async fn main() { if let Some(hook) = &persist { hook().await; } + // ...and an account that just joined the root OU can + // be a stack set's auto-deployment target. + if !was_member { + changed.fire_if_membership_changed(&orgs).await; + } axum::Json(resp) } } diff --git a/website/content/docs/services/cloudformation.md b/website/content/docs/services/cloudformation.md index 20cb65f7f..bbc16395a 100644 --- a/website/content/docs/services/cloudformation.md +++ b/website/content/docs/services/cloudformation.md @@ -57,6 +57,7 @@ Stack instances are **real stacks**. `CreateStackInstances` creates a `StackSet- - **Operations** - as in AWS, a mutating call returns its `OperationId` straight away and the deployment runs in the background; poll `DescribeStackSetOperation` until it leaves `RUNNING`. Every operation records a per-target result (`Account`, `Region`, `Status`, `StatusReason`, `AccountGateResult`). Targets deploy in `RegionOrder` then request order. A failing target counts against `FailureToleranceCount` / `FailureTolerancePercentage` per region; once the tolerance is exceeded the remaining targets are `CANCELLED` and the operation ends `FAILED`. Stacks that provision asynchronously (templates with custom resources) leave the operation `RUNNING` until they settle; `StopStackSetOperation` cancels the targets that have not started. A second operation while one is running is `OperationInProgressException`, and a reused `OperationId` is `OperationIdAlreadyExistsException`. An operation cut short by a restart is settled as `FAILED` when state is loaded, so it does not block the stack set. - **Account gate** - when a target account has a Lambda named `AWSCloudFormationStackSetAccountGate`, it is invoked before deploying and the deployment only proceeds if it returns `{"Status": "SUCCEEDED"}`. Without the function the gate is `SKIPPED`. - **Service-managed** - `PermissionModel=SERVICE_MANAGED` needs an organization with StackSets trusted access (`ActivateOrganizationsAccess`, or Organizations `EnableAWSServiceAccess` for `member.org.stacksets.cloudformation.amazonaws.com`). `DeploymentTargets.OrganizationalUnitIds` resolve to the accounts in those OUs and every OU nested below them, never the management account; `AccountFilterType` (`INTERSECTION`, `DIFFERENCE`, `UNION`, `NONE`) combines them with `DeploymentTargets.Accounts` or an `AccountsUrl` file in S3. Suspended accounts are recorded as `SKIPPED_SUSPENDED_ACCOUNT`. `CallAs=DELEGATED_ADMIN` works from an account registered as a StackSets delegated administrator and acts on the management account's stack sets. +- **Auto-deployment** - with `AutoDeployment.Enabled=true`, a service-managed stack set follows the organization. An account that joins one of the OUs the stack set is deployed to (directly, by being moved there, created there, or invited into the organization) gains that stack set's instances in that OU's regions, recorded as an ordinary `CREATE` operation. An account that leaves loses them: its instances are deleted with their stacks, or, with `RetainStacksOnAccountRemoval=true`, only the instances are dropped and the stacks stay behind in the account. The OUs and regions are the stack set's own record of where it deploys, set by `CreateStackInstances` and cleared when `DeleteStackInstances` empties an OU of a region, so an OU that momentarily holds no accounts stays a target; `ListStackSetAutoDeploymentTargets` reports them. Instances left out on purpose stay out: an account an `AccountFilterType` excluded when the instances were created, or one whose instance was removed with `DeleteStackInstances`, is not re-added while it stays in the OU that decision was made under (leaving, or moving to another target OU, deploys to it again). The decision is per OU and per region, so removing an instance in one region does not stop the OU's other regions deploying to that account. A stack that creates an account itself (`AWS::Organizations::Account`) triggers the same deployment. Deployment runs before the Organizations call that triggered it returns, so an account is fully provisioned by the time `MoveAccount`, `AcceptHandshake` or `RemoveAccountFromOrganization` answers (`CreateAccount` enrolls the account asynchronously, as in AWS, so its deployment follows the request reaching `SUCCEEDED` rather than preceding it); a stack set that is busy with another operation is never waited on inline — it is re-planned in the background once that operation finishes. A template whose resources take a long time to provision makes the triggering Organizations call wait too, but only up to 20 seconds; past that the deployment carries on in the background, as AWS does from the start. - **Import** - `ImportStacksToStackSet` adopts existing stacks (by `StackIds` or a `StackIdsUrl` file) as instances without redeploying them. A stack already managed by a stack set is refused, and a stack whose template differs from the stack set's is recorded as `FAILED_IMPORT`. `CreateStackSet` with `StackId` starts a stack set from a stack's template and parameters. - **Drift** - `DetectStackSetDrift` checks each instance's stack resources against the live backing services, sets each instance's `DriftStatus`, and reports the counts in `StackSetDriftDetectionDetails`; `ListStackInstanceResourceDrifts` lists the per-resource results for an operation. diff --git a/website/content/docs/services/organizations.md b/website/content/docs/services/organizations.md index c4de8ab41..64fc4710f 100644 --- a/website/content/docs/services/organizations.md +++ b/website/content/docs/services/organizations.md @@ -17,6 +17,7 @@ fakecloud implements **63 of 63** AWS Organizations operations at 100% Smithy co - `DescribeAccount`, `ListAccounts`, `ListAccountsForParent` paginate the directory. - `InviteAccountToOrganization` + `AcceptHandshake` / `DeclineHandshake` / `CancelHandshake` / `DescribeHandshake` + `ListHandshakesForAccount` / `ListHandshakesForOrganization` run the real handshake state machine — only the invited account can accept/decline, only the inviter can cancel, and expired handshakes flip to `EXPIRED`. - **Organizational Units** — `CreateOrganizationalUnit`, `UpdateOrganizationalUnit`, `DeleteOrganizationalUnit`, `DescribeOrganizationalUnit`, `ListOrganizationalUnitsForParent`, `ListChildren`, `ListParents`, `ListRoots`, `MoveAccount`. The hierarchy is enforced — non-empty OUs cannot be deleted, and `MoveAccount` validates both source and destination parents. +- **StackSets auto-deployment** — a membership change (an account created, invited, moved between OUs, removed or closed) reconciles every service-managed CloudFormation stack set that has `AutoDeployment.Enabled`, so an account that joins a targeted OU is provisioned with that stack set's stacks before the Organizations call returns. See [CloudFormation](/docs/services/cloudformation/#stack-sets). - **Policies** — `CreatePolicy`, `UpdatePolicy`, `DeletePolicy`, `DescribePolicy`, `ListPolicies`, `ListPoliciesForTarget`, `ListTargetsForPolicy`, `AttachPolicy`, `DetachPolicy`, `EnablePolicyType`, `DisablePolicyType`, `DescribeEffectivePolicy`. The full `PolicyType` enum is accepted on the list filters; the four types fakecloud manages (`SERVICE_CONTROL_POLICY`, `TAG_POLICY`, `BACKUP_POLICY`, `AISERVICES_OPT_OUT_POLICY`) can be created — others return `PolicyTypeNotAvailableForOrganizationException`, an out-of-enum value returns `InvalidInputException`. Policy documents are JSON-validated on create/update — malformed content is rejected with `MalformedPolicyDocumentException`. - **Effective-policy validation** — `ListAccountsWithInvalidEffectivePolicy` and `ListEffectivePolicyValidationErrors` return the honest empty result: fakecloud stores only well-formed policies, so no account ever has an invalid effective policy. - **Billing responsibility transfers** — `InviteOrganizationToTransferResponsibility` opens a handshake-backed `BILLING` transfer; `DescribeResponsibilityTransfer`, `UpdateResponsibilityTransfer` (rename), `TerminateResponsibilityTransfer` (-> `WITHDRAWN`), and `ListInboundResponsibilityTransfers` / `ListOutboundResponsibilityTransfers` operate over the transfer records, filtered by direction.