Enable unattended-upgrades fleet-wide; keep Elasticsearch undisturbed

debian/changelog

@@ -1,3 +1,15 @@
+puppet-code (0.1.0-1build310) noble; urgency=medium
+
+  * commit event. see changes history in git log
+
+ -- root <packager@infrahouse.com>  Sun, 28 Jun 2026 16:31:11 +0000
+
+puppet-code (0.1.0-1build309) noble; urgency=medium
+
+  * commit event. see changes history in git log
+
+ -- root <packager@infrahouse.com>  Sun, 28 Jun 2026 16:21:13 +0000
+
 puppet-code (0.1.0-1build308) noble; urgency=medium
 
   * commit event. see changes history in git log

environments/development/modules/profile/manifests/base.pp

@@ -9,6 +9,7 @@
   include 'profile::infrahouse_toolkit'
   include 'profile::puppet_apply'
   include 'profile::swap'
+  include 'profile::unattended_upgrades'
   include '::accounts'
   include '::sudo'
 

environments/development/modules/profile/manifests/elastic/service.pp

@@ -50,8 +50,35 @@
     ]
   }
 
-  service { 'unattended-upgrades.service':
-    ensure => stopped,
+  # Security updates are applied fleet-wide by profile::unattended_upgrades.
+  # On Elasticsearch nodes we keep that running but must never let it bounce
+  # the ES process: the elasticsearch package is blacklisted from automatic
+  # upgrades and needrestart is set to list-only, so library security fixes
+  # land on disk without an automatic restart. Elasticsearch is restarted only
+  # via instance refresh, guarded by the decommission-node cron below.
+  #
+  # The blacklist is baked in here (not Hiera) so an Elasticsearch node can
+  # never come up without it. The entry appends to the list defined by
+  # profile::unattended_upgrades.
+  file { '/etc/apt/apt.conf.d/53-elasticsearch-blacklist':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/elasticsearch/unattended-upgrades-blacklist.conf.erb'),
+  }
+
+  package { 'needrestart':
+    ensure => present,
+  }
+
+  file { '/etc/needrestart/conf.d/90-elastic-no-auto-restart.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/elasticsearch/needrestart-no-auto-restart.conf.erb'),
+    require => Package['needrestart'],
   }
 
   # If node is about to be replaced by instance refresh

environments/development/modules/profile/manifests/unattended_upgrades.pp

@@ -0,0 +1,75 @@
+# @summary: Enable unattended-upgrades fleet-wide for automatic security updates.
+#
+# Included by profile::base so every host receives security fixes. The base
+# AMI ships the relevant systemd units masked; this profile unmasks and enables
+# them and owns their configuration via Puppet.
+#
+# Hosts that must not be disrupted by an automatic service restart (e.g.
+# Elasticsearch nodes) keep unattended-upgrades running but drop their own
+# apt.conf.d blacklist entry (which appends to the list) and suppress automatic
+# restarts separately (see profile::elastic::service).
+class profile::unattended_upgrades (
+  Boolean $automatic_reboot = lookup('profile::unattended_upgrades::automatic_reboot', undef, undef, false),
+) {
+
+  package { 'unattended-upgrades':
+    ensure => present,
+  }
+
+  # Units that drive automatic upgrades:
+  #   - the timers run the periodic download + upgrade
+  #   - unattended-upgrades.service applies pending upgrades on shutdown/boot
+  # The base AMI ships these masked. A masked apt-daily.service also prevents
+  # its timer from starting ("unit to trigger not loaded"), so the trigger
+  # .service units must be unmasked too even though we never run them directly.
+  # Puppet's service provider cannot unmask a unit, hence the execs.
+  $unmask_units = [
+    'unattended-upgrades.service',
+    'apt-daily.service',
+    'apt-daily.timer',
+    'apt-daily-upgrade.service',
+    'apt-daily-upgrade.timer',
+  ]
+
+  # Only the timers (periodic runs) and unattended-upgrades.service (apply on
+  # shutdown) are actively enabled; apt-daily*.service are oneshots triggered
+  # by their timers.
+  $enabled_units = [
+    'unattended-upgrades.service',
+    'apt-daily.timer',
+    'apt-daily-upgrade.timer',
+  ]
+
+  $unmask_units.each |$unit| {
+    exec { "unmask-${unit}":
+      command => "systemctl unmask ${unit}",
+      path    => '/bin:/usr/bin:/sbin:/usr/sbin',
+      onlyif  => "systemctl is-enabled ${unit} 2>/dev/null | grep -qx masked",
+      before  => Service[$enabled_units],
+    }
+  }
+
+  service { $enabled_units:
+    ensure  => running,
+    enable  => true,
+    require => Package['unattended-upgrades'],
+  }
+
+  file { '/etc/apt/apt.conf.d/20auto-upgrades':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/unattended_upgrades/20auto-upgrades.erb'),
+    require => Package['unattended-upgrades'],
+  }
+
+  file { '/etc/apt/apt.conf.d/52unattended-upgrades-infrahouse':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/unattended_upgrades/52unattended-upgrades-infrahouse.erb'),
+    require => Package['unattended-upgrades'],
+  }
+}

environments/development/modules/profile/templates/elasticsearch/needrestart-no-auto-restart.conf.erb

@@ -0,0 +1,5 @@
+# Managed by Puppet (profile::elastic::service). Do not edit.
+# Do not auto-restart services after package/library upgrades. Elasticsearch is
+# restarted only via controlled instance refresh, so unattended-upgrades must
+# never bounce it. 'l' = list pending restarts only, never act.
+$nrconf{restart} = 'l';

environments/development/modules/profile/templates/elasticsearch/unattended-upgrades-blacklist.conf.erb

@@ -0,0 +1,6 @@
+// Managed by Puppet (profile::elastic::service). Do not edit.
+// Never auto-upgrade Elasticsearch; version changes go via instance refresh.
+// Appends to Unattended-Upgrade::Package-Blacklist (see 52unattended-upgrades).
+Unattended-Upgrade::Package-Blacklist {
+    "elasticsearch";
+};

environments/development/modules/profile/templates/unattended_upgrades/20auto-upgrades.erb

@@ -0,0 +1,3 @@
+// Managed by Puppet (profile::unattended_upgrades). Do not edit.
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Unattended-Upgrade "1";

environments/development/modules/profile/templates/unattended_upgrades/52unattended-upgrades-infrahouse.erb

@@ -0,0 +1,7 @@
+// Managed by Puppet (profile::unattended_upgrades). Do not edit.
+// Augments the distro default /etc/apt/apt.conf.d/50unattended-upgrades: the
+// -security origin stays enabled there; here we only pin reboot behaviour.
+// Per-host package blacklisting is done by the relevant profile via its own
+// apt.conf.d drop-in (entries append to Unattended-Upgrade::Package-Blacklist).
+
+Unattended-Upgrade::Automatic-Reboot "<%= @automatic_reboot %>";

environments/sandbox/modules/profile/manifests/base.pp

@@ -9,6 +9,7 @@
   include 'profile::infrahouse_toolkit'
   include 'profile::puppet_apply'
   include 'profile::swap'
+  include 'profile::unattended_upgrades'
   include '::accounts'
   include '::sudo'
 

environments/sandbox/modules/profile/manifests/elastic/service.pp

@@ -50,8 +50,35 @@
     ]
   }
 
-  service { 'unattended-upgrades.service':
-    ensure => stopped,
+  # Security updates are applied fleet-wide by profile::unattended_upgrades.
+  # On Elasticsearch nodes we keep that running but must never let it bounce
+  # the ES process: the elasticsearch package is blacklisted from automatic
+  # upgrades and needrestart is set to list-only, so library security fixes
+  # land on disk without an automatic restart. Elasticsearch is restarted only
+  # via instance refresh, guarded by the decommission-node cron below.
+  #
+  # The blacklist is baked in here (not Hiera) so an Elasticsearch node can
+  # never come up without it. The entry appends to the list defined by
+  # profile::unattended_upgrades.
+  file { '/etc/apt/apt.conf.d/53-elasticsearch-blacklist':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/elasticsearch/unattended-upgrades-blacklist.conf.erb'),
+  }
+
+  package { 'needrestart':
+    ensure => present,
+  }
+
+  file { '/etc/needrestart/conf.d/90-elastic-no-auto-restart.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/elasticsearch/needrestart-no-auto-restart.conf.erb'),
+    require => Package['needrestart'],
   }
 
   # If node is about to be replaced by instance refresh

environments/sandbox/modules/profile/manifests/unattended_upgrades.pp

@@ -0,0 +1,75 @@
+# @summary: Enable unattended-upgrades fleet-wide for automatic security updates.
+#
+# Included by profile::base so every host receives security fixes. The base
+# AMI ships the relevant systemd units masked; this profile unmasks and enables
+# them and owns their configuration via Puppet.
+#
+# Hosts that must not be disrupted by an automatic service restart (e.g.
+# Elasticsearch nodes) keep unattended-upgrades running but drop their own
+# apt.conf.d blacklist entry (which appends to the list) and suppress automatic
+# restarts separately (see profile::elastic::service).
+class profile::unattended_upgrades (
+  Boolean $automatic_reboot = lookup('profile::unattended_upgrades::automatic_reboot', undef, undef, false),
+) {
+
+  package { 'unattended-upgrades':
+    ensure => present,
+  }
+
+  # Units that drive automatic upgrades:
+  #   - the timers run the periodic download + upgrade
+  #   - unattended-upgrades.service applies pending upgrades on shutdown/boot
+  # The base AMI ships these masked. A masked apt-daily.service also prevents
+  # its timer from starting ("unit to trigger not loaded"), so the trigger
+  # .service units must be unmasked too even though we never run them directly.
+  # Puppet's service provider cannot unmask a unit, hence the execs.
+  $unmask_units = [
+    'unattended-upgrades.service',
+    'apt-daily.service',
+    'apt-daily.timer',
+    'apt-daily-upgrade.service',
+    'apt-daily-upgrade.timer',
+  ]
+
+  # Only the timers (periodic runs) and unattended-upgrades.service (apply on
+  # shutdown) are actively enabled; apt-daily*.service are oneshots triggered
+  # by their timers.
+  $enabled_units = [
+    'unattended-upgrades.service',
+    'apt-daily.timer',
+    'apt-daily-upgrade.timer',
+  ]
+
+  $unmask_units.each |$unit| {
+    exec { "unmask-${unit}":
+      command => "systemctl unmask ${unit}",
+      path    => '/bin:/usr/bin:/sbin:/usr/sbin',
+      onlyif  => "systemctl is-enabled ${unit} 2>/dev/null | grep -qx masked",
+      before  => Service[$enabled_units],
+    }
+  }
+
+  service { $enabled_units:
+    ensure  => running,
+    enable  => true,
+    require => Package['unattended-upgrades'],
+  }
+
+  file { '/etc/apt/apt.conf.d/20auto-upgrades':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/unattended_upgrades/20auto-upgrades.erb'),
+    require => Package['unattended-upgrades'],
+  }
+
+  file { '/etc/apt/apt.conf.d/52unattended-upgrades-infrahouse':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/unattended_upgrades/52unattended-upgrades-infrahouse.erb'),
+    require => Package['unattended-upgrades'],
+  }
+}

environments/sandbox/modules/profile/templates/elasticsearch/needrestart-no-auto-restart.conf.erb

@@ -0,0 +1,5 @@
+# Managed by Puppet (profile::elastic::service). Do not edit.
+# Do not auto-restart services after package/library upgrades. Elasticsearch is
+# restarted only via controlled instance refresh, so unattended-upgrades must
+# never bounce it. 'l' = list pending restarts only, never act.
+$nrconf{restart} = 'l';

environments/sandbox/modules/profile/templates/elasticsearch/unattended-upgrades-blacklist.conf.erb

@@ -0,0 +1,6 @@
+// Managed by Puppet (profile::elastic::service). Do not edit.
+// Never auto-upgrade Elasticsearch; version changes go via instance refresh.
+// Appends to Unattended-Upgrade::Package-Blacklist (see 52unattended-upgrades).
+Unattended-Upgrade::Package-Blacklist {
+    "elasticsearch";
+};

environments/sandbox/modules/profile/templates/unattended_upgrades/20auto-upgrades.erb

@@ -0,0 +1,3 @@
+// Managed by Puppet (profile::unattended_upgrades). Do not edit.
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Unattended-Upgrade "1";

environments/sandbox/modules/profile/templates/unattended_upgrades/52unattended-upgrades-infrahouse.erb

@@ -0,0 +1,7 @@
+// Managed by Puppet (profile::unattended_upgrades). Do not edit.
+// Augments the distro default /etc/apt/apt.conf.d/50unattended-upgrades: the
+// -security origin stays enabled there; here we only pin reboot behaviour.
+// Per-host package blacklisting is done by the relevant profile via its own
+// apt.conf.d drop-in (entries append to Unattended-Upgrade::Package-Blacklist).
+
+Unattended-Upgrade::Automatic-Reboot "<%= @automatic_reboot %>";

modules/profile/manifests/base.pp

@@ -9,6 +9,7 @@
   include 'profile::infrahouse_toolkit'
   include 'profile::puppet_apply'
   include 'profile::swap'
+  include 'profile::unattended_upgrades'
   include '::accounts'
   include '::sudo'
 

modules/profile/manifests/elastic/service.pp

@@ -50,8 +50,35 @@
     ]
   }
 
-  service { 'unattended-upgrades.service':
-    ensure => stopped,
+  # Security updates are applied fleet-wide by profile::unattended_upgrades.
+  # On Elasticsearch nodes we keep that running but must never let it bounce
+  # the ES process: the elasticsearch package is blacklisted from automatic
+  # upgrades and needrestart is set to list-only, so library security fixes
+  # land on disk without an automatic restart. Elasticsearch is restarted only
+  # via instance refresh, guarded by the decommission-node cron below.
+  #
+  # The blacklist is baked in here (not Hiera) so an Elasticsearch node can
+  # never come up without it. The entry appends to the list defined by
+  # profile::unattended_upgrades.
+  file { '/etc/apt/apt.conf.d/53-elasticsearch-blacklist':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/elasticsearch/unattended-upgrades-blacklist.conf.erb'),
+  }
+
+  package { 'needrestart':
+    ensure => present,
+  }
+
+  file { '/etc/needrestart/conf.d/90-elastic-no-auto-restart.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profile/elasticsearch/needrestart-no-auto-restart.conf.erb'),
+    require => Package['needrestart'],
   }
 
   # If node is about to be replaced by instance refresh