Promote InfraHouse repo/key convergence to global (production)

.claude/plans/infrahouse-keyring-convergence.md

@@ -102,7 +102,10 @@ Test node: `jumphost-sincere-crawdad.rmdbkn.ci-cd.infrahouse.com`
 - [x] Promote: copy the change into `environments/sandbox/modules/profile` (byte-identical to dev:
   `infrahouse_repo.pp` + `files/converge-infrahouse-keyring.sh` + `base.pp` include).
 - [x] Cut a release — PR [#279](https://github.com/infrahouse/puppet-code/pull/279) **merged** → CD.
-- [ ] Watch sandbox nodes across roles: keyring converges, `apt-get update` clean, no drift
+- [x] Watch sandbox nodes across roles: keyring converges, `apt-get update` clean, no drift.
+  Verified 2026-07-04 via SSM: `elastic_master` (`i-0f554096e62f32ea0`) `puppet_exit=0` +
+  `CONVERGED_OK` (regression below fixed), and `jumphost` (`i-04c4cd399cd73d6a0`) `puppet_exit=0` +
+  `CONVERGED_OK` + apt `Hit … noble InRelease`. Source list md5 `627951c5…` on both.
   - **Regression found on `elastic_master` (sandbox `ip-10-0-2-191`):** `profile::infrahouse_repo`
     is in `base` (every node) and `ensure_packages`'d `ca-certificates`, which collided with
     `profile::elastic::tls`'s **native** `package { 'ca-certificates' }` → duplicate declaration,
@@ -111,7 +114,10 @@ Test node: `jumphost-sincere-crawdad.rmdbkn.ci-cd.infrahouse.com`
     it (only jumphost was tested). Applied to dev + sandbox; global gets it with the Phase-3 promo.
 
 ### Phase 3: Production (global modules)
-- [ ] Promote: move the change into global `modules/profile` (production has no env-local override)
+- [x] Promote: move the change into global `modules/profile` (production has no env-local override).
+  Bundled with the `elastic::tls` ca-certificates fix so prod's elastic nodes never see the broken
+  intermediate state; all 4 touched files now byte-identical across global/dev/sandbox (invariant
+  from #276 restored). PR pending.
 - [ ] Cut a release (merge → CD); deploy fleet-wide
 - [ ] Watch production; spot-check long-lived instances across services (openvpn, jumphost, etc.)
 - [ ] **GATE:** fleet trusts the current bundle before any signing-key retire happens upstream

debian/changelog

@@ -1,3 +1,9 @@
+puppet-code (0.1.0-1build314) noble; urgency=medium
+
+  * commit event. see changes history in git log
+
+ -- root <packager@infrahouse.com>  Sat, 04 Jul 2026 12:50:31 +0000
+
 puppet-code (0.1.0-1build313) noble; urgency=medium
 
   * commit event. see changes history in git log

modules/profile/files/converge-infrahouse-keyring.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Managed by Puppet (profile::infrahouse_repo). Do not edit locally.
+#
+# Converge /etc/apt/keyrings/infrahouse.gpg from the published InfraHouse
+# signing-key bundle so that long-lived instances pick up rotated GPG keys
+# without being reprovisioned. Trust is anchored on the TLS channel to
+# release-<codename>.infrahouse.com (no fingerprint pinning), consistent with
+# cloud-init. The (possibly concatenated multi-key) armored bundle is fetched
+# each run and dearmored into the keyring.
+#
+# Usage:
+#   converge-infrahouse-keyring.sh check <codename>
+#       exit 0 if the installed keyring already matches the published bundle,
+#       exit 1 otherwise (including when the bundle cannot be fetched, so the
+#       caller re-runs 'apply' and surfaces the underlying error).
+#   converge-infrahouse-keyring.sh apply <codename>
+#       fetch + dearmor + install the keyring atomically. A fetch/dearmor
+#       failure leaves the existing keyring intact. Refreshing apt is left to
+#       the Puppet-managed 'apt-get update' that this exec notifies.
+#
+set -euo pipefail
+
+export PATH=/usr/sbin:/usr/bin:/sbin:/bin
+
+KEYRING=/etc/apt/keyrings/infrahouse.gpg
+
+usage() {
+  echo "usage: $0 {check|apply} <codename>" >&2
+  exit 2
+}
+
+[ "$#" -eq 2 ] || usage
+action=$1
+codename=$2
+url="https://release-${codename}.infrahouse.com/DEB-GPG-KEY-release-${codename}.infrahouse.com"
+
+# Fetch the armored bundle over TLS and dearmor it into $1. Returns non-zero
+# without touching $KEYRING on any failure (fetch, dearmor, or empty result).
+fetch_dearmored() {
+  local out=$1 armored
+  armored=$(mktemp)
+  # shellcheck disable=SC2064
+  trap "rm -f '${armored}'" RETURN
+  curl --fail --silent --show-error --location \
+    --retry 3 --retry-delay 2 --max-time 30 \
+    --output "${armored}" "${url}"
+  gpg --dearmor <"${armored}" >"${out}"
+  # Refuse to install an empty/garbage keyring.
+  test -s "${out}"
+}
+
+candidate=$(mktemp)
+trap 'rm -f "${candidate}"' EXIT
+
+case "${action}" in
+  check)
+    fetch_dearmored "${candidate}" || exit 1
+    # 0 = already converged, 1 = differs or keyring missing.
+    cmp -s "${candidate}" "${KEYRING}"
+    ;;
+  apply)
+    fetch_dearmored "${candidate}"
+    if cmp -s "${candidate}" "${KEYRING}"; then
+      exit 0
+    fi
+    install -D -o root -g root -m 0644 "${candidate}" "${KEYRING}"
+    ;;
+  *)
+    usage
+    ;;
+esac

modules/profile/manifests/base.pp

@@ -5,6 +5,7 @@
 
   include 'profile::ntp'
   include 'profile::repos'
+  include 'profile::infrahouse_repo'
   include 'profile::packages'
   include 'profile::infrahouse_toolkit'
   include 'profile::puppet_apply'

modules/profile/manifests/elastic/tls.pp

@@ -93,10 +93,9 @@
     mode    => '0600',
     require => Exec[generate_node_pem],
   }
-# Let's encrypt root certificates
-  package {'ca-certificates':
-    ensure => present,
-  }
+  # Let's encrypt root certificates. The ca-certificates package (which provides
+  # the Mozilla/ISRG roots linked below) is installed fleet-wide by profile::base
+  # -> profile::infrahouse_repo; the file resources below just require it.
   file { '/etc/elasticsearch/tls/ISRG_Root_X1.crt':
     ensure  => present,
     owner   => 'elasticsearch',

modules/profile/manifests/infrahouse_repo.pp

@@ -0,0 +1,67 @@
+# @summary: Converges the InfraHouse APT repository (source list + signing key).
+#
+# cloud-init seeds both `/etc/apt/sources.list.d/50-infrahouse.list` and
+# `/etc/apt/keyrings/infrahouse.gpg` at **first boot only** — it must, because the
+# InfraHouse toolkit (ih-secrets) has to be installable before Puppet ever runs.
+# This class **converges** both on every Puppet run so long-lived instances pick
+# up a rotated signing key (the urgent case: the key expires) or a changed repo
+# line without being reprovisioned.
+#
+# Ownership is intentionally shared: cloud-init writes the seed, Puppet keeps it
+# in sync. The rendered source line is kept byte-identical to cloud-init's seed so
+# the two never fight. Trust is anchored on the TLS channel to
+# release-<codename>.infrahouse.com (no fingerprint pinning), consistent with
+# cloud-init. On a fetch failure the previously installed keyring is left intact.
+#
+# @param codename APT codename whose repo/bundle to converge; derived from the
+#   node's OS facts by default.
+class profile::infrahouse_repo (
+  String[1] $codename = $facts['os']['distro']['codename'],
+) {
+  $script      = '/usr/local/sbin/converge-infrahouse-keyring.sh'
+  $keyring     = '/etc/apt/keyrings/infrahouse.gpg'
+  $source_list = '/etc/apt/sources.list.d/50-infrahouse.list'
+
+  stdlib::ensure_packages(['curl', 'gnupg', 'ca-certificates'])
+
+  file { $script:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/profile/converge-infrahouse-keyring.sh',
+  }
+
+  # Converge the keyring: 'check' short-circuits when already in sync, so steady
+  # state is a no-op and the shared 'apt-get update' only fires on a key change.
+  exec { 'profile::infrahouse_repo::converge':
+    command => "${script} apply ${codename}",
+    unless  => "${script} check ${codename}",
+    path    => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'],
+    require => [
+      File[$script],
+      Package['curl'],
+      Package['gnupg'],
+    ],
+    notify  => Exec['profile::infrahouse_repo::apt_update'],
+  }
+
+  # Converge the repo line. Kept byte-identical to cloud-init's first-boot seed;
+  # $codename fills both the host (release-<codename>) and the suite.
+  file { $source_list:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => "deb [signed-by=${keyring}] https://release-${codename}.infrahouse.com/ ${codename} main\n",
+    require => Exec['profile::infrahouse_repo::converge'],
+    notify  => Exec['profile::infrahouse_repo::apt_update'],
+  }
+
+  # Single refresh point for both the keyring and the source list.
+  exec { 'profile::infrahouse_repo::apt_update':
+    command     => 'apt-get update',
+    path        => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'],
+    refreshonly => true,
+  }
+}