diff --git a/debian/changelog b/debian/changelog index b0bb862..12fd44c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +puppet-code (0.1.0-1build303) noble; urgency=medium + + * commit event. see changes history in git log + + -- root Sun, 19 Apr 2026 16:47:45 +0000 + puppet-code (0.1.0-1build302) noble; urgency=medium * commit event. see changes history in git log diff --git a/modules/profile/files/github_runner/gha-lifecycle-heartbeater.sh b/modules/profile/files/github_runner/gha-lifecycle-heartbeater.sh new file mode 100644 index 0000000..fb183ff --- /dev/null +++ b/modules/profile/files/github_runner/gha-lifecycle-heartbeater.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# No-op unless the instance is in Terminating:Wait. Fire-and-forget. +set -eu + +hook_name="${DEREGISTRATION_HOOK_NAME:-}" +[[ -z "$hook_name" ]] && exit 0 + +instance_id=$(ec2metadata --instance-id) +state=$(aws autoscaling describe-auto-scaling-instances \ + --instance-ids "$instance_id" \ + --query 'AutoScalingInstances[0].LifecycleState' --output text 2>/dev/null || echo "") + +if [[ "$state" == "Terminating:Wait" ]]; then + asg=$(ih-ec2 tags | jq -r '."aws:autoscaling:groupName"') + aws autoscaling record-lifecycle-action-heartbeat \ + --auto-scaling-group-name "$asg" \ + --lifecycle-hook-name "$hook_name" \ + --instance-id "$instance_id" +fi diff --git a/modules/profile/files/github_runner/gha-lifecycle-heartbeater.timer b/modules/profile/files/github_runner/gha-lifecycle-heartbeater.timer new file mode 100644 index 0000000..0ea5e62 --- /dev/null +++ b/modules/profile/files/github_runner/gha-lifecycle-heartbeater.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Every 10 minutes, heartbeat the deregistration lifecycle hook if needed + +[Timer] +OnBootSec=2min +OnUnitActiveSec=10min +Unit=gha-lifecycle-heartbeater.service + +[Install] +WantedBy=timers.target diff --git a/modules/profile/files/github_runner/gha-on-runner-exit.sh b/modules/profile/files/github_runner/gha-on-runner-exit.sh new file mode 100644 index 0000000..a1b9834 --- /dev/null +++ b/modules/profile/files/github_runner/gha-on-runner-exit.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Called by systemd's ExecStopPost when actions-runner.service stops. +# If the ASG wants this instance terminated, complete the deregistration +# lifecycle hook now so the instance can go away cleanly. +set -eu + +hook_name="${DEREGISTRATION_HOOK_NAME:-}" +[[ -z "$hook_name" ]] && exit 0 + +instance_id=$(ec2metadata --instance-id) +state=$(aws autoscaling describe-auto-scaling-instances \ + --instance-ids "$instance_id" \ + --query 'AutoScalingInstances[0].LifecycleState' --output text 2>/dev/null || echo "") + +case "$state" in + Terminating:Wait|Terminating:Proceed) + /usr/local/bin/ih-aws autoscaling complete --result CONTINUE "$hook_name" + ;; +esac diff --git a/modules/profile/files/github_runner/gha_prerun.sh b/modules/profile/files/github_runner/gha_prerun.sh index 8364a4f..ed5e142 100644 --- a/modules/profile/files/github_runner/gha_prerun.sh +++ b/modules/profile/files/github_runner/gha_prerun.sh @@ -3,4 +3,22 @@ set -eu sudo chown -R "$USER" "$GITHUB_WORKSPACE" -/usr/local/bin/ih-aws autoscaling scale-in enable-protection + +# Try to protect this instance from scale-in. If the ASG has already +# decided to terminate us, protection is meaningless; let the job run +# and let the deprovisioning path finish us off cleanly. +if ! /usr/local/bin/ih-aws autoscaling scale-in enable-protection 2>/tmp/prerun_err; then + instance_id=$(ec2metadata --instance-id) + state=$(aws autoscaling describe-auto-scaling-instances \ + --instance-ids "$instance_id" \ + --query 'AutoScalingInstances[0].LifecycleState' --output text 2>/dev/null || echo "") + case "$state" in + Terminating:Wait|Terminating:Proceed) + echo "prerun: instance is in $state — skipping protect, job will proceed" >&2 + ;; + *) + cat /tmp/prerun_err >&2 + exit 1 + ;; + esac +fi diff --git a/modules/profile/manifests/github_runner/register.pp b/modules/profile/manifests/github_runner/register.pp index c0f3044..490cda3 100644 --- a/modules/profile/manifests/github_runner/register.pp +++ b/modules/profile/manifests/github_runner/register.pp @@ -25,6 +25,14 @@ creates => "${runner_package_directory}/.credentials", require => [ Exec[extract_runner_package] - ] + ], + notify => Exec['delete_registration_token'], + } + + exec { 'delete_registration_token': + user => $user, + path => '/usr/bin:/usr/local/bin', + command => "aws secretsmanager delete-secret --secret-id ${token_secret} --force-delete-without-recovery", + refreshonly => true, } } diff --git a/modules/profile/manifests/github_runner/service.pp b/modules/profile/manifests/github_runner/service.pp index 14f75db..6ff6018 100644 --- a/modules/profile/manifests/github_runner/service.pp +++ b/modules/profile/manifests/github_runner/service.pp @@ -18,6 +18,11 @@ $env_file = "${runner_package_directory}/.env" $prerun_path = '/usr/local/bin/gha_prerun.sh' $postrun_path = '/usr/local/bin/gha_postrun.sh' + $on_exit_path = '/usr/local/bin/gha-on-runner-exit.sh' + $heartbeater_script = '/usr/local/bin/gha-lifecycle-heartbeater.sh' + $heartbeater_service = '/etc/systemd/system/gha-lifecycle-heartbeater.service' + $heartbeater_timer = '/etc/systemd/system/gha-lifecycle-heartbeater.timer' + $deregistration_hookname = pick_default($facts['deregistration_hookname'], '') file { $env_file: ensure => file, @@ -56,6 +61,22 @@ mode => '0755', } + file { $on_exit_path: + ensure => file, + source => 'puppet:///modules/profile/github_runner/gha-on-runner-exit.sh', + owner => 'root', + group => 'root', + mode => '0755', + } + + file { $heartbeater_script: + ensure => file, + source => 'puppet:///modules/profile/github_runner/gha-lifecycle-heartbeater.sh', + owner => 'root', + group => 'root', + mode => '0755', + } + file { $start_script: ensure => file, content => template('profile/github_runner/start-actions-runner.sh.erb'), @@ -73,6 +94,24 @@ notify => Exec['daemon-reload'], } + file { $heartbeater_service: + ensure => file, + content => template('profile/github_runner/gha-lifecycle-heartbeater.service.erb'), + owner => 'root', + group => 'root', + mode => '0644', + notify => Exec['daemon-reload'], + } + + file { $heartbeater_timer: + ensure => file, + source => 'puppet:///modules/profile/github_runner/gha-lifecycle-heartbeater.timer', + owner => 'root', + group => 'root', + mode => '0644', + notify => Exec['daemon-reload'], + } + exec { 'daemon-reload': command => '/usr/bin/systemctl daemon-reload', refreshonly => true, @@ -88,6 +127,17 @@ ] } + service { 'gha-lifecycle-heartbeater.timer': + ensure => running, + enable => true, + require => [ + File[$heartbeater_script], + File[$heartbeater_service], + File[$heartbeater_timer], + Exec['daemon-reload'], + ] + } + cron { 'check-health': command => [ 'ih-github', diff --git a/modules/profile/templates/github_runner/actions-runner.service.erb b/modules/profile/templates/github_runner/actions-runner.service.erb index d3d7e22..162edb8 100644 --- a/modules/profile/templates/github_runner/actions-runner.service.erb +++ b/modules/profile/templates/github_runner/actions-runner.service.erb @@ -1,13 +1,19 @@ [Unit] Description=GitHub self-hosted runner -After=network.target +After=network-online.target +Wants=network-online.target [Service] Type=simple ExecStart=<%= @start_script %> +ExecStopPost=/usr/local/bin/gha-on-runner-exit.sh +Environment=DEREGISTRATION_HOOK_NAME=<%= @deregistration_hookname %> WorkingDirectory=<%= @runner_package_directory %> User=<%= @github_runner_user %> Group=<%= @github_runner_group %> +KillMode=process +KillSignal=SIGTERM +TimeoutStopSec=21600 Restart=on-failure [Install] diff --git a/modules/profile/templates/github_runner/gha-lifecycle-heartbeater.service.erb b/modules/profile/templates/github_runner/gha-lifecycle-heartbeater.service.erb new file mode 100644 index 0000000..7c42663 --- /dev/null +++ b/modules/profile/templates/github_runner/gha-lifecycle-heartbeater.service.erb @@ -0,0 +1,8 @@ +[Unit] +Description=Extend deregistration lifecycle hook while this instance is terminating + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/gha-lifecycle-heartbeater.sh +Environment=DEREGISTRATION_HOOK_NAME=<%= @deregistration_hookname %> +Restart=on-failure diff --git a/modules/profile/templates/github_runner/start-actions-runner.sh.erb b/modules/profile/templates/github_runner/start-actions-runner.sh.erb index a6ad190..05e145e 100644 --- a/modules/profile/templates/github_runner/start-actions-runner.sh.erb +++ b/modules/profile/templates/github_runner/start-actions-runner.sh.erb @@ -2,9 +2,13 @@ set -eu +instance_id=$(ec2metadata --instance-id) + while true do - state="$(aws autoscaling describe-auto-scaling-instances --instance-ids "$(ec2metadata --instance-id)" | jq -r .AutoScalingInstances[0].LifecycleState)" + state=$(aws autoscaling describe-auto-scaling-instances \ + --instance-ids "$instance_id" \ + --query 'AutoScalingInstances[0].LifecycleState' --output text) if [[ "$state" == "InService" ]]; then break else @@ -13,5 +17,4 @@ do fi done -# Start actions-runner -<%= @runner_package_directory %>/run.sh +exec <%= @runner_package_directory %>/bin/runsvc.sh