feat: supervise user-defined workspace services

.github/workflows/test-image.yaml

@@ -38,3 +38,21 @@ jobs:
       tailscale_oauth_secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
       build_secrets: |
         FETCH_GH_TOKEN=${{ secrets.GITHUB_TOKEN }}
+
+  integration:
+    needs: build
+    uses: ./.github/workflows/test-template.yaml
+    with:
+      image_repository: ${{ vars.CONTAINER_REGISTRY_PATH }}/coder-workspace
+      image_tag: ${{ needs.build.outputs.image_tag }}
+      service_commands: >-
+        ["test -f /home/coder/.local/state/dotfiles/applied || { touch /tmp/service-started-too-early; exit 1; };
+        printf x >> /tmp/service-restarts; sleep 2; exit 1",
+        "test -f /home/coder/.local/state/dotfiles/applied || { touch /tmp/service-started-too-early; exit 1; };
+        touch /tmp/service-steady; exec sleep 300"]
+    secrets:
+      private_registry: ${{ secrets.CONTAINER_REGISTRY }}
+      private_registry_username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+      private_registry_token: ${{ secrets.CONTAINER_REGISTRY_PASSWORD }}
+      tailscale_oauth_client_id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+      tailscale_oauth_secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }}

.github/workflows/test-template.yaml

@@ -8,9 +8,33 @@ on:
     - .github/compose/**
     - .github/workflows/test-template.yaml
     - templates/kubernetes/homelab-workspace/**
+  workflow_call:
+    inputs:
+      image_repository:
+        type: string
+        default: ""
+      image_tag:
+        type: string
+        default: ""
+      service_commands:
+        type: string
+        default: "[]"
+    secrets:
+      private_registry:
+        required: false
+      private_registry_username:
+        required: false
+      private_registry_token:
+        required: false
+      tailscale_oauth_client_id:
+        required: false
+      tailscale_oauth_secret:
+        required: false
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  # A called workflow inherits the caller's github.workflow and run ID. Include
+  # its image input so it cannot cancel the test-image run that invoked it.
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-${{ inputs.image_tag || 'released' }}
   cancel-in-progress: true
 
 permissions:
@@ -50,6 +74,36 @@ jobs:
       # created explicitly before the template can be applied.
       run: kubectl create namespace coder
 
+    - name: Connect to private registry network
+      if: inputs.image_tag != ''
+      uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3
+      with:
+        oauth-client-id: ${{ secrets.tailscale_oauth_client_id }}
+        oauth-secret: ${{ secrets.tailscale_oauth_secret }}
+        tags: tag:github-action-ci-runner
+
+    - name: Log in to private registry
+      if: inputs.image_tag != ''
+      uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+      with:
+        registry: ${{ secrets.private_registry }}
+        username: ${{ secrets.private_registry_username }}
+        password: ${{ secrets.private_registry_token }}
+
+    - name: Load pull request image into test cluster
+      if: inputs.image_tag != ''
+      id: pull_request_image
+      env:
+        IMAGE_REGISTRY: ${{ secrets.private_registry }}
+        IMAGE_REPOSITORY: ${{ inputs.image_repository }}
+        IMAGE_TAG: ${{ inputs.image_tag }}
+      shell: bash
+      run: |
+        workspace_image="${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG}"
+        docker pull "${workspace_image}"
+        kind load docker-image --name "${KIND_CLUSTER}" "${workspace_image}"
+        echo "workspace_image=${workspace_image}" >> "${GITHUB_OUTPUT}"
+
     - name: Start local Coder
       env:
         CODER_KUBECONFIG: ${{ runner.temp }}/kubeconfig-coder
@@ -99,7 +153,8 @@ jobs:
         coder_password: ci-password
 
     - name: Determine latest released workspace image
-      id: image
+      if: inputs.image_tag == ''
+      id: released_image
       env:
         GH_TOKEN: ${{ github.token }}
       shell: bash
@@ -113,15 +168,20 @@ jobs:
         template_dir: current/templates/kubernetes/homelab-workspace
         template_name: ${{ env.TEMPLATE_NAME }}
         template_version: ${{ github.sha }}
-        workspace_image: ${{ steps.image.outputs.workspace_image }}
+        workspace_image: ${{ steps.pull_request_image.outputs.workspace_image || steps.released_image.outputs.workspace_image }}
         test_mode: "true"
 
     - name: Start workspace
+      env:
+        SERVICE_COMMANDS: ${{ inputs.service_commands || '[]' }}
       shell: bash
       run: |
+        service_parameter="$(jq --null-input --raw-output \
+        --arg value "service_commands=${SERVICE_COMMANDS}" '[$value] | @csv')"
         coder create "${WORKSPACE_NAME}" --template "${TEMPLATE_NAME}" --no-wait --yes \
         --parameter dotfiles_url=https://github.com/ppat/dotfiles.git \
-        --parameter memory=4 --parameter preferred_nodes='[]' --parameter memory_watchdog_mode=enforce
+        --parameter memory=4 --parameter preferred_nodes='[]' --parameter memory_watchdog_mode=enforce \
+        --parameter "${service_parameter}"
 
     - name: Ping workspace agent
       shell: bash
@@ -150,6 +210,30 @@ jobs:
         origin="$(coder ssh "${WORKSPACE_NAME}" -- git -C /home/coder/.local/share/chezmoi remote get-url origin)"
         [[ "${origin}" == "https://github.com/ppat/dotfiles.git" ]]
 
+    - name: Confirm supervised service starts after dotfiles and restarts
+      if: inputs.service_commands != '' && inputs.service_commands != '[]'
+      shell: bash
+      run: |
+        # Expand the positional parameter inside the runner-owned polling shell.
+        # shellcheck disable=SC2016
+        if ! timeout 2m bash -c \
+        'until count="$(coder ssh "$1" -- stat --format=%s /tmp/service-restarts 2>/dev/null)" &&
+        [ "$count" -ge 2 ]; do sleep 2; done' \
+        _ "${WORKSPACE_NAME}"; then
+        coder ssh "${WORKSPACE_NAME}" -- \
+        supervisorctl --configuration /supervisord.conf status || true
+        coder ssh "${WORKSPACE_NAME}" -- ls -la /home/coder/.local/state/supervisor /tmp/service-* || true
+        coder ssh "${WORKSPACE_NAME}" -- tail -n 100 /home/coder/.local/state/supervisor/service-0.log || true
+        coder ssh "${WORKSPACE_NAME}" -- tail -n 100 /home/coder/.local/state/supervisor/service-1.log || true
+        exit 1
+        fi
+        coder ssh "${WORKSPACE_NAME}" -- test ! -e /tmp/service-started-too-early
+        coder ssh "${WORKSPACE_NAME}" -- test -e /tmp/service-steady
+        coder ssh "${WORKSPACE_NAME}" -- \
+        supervisorctl --configuration /supervisord.conf status service:service-0
+        coder ssh "${WORKSPACE_NAME}" -- \
+        supervisorctl --configuration /supervisord.conf status service:service-1
+
     - name: Show dotfiles log
       if: always()
       run: coder ssh "${WORKSPACE_NAME}" -- cat /home/coder/.local/state/dotfiles/run.log

DESIGN.md

@@ -69,6 +69,8 @@ The rule that ties the layers together: a package or tool belongs in the *lowest
 
 **Unprivileged by default.** The workspace itself runs as an unprivileged, non-root, fixed-identity container. Anything that genuinely needs elevated privilege (installing packages, preparing shared volume state) is scoped to a narrow, short-lived setup step that runs before the workspace shell exists, not to something the workspace user can reach into.
 
+**User-defined services start after dotfiles, under an unprivileged supervisor.** The image includes Supervisor because every workspace that declares a service needs the same stable process manager. The template accepts service commands as a list and starts one independently supervised program for each item, with automatic restart and per-program logs under the user's state directory. `supervisord.conf` is a native, mounted configuration file; its `numprocs` expands the command count from the environment, and each process resolves its command from the original JSON by index. No shell generates Supervisor configuration and command text is never reparsed as INI. Supervisor is launched by the external dotfiles orchestration script only after Chezmoi has successfully applied (or the explicit no-repository path has completed); a failed apply starts no services. It runs as the `coder` user, not PID 1 and not root, preserving the workspace's privilege boundary while leaving the Coder agent responsible for the container lifecycle.
+
 **The Deployment name is a Prometheus identity, not just a Kubernetes identifier.** `deployment.tf` names the workspace Deployment `coder-workspace-<owner>-<workspace-name>` (`local.workload_name` in `main.tf`) rather than the workspace UUID it used before. cAdvisor's `container_*` series carry no Kubernetes labels — they come from the cgroup filesystem, with no API-server connection — so per-workspace CPU/memory/PSI/OOM can only be attributed to a human-readable identity through the cluster's existing `namespace_workload_pod:kube_pod_owner:relabel` recording rule, which resolves pod → ReplicaSet → Deployment into a `workload` label. That rule already runs for free; naming the Deployment meaningfully is the only lever this repo has to make its output meaningful, at zero added Prometheus series and no PromQL join. Three things shape the exact scheme:
 
 - *Owner is included* even though this is a single-operator homelab today, because Coder workspace names are unique per-owner, not cluster-wide — two owners could otherwise pick the same workspace name and collide. Cheap to include now; expensive to retrofit after a second migration.

TESTING.md

@@ -17,11 +17,13 @@ the registries, and pushes the live Coder template with that release tag.
 PR workflows are scoped directly by changed paths:
 
 - `test-image.yaml` runs for image changes. It delegates the multi-architecture build and private-registry cache to
-  the shared image-build workflow, which connects to Tailscale for that registry.
+  the shared image-build workflow, then calls the template integration workflow with that branch image. The image is
+  built once: the integration job pulls the published branch artifact and loads it into Kind.
 - `test-template.yaml` runs for template changes. It creates a local Coder/Postgres compose deployment, gives Coder
   the test-cluster kubeconfig and an OpenTofu binary bind-mounted over its bundled `terraform`, publishes the
   template with the latest released GHCR workspace image, creates a workspace, pings its agent three times with a
-  timeout, and verifies SSH by running `env`.
+  timeout, and verifies SSH by running `env`. As a reusable workflow called by `test-image.yaml`, it instead uses the
+  exact branch image and exercises the contract between image and template.
 
 This test control plane is where this repo proves OpenTofu is the thing that actually *applies* the template —
 the live deployment's provisioner is a separate repo's decision, out of this repo's control (see the
@@ -42,7 +44,8 @@ pinning the Dockerfile's `FROM` line gets.
 
 ## Running the template test locally
 
-`test-template.yaml` needs nothing CI has that a laptop doesn't — no secrets, no Tailscale, no private registry. With
+The standalone `test-template.yaml` path needs nothing CI has that a laptop doesn't — no secrets, no Tailscale, no
+private registry. With
 `kind`, `docker compose`, `kubectl`, the `coder` CLI, and `tofu` (`mise install`, pinned in `mise.toml`) on `PATH`,
 from the repo root:
 
@@ -70,7 +73,8 @@ coder template push --directory templates/kubernetes/homelab-workspace \
   --var workspace_image=ghcr.io/ppat/coder-workspace:<a-released-tag> --var test_mode=true \
   --name local --yes homelab-workspace-test
 coder create local-test --template homelab-workspace-test --no-wait --yes \
-  --parameter memory=4 --parameter preferred_nodes='[]' --parameter memory_watchdog_mode=enforce
+  --parameter memory=4 --parameter preferred_nodes='[]' --parameter memory_watchdog_mode=enforce \
+  --parameter dotfiles_url='' --parameter service_commands='[]'
 coder ping --num 3 --timeout 30s local-test
 coder ssh local-test -- env
 ```
@@ -89,7 +93,12 @@ no `-f` needed.
 
 1. **Image build** — an image change exercises both published architectures and the existing private cache through
    the shared image-build workflow.
-2. **Template runtime** — a template-only PR applies against fresh Coder, Postgres, and Kubernetes state, using the
+2. **Image/template integration** — the image workflow passes its published branch tag to the reusable template
+   workflow. Two service commands record if either starts before Chezmoi completes; one exits repeatedly and must be
+   invoked at least twice, while the other must remain running as an independent Supervisor process. This proves the
+   changed image supplies Supervisor, the changed template starts it only after dotfiles, and Supervisor expands and
+   restarts the commands. The template workflow loads the existing image into Kind; it does not build another one.
+3. **Template runtime** — a template-only PR applies against fresh Coder, Postgres, and Kubernetes state, using the
    last released GHCR image. The workspace start, agent ping, and SSH command prove the rendered template's runtime
    path rather than merely its Terraform syntax.
 

images/homelab-workspace/Dockerfile

@@ -79,6 +79,7 @@ RUN --mount=type=cache,target=/var/cache/apt,id=cache-apt-${TARGETARCH},sharing=
       ssh-askpass \
       strace \
       sudo \
+      supervisor \
       sysstat \
       tmux \
       traceroute \

templates/kubernetes/homelab-workspace/configmap.tf

@@ -8,11 +8,16 @@ resource "kubernetes_config_map_v1" "workspace_scripts" {
   }
 
   data = {
-    agent_startup_script        = file("${path.cwd}/script-agent-startup.sh")
-    container_entrypoint_script = file("${path.cwd}/script-container-entrypoint.sh")
-    memory_watchdog_script      = file("${path.cwd}/script-memory-watchdog.sh")
-    prepare_workspace_script    = file("${path.cwd}/script-prepare-workspace.sh")
-    vscode_server_gc_script     = file("${path.cwd}/script-vscode-server-gc.sh")
-    workspace_init_script       = coder_agent.main.init_script
+    agent_startup_script         = file("${path.cwd}/script-agent-startup.sh")
+    container_entrypoint_script  = file("${path.cwd}/script-container-entrypoint.sh")
+    dotfiles_script              = file("${path.cwd}/script-dotfiles.sh")
+    memory_watchdog_script       = file("${path.cwd}/script-memory-watchdog.sh")
+    memory_watchdog_start_script = file("${path.cwd}/script-memory-watchdog-start.sh")
+    prepare_workspace_script     = file("${path.cwd}/script-prepare-workspace.sh")
+    service_command_script       = file("${path.cwd}/script-service-command.sh")
+    start_services_script        = file("${path.cwd}/script-start-services.sh")
+    supervisor_config            = file("${path.cwd}/supervisord.conf")
+    vscode_server_gc_script      = file("${path.cwd}/script-vscode-server-gc.sh")
+    workspace_init_script        = coder_agent.main.init_script
   }
 }

templates/kubernetes/homelab-workspace/deployment.tf

@@ -153,11 +153,36 @@ resource "kubernetes_deployment_v1" "deployment" {
             name       = "coder-scripts"
             sub_path   = "container_entrypoint_script"
           }
+          volume_mount {
+            mount_path = "/dotfiles.sh"
+            name       = "coder-scripts"
+            sub_path   = "dotfiles_script"
+          }
           volume_mount {
             mount_path = "/memory-watchdog.sh"
             name       = "coder-scripts"
             sub_path   = "memory_watchdog_script"
           }
+          volume_mount {
+            mount_path = "/memory-watchdog-start.sh"
+            name       = "coder-scripts"
+            sub_path   = "memory_watchdog_start_script"
+          }
+          volume_mount {
+            mount_path = "/service-command.sh"
+            name       = "coder-scripts"
+            sub_path   = "service_command_script"
+          }
+          volume_mount {
+            mount_path = "/start-services.sh"
+            name       = "coder-scripts"
+            sub_path   = "start_services_script"
+          }
+          volume_mount {
+            mount_path = "/supervisord.conf"
+            name       = "coder-scripts"
+            sub_path   = "supervisor_config"
+          }
           volume_mount {
             mount_path = "/vscode-server-gc.sh"
             name       = "coder-scripts"

templates/kubernetes/homelab-workspace/env.tf

@@ -28,6 +28,24 @@ resource "coder_env" "dotfiles_coder_username" {
   value    = data.coder_workspace_owner.me.name
 }
 
+resource "coder_env" "service_commands" {
+  agent_id = coder_agent.main.id
+  name     = "SUPERVISOR_SERVICE_COMMANDS"
+  value    = jsonencode(local.validated_service_commands)
+}
+
+resource "coder_env" "service_count" {
+  agent_id = coder_agent.main.id
+  name     = "SUPERVISOR_SERVICE_COUNT"
+  value    = tostring(length(local.validated_service_commands))
+}
+
+resource "coder_env" "template_test_mode" {
+  agent_id = coder_agent.main.id
+  name     = "TEMPLATE_TEST_MODE"
+  value    = tostring(var.test_mode)
+}
+
 # The switch that arms the memory watchdog. "observe" measures and records what
 # it would have done; "enforce" kills any process that has been over its share of
 # the 2048 MiB VS Code envelope for ten minutes and had previously been seen

templates/kubernetes/homelab-workspace/parameters.tf

@@ -42,6 +42,17 @@ data "coder_parameter" "dotfiles_url" {
   }
 }
 
+data "coder_parameter" "service_commands" {
+  name = "service_commands"
+
+  default      = jsonencode([])
+  display_name = "Service Commands"
+  description  = "Commands to keep running with Supervisor after dotfiles have been applied"
+  icon         = "/icon/terminal.svg"
+  mutable      = true
+  type         = "list(string)"
+}
+
 data "coder_parameter" "memory_watchdog_mode" {
   name = "memory_watchdog_mode"
 
@@ -92,6 +103,11 @@ locals {
     str if length(regexall("[^a-zA-Z0-9-]", str)) == 0
   ] : []
 
+  validated_service_commands = (data.coder_parameter.service_commands.value != "") ? [
+    for command in jsondecode(data.coder_parameter.service_commands.value) :
+    command if trimspace(command) != ""
+  ] : []
+
   # Keep an invalid value inert even if it came from stored state created
   # before the parameter acquired its form validation.
   validated_dotfiles_url = length(regexall(

templates/kubernetes/homelab-workspace/script-dotfiles.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -euo pipefail
+
+state_dir="${HOME}/.local/state/dotfiles"
+mkdir -p "${state_dir}"
+rm -f "${state_dir}/applied" "${state_dir}/failed"
+exec > >(/usr/bin/tee "${state_dir}/run.log") 2>&1
+record_failure() {
+  local status="$?"
+  if (( status != 0 )); then
+    printf '%s\n' "${status}" >"${state_dir}/failed"
+  fi
+}
+trap record_failure EXIT
+
+if [[ -z "${DOTFILES_URL}" ]]; then
+  echo "no dotfiles repository configured"
+else
+  chezmoi_bin="${HOMEBREW_PREFIX}/bin/chezmoi"
+  if [[ ! -x "${chezmoi_bin}" ]]; then
+    "${HOMEBREW_PREFIX}/bin/brew" install chezmoi
+  fi
+
+  config_dir="${XDG_CONFIG_HOME:-${HOME}/.config}/chezmoi"
+  config_file="${config_dir}/chezmoi.toml"
+  if [[ ! -e "${config_file}" ]]; then
+    mkdir -p "${config_dir}"
+    escape_toml() {
+      local value="${1//\\/\\\\}"
+      printf '%s' "${value//\"/\\\"}"
+    }
+    {
+      printf '[data]\n'
+      printf 'name = "%s"\n' "$(escape_toml "${DOTFILES_OWNER_NAME}")"
+      printf 'email = "%s"\n' "$(escape_toml "${DOTFILES_OWNER_EMAIL}")"
+      printf 'coderUsername = "%s"\n' "$(escape_toml "${DOTFILES_CODER_USERNAME}")"
+      printf 'bwsAccessToken = ""\n'
+    } >"${config_file}"
+  fi
+
+  source_dir="$(${chezmoi_bin} source-path)"
+  if [[ -d "${source_dir}/.git" ]]; then
+    "${chezmoi_bin}" update --skip-secrets
+  else
+    mkdir -p "$(dirname "${source_dir}")"
+    git clone -- "${DOTFILES_URL}" "${source_dir}"
+    if [[ "${TEMPLATE_TEST_MODE}" == "true" ]]; then
+      # The integration test verifies this template's Chezmoi orchestration;
+      # repository-owned workstation bootstrap scripts are outside its scope.
+      "${chezmoi_bin}" init --apply --skip-secrets --exclude=scripts
+    else
+      "${chezmoi_bin}" init --apply --skip-secrets
+    fi
+  fi
+fi
+
+touch "${state_dir}/applied"
+/bin/bash /start-services.sh