From 7fda778319df43d76e1c5baa14a947de2cc34f97 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Wed, 16 Sep 2026 19:31:54 +0000 Subject: [PATCH] refactor: simplify template storage and runtime tests Move Supervisor service-command coverage into the template integration job and use the released workspace image. Replace test mode with explicit controls, mount scripts as one directory, and expose mutable storage sizes. Test parameter metadata, script availability, service restarts, home lifecycle, and tmp PVC reprovisioning. --- .github/workflows/publish.yaml | 3 +- .github/workflows/test-image.yaml | 18 -- .github/workflows/test-template.yaml | 186 +++++++++++------- CLAUDE.md | 10 +- TESTING.md | 35 ++-- actions/coder-template-push/action.yaml | 14 +- .../homelab-workspace/coder-agent.tf | 2 +- .../kubernetes/homelab-workspace/configmap.tf | 22 +-- .../homelab-workspace/deployment.tf | 105 +++------- templates/kubernetes/homelab-workspace/env.tf | 6 +- .../homelab-workspace/parameters.tf | 20 +- .../script-container-entrypoint.sh | 4 +- .../homelab-workspace/script-dotfiles.sh | 4 +- .../script-memory-watchdog-start.sh | 2 +- .../script-start-services.sh | 2 +- .../script-vscode-server-gc.sh | 2 +- .../kubernetes/homelab-workspace/scripts.tf | 6 +- .../homelab-workspace/supervisord.conf | 2 +- .../kubernetes/homelab-workspace/terraform.tf | 2 +- .../kubernetes/homelab-workspace/variables.tf | 6 +- 20 files changed, 227 insertions(+), 224 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index bbaf4561..a5a2259a 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -82,7 +82,8 @@ jobs: workspace_image: "ghcr.io/ppat/coder-workspace:${{ needs.publish-image.outputs.image_tag }}" home_pvc_storage_class: sc-longhorn-replicated tmp_pvc_storage_class: sc-longhorn-local-non-replicated-ephemeral - test_mode: "false" + kubernetes_config_path: "" + skip_dotfiles_scripts: "false" message: "[Release Notes](https://github.com/${{ github.repository }}/releases/tag/${{ env.source_sha }})" - name: Tailscale Disconnect diff --git a/.github/workflows/test-image.yaml b/.github/workflows/test-image.yaml index e8015a50..3c8caf50 100644 --- a/.github/workflows/test-image.yaml +++ b/.github/workflows/test-image.yaml @@ -38,21 +38,3 @@ jobs: tailscale_oauth_secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} build_secrets: | FETCH_GH_TOKEN=${{ secrets.GITHUB_TOKEN }} - - integration: - needs: build - uses: ./.github/workflows/test-template.yaml - with: - image_repository: ${{ vars.CONTAINER_REGISTRY_PATH }}/coder-workspace - image_tag: ${{ needs.build.outputs.image_tag }} - service_commands: >- - ["test -f /home/coder/.local/state/dotfiles/applied || { touch /tmp/service-started-too-early; exit 1; }; - printf x >> /tmp/service-restarts; sleep 2; exit 1", - "test -f /home/coder/.local/state/dotfiles/applied || { touch /tmp/service-started-too-early; exit 1; }; - touch /tmp/service-steady; exec sleep 300"] - secrets: - private_registry: ${{ secrets.CONTAINER_REGISTRY }} - private_registry_username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - private_registry_token: ${{ secrets.CONTAINER_REGISTRY_PASSWORD }} - tailscale_oauth_client_id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }} - tailscale_oauth_secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} diff --git a/.github/workflows/test-template.yaml b/.github/workflows/test-template.yaml index ee218aa2..5601fc01 100644 --- a/.github/workflows/test-template.yaml +++ b/.github/workflows/test-template.yaml @@ -5,36 +5,13 @@ name: test template on: pull_request: paths: + - actions/coder-template-push/** - .github/compose/** - .github/workflows/test-template.yaml - templates/kubernetes/homelab-workspace/** - workflow_call: - inputs: - image_repository: - type: string - default: "" - image_tag: - type: string - default: "" - service_commands: - type: string - default: "[]" - secrets: - private_registry: - required: false - private_registry_username: - required: false - private_registry_token: - required: false - tailscale_oauth_client_id: - required: false - tailscale_oauth_secret: - required: false concurrency: - # A called workflow inherits the caller's github.workflow and run ID. Include - # its image input so it cannot cancel the test-image run that invoked it. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-${{ inputs.image_tag || 'released' }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true permissions: @@ -74,36 +51,6 @@ jobs: # created explicitly before the template can be applied. run: kubectl create namespace coder - - name: Connect to private registry network - if: inputs.image_tag != '' - uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 - with: - oauth-client-id: ${{ secrets.tailscale_oauth_client_id }} - oauth-secret: ${{ secrets.tailscale_oauth_secret }} - tags: tag:github-action-ci-runner - - - name: Log in to private registry - if: inputs.image_tag != '' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ${{ secrets.private_registry }} - username: ${{ secrets.private_registry_username }} - password: ${{ secrets.private_registry_token }} - - - name: Load pull request image into test cluster - if: inputs.image_tag != '' - id: pull_request_image - env: - IMAGE_REGISTRY: ${{ secrets.private_registry }} - IMAGE_REPOSITORY: ${{ inputs.image_repository }} - IMAGE_TAG: ${{ inputs.image_tag }} - shell: bash - run: | - workspace_image="${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG}" - docker pull "${workspace_image}" - kind load docker-image --name "${KIND_CLUSTER}" "${workspace_image}" - echo "workspace_image=${workspace_image}" >> "${GITHUB_OUTPUT}" - - name: Start local Coder env: CODER_KUBECONFIG: ${{ runner.temp }}/kubeconfig-coder @@ -153,7 +100,6 @@ jobs: coder_password: ci-password - name: Determine latest released workspace image - if: inputs.image_tag == '' id: released_image env: GH_TOKEN: ${{ github.token }} @@ -168,21 +114,49 @@ jobs: template_dir: current/templates/kubernetes/homelab-workspace template_name: ${{ env.TEMPLATE_NAME }} template_version: ${{ github.sha }} - workspace_image: ${{ steps.pull_request_image.outputs.workspace_image || steps.released_image.outputs.workspace_image }} + workspace_image: ${{ steps.released_image.outputs.workspace_image }} home_pvc_storage_class: standard tmp_pvc_storage_class: standard - test_mode: "true" + kubernetes_config_path: /home/coder/.kube/config + skip_dotfiles_scripts: "true" + + - name: Confirm storage parameter contracts + shell: bash + run: | + version_id="$(coder templates versions list "${TEMPLATE_NAME}" --output json \ + | jq --raw-output '.[0].TemplateVersion.id')" + [[ "${version_id}" =~ ^[0-9a-f-]{36}$ ]] + session_token="$(<"${HOME}/.config/coderv2/session")" + owner_id="$(curl --fail --silent --show-error \ + --header "Coder-Session-Token: ${session_token}" "${CODER_URL}/api/v2/users/me" | jq --raw-output .id)" + parameters="$(curl --fail --silent --show-error --request POST \ + --header "Coder-Session-Token: ${session_token}" --header 'Content-Type: application/json' \ + --data "$(jq --null-input --arg owner_id "${owner_id}" \ + '{id: 0, owner_id: $owner_id, inputs: {use_existing_home_pvc: "false"}}')" \ + "${CODER_URL}/api/v2/templateversions/${version_id}/dynamic-parameters/evaluate")" + jq --exit-status \ + 'any(.parameters[]; .name == "home_pvc_size" and .mutable == true + and any(.validations[]; .validation_max == 256))' \ + <<<"${parameters}" + jq --exit-status \ + 'any(.parameters[]; .name == "tmp_pvc_size" and .mutable == true and .default_value.value == "1" + and any(.validations[]; .validation_min == 1 and .validation_max == 50))' \ + <<<"${parameters}" - name: Start workspace env: - SERVICE_COMMANDS: ${{ inputs.service_commands || '[]' }} + SERVICE_COMMANDS: >- + ["test -f /home/coder/.local/state/dotfiles/applied || { touch /tmp/service-started-too-early; exit 1; }; + printf x >> /tmp/service-restarts; sleep 2; exit 1", + "test -f /home/coder/.local/state/dotfiles/applied || { touch /tmp/service-started-too-early; exit 1; }; + touch /tmp/service-steady; exec sleep 300"] shell: bash run: | service_parameter="$(jq --null-input --raw-output \ --arg value "service_commands=${SERVICE_COMMANDS}" '[$value] | @csv')" coder create "${WORKSPACE_NAME}" --template "${TEMPLATE_NAME}" --no-wait --yes \ --parameter dotfiles_url=https://github.com/ppat/dotfiles.git \ - --parameter use_existing_home_pvc=false --parameter home_pvc_size=2 \ + --parameter use_existing_home_pvc=false --parameter home_pvc_size=2 --parameter tmp_pvc_size=1 \ --parameter memory=4 --parameter preferred_nodes='[]' --parameter memory_watchdog_mode=enforce \ --parameter "${service_parameter}" @@ -195,6 +169,27 @@ jobs: run: | coder ssh "${WORKSPACE_NAME}" -- env + - name: Confirm script bundle and temporary storage + shell: bash + run: | + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-agent-startup.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-container-entrypoint.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-dotfiles.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-memory-watchdog.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-memory-watchdog-start.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-prepare-workspace.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-service-command.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-start-services.sh + coder ssh "${WORKSPACE_NAME}" -- test -f /scripts/supervisord.conf + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/script-vscode-server-gc.sh + coder ssh "${WORKSPACE_NAME}" -- test -x /scripts/workspace-init.sh + deployment_name="$(kubectl --namespace coder get deployment \ + --selector "com.coder.workspace.name=${WORKSPACE_NAME}" \ + --output jsonpath='{.items[0].metadata.name}')" + [[ "$(kubectl --namespace coder get "deployment/${deployment_name}" \ + --output jsonpath='{.spec.template.spec.volumes[?(@.name=="tmp")].ephemeral.volumeClaimTemplate.spec.resources.requests.storage}')" \ + == "1Gi" ]] + - name: Confirm dotfiles were applied shell: bash run: | @@ -214,7 +209,6 @@ jobs: [[ "${origin}" == "https://github.com/ppat/dotfiles.git" ]] - name: Confirm supervised service starts after dotfiles and restarts - if: inputs.service_commands != '' && inputs.service_commands != '[]' shell: bash run: | # Expand the positional parameter inside the runner-owned polling shell. @@ -224,7 +218,7 @@ jobs: [ "$count" -ge 2 ]; do sleep 2; done' \ _ "${WORKSPACE_NAME}"; then coder ssh "${WORKSPACE_NAME}" -- \ - supervisorctl --configuration /supervisord.conf status || true + supervisorctl --configuration /scripts/supervisord.conf status || true coder ssh "${WORKSPACE_NAME}" -- ls -la /home/coder/.local/state/supervisor /tmp/service-* || true coder ssh "${WORKSPACE_NAME}" -- tail -n 100 /home/coder/.local/state/supervisor/service-0.log || true coder ssh "${WORKSPACE_NAME}" -- tail -n 100 /home/coder/.local/state/supervisor/service-1.log || true @@ -233,13 +227,13 @@ jobs: coder ssh "${WORKSPACE_NAME}" -- test ! -e /tmp/service-started-too-early coder ssh "${WORKSPACE_NAME}" -- test -e /tmp/service-steady coder ssh "${WORKSPACE_NAME}" -- \ - supervisorctl --configuration /supervisord.conf status service:service-0 + supervisorctl --configuration /scripts/supervisord.conf status service:service-0 coder ssh "${WORKSPACE_NAME}" -- \ - supervisorctl --configuration /supervisord.conf status service:service-1 + supervisorctl --configuration /scripts/supervisord.conf status service:service-1 - name: Show dotfiles log if: always() - run: coder ssh "${WORKSPACE_NAME}" -- cat /home/coder/.local/state/dotfiles/run.log + run: coder ssh "${WORKSPACE_NAME}" -- cat /home/coder/.local/state/dotfiles/run.log || true - name: Confirm workspace-owned home PVC lifecycle shell: bash @@ -248,15 +242,75 @@ jobs: --selector "com.coder.workspace.name=${WORKSPACE_NAME}" \ --output jsonpath='{.items[0].metadata.name}')" [[ -n "${pvc_name}" ]] + pod_uid="$(kubectl --namespace coder get pod \ + --selector "com.coder.workspace.name=${WORKSPACE_NAME}" \ + --output jsonpath='{.items[0].metadata.uid}')" + tmp_pvc_name="$(kubectl --namespace coder get pvc --output json | jq --raw-output --arg uid "${pod_uid}" \ + '.items[] | select(any(.metadata.ownerReferences[]?; .uid == $uid)) | .metadata.name')" + [[ -n "${tmp_pvc_name}" ]] coder ssh "${WORKSPACE_NAME}" -- touch /home/coder/.home-pvc-lifecycle coder stop "${WORKSPACE_NAME}" --yes kubectl --namespace coder get "persistentvolumeclaim/${pvc_name}" + kubectl --namespace coder wait --for=delete \ + "persistentvolumeclaim/${tmp_pvc_name}" --timeout=5m coder start "${WORKSPACE_NAME}" --no-wait --yes coder ping --num 3 --timeout 30s "${WORKSPACE_NAME}" - coder ssh "${WORKSPACE_NAME}" -- test -f /home/coder/.home-pvc-lifecycle + coder ssh "${WORKSPACE_NAME}" -- test -f /home/coder/.home-pvc-lifecycle || { + echo "::error::Home PVC did not preserve the lifecycle marker across stop/start." + exit 1 + } + home_size="$(kubectl --namespace coder get "persistentvolumeclaim/${pvc_name}" \ + --output jsonpath='{.spec.resources.requests.storage}')" + [[ "${home_size}" == "2Gi" ]] || { + echo "::error::Expected home PVC ${pvc_name} to remain 2Gi, observed ${home_size}." + exit 1 + } + new_pod_uid="$(kubectl --namespace coder get pod \ + --selector "com.coder.workspace.name=${WORKSPACE_NAME}" \ + --output jsonpath='{.items[0].metadata.uid}')" + new_tmp_pvc_name="$(kubectl --namespace coder get pvc --output json | jq --raw-output --arg uid "${new_pod_uid}" \ + '.items[] | select(any(.metadata.ownerReferences[]?; .uid == $uid)) | .metadata.name')" + [[ -n "${new_tmp_pvc_name}" && "${new_tmp_pvc_name}" != "${tmp_pvc_name}" ]] || { + echo "::error::Expected a replacement tmp PVC; old=${tmp_pvc_name}, new=${new_tmp_pvc_name:-missing}." + exit 1 + } + tmp_request="$(kubectl --namespace coder get "persistentvolumeclaim/${new_tmp_pvc_name}" \ + --output jsonpath='{.spec.resources.requests.storage}')" + tmp_capacity="$(kubectl --namespace coder get "persistentvolumeclaim/${new_tmp_pvc_name}" \ + --output jsonpath='{.status.capacity.storage}')" + [[ "${tmp_request}" == "1Gi" && "${tmp_capacity}" == "1Gi" ]] || { + echo "::error::Expected tmp PVC ${new_tmp_pvc_name} to request and bind 1Gi; request=${tmp_request}, capacity=${tmp_capacity}." + exit 1 + } coder delete "${WORKSPACE_NAME}" --yes kubectl --namespace coder wait --for=delete "persistentvolumeclaim/${pvc_name}" --timeout=5m + - name: Confirm custom temporary storage size + env: + SIZED_WORKSPACE_NAME: template-test-tmp-size + shell: bash + run: | + coder create "${SIZED_WORKSPACE_NAME}" --template "${TEMPLATE_NAME}" --no-wait --yes \ + --parameter dotfiles_url='' --parameter use_existing_home_pvc=false \ + --parameter home_pvc_size=1 --parameter tmp_pvc_size=2 --parameter memory=4 \ + --parameter preferred_nodes='[]' --parameter service_commands='[]' \ + --parameter memory_watchdog_mode=enforce + coder ping --num 3 --timeout 30s "${SIZED_WORKSPACE_NAME}" + pod_uid="$(kubectl --namespace coder get pod \ + --selector "com.coder.workspace.name=${SIZED_WORKSPACE_NAME}" \ + --output jsonpath='{.items[0].metadata.uid}')" + tmp_pvc_name="$(kubectl --namespace coder get pvc --output json | jq --raw-output --arg uid "${pod_uid}" \ + '.items[] | select(any(.metadata.ownerReferences[]?; .uid == $uid)) | .metadata.name')" + tmp_request="$(kubectl --namespace coder get "persistentvolumeclaim/${tmp_pvc_name}" \ + --output jsonpath='{.spec.resources.requests.storage}')" + tmp_capacity="$(kubectl --namespace coder get "persistentvolumeclaim/${tmp_pvc_name}" \ + --output jsonpath='{.status.capacity.storage}')" + [[ "${tmp_request}" == "2Gi" && "${tmp_capacity}" == "2Gi" ]] || { + echo "::error::Expected tmp PVC ${tmp_pvc_name} to request and bind 2Gi; request=${tmp_request}, capacity=${tmp_capacity}." + exit 1 + } + coder delete "${SIZED_WORKSPACE_NAME}" --yes + - name: Show Coder logs on failure if: failure() run: docker compose -f "${COMPOSE_FILE}" logs diff --git a/CLAUDE.md b/CLAUDE.md index 99c58d7b..dea27d8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,7 +38,7 @@ A separate workflow, `.github/workflows/test-watchdog.yaml`, runs the one thing ./templates/kubernetes/homelab-workspace/script-memory-watchdog-test.sh ``` -There is no local way to build/publish the image against the private registry cache (`test-image.yaml` needs the Tailscale-routed credentials CI has), but the template test path needs neither secrets nor CI: `test-template.yaml`'s Kind/Compose/`coder` sequence runs the same on a laptop — see [TESTING.md](TESTING.md) for the local recipe and for how a change actually gets exercised (including the `test_mode` flow), and the **Release flow** section below for how it ships for real. +There is no local way to build/publish the image against the private registry cache (`test-image.yaml` needs the Tailscale-routed credentials CI has), but the template test path needs neither secrets nor CI: `test-template.yaml`'s Kind/Compose/`coder` sequence runs the same on a laptop — see [TESTING.md](TESTING.md) for the local recipe and for how a change actually gets exercised, and the **Release flow** section below for how it ships for real. ## Commit messages @@ -76,9 +76,9 @@ Quick orientation map — for what each piece is *for* and the decisions behind | `deployment.tf` / `configmap.tf` | Kubernetes Pod spec, volumes, ConfigMap | | `env.tf` | `coder_env` resources exposed to the agent | | `scripts.tf` | `coder_script` resources — the memory watchdog daemon and the weekly `vscode-server` GC schedule | -| `variables.tf` | `workspace_image`, `test_mode` — both supplied by the release workflow | +| `variables.tf` | Publish-time image, storage-class, and dotfiles-script controls | | `script-agent-startup.sh` / `script-prepare-workspace.sh` | Scripts run on agent/workspace startup | -| `script-container-entrypoint.sh` | The workspace container's `command`. Wipes `/tmp` and `exec`s Coder's generated `/workspace-init.sh` — the wipe must precede the agent, see the gotcha below | +| `script-container-entrypoint.sh` | The workspace container's `command`. Wipes `/tmp` and `exec`s Coder's generated `/scripts/workspace-init.sh` — the wipe must precede the agent, see the gotcha below | | `script-memory-watchdog.sh` | Userspace memory watchdog — see [DESIGN.md](DESIGN.md#design-tensions-and-decisions). It bounds the **standing population of restartable helpers** against a fixed **2048 MiB envelope** for the VS Code tree (per-role shares of it, PSS, ten-minute dwell) and records every per-process sweep. It does **not** try to prevent an acute OOM. `memory_watchdog_mode` selects `observe` / `enforce` (the default; arms every role). `enforce-all` is retired and honoured as `enforce` | | `script-memory-watchdog-test.sh` | Fixture tests for the watchdog's envelope and shares, process selection, dwell, the oversize rule and the kill-rate report. Run by hand (`./script-memory-watchdog-test.sh`) and by the `watchdog` job in `.github/workflows/test-watchdog.yaml`. `kill` is shadowed by a function throughout — the fixture pids are real pids in whatever container runs the suite | | `script-vscode-server-gc.sh` | Weekly GC of `~/.vscode-server` (interrupted downloads, superseded server versions/extensions, orphaned CLI binaries — see the script's own header for the exact signal per class, and the `coder_script.vscode_server_gc` comment in `scripts.tf` for why it's template-owned rather than dotfiles-owned) | @@ -125,7 +125,7 @@ Things that look arbitrary in the code but are load-bearing (full reasoning in [ - `parameters.tf`'s `memory_watchdog_mode` also goes through the `local.validated_*` treatment: Coder constrains the value server-side, but it is the single switch deciding whether the watchdog may signal processes, so an unrecognised value falls back to the inert `observe` rather than being passed through. The retired `enforce-all` is the one exception — it is mapped to `enforce` in both the Terraform local and `normalise_mode`, because a workspace still carrying that stored value had asked for *more* enforcement and must not silently get none. - Adding a package/tool has three possible homes, and picking the wrong one is a real mistake, not a style choice — route by the rule in [DESIGN.md](DESIGN.md#where-the-workspace-environment-comes-from): universal + stable → image (`Dockerfile`); occasionally-needed + apt-only + too heavy to bake in → the template's `system_packages` parameter; personal, fast-moving, or not an apt package → the operator's dotfiles (a *different* repo — see below), never this one. - `deployment.tf` mounts `/tmp` on its own ephemeral Longhorn volume, not the node's root filesystem and not the NFS-backed home PVC - see [DESIGN.md](DESIGN.md#design-tensions-and-decisions) for why both of those are wrong for it. Its lifecycle is per-Pod, the same as the `system` volume, so it is *not* wiped by a container-only restart within a live Pod - `script-container-entrypoint.sh` wipes it explicitly on every container start instead. Anything relying on `/tmp` persisting across a container restart was already wrong before this (the same was true for free when it was the container's writable overlay). -- **The `/tmp` wipe belongs in the container entrypoint and nowhere later - this is a fixed regression, not a style preference.** The workspace container's `command` is `script-container-entrypoint.sh`, which wipes `/tmp` and then `exec`s Coder's generated `/workspace-init.sh`. That bootstrap unpacks the agent CLI into a per-boot `mktemp` directory under `/tmp`, chdirs into it, appends it to the PATH of every session and script the agent runs, and only then runs the agent startup script - so wiping `/tmp` from `script-agent-startup.sh` deletes the CLI the agent installed seconds earlier and every `coder stat` metadata panel reports `coder: command not found`. Do not "fix" a future collision here with an exclusion list: the agent's `/tmp` paths are version-dependent and not consistently prefixed (v2.35.3 owns `coder.XXXXXX/`, `coder-agent.sock`, `coder-agent*.log`, `coder-script-data/`, `coder-screen/` *and* `boundary-audit.sock`), so an allowlist rots silently. `script-agent-startup.sh` asserts the CLI resolves and runs, which is what makes a recurrence loud. +- **The `/tmp` wipe belongs in the container entrypoint and nowhere later - this is a fixed regression, not a style preference.** The workspace container's `command` is `script-container-entrypoint.sh`, which wipes `/tmp` and then `exec`s Coder's generated `/scripts/workspace-init.sh`. That bootstrap unpacks the agent CLI into a per-boot `mktemp` directory under `/tmp`, chdirs into it, appends it to the PATH of every session and script the agent runs, and only then runs the agent startup script - so wiping `/tmp` from `script-agent-startup.sh` deletes the CLI the agent installed seconds earlier and every `coder stat` metadata panel reports `coder: command not found`. Do not "fix" a future collision here with an exclusion list: the agent's `/tmp` paths are version-dependent and not consistently prefixed (v2.35.3 owns `coder.XXXXXX/`, `coder-agent.sock`, `coder-agent*.log`, `coder-script-data/`, `coder-screen/` *and* `boundary-audit.sock`), so an allowlist rots silently. `script-agent-startup.sh` asserts the CLI resolves and runs, which is what makes a recurrence loud. - A `coder_script` in `scripts.tf` must never gate its actual work behind `[ -x ] && ... || true` (or equivalent) when `` is supplied by something outside the template, e.g. a dotfiles-installed binary. That guard makes "the payload isn't there" indistinguishable from "the payload ran and had nothing to do" — both report success on the schedule's weekly cron. This happened: `vscode_server_gc` was originally written that way, expecting `$HOME/.local/bin/vscode-server-gc` from dotfiles, and would have silently no-opped forever on any workspace dotfiles hadn't been applied to (confirmed on the `test` workspace, which reached ~11 GB of `~/.vscode-server` with dotfiles never applied). Fixed by making `script-vscode-server-gc.sh` a template-owned script mounted via `configmap.tf`/`deployment.tf`, so `scripts.tf` invokes it directly and an actual failure now surfaces as a failed run in the Coder UI instead of vanishing into `|| true`. - `deployment.tf`'s Deployment `metadata.name` (`local.workload_name` in `main.tf`) is not cosmetic: the cluster's Prometheus resolves pod → ReplicaSet → Deployment via an existing `kube_pod_owner` recording rule and exposes the result as a `workload` label with no other join needed, so whatever this Deployment is named *is* the identity CPU/memory/PSI/OOM metrics get attributed to. Don't revert it to an opaque identifier (e.g. the workspace UUID) without re-breaking that attribution — see [DESIGN.md](DESIGN.md#design-tensions-and-decisions). @@ -139,4 +139,4 @@ This repo is only the image+template layer. When a task's real cause is above or ## Working on a template or image change 1. Make the change, run `pre-commit run --all-files` and the scoped `tofu validate`/`tflint` commands above. -2. Follow [TESTING.md](TESTING.md) to exercise it via `test_mode` before merging — merging to `main` publishes to the real template/image with no separate promotion step. +2. Follow [TESTING.md](TESTING.md) to exercise it against the disposable Coder/Kind control plane before merging — merging to `main` publishes to the real template/image with no separate promotion step. diff --git a/TESTING.md b/TESTING.md index ac18e519..451ebdca 100644 --- a/TESTING.md +++ b/TESTING.md @@ -17,13 +17,11 @@ the registries, and pushes the live Coder template with that release tag. PR workflows are scoped directly by changed paths: - `test-image.yaml` runs for image changes. It delegates the multi-architecture build and private-registry cache to - the shared image-build workflow, then calls the template integration workflow with that branch image. The image is - built once: the integration job pulls the published branch artifact and loads it into Kind. + the shared image-build workflow. - `test-template.yaml` runs for template changes. It creates a local Coder/Postgres compose deployment, gives Coder the test-cluster kubeconfig and an OpenTofu binary bind-mounted over its bundled `terraform`, publishes the - template with the latest released GHCR workspace image, creates a workspace, pings its agent three times with a - timeout, and verifies SSH by running `env`. As a reusable workflow called by `test-image.yaml`, it instead uses the - exact branch image and exercises the contract between image and template. + template with the latest released GHCR workspace image, creates a workspace, and exercises the script bundle, + dotfiles-to-services ordering, mutable storage sizes, stop/start persistence, and deletion lifecycle. This test control plane is where this repo proves OpenTofu is the thing that actually *applies* the template — the live deployment's provisioner is a separate repo's decision, out of this repo's control (see the @@ -33,8 +31,8 @@ repo verified rather than carrying an untested assumption that OpenTofu behaves The temporary Coder deployment has no `CODER_ACCESS_URL`, so Coder creates its development tunnel. This is the workspace agent's route back from Kind; the runner continues to call Coder on localhost and template testing needs -neither Tailscale nor the private registry. Test mode omits the production `/tmp` volume and uses a 2 GiB home volume; -live workspaces retain their production storage. The Compose Coder version matches the live deployment's Helm chart +neither Tailscale nor the private registry. The test uses Kind's `standard` class for both PVC types and the same +generic ephemeral `/tmp` volume the production template renders. The Compose Coder version matches the live deployment's Helm chart version and the Postgres version matches the live CloudNativePG cluster's (see `homelab-ops-kubernetes-apps`'s `apps/subsystems/coder/`) — the Postgres *image* itself is the plain upstream one rather than CloudNativePG's, since this Compose stack isn't standing in for CloudNativePG, just a same-version Postgres for Coder to talk to. Both @@ -71,10 +69,12 @@ CODER_SESSION_TOKEN="${CODER_SESSION_TOKEN}" coder login --use-token-as-session coder template push --directory templates/kubernetes/homelab-workspace \ --var workspace_image=ghcr.io/ppat/coder-workspace: \ - --var home_pvc_storage_class=standard --var tmp_pvc_storage_class=standard --var test_mode=true \ + --var home_pvc_storage_class=standard --var tmp_pvc_storage_class=standard \ + --var kubernetes_config_path=/home/coder/.kube/config \ + --var skip_dotfiles_scripts=true \ --name local --yes homelab-workspace-test coder create local-test --template homelab-workspace-test --no-wait --yes \ - --parameter use_existing_home_pvc=false --parameter home_pvc_size=2 \ + --parameter use_existing_home_pvc=false --parameter home_pvc_size=2 --parameter tmp_pvc_size=1 \ --parameter memory=4 --parameter preferred_nodes='[]' --parameter memory_watchdog_mode=enforce \ --parameter dotfiles_url='' --parameter service_commands='[]' coder ping --num 3 --timeout 30s local-test @@ -95,21 +95,18 @@ no `-f` needed. 1. **Image build** — an image change exercises both published architectures and the existing private cache through the shared image-build workflow. -2. **Image/template integration** — the image workflow passes its published branch tag to the reusable template - workflow. Two service commands record if either starts before Chezmoi completes; one exits repeatedly and must be - invoked at least twice, while the other must remain running as an independent Supervisor process. This proves the - changed image supplies Supervisor, the changed template starts it only after dotfiles, and Supervisor expands and - restarts the commands. The template workflow loads the existing image into Kind; it does not build another one. -3. **Template runtime** — a template-only PR applies against fresh Coder, Postgres, and Kubernetes state, using the - last released GHCR image. The workspace start, agent ping, and SSH command prove the rendered template's runtime - path rather than merely its Terraform syntax. +2. **Template runtime** — a template PR applies against fresh Coder, Postgres, and Kubernetes state using the last + released GHCR image. Two service commands record if either starts before Chezmoi completes; one exits repeatedly + and must be invoked at least twice, while the other remains running independently. The same job proves the mounted + script bundle, both PVC size parameters, stop/start persistence, and deletion ownership rather than merely parsing + the rendered HCL. ## The memory watchdog is the one part with runtime behaviour Everything else here is declarative and is covered by the stages above. `script-memory-watchdog.sh` decides at runtime whether to kill a process, so it gets two things neither lint nor a template push can provide: - **Fixtures** — `./templates/kubernetes/homelab-workspace/script-memory-watchdog-test.sh`, also run by the `watchdog` job in `.github/workflows/test-watchdog.yaml`. The suite is built around negative assertions paired with the mutation that must flip them: "it did not kill the agent session" proves nothing unless removing one rule makes it kill the agent session. Two of its own safety properties matter as much as its assertions: `kill` is shadowed by a function throughout, because the fixture pids name real processes in whatever container runs the suite; and every load of the watchdog redirects its stdout emission to a temporary file, with the suite exiting outright if that seam did not take — the real default is the container's own stdout, and a run without the seam has already put fixture kill lines into a live workspace's log stream. -- **A live drill on a disposable `test_mode` workspace**, which has disposable storage and can be wrecked freely. Fixtures cannot answer whether a real process tree classifies correctly, whether a supervisor really does respawn what was killed, or whether the respawned process comes back inside its share (a drift the watchdog should keep policing) or above it (an `oversize` it should report and leave alone). The drill that has been run: a stand-in session root (`exec -a claude bash`, to exercise the shebang branch of the claude-root guard rather than the trivial compiled-binary one) with an over-budget helper that a supervisor respawns, a second helper inside its budget, and a process detached into its own session with `setsid` — the same mechanism Claude Code uses to detach a Bash tool call — larger than both. +- **A live drill on a disposable workspace with workspace-owned storage**, which can be wrecked freely. Fixtures cannot answer whether a real process tree classifies correctly, whether a supervisor really does respawn what was killed, or whether the respawned process comes back inside its share (a drift the watchdog should keep policing) or above it (an `oversize` it should report and leave alone). The drill that has been run: a stand-in session root (`exec -a claude bash`, to exercise the shebang branch of the claude-root guard rather than the trivial compiled-binary one) with an over-budget helper that a supervisor respawns, a second helper inside its budget, and a process detached into its own session with `setsid` — the same mechanism Claude Code uses to detach a Bash tool call — larger than both. Neither replaces the other, and the live one is where every defect that mattered in this component has been found. @@ -144,7 +141,7 @@ Not exercised by this drill, and not claimed to be: production timings (everythi An `event=kill`/`event=oversize` line, unlike the sweep log, does leave the pod (see [DESIGN.md](DESIGN.md#design-tensions-and-decisions)), so a claim like the one above is checkable: `{namespace="coder"} |= "component=memory-watchdog" |~ "event=(kill|would-kill|oversize|refused|kill-rate)"` against Loki finds exactly the six lines quoted above, timestamped and labelled with the `watchdog-drill` pod. -The same query, run over the full retained history, is also what falsifies a different claim: PR [#865](https://github.com/ppat/coder/pull/865) (merged 2026-08-18), which introduced this drill practice, described one run in its body as killing "a drifted 739 MB helper... three times" with the role disarming "on the third kill," and stated "the daemon's own lines from the test workspace are in Loki." Neither `event=kill` nor `event=disarmed` nor the string `739` appears anywhere in the retained record for any workspace at any time before this run. The only kill/disarm lines anywhere near that PR's merge time are on the operator's live workspace at 2026-08-18T01:01–01:08, and the daemon logged its own correction for them at the time: `event=note reason=fixture-leak`, naming fixture pids 41/61 at 1907/1583 MB — the same incident this repo's docs already describe as the fixture suite's stdout seam leaking into production before it was guarded. No `event=kill` or `event=disarmed` line from a workspace named `test` exists in the record at all. `event=budget role=extensionHost budget_mb=1069 pod_share_mb=512 floored=1` — the figures the PR body also cites as drill evidence — is not distinguishing evidence either: every disposable `test_mode` workspace since has printed exactly that line at startup, regardless of whether anything was ever killed, because `floored=1` was a constant at that pod size — the bug PR #881 fixes. +The same query, run over the full retained history, is also what falsifies a different claim: PR [#865](https://github.com/ppat/coder/pull/865) (merged 2026-08-18), which introduced this drill practice, described one run in its body as killing "a drifted 739 MB helper... three times" with the role disarming "on the third kill," and stated "the daemon's own lines from the test workspace are in Loki." Neither `event=kill` nor `event=disarmed` nor the string `739` appears anywhere in the retained record for any workspace at any time before this run. The only kill/disarm lines anywhere near that PR's merge time are on the operator's live workspace at 2026-08-18T01:01–01:08, and the daemon logged its own correction for them at the time: `event=note reason=fixture-leak`, naming fixture pids 41/61 at 1907/1583 MB — the same incident this repo's docs already describe as the fixture suite's stdout seam leaking into production before it was guarded. No `event=kill` or `event=disarmed` line from a workspace named `test` exists in the record at all. `event=budget role=extensionHost budget_mb=1069 pod_share_mb=512 floored=1` — the figures the PR body also cites as drill evidence — is not distinguishing evidence either: every disposable test workspace then printed exactly that line at startup, regardless of whether anything was ever killed, because `floored=1` was a constant at that pod size — the bug PR #881 fixes. None of this proves the described drill did not happen — a drill run through the fixture harness's required `WATCHDOG_STDOUT_PATH` seam would, correctly, leave nothing in Loki to find, which is indistinguishable in the record from a drill that never ran. What the record supports is narrower than what was claimed: the specific figures and the "verified end to end... in Loki" line cannot be checked, and the only Loki-verifiable kill/disarm evidence tied to that PR's timeframe is the fixture leak, which was not a drill result. The verified account above is offered in its place. diff --git a/actions/coder-template-push/action.yaml b/actions/coder-template-push/action.yaml index bc7188f4..807c9347 100644 --- a/actions/coder-template-push/action.yaml +++ b/actions/coder-template-push/action.yaml @@ -24,8 +24,12 @@ inputs: tmp_pvc_storage_class: description: Storage class used for ephemeral tmp PVCs. required: true - test_mode: - description: Value passed to the template's test_mode variable. + kubernetes_config_path: + description: Provisioner-local kubeconfig path, or empty for in-cluster authentication. + required: true + default: "" + skip_dotfiles_scripts: + description: Whether Chezmoi should skip repository-owned bootstrap scripts. required: true default: "false" message: @@ -45,7 +49,8 @@ runs: WORKSPACE_IMAGE: ${{ inputs.workspace_image }} HOME_PVC_STORAGE_CLASS: ${{ inputs.home_pvc_storage_class }} TMP_PVC_STORAGE_CLASS: ${{ inputs.tmp_pvc_storage_class }} - TEST_MODE: ${{ inputs.test_mode }} + KUBERNETES_CONFIG_PATH: ${{ inputs.kubernetes_config_path }} + SKIP_DOTFILES_SCRIPTS: ${{ inputs.skip_dotfiles_scripts }} TEMPLATE_MESSAGE: ${{ inputs.message }} # yamllint disable-line rule:indentation run: | @@ -54,7 +59,8 @@ runs: --var "workspace_image=${WORKSPACE_IMAGE}" --var "home_pvc_storage_class=${HOME_PVC_STORAGE_CLASS}" --var "tmp_pvc_storage_class=${TMP_PVC_STORAGE_CLASS}" - --var "test_mode=${TEST_MODE}" + --var "kubernetes_config_path=${KUBERNETES_CONFIG_PATH}" + --var "skip_dotfiles_scripts=${SKIP_DOTFILES_SCRIPTS}" --name "${TEMPLATE_VERSION}" --yes ) diff --git a/templates/kubernetes/homelab-workspace/coder-agent.tf b/templates/kubernetes/homelab-workspace/coder-agent.tf index 5e57f998..81842a1a 100644 --- a/templates/kubernetes/homelab-workspace/coder-agent.tf +++ b/templates/kubernetes/homelab-workspace/coder-agent.tf @@ -1,7 +1,7 @@ resource "coder_agent" "main" { arch = "amd64" os = "linux" - startup_script = "/bin/bash --noprofile --norc /agent-startup.sh" + startup_script = "/bin/bash --noprofile --norc /scripts/script-agent-startup.sh" startup_script_behavior = "blocking" metadata { diff --git a/templates/kubernetes/homelab-workspace/configmap.tf b/templates/kubernetes/homelab-workspace/configmap.tf index c64fbfbe..890e880d 100644 --- a/templates/kubernetes/homelab-workspace/configmap.tf +++ b/templates/kubernetes/homelab-workspace/configmap.tf @@ -8,16 +8,16 @@ resource "kubernetes_config_map_v1" "workspace_scripts" { } data = { - agent_startup_script = file("${path.cwd}/script-agent-startup.sh") - container_entrypoint_script = file("${path.cwd}/script-container-entrypoint.sh") - dotfiles_script = file("${path.cwd}/script-dotfiles.sh") - memory_watchdog_script = file("${path.cwd}/script-memory-watchdog.sh") - memory_watchdog_start_script = file("${path.cwd}/script-memory-watchdog-start.sh") - prepare_workspace_script = file("${path.cwd}/script-prepare-workspace.sh") - service_command_script = file("${path.cwd}/script-service-command.sh") - start_services_script = file("${path.cwd}/script-start-services.sh") - supervisor_config = file("${path.cwd}/supervisord.conf") - vscode_server_gc_script = file("${path.cwd}/script-vscode-server-gc.sh") - workspace_init_script = coder_agent.main.init_script + "script-agent-startup.sh" = file("${path.cwd}/script-agent-startup.sh") + "script-container-entrypoint.sh" = file("${path.cwd}/script-container-entrypoint.sh") + "script-dotfiles.sh" = file("${path.cwd}/script-dotfiles.sh") + "script-memory-watchdog.sh" = file("${path.cwd}/script-memory-watchdog.sh") + "script-memory-watchdog-start.sh" = file("${path.cwd}/script-memory-watchdog-start.sh") + "script-prepare-workspace.sh" = file("${path.cwd}/script-prepare-workspace.sh") + "script-service-command.sh" = file("${path.cwd}/script-service-command.sh") + "script-start-services.sh" = file("${path.cwd}/script-start-services.sh") + "supervisord.conf" = file("${path.cwd}/supervisord.conf") + "script-vscode-server-gc.sh" = file("${path.cwd}/script-vscode-server-gc.sh") + "workspace-init.sh" = coder_agent.main.init_script } } diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index 4b948785..e90d3424 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -48,18 +48,15 @@ resource "kubernetes_deployment_v1" "deployment" { automount_service_account_token = false init_container { name = "prepare-workspace" - command = ["/bin/bash", "/prepare-workspace-script.sh"] + command = ["/bin/bash", "/scripts/script-prepare-workspace.sh"] image = var.workspace_image env { name = "HOMEBREW_PREFIX" value = local.homebrew_directory } - dynamic "env" { - for_each = var.test_mode ? [] : toset(["normalise-tmp-permissions"]) - content { - name = "NORMALISE_TMP_PERMISSIONS" - value = "true" - } + env { + name = "NORMALISE_TMP_PERMISSIONS" + value = "true" } volume_mount { mount_path = local.home_directory @@ -72,19 +69,15 @@ resource "kubernetes_deployment_v1" "deployment" { sub_path = "${data.coder_workspace.me.name}/.linuxbrew" } volume_mount { - mount_path = "/prepare-workspace-script.sh" + mount_path = "/scripts" name = "coder-scripts" - sub_path = "prepare_workspace_script" } # fsGroup makes the production volume group-writable. Prepare its # root as a standard shared temporary directory before the # unprivileged workspace container starts. - dynamic "volume_mount" { - for_each = var.test_mode ? [] : toset(["tmp"]) - content { - mount_path = "/tmp" - name = volume_mount.value - } + volume_mount { + mount_path = "/tmp" + name = "tmp" } security_context { run_as_user = 0 @@ -92,13 +85,13 @@ resource "kubernetes_deployment_v1" "deployment" { } container { name = "workspace" - # Not Coder's generated /workspace-init.sh directly: the entrypoint + # Not Coder's generated /scripts/workspace-init.sh directly: the entrypoint # wipes /tmp and then execs it. The wipe has to happen before the # agent unpacks its CLI into /tmp, and this is the only hook that runs # on a container-only restart within a live Pod (init containers do # not) - see script-container-entrypoint.sh and the "tmp" volume # below. - command = ["/bin/bash", "/container-entrypoint.sh"] + command = ["/bin/bash", "/scripts/script-container-entrypoint.sh"] image = var.workspace_image env { name = "CODER_AGENT_TOKEN" @@ -144,61 +137,12 @@ resource "kubernetes_deployment_v1" "deployment" { name = "home" } volume_mount { - mount_path = "/agent-startup.sh" + mount_path = "/scripts" name = "coder-scripts" - sub_path = "agent_startup_script" } volume_mount { - mount_path = "/container-entrypoint.sh" - name = "coder-scripts" - sub_path = "container_entrypoint_script" - } - volume_mount { - mount_path = "/dotfiles.sh" - name = "coder-scripts" - sub_path = "dotfiles_script" - } - volume_mount { - mount_path = "/memory-watchdog.sh" - name = "coder-scripts" - sub_path = "memory_watchdog_script" - } - volume_mount { - mount_path = "/memory-watchdog-start.sh" - name = "coder-scripts" - sub_path = "memory_watchdog_start_script" - } - volume_mount { - mount_path = "/service-command.sh" - name = "coder-scripts" - sub_path = "service_command_script" - } - volume_mount { - mount_path = "/start-services.sh" - name = "coder-scripts" - sub_path = "start_services_script" - } - volume_mount { - mount_path = "/supervisord.conf" - name = "coder-scripts" - sub_path = "supervisor_config" - } - volume_mount { - mount_path = "/vscode-server-gc.sh" - name = "coder-scripts" - sub_path = "vscode_server_gc_script" - } - volume_mount { - mount_path = "/workspace-init.sh" - name = "coder-scripts" - sub_path = "workspace_init_script" - } - dynamic "volume_mount" { - for_each = var.test_mode ? [] : toset(["tmp"]) - content { - mount_path = "/tmp" - name = volume_mount.value - } + mount_path = "/tmp" + name = "tmp" } } enable_service_links = false @@ -250,19 +194,16 @@ resource "kubernetes_deployment_v1" "deployment" { # the entrypoint and nowhere later: the Coder agent unpacks its own CLI # into /tmp before it runs anything else, so a wipe from the agent # startup script deletes it. - dynamic "volume" { - for_each = var.test_mode ? [] : toset(["tmp"]) - content { - name = volume.value - ephemeral { - volume_claim_template { - spec { - access_modes = ["ReadWriteOnce"] - storage_class_name = var.tmp_pvc_storage_class - resources { - requests = { - storage = "20Gi" - } + volume { + name = "tmp" + ephemeral { + volume_claim_template { + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = var.tmp_pvc_storage_class + resources { + requests = { + storage = "${data.coder_parameter.tmp_pvc_size.value}Gi" } } } diff --git a/templates/kubernetes/homelab-workspace/env.tf b/templates/kubernetes/homelab-workspace/env.tf index b6a9907b..6069e22a 100644 --- a/templates/kubernetes/homelab-workspace/env.tf +++ b/templates/kubernetes/homelab-workspace/env.tf @@ -40,10 +40,10 @@ resource "coder_env" "service_count" { value = tostring(length(local.validated_service_commands)) } -resource "coder_env" "template_test_mode" { +resource "coder_env" "skip_dotfiles_scripts" { agent_id = coder_agent.main.id - name = "TEMPLATE_TEST_MODE" - value = tostring(var.test_mode) + name = "SKIP_DOTFILES_SCRIPTS" + value = tostring(var.skip_dotfiles_scripts) } # The switch that arms the memory watchdog. "observe" measures and records what diff --git a/templates/kubernetes/homelab-workspace/parameters.tf b/templates/kubernetes/homelab-workspace/parameters.tf index 77cf5c4a..185a955f 100644 --- a/templates/kubernetes/homelab-workspace/parameters.tf +++ b/templates/kubernetes/homelab-workspace/parameters.tf @@ -40,7 +40,7 @@ data "coder_parameter" "home_pvc_size" { display_name = "Home PVC Size" description = "Size of the workspace-owned home persistent volume claim in GiB" icon = "/icon/database.svg" - mutable = false + mutable = true order = 3 type = "number" form_type = "input" @@ -51,6 +51,24 @@ data "coder_parameter" "home_pvc_size" { } } +data "coder_parameter" "tmp_pvc_size" { + name = "tmp_pvc_size" + + default = 1 + display_name = "Temporary Storage Size" + description = "Size of the ephemeral temporary-storage persistent volume claim in GiB" + icon = "/icon/database.svg" + mutable = true + order = 4 + type = "number" + form_type = "slider" + + validation { + min = 1 + max = 50 + } +} + data "coder_parameter" "resources_memory" { name = "memory" diff --git a/templates/kubernetes/homelab-workspace/script-container-entrypoint.sh b/templates/kubernetes/homelab-workspace/script-container-entrypoint.sh index 4c3bc463..5ff41930 100644 --- a/templates/kubernetes/homelab-workspace/script-container-entrypoint.sh +++ b/templates/kubernetes/homelab-workspace/script-container-entrypoint.sh @@ -17,7 +17,7 @@ TMP_WIPE_STATUS_FILE="/tmp/.tmp-wipe-status" # # This has to run in the container entrypoint, before the Coder agent exists, # and not in the agent startup script. Coder's own bootstrap (the generated -# /workspace-init.sh) unpacks the agent CLI into a per-boot mktemp directory +# /scripts/workspace-init.sh) unpacks the agent CLI into a per-boot mktemp directory # under /tmp, chdirs into it, appends it to the PATH of every session and script # the agent runs, and only then runs the startup script - so a wipe from the # startup script deletes the CLI the agent installed moments earlier, which is @@ -57,7 +57,7 @@ main() { # than spawning it: the agent has to stay PID 1, both because it reaps orphans # in this container and because the Deployment's liveness probe pgreps for it. echo "Starting Coder agent..." - exec /bin/bash /workspace-init.sh + exec /bin/bash /scripts/workspace-init.sh } main diff --git a/templates/kubernetes/homelab-workspace/script-dotfiles.sh b/templates/kubernetes/homelab-workspace/script-dotfiles.sh index b10ebff7..a93a0f28 100755 --- a/templates/kubernetes/homelab-workspace/script-dotfiles.sh +++ b/templates/kubernetes/homelab-workspace/script-dotfiles.sh @@ -44,7 +44,7 @@ else else mkdir -p "$(dirname "${source_dir}")" git clone -- "${DOTFILES_URL}" "${source_dir}" - if [[ "${TEMPLATE_TEST_MODE}" == "true" ]]; then + if [[ "${SKIP_DOTFILES_SCRIPTS}" == "true" ]]; then # The integration test verifies this template's Chezmoi orchestration; # repository-owned workstation bootstrap scripts are outside its scope. "${chezmoi_bin}" init --apply --skip-secrets --exclude=scripts @@ -55,4 +55,4 @@ else fi touch "${state_dir}/applied" -/bin/bash /start-services.sh +/bin/bash /scripts/script-start-services.sh diff --git a/templates/kubernetes/homelab-workspace/script-memory-watchdog-start.sh b/templates/kubernetes/homelab-workspace/script-memory-watchdog-start.sh index b6d9e3ea..74114e94 100755 --- a/templates/kubernetes/homelab-workspace/script-memory-watchdog-start.sh +++ b/templates/kubernetes/homelab-workspace/script-memory-watchdog-start.sh @@ -3,6 +3,6 @@ set -u state_dir="${HOME}/.local/state/vscode-memory-watchdog" mkdir -p "${state_dir}" -/usr/bin/setsid --fork /bin/bash /memory-watchdog.sh \ +/usr/bin/setsid --fork /bin/bash /scripts/script-memory-watchdog.sh \ >"${state_dir}/boot.log" 2>&1 echo "memory watchdog started in ${WATCHDOG_MODE:-observe} mode; state in ${state_dir}" diff --git a/templates/kubernetes/homelab-workspace/script-start-services.sh b/templates/kubernetes/homelab-workspace/script-start-services.sh index d3b65456..6eb10e7e 100755 --- a/templates/kubernetes/homelab-workspace/script-start-services.sh +++ b/templates/kubernetes/homelab-workspace/script-start-services.sh @@ -18,5 +18,5 @@ if [[ -s "${state_dir}/supervisord.pid" ]]; then rm -f "${state_dir}/supervisord.pid" "${state_dir}/supervisor.sock" fi -/usr/bin/supervisord --configuration /supervisord.conf +/usr/bin/supervisord --configuration /scripts/supervisord.conf echo "started ${SUPERVISOR_SERVICE_COUNT} supervised service command(s); state in ${state_dir}" diff --git a/templates/kubernetes/homelab-workspace/script-vscode-server-gc.sh b/templates/kubernetes/homelab-workspace/script-vscode-server-gc.sh index eafe758b..f2ac8468 100644 --- a/templates/kubernetes/homelab-workspace/script-vscode-server-gc.sh +++ b/templates/kubernetes/homelab-workspace/script-vscode-server-gc.sh @@ -25,7 +25,7 @@ set -euo pipefail # cleanup script that only exists via a dotfiles deploy can't cover the case # that motivated writing it. See configmap.tf (mounts this into the # ConfigMap the pod reads from) and deployment.tf (mounts it into the -# workspace container at /vscode-server-gc.sh); scripts.tf's coder_script +# workspace container at /scripts/script-vscode-server-gc.sh); scripts.tf's coder_script # "vscode_server_gc" invokes it directly on its weekly cron -- no existence # check, because the ConfigMap mount guarantees it's there whenever the pod # is, and see that resource's comment for the silent-no-op failure mode this diff --git a/templates/kubernetes/homelab-workspace/scripts.tf b/templates/kubernetes/homelab-workspace/scripts.tf index 6065158d..ad2ac228 100644 --- a/templates/kubernetes/homelab-workspace/scripts.tf +++ b/templates/kubernetes/homelab-workspace/scripts.tf @@ -13,7 +13,7 @@ resource "coder_script" "dotfiles" { icon = "/icon/terminal.svg" run_on_start = true start_blocks_login = false - script = "/bin/bash /dotfiles.sh" + script = "/bin/bash /scripts/script-dotfiles.sh" } # Starts the memory watchdog, which bounds the standing population of @@ -31,7 +31,7 @@ resource "coder_script" "memory_watchdog" { icon = "/icon/memory.svg" run_on_start = true start_blocks_login = false - script = "/bin/bash /memory-watchdog-start.sh" + script = "/bin/bash /scripts/script-memory-watchdog-start.sh" } # Weekly garbage collection of ~/.vscode-server, which grows without bound and @@ -55,5 +55,5 @@ resource "coder_script" "vscode_server_gc" { icon = "/icon/code.svg" # Coder's cron is 6-field (seconds first), not the usual 5. Sundays at 04:00. cron = "0 0 4 * * 0" - script = "/bin/bash /vscode-server-gc.sh" + script = "/bin/bash /scripts/script-vscode-server-gc.sh" } diff --git a/templates/kubernetes/homelab-workspace/supervisord.conf b/templates/kubernetes/homelab-workspace/supervisord.conf index d91670d8..97636678 100644 --- a/templates/kubernetes/homelab-workspace/supervisord.conf +++ b/templates/kubernetes/homelab-workspace/supervisord.conf @@ -17,7 +17,7 @@ supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface process_name=service-%(process_num)s numprocs=%(ENV_SUPERVISOR_SERVICE_COUNT)s numprocs_start=0 -command=/bin/bash /service-command.sh %(process_num)s +command=/bin/bash /scripts/script-service-command.sh %(process_num)s directory=%(ENV_HOME)s autostart=true autorestart=true diff --git a/templates/kubernetes/homelab-workspace/terraform.tf b/templates/kubernetes/homelab-workspace/terraform.tf index 034a08fe..197e5d9e 100644 --- a/templates/kubernetes/homelab-workspace/terraform.tf +++ b/templates/kubernetes/homelab-workspace/terraform.tf @@ -17,5 +17,5 @@ provider "coder" { } provider "kubernetes" { - config_path = var.test_mode ? "/home/coder/.kube/config" : null + config_path = var.kubernetes_config_path != "" ? var.kubernetes_config_path : null } diff --git a/templates/kubernetes/homelab-workspace/variables.tf b/templates/kubernetes/homelab-workspace/variables.tf index 50fc434c..a9ad08a7 100644 --- a/templates/kubernetes/homelab-workspace/variables.tf +++ b/templates/kubernetes/homelab-workspace/variables.tf @@ -10,6 +10,10 @@ variable "tmp_pvc_storage_class" { type = string } -variable "test_mode" { +variable "kubernetes_config_path" { + type = string +} + +variable "skip_dotfiles_scripts" { type = bool }