Speed up macOS VM rebuilds and add a --no-secrets opt-out

Taskfile.yml

@@ -380,6 +380,14 @@ tasks:
     env:
       DOCKER_HOST: '{{.DOCKER_SOCK}}'
 
+  test:tart:
+    desc: "Host-side diagnostics for the tart (--os macos) path: env, flake metadata, aarch64-darwin platform preflight. Artifacts → test/results/"
+    platforms: [darwin]
+    silent: true
+    dir: "{{.TASKFILE_DIR}}"
+    cmds:
+      - go test -v -count=1 -timeout 600s -run TestTartDarwinIntegration ./cmd/ {{.CLI_ARGS}}
+
   test:vagrant:
     desc: Run E2E install test in a clean Debian VM (QEMU). Requires vagrant + vagrant-qemu plugin.
     dir: "{{.TASKFILE_DIR}}/test/vagrant"
@@ -535,6 +543,68 @@ tasks:
   debug:
     cmds:
       - task: debug:macos
+
+  # ── macOS tart VM /nix + s6 diagnostics (read-only) → .scratch/debug/macos-nix.log ─────
+  debug:macos:nix:
+    desc: "Probe the tart session VM: /nix mounts, darwin-store daemon, fstab, s6 tree → .scratch/debug/macos-nix.log"
+    platforms: [darwin]
+    silent: true
+    vars:
+      VM: '{{.VM | default "CELL-tart"}}'
+      LOG: '{{.TASKFILE_DIR}}/.scratch/debug/macos-nix.log'
+      NIX_IMG: '{{.HOME}}/.devcell/darwin/nix.img'
+      CELL_HOME: '{{.HOME}}/.devcell/CELL'
+    cmds:
+      - mkdir -p {{.TASKFILE_DIR}}/.scratch/debug
+      - |
+        {
+          echo "=== debug:macos:nix $(date -u +%Y%m%dT%H%M%SZ) VM={{.VM}} ==="
+          echo "--- host: tart list ---"
+          tart list 2>&1 || true
+
+          if ! tart list 2>/dev/null | grep -E "^local[[:space:]]+{{.VM}}[[:space:]]" | grep -q running; then
+            echo "--- VM not running — booting in background (left running afterwards) ---"
+            nohup tart run --no-graphics \
+              --dir home:{{.CELL_HOME}} \
+              --dir project:{{.TASKFILE_DIR}} \
+              --disk {{.NIX_IMG}} \
+              {{.VM}} >/dev/null 2>&1 &
+          fi
+          echo "--- waiting for guest agent (max 120s) ---"
+          ok=""
+          for i in $(seq 1 40); do
+            if tart exec {{.VM}} true 2>/dev/null; then ok=1; break; fi
+            sleep 3
+          done
+          if [ -z "$ok" ]; then echo "FATAL: guest agent never came up"; exit 1; fi
+
+          probe() { echo ""; echo "--- $1 ---"; shift; tart exec {{.VM}} bash -c "$*" 2>&1 || true; }
+
+          probe "/nix mount table (order matters: last line shadows)"  '/sbin/mount | grep -n /nix'
+          probe "df /nix (which device actually serves /nix)"          'df /nix'
+          probe "diskutil list (all disks/volumes)"                    'diskutil list'
+          probe "fstab (is the installer APFS entry really gone?)"     'cat /etc/fstab'
+          probe "synthetic.conf"                                       'cat /etc/synthetic.conf'
+          probe "LaunchDaemons on disk (darwin-store plist still present?)" 'ls -la /Library/LaunchDaemons/'
+          probe "launchctl: org.nixos.darwin-store"                    'sudo launchctl print system/org.nixos.darwin-store 2>&1 | head -25'
+          probe "launchctl: com.devcell.mount-nix"                     'sudo launchctl print system/com.devcell.mount-nix 2>&1 | head -25'
+          probe "launchctl: org.nixos.nix-daemon"                      'sudo launchctl print system/org.nixos.nix-daemon 2>&1 | head -25'
+          probe "launchctl: com.devcell.s6-svscan"                     'sudo launchctl print system/com.devcell.s6-svscan 2>&1 | head -25'
+          probe "s6-svscan plist on disk"                              'ls -la /Library/LaunchDaemons/ | grep -i s6; cat /Library/LaunchDaemons/*s6*.plist 2>/dev/null'
+          probe "boot log: mount-nix vs darwin-store race"             'log show --last boot --predicate "process == \"diskarbitrationd\" OR eventMessage CONTAINS \"DevcellNix\" OR eventMessage CONTAINS \"Nix Store\"" 2>/dev/null | tail -40'
+          probe "visible /nix top-level"                               'ls -la /nix/ | head -15'
+          probe "visible /nix/var/nix/profiles"                        'ls -la /nix/var/nix/profiles/ 2>&1'
+          probe "system profile + current-system"                      'ls -la /nix/var/nix/profiles/system 2>&1; readlink /run/current-system 2>&1'
+          probe "s6 in system PATH"                                    'ls /run/current-system/sw/bin/ 2>/dev/null | grep -i ^s6 | head; command -v s6-rc s6-svscan 2>&1'
+          probe "/etc/s6 tree"                                         'ls -laR /etc/s6/ 2>&1 | head -60'
+          probe "per-user devcell profile"                             'ls /etc/profiles/per-user/devcell/bin 2>&1 | head -20'
+          probe "store roots that prove JHFS+ content exists"          'ls /nix/store/ 2>/dev/null | head -10; ls /nix/store/ 2>/dev/null | wc -l'
+
+          echo ""
+          echo "=== done (VM {{.VM}} left running) ==="
+        } 2>&1 | tee {{.LOG}}
+        echo "full log: {{.LOG}}"
+
   # ── Forced QEMU Windows autobuild with full log capture (CELL-428/429) ─────
   debug:autobuild:
     desc: "Forced QEMU Windows template build with full log capture → .scratch/debug/autobuild.log"

cmd/build.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/DimmKirr/devcell/internal/cfg"
 	"github.com/DimmKirr/devcell/internal/config"
@@ -35,6 +36,7 @@ func init() {
 	buildCmd.Flags().String("image", "", "override the built image tag (e.g. devcell-user:dev-thin); env DEVCELL_BUILD_IMAGE has lower precedence")
 	buildCmd.Flags().Bool("force", false, "recreate VM even if it already exists (tart only)")
 	buildCmd.Flags().Bool("no-cache", false, "re-download OCI image, bypassing tart cache (tart only)")
+	buildCmd.Flags().String("stage", "full", `build stage: "base" (infra only) or "full" (default, includes stack activation) (tart only)`)
 }
 
 func runBuild(cmd *cobra.Command, _ []string) error {
@@ -57,6 +59,7 @@ func runBuild(cmd *cobra.Command, _ []string) error {
 		"update":     scanFlag("--update"),
 		"no_cache":   scanFlag("--no-cache"),
 		"force":      scanFlag("--force"),
+		"stage":      cmd.Flags().Lookup("stage").Value.String(),
 	})
 
 	// ── tart engine ──────────────────────────────────────────────────────────
@@ -71,8 +74,16 @@ func runBuild(cmd *cobra.Command, _ []string) error {
 		}
 		force, _ := cmd.Flags().GetBool("force")
 		noCache, _ := cmd.Flags().GetBool("no-cache")
+		stage := cmd.Flags().Lookup("stage").Value.String()
+		if stage != "base" && stage != "full" {
+			return fmt.Errorf("--stage must be \"base\" or \"full\", got %q", stage)
+		}
+		update, _ := cmd.Flags().GetBool("update")
+		if update {
+			force = true
+		}
 		tartOCIImage := cellCfgTart.Cell.ResolvedTartOCIImage()
-		return runBuildTart(c.CellName, c.HostHome, c.BaseDir, stack, nil, force, noCache, scanFlag("--dry-run"), tartOCIImage)
+		return runBuildTart(c.CellName, c.HostHome, c.BaseDir, stack, nil, force, noCache, scanFlag("--dry-run"), tartOCIImage, stage)
 	}
 
 	// ── qemu engine ─────────────────────────────────────────────────────────
@@ -254,7 +265,7 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec
 	coreImage := cellCfg.Nix.ResolvedImage()
 	tag := runner.ResolveBuildTag(imageOverride, runner.UserImageTagThin())
 	volumeName := runner.ThinStoreVolume()
-	containerName := "devcell-thin-builder"
+	containerName := runner.ThinBuilderContainerName(c.AppName)
 
 	// ── Ensure core image exists for target platform ───────────────────────
 	targetPlatform := runner.DockerPlatform(runner.DetectArch())
@@ -281,7 +292,18 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec
 	buildLabel := runner.BuildLabel("Building thin image", stack, explicitStack)
 	sp := ux.NewProgressSpinner(buildLabel)
 
-	_ = exec.CommandContext(ctx, "docker", "rm", "-f", containerName).Run()
+	// Reclaim this app's builder slot. A builder left behind by a crashed or
+	// interrupted run is removed; a *running* one is never killed — builds
+	// for other apps have their own name and are untouched either way.
+	exists, running := runner.BuilderContainerState(ctx, containerName)
+	remove, err := runner.ReclaimBuilderSlot(containerName, exists, running)
+	if err != nil {
+		sp.Fail(buildLabel + " failed")
+		return err
+	}
+	if remove {
+		_ = exec.CommandContext(ctx, "docker", "rm", "-f", containerName).Run()
+	}
 
 	// CELL-41: pass the real user-facing stack name + modules CSV so the
 	// container's metadata.json reports them truthfully. The HM target stays
@@ -358,6 +380,12 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec
 	cmd.Stderr = out
 	if err := cmd.Run(); err != nil {
 		sp.Fail(buildLabel + " failed")
+		if runner.BuilderOrphanedByCancel(err, ctx.Err()) {
+			// ctx is already cancelled; use a fresh one so the cleanup runs.
+			rmCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			_ = exec.CommandContext(rmCtx, "docker", "rm", "-f", containerName).Run()
+			cancel()
+		}
 		if !ux.Verbose && buf.Len() > 0 {
 			fmt.Fprint(os.Stderr, buf.String())
 		}
@@ -371,4 +399,3 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec
 	sp.Success(successLabel)
 	return nil
 }
-

cmd/build_tart_darwin.go

@@ -23,7 +23,7 @@ import (
 // Mirrors the Docker build flow: init scaffolds config/keys (no images),
 // build creates and provisions the image. The VM is booted for provisioning
 // and shut down when done — cell shell starts it again for the session.
-func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage string) error {
+func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage, stage string) error {
 	cfg := tart.BuildConfig{
 		CellName: cellName,
 		HomeDir:  hostHome,
@@ -35,19 +35,40 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string
 		return err
 	}
 
-	templateName := tart.TemplateVMName(stack, modules)
+	// Determine template name and clone source based on stage.
+	var templateName string
+	var cloneSource string
+	var cloneFromBase bool
+
+	switch stage {
+	case "base":
+		templateName = tart.BaseTemplateName
+		cloneSource = tartOCIImage
+	default: // "full"
+		templateName = tart.TemplateVMName(stack, modules)
+		if _, err := tart.TartGet(context.Background(), tart.BaseTemplateName); err == nil {
+			cloneSource = tart.BaseTemplateName
+			cloneFromBase = true
+			ux.Debugf("base template %s found: will clone locally (fast path)", tart.BaseTemplateName)
+		} else {
+			cloneSource = tartOCIImage
+			ux.Debugf("no base template: full build from OCI")
+		}
+	}
+
 	buildVM := "devcell-build-tmp"
 
 	nixhomeRef := runner.ResolveNixhomeRef(version.Version)
 
-	ux.Debugf("build config: cell=%s stack=%s cpus=%d mem=%dGB sshPort=%d",
-		cfg.CellName, cfg.Stack, cfg.CPUs, cfg.MemoryGB, cfg.SSHPort)
-	ux.Debugf("template: %s  buildVM: %s  force=%v noCache=%v", templateName, buildVM, force, noCache)
+	ux.Debugf("build config: cell=%s stack=%s stage=%s cpus=%d mem=%dGB sshPort=%d",
+		cfg.CellName, cfg.Stack, stage, cfg.CPUs, cfg.MemoryGB, cfg.SSHPort)
+	ux.Debugf("template: %s  cloneSource: %s  buildVM: %s  force=%v noCache=%v", templateName, cloneSource, buildVM, force, noCache)
 	ux.Debugf("nixhome: %s  projectDir: %s", nixhomeRef, projectDir)
 
 	if dryRun {
 		fmt.Printf("Would build macOS VM template: %s\n", templateName)
-		fmt.Printf("  OCI image: %s\n", tartOCIImage)
+		fmt.Printf("  Stage: %s\n", stage)
+		fmt.Printf("  Clone source: %s\n", cloneSource)
 		fmt.Printf("  Stack: %s\n", cfg.Stack)
 		fmt.Printf("  CPUs: %d  Memory: %dGB\n", cfg.CPUs, cfg.MemoryGB)
 		if len(cfg.Modules) > 0 {
@@ -122,8 +143,12 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string
 		return err
 	}
 
-	// --- Phase 2: Clone OCI image → build VM ---
-	if err := pr.PhaseDetailed("Cloning VM from OCI image", func() (string, error) {
+	// --- Phase 2: Clone source → build VM ---
+	cloneLabel := "Cloning VM from OCI image"
+	if cloneFromBase {
+		cloneLabel = "Cloning VM from base template"
+	}
+	if err := pr.PhaseDetailed(cloneLabel, func() (string, error) {
 		if _, getErr := tart.TartGet(ctx, templateName); getErr == nil {
 			if !force {
 				return "", fmt.Errorf("template %s already exists — use --force to rebuild", templateName)
@@ -140,12 +165,12 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string
 			_ = tart.TartDelete(ctx, buildVM)
 		}
 
-		ux.Debugf("cloning %s → %s (noCache=%v)", tartOCIImage, buildVM, noCache)
+		ux.Debugf("cloning %s → %s (noCache=%v)", cloneSource, buildVM, noCache)
 		args := []string{"clone"}
-		if noCache {
+		if noCache && !cloneFromBase {
 			args = append(args, "--no-cache")
 		}
-		args = append(args, tartOCIImage, buildVM)
+		args = append(args, cloneSource, buildVM)
 		cmd := exec.CommandContext(ctx, "tart", args...)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
@@ -185,11 +210,22 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string
 		getOut, _ := exec.CommandContext(ctx, "tart", "get", buildVM).CombinedOutput()
 		ux.Debugf("tart get %s (pre-boot):\n%s", buildVM, string(getOut))
 	}
+	// --- Detect host nix store for caching ---
+	hostNixPath := tart.DetectHostNixStore()
+	if hostNixPath != "" {
+		ux.Debugf("host nix store detected at %s (valid: store/ + db.sqlite present) — will share as read-only substituter", hostNixPath)
+	} else {
+		ux.Debugf("no host nix store found at /nix — VM will download from cache.nixos.org")
+	}
+
 	// --- Phase 4: Boot VM ---
 	sharedDirs := map[string]string{
 		"nixhome": nixhomeRef,
 		"home":    cellHome,
 	}
+	if hostNixPath != "" {
+		sharedDirs["hostnix"] = hostNixPath + ":ro"
+	}
 	disks := []string{nixVolumePath}
 	ux.Debugf("booting VM %s with shared dirs: %v, disks: %v", buildVM, sharedDirs, disks)
 
@@ -226,20 +262,24 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string
 		return err
 	}
 
-	// --- Phase 6: Bootstrap passwordless sudo ---
-	if err := pr.PhaseDetailed("Bootstrapping passwordless sudo", func() (string, error) {
-		bootstrapCmd := fmt.Sprintf(
-			"echo '%s' | sudo -S sh -c \"mkdir -p /etc/sudoers.d && echo '%s ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/%s && chmod 440 /etc/sudoers.d/%s\"",
-			tartImagePassword, tartImageUser, tartImageUser, tartImageUser,
-		)
-		ux.Debugf("bootstrap: configuring passwordless sudo for %s", tartImageUser)
-		if err := tart.TartExec(ctx, buildVM, []string{"bash", "-l", "-c", bootstrapCmd}, os.Stdout, os.Stderr); err != nil {
-			return "", fmt.Errorf("bootstrap sudo: %w", err)
+	// --- Phase 6: Bootstrap passwordless sudo (skip when cloning from base) ---
+	if !cloneFromBase {
+		if err := pr.PhaseDetailed("Bootstrapping passwordless sudo", func() (string, error) {
+			bootstrapCmd := fmt.Sprintf(
+				"echo '%s' | sudo -S sh -c \"mkdir -p /etc/sudoers.d && echo '%s ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/%s && chmod 440 /etc/sudoers.d/%s\"",
+				tartImagePassword, tartImageUser, tartImageUser, tartImageUser,
+			)
+			ux.Debugf("bootstrap: configuring passwordless sudo for %s", tartImageUser)
+			if err := tart.TartExec(ctx, buildVM, []string{"bash", "-l", "-c", bootstrapCmd}, os.Stdout, os.Stderr); err != nil {
+				return "", fmt.Errorf("bootstrap sudo: %w", err)
+			}
+			return tartImageUser, nil
+		}); err != nil {
+			stopVM()
+			return err
 		}
-		return tartImageUser, nil
-	}); err != nil {
-		stopVM()
-		return err
+	} else {
+		ux.Debugf("skipping sudo bootstrap: base template already has passwordless sudo")
 	}
 
 	// --- Diagnostic: host-side post-boot checks ---
@@ -283,17 +323,27 @@ echo "=== END GUEST DIAGNOSTICS ==="`
 		}
 	}
 
-	// --- Phase 7: Full provisioning via tart exec ---
+	// --- Phase 7: Provisioning via tart exec ---
 	initCfg := tart.InitConfig{
-		CellName: cellName,
-		HomeDir:  hostHome,
-		Stack:    stack,
-		Username: tartImageUser,
-		Password: tartImagePassword,
+		CellName:   cellName,
+		HomeDir:    hostHome,
+		Stack:      stack,
+		Username:   tartImageUser,
+		Password:   tartImagePassword,
+		HasHostNix: hostNixPath != "",
 	}
 	initCfg.ApplyDefaults()
-	steps := tart.ProvisionSteps(initCfg, pubKey, false)
-	ux.Debugf("provisioning: %d steps via tart exec", len(steps))
+
+	var steps []tart.ProvisionStep
+	switch {
+	case stage == "base":
+		steps = tart.BaseProvisionSteps(initCfg, pubKey)
+	case cloneFromBase:
+		steps = tart.StackProvisionSteps(initCfg)
+	default:
+		steps = tart.ProvisionSteps(initCfg, pubKey, false)
+	}
+	ux.Debugf("provisioning: %d steps via tart exec (stage=%s, cloneFromBase=%v)", len(steps), stage, cloneFromBase)
 
 	const reformatMarker = "DEVCELL_REFORMAT_NEEDED:"
 	for i, step := range steps {

cmd/build_tart_stub.go

@@ -4,6 +4,6 @@ package main
 
 import "fmt"
 
-func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage string) error {
+func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage, stage string) error {
 	return fmt.Errorf("cell build --engine=tart requires macOS on Apple Silicon (darwin/arm64)")
 }