diff --git a/Taskfile.yml b/Taskfile.yml index 5a4181f8..b3caf27d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -380,6 +380,14 @@ tasks: env: DOCKER_HOST: '{{.DOCKER_SOCK}}' + test:tart: + desc: "Host-side diagnostics for the tart (--os macos) path: env, flake metadata, aarch64-darwin platform preflight. Artifacts → test/results/" + platforms: [darwin] + silent: true + dir: "{{.TASKFILE_DIR}}" + cmds: + - go test -v -count=1 -timeout 600s -run TestTartDarwinIntegration ./cmd/ {{.CLI_ARGS}} + test:vagrant: desc: Run E2E install test in a clean Debian VM (QEMU). Requires vagrant + vagrant-qemu plugin. dir: "{{.TASKFILE_DIR}}/test/vagrant" @@ -535,6 +543,68 @@ tasks: debug: cmds: - task: debug:macos + + # ── macOS tart VM /nix + s6 diagnostics (read-only) → .scratch/debug/macos-nix.log ───── + debug:macos:nix: + desc: "Probe the tart session VM: /nix mounts, darwin-store daemon, fstab, s6 tree → .scratch/debug/macos-nix.log" + platforms: [darwin] + silent: true + vars: + VM: '{{.VM | default "CELL-tart"}}' + LOG: '{{.TASKFILE_DIR}}/.scratch/debug/macos-nix.log' + NIX_IMG: '{{.HOME}}/.devcell/darwin/nix.img' + CELL_HOME: '{{.HOME}}/.devcell/CELL' + cmds: + - mkdir -p {{.TASKFILE_DIR}}/.scratch/debug + - | + { + echo "=== debug:macos:nix $(date -u +%Y%m%dT%H%M%SZ) VM={{.VM}} ===" + echo "--- host: tart list ---" + tart list 2>&1 || true + + if ! tart list 2>/dev/null | grep -E "^local[[:space:]]+{{.VM}}[[:space:]]" | grep -q running; then + echo "--- VM not running — booting in background (left running afterwards) ---" + nohup tart run --no-graphics \ + --dir home:{{.CELL_HOME}} \ + --dir project:{{.TASKFILE_DIR}} \ + --disk {{.NIX_IMG}} \ + {{.VM}} >/dev/null 2>&1 & + fi + echo "--- waiting for guest agent (max 120s) ---" + ok="" + for i in $(seq 1 40); do + if tart exec {{.VM}} true 2>/dev/null; then ok=1; break; fi + sleep 3 + done + if [ -z "$ok" ]; then echo "FATAL: guest agent never came up"; exit 1; fi + + probe() { echo ""; echo "--- $1 ---"; shift; tart exec {{.VM}} bash -c "$*" 2>&1 || true; } + + probe "/nix mount table (order matters: last line shadows)" '/sbin/mount | grep -n /nix' + probe "df /nix (which device actually serves /nix)" 'df /nix' + probe "diskutil list (all disks/volumes)" 'diskutil list' + probe "fstab (is the installer APFS entry really gone?)" 'cat /etc/fstab' + probe "synthetic.conf" 'cat /etc/synthetic.conf' + probe "LaunchDaemons on disk (darwin-store plist still present?)" 'ls -la /Library/LaunchDaemons/' + probe "launchctl: org.nixos.darwin-store" 'sudo launchctl print system/org.nixos.darwin-store 2>&1 | head -25' + probe "launchctl: com.devcell.mount-nix" 'sudo launchctl print system/com.devcell.mount-nix 2>&1 | head -25' + probe "launchctl: org.nixos.nix-daemon" 'sudo launchctl print system/org.nixos.nix-daemon 2>&1 | head -25' + probe "launchctl: com.devcell.s6-svscan" 'sudo launchctl print system/com.devcell.s6-svscan 2>&1 | head -25' + probe "s6-svscan plist on disk" 'ls -la /Library/LaunchDaemons/ | grep -i s6; cat /Library/LaunchDaemons/*s6*.plist 2>/dev/null' + probe "boot log: mount-nix vs darwin-store race" 'log show --last boot --predicate "process == \"diskarbitrationd\" OR eventMessage CONTAINS \"DevcellNix\" OR eventMessage CONTAINS \"Nix Store\"" 2>/dev/null | tail -40' + probe "visible /nix top-level" 'ls -la /nix/ | head -15' + probe "visible /nix/var/nix/profiles" 'ls -la /nix/var/nix/profiles/ 2>&1' + probe "system profile + current-system" 'ls -la /nix/var/nix/profiles/system 2>&1; readlink /run/current-system 2>&1' + probe "s6 in system PATH" 'ls /run/current-system/sw/bin/ 2>/dev/null | grep -i ^s6 | head; command -v s6-rc s6-svscan 2>&1' + probe "/etc/s6 tree" 'ls -laR /etc/s6/ 2>&1 | head -60' + probe "per-user devcell profile" 'ls /etc/profiles/per-user/devcell/bin 2>&1 | head -20' + probe "store roots that prove JHFS+ content exists" 'ls /nix/store/ 2>/dev/null | head -10; ls /nix/store/ 2>/dev/null | wc -l' + + echo "" + echo "=== done (VM {{.VM}} left running) ===" + } 2>&1 | tee {{.LOG}} + echo "full log: {{.LOG}}" + # ── Forced QEMU Windows autobuild with full log capture (CELL-428/429) ───── debug:autobuild: desc: "Forced QEMU Windows template build with full log capture → .scratch/debug/autobuild.log" diff --git a/cmd/build.go b/cmd/build.go index adda5093..8d9407c7 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/DimmKirr/devcell/internal/cfg" "github.com/DimmKirr/devcell/internal/config" @@ -35,6 +36,7 @@ func init() { buildCmd.Flags().String("image", "", "override the built image tag (e.g. devcell-user:dev-thin); env DEVCELL_BUILD_IMAGE has lower precedence") buildCmd.Flags().Bool("force", false, "recreate VM even if it already exists (tart only)") buildCmd.Flags().Bool("no-cache", false, "re-download OCI image, bypassing tart cache (tart only)") + buildCmd.Flags().String("stage", "full", `build stage: "base" (infra only) or "full" (default, includes stack activation) (tart only)`) } func runBuild(cmd *cobra.Command, _ []string) error { @@ -57,6 +59,7 @@ func runBuild(cmd *cobra.Command, _ []string) error { "update": scanFlag("--update"), "no_cache": scanFlag("--no-cache"), "force": scanFlag("--force"), + "stage": cmd.Flags().Lookup("stage").Value.String(), }) // ── tart engine ────────────────────────────────────────────────────────── @@ -71,8 +74,16 @@ func runBuild(cmd *cobra.Command, _ []string) error { } force, _ := cmd.Flags().GetBool("force") noCache, _ := cmd.Flags().GetBool("no-cache") + stage := cmd.Flags().Lookup("stage").Value.String() + if stage != "base" && stage != "full" { + return fmt.Errorf("--stage must be \"base\" or \"full\", got %q", stage) + } + update, _ := cmd.Flags().GetBool("update") + if update { + force = true + } tartOCIImage := cellCfgTart.Cell.ResolvedTartOCIImage() - return runBuildTart(c.CellName, c.HostHome, c.BaseDir, stack, nil, force, noCache, scanFlag("--dry-run"), tartOCIImage) + return runBuildTart(c.CellName, c.HostHome, c.BaseDir, stack, nil, force, noCache, scanFlag("--dry-run"), tartOCIImage, stage) } // ── qemu engine ───────────────────────────────────────────────────────── @@ -254,7 +265,7 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec coreImage := cellCfg.Nix.ResolvedImage() tag := runner.ResolveBuildTag(imageOverride, runner.UserImageTagThin()) volumeName := runner.ThinStoreVolume() - containerName := "devcell-thin-builder" + containerName := runner.ThinBuilderContainerName(c.AppName) // ── Ensure core image exists for target platform ─────────────────────── targetPlatform := runner.DockerPlatform(runner.DetectArch()) @@ -281,7 +292,18 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec buildLabel := runner.BuildLabel("Building thin image", stack, explicitStack) sp := ux.NewProgressSpinner(buildLabel) - _ = exec.CommandContext(ctx, "docker", "rm", "-f", containerName).Run() + // Reclaim this app's builder slot. A builder left behind by a crashed or + // interrupted run is removed; a *running* one is never killed — builds + // for other apps have their own name and are untouched either way. + exists, running := runner.BuilderContainerState(ctx, containerName) + remove, err := runner.ReclaimBuilderSlot(containerName, exists, running) + if err != nil { + sp.Fail(buildLabel + " failed") + return err + } + if remove { + _ = exec.CommandContext(ctx, "docker", "rm", "-f", containerName).Run() + } // CELL-41: pass the real user-facing stack name + modules CSV so the // container's metadata.json reports them truthfully. The HM target stays @@ -358,6 +380,12 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec cmd.Stderr = out if err := cmd.Run(); err != nil { sp.Fail(buildLabel + " failed") + if runner.BuilderOrphanedByCancel(err, ctx.Err()) { + // ctx is already cancelled; use a fresh one so the cleanup runs. + rmCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _ = exec.CommandContext(rmCtx, "docker", "rm", "-f", containerName).Run() + cancel() + } if !ux.Verbose && buf.Len() > 0 { fmt.Fprint(os.Stderr, buf.String()) } @@ -371,4 +399,3 @@ func runBuildThin(c config.Config, stackOverride, imageOverride string, forceRec sp.Success(successLabel) return nil } - diff --git a/cmd/build_tart_darwin.go b/cmd/build_tart_darwin.go index a451c10e..62fdcaf3 100644 --- a/cmd/build_tart_darwin.go +++ b/cmd/build_tart_darwin.go @@ -23,7 +23,7 @@ import ( // Mirrors the Docker build flow: init scaffolds config/keys (no images), // build creates and provisions the image. The VM is booted for provisioning // and shut down when done — cell shell starts it again for the session. -func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage string) error { +func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage, stage string) error { cfg := tart.BuildConfig{ CellName: cellName, HomeDir: hostHome, @@ -35,19 +35,40 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string return err } - templateName := tart.TemplateVMName(stack, modules) + // Determine template name and clone source based on stage. + var templateName string + var cloneSource string + var cloneFromBase bool + + switch stage { + case "base": + templateName = tart.BaseTemplateName + cloneSource = tartOCIImage + default: // "full" + templateName = tart.TemplateVMName(stack, modules) + if _, err := tart.TartGet(context.Background(), tart.BaseTemplateName); err == nil { + cloneSource = tart.BaseTemplateName + cloneFromBase = true + ux.Debugf("base template %s found: will clone locally (fast path)", tart.BaseTemplateName) + } else { + cloneSource = tartOCIImage + ux.Debugf("no base template: full build from OCI") + } + } + buildVM := "devcell-build-tmp" nixhomeRef := runner.ResolveNixhomeRef(version.Version) - ux.Debugf("build config: cell=%s stack=%s cpus=%d mem=%dGB sshPort=%d", - cfg.CellName, cfg.Stack, cfg.CPUs, cfg.MemoryGB, cfg.SSHPort) - ux.Debugf("template: %s buildVM: %s force=%v noCache=%v", templateName, buildVM, force, noCache) + ux.Debugf("build config: cell=%s stack=%s stage=%s cpus=%d mem=%dGB sshPort=%d", + cfg.CellName, cfg.Stack, stage, cfg.CPUs, cfg.MemoryGB, cfg.SSHPort) + ux.Debugf("template: %s cloneSource: %s buildVM: %s force=%v noCache=%v", templateName, cloneSource, buildVM, force, noCache) ux.Debugf("nixhome: %s projectDir: %s", nixhomeRef, projectDir) if dryRun { fmt.Printf("Would build macOS VM template: %s\n", templateName) - fmt.Printf(" OCI image: %s\n", tartOCIImage) + fmt.Printf(" Stage: %s\n", stage) + fmt.Printf(" Clone source: %s\n", cloneSource) fmt.Printf(" Stack: %s\n", cfg.Stack) fmt.Printf(" CPUs: %d Memory: %dGB\n", cfg.CPUs, cfg.MemoryGB) if len(cfg.Modules) > 0 { @@ -122,8 +143,12 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string return err } - // --- Phase 2: Clone OCI image → build VM --- - if err := pr.PhaseDetailed("Cloning VM from OCI image", func() (string, error) { + // --- Phase 2: Clone source → build VM --- + cloneLabel := "Cloning VM from OCI image" + if cloneFromBase { + cloneLabel = "Cloning VM from base template" + } + if err := pr.PhaseDetailed(cloneLabel, func() (string, error) { if _, getErr := tart.TartGet(ctx, templateName); getErr == nil { if !force { return "", fmt.Errorf("template %s already exists — use --force to rebuild", templateName) @@ -140,12 +165,12 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string _ = tart.TartDelete(ctx, buildVM) } - ux.Debugf("cloning %s → %s (noCache=%v)", tartOCIImage, buildVM, noCache) + ux.Debugf("cloning %s → %s (noCache=%v)", cloneSource, buildVM, noCache) args := []string{"clone"} - if noCache { + if noCache && !cloneFromBase { args = append(args, "--no-cache") } - args = append(args, tartOCIImage, buildVM) + args = append(args, cloneSource, buildVM) cmd := exec.CommandContext(ctx, "tart", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -185,11 +210,22 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string getOut, _ := exec.CommandContext(ctx, "tart", "get", buildVM).CombinedOutput() ux.Debugf("tart get %s (pre-boot):\n%s", buildVM, string(getOut)) } + // --- Detect host nix store for caching --- + hostNixPath := tart.DetectHostNixStore() + if hostNixPath != "" { + ux.Debugf("host nix store detected at %s (valid: store/ + db.sqlite present) — will share as read-only substituter", hostNixPath) + } else { + ux.Debugf("no host nix store found at /nix — VM will download from cache.nixos.org") + } + // --- Phase 4: Boot VM --- sharedDirs := map[string]string{ "nixhome": nixhomeRef, "home": cellHome, } + if hostNixPath != "" { + sharedDirs["hostnix"] = hostNixPath + ":ro" + } disks := []string{nixVolumePath} ux.Debugf("booting VM %s with shared dirs: %v, disks: %v", buildVM, sharedDirs, disks) @@ -226,20 +262,24 @@ func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string return err } - // --- Phase 6: Bootstrap passwordless sudo --- - if err := pr.PhaseDetailed("Bootstrapping passwordless sudo", func() (string, error) { - bootstrapCmd := fmt.Sprintf( - "echo '%s' | sudo -S sh -c \"mkdir -p /etc/sudoers.d && echo '%s ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/%s && chmod 440 /etc/sudoers.d/%s\"", - tartImagePassword, tartImageUser, tartImageUser, tartImageUser, - ) - ux.Debugf("bootstrap: configuring passwordless sudo for %s", tartImageUser) - if err := tart.TartExec(ctx, buildVM, []string{"bash", "-l", "-c", bootstrapCmd}, os.Stdout, os.Stderr); err != nil { - return "", fmt.Errorf("bootstrap sudo: %w", err) + // --- Phase 6: Bootstrap passwordless sudo (skip when cloning from base) --- + if !cloneFromBase { + if err := pr.PhaseDetailed("Bootstrapping passwordless sudo", func() (string, error) { + bootstrapCmd := fmt.Sprintf( + "echo '%s' | sudo -S sh -c \"mkdir -p /etc/sudoers.d && echo '%s ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/%s && chmod 440 /etc/sudoers.d/%s\"", + tartImagePassword, tartImageUser, tartImageUser, tartImageUser, + ) + ux.Debugf("bootstrap: configuring passwordless sudo for %s", tartImageUser) + if err := tart.TartExec(ctx, buildVM, []string{"bash", "-l", "-c", bootstrapCmd}, os.Stdout, os.Stderr); err != nil { + return "", fmt.Errorf("bootstrap sudo: %w", err) + } + return tartImageUser, nil + }); err != nil { + stopVM() + return err } - return tartImageUser, nil - }); err != nil { - stopVM() - return err + } else { + ux.Debugf("skipping sudo bootstrap: base template already has passwordless sudo") } // --- Diagnostic: host-side post-boot checks --- @@ -283,17 +323,27 @@ echo "=== END GUEST DIAGNOSTICS ==="` } } - // --- Phase 7: Full provisioning via tart exec --- + // --- Phase 7: Provisioning via tart exec --- initCfg := tart.InitConfig{ - CellName: cellName, - HomeDir: hostHome, - Stack: stack, - Username: tartImageUser, - Password: tartImagePassword, + CellName: cellName, + HomeDir: hostHome, + Stack: stack, + Username: tartImageUser, + Password: tartImagePassword, + HasHostNix: hostNixPath != "", } initCfg.ApplyDefaults() - steps := tart.ProvisionSteps(initCfg, pubKey, false) - ux.Debugf("provisioning: %d steps via tart exec", len(steps)) + + var steps []tart.ProvisionStep + switch { + case stage == "base": + steps = tart.BaseProvisionSteps(initCfg, pubKey) + case cloneFromBase: + steps = tart.StackProvisionSteps(initCfg) + default: + steps = tart.ProvisionSteps(initCfg, pubKey, false) + } + ux.Debugf("provisioning: %d steps via tart exec (stage=%s, cloneFromBase=%v)", len(steps), stage, cloneFromBase) const reformatMarker = "DEVCELL_REFORMAT_NEEDED:" for i, step := range steps { diff --git a/cmd/build_tart_stub.go b/cmd/build_tart_stub.go index 9014b15e..ac2df78b 100644 --- a/cmd/build_tart_stub.go +++ b/cmd/build_tart_stub.go @@ -4,6 +4,6 @@ package main import "fmt" -func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage string) error { +func runBuildTart(cellName, hostHome, projectDir, stack string, modules []string, force, noCache, dryRun bool, tartOCIImage, stage string) error { return fmt.Errorf("cell build --engine=tart requires macOS on Apple Silicon (darwin/arm64)") } diff --git a/cmd/default_command_test.go b/cmd/default_command_test.go index 2e3e52b9..10e2e199 100644 --- a/cmd/default_command_test.go +++ b/cmd/default_command_test.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "reflect" + "strings" "testing" ) @@ -57,6 +59,36 @@ func TestRewriteDefaultCommand_ExplicitSubcommandWins(t *testing.T) { } } +func TestRewriteDefaultCommand_UnknownPositionalNotRewritten(t *testing.T) { + // `cell abc` must NOT become `cell claude abc` — an unknown positional + // falls through to rootCmd.RunE, which reports "unknown command". + for _, args := range [][]string{ + {"abc"}, + {"abc", "-c"}, + } { + got := rewriteDefaultCommand(args, "claude", testKnownCmds()) + if !reflect.DeepEqual(got, args) { + t.Errorf("unknown positional %v must be left alone: got %v", args, got) + } + } +} + +func TestRootRunE_UnknownCommandShowsHelp(t *testing.T) { + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + defer rootCmd.SetOut(nil) + defer rootCmd.SetErr(nil) + + err := rootCmd.RunE(rootCmd, []string{"abc"}) + if err == nil || !strings.Contains(err.Error(), `unknown command "abc"`) { + t.Fatalf("want unknown-command error, got %v", err) + } + if !strings.Contains(out.String(), "Available Commands:") { + t.Errorf("help must be printed with the error, got output:\n%s", out.String()) + } +} + func TestRewriteDefaultCommand_HelpAndVersionUntouched(t *testing.T) { for _, args := range [][]string{ {"--help"}, {"-h"}, {"--version"}, {"help"}, diff --git a/cmd/root.go b/cmd/root.go index 0fe26b47..81f56b51 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -58,7 +58,8 @@ tools inside a consistent Docker dev environment.`, }, RunE: func(cmd *cobra.Command, args []string) error { if len(args) > 0 { - return fmt.Errorf("unknown command %q — run 'cell --help' for usage", args[0]) + _ = cmd.Help() + return fmt.Errorf("unknown command %q", args[0]) } // A valid default_command never reaches here — applyDefaultCommand // rewrites os.Args before Execute, so cobra dispatches to the @@ -96,6 +97,13 @@ func rewriteDefaultCommand(args []string, defaultCmd string, knownCmds map[strin case "--help", "-h", "--version", "help", "completion", "__complete", "__completeNoDesc": return args } + // An unknown positional (not a flag) is a typo'd subcommand, not + // input for the default command — leave it for rootCmd.RunE to + // report as "unknown command" instead of silently launching the + // default agent with it. + if !strings.HasPrefix(first, "-") { + return args + } } return append([]string{defaultCmd}, args...) } @@ -233,7 +241,9 @@ var cellBoolFlags = map[string]bool{ "--thin": true, // thin image mode (default) "--no-thin": true, // legacy, ignored "--thick": true, // legacy, ignored - "--no-1password": true, // skip [op] documents resolution at cell-open (CELL-42) + "--no-secrets": true, // skip all secrets injection: op item get + op run -- prefix + "--skip-secrets": true, // alias for --no-secrets + "--no-1password": true, // alias for --no-secrets (legacy, CELL-42) "--local": true, // pin --engine=qemu to the in-container path (CELL-378) "--auto-cleanup": true, // run the CELL-334 root reaper at cell start (CELL-390) "--use-flake": true, // opt-in to project-level flake.nix install (CELL-447) @@ -692,13 +702,13 @@ func runAgent(binary string, defaultFlags, userArgs []string, extraEnv map[strin // Loading secrets — CELL-261 phase, now expressed through PhaseRunner. // Suppressed entirely when no [op].documents are configured, or when the - // user opted out via --no-1password / DEVCELL_NO_1PASSWORD. + // user opted out via --no-secrets / --no-1password / DEVCELL_NO_SECRETS / DEVCELL_NO_1PASSWORD. var inheritEnv []string opDocs := cellCfg.Op.ResolvedDocuments() - skipOp := scanFlag("--no-1password") - noOpEnv := os.Getenv("DEVCELL_NO_1PASSWORD") + skipSecrets := scanFlag("--no-secrets") || scanFlag("--skip-secrets") || scanFlag("--no-1password") + noSecretsEnv := firstNonEmpty(os.Getenv("DEVCELL_NO_SECRETS"), os.Getenv("DEVCELL_NO_1PASSWORD")) switch { - case op.ShouldResolve(skipOp, noOpEnv, opDocs): + case op.ShouldResolve(skipSecrets, noSecretsEnv, opDocs): ux.Debugf("1Password: resolving %d document(s): %v", len(opDocs), opDocs) _ = pr.PhaseDetailedRunning("Loading secrets (please authorize 1Password)", "Loaded secrets", func() (string, error) { if _, err := exec.LookPath("op"); err != nil { @@ -728,8 +738,8 @@ func runAgent(binary string, defaultFlags, userArgs []string, extraEnv map[strin } return ux.FormatSecretsPhase(len(resolved), len(errs)), nil }) - case len(opDocs) > 0 && (skipOp || noOpEnv != ""): - ux.Debugf("1Password: skipped (--no-1password / DEVCELL_NO_1PASSWORD)") + case len(opDocs) > 0 && (skipSecrets || noSecretsEnv != ""): + ux.Debugf("1Password: skipped (--no-secrets / DEVCELL_NO_SECRETS)") } // Resolve deferred API keys that depend on 1Password secrets. @@ -828,6 +838,7 @@ func runAgent(binary string, defaultFlags, userArgs []string, extraEnv map[strin BootDir: bootDirEnv, TTY: isatty.IsTerminal(os.Stdin.Fd()), Detach: startDetach, + NoSecrets: skipSecrets, } argv := runner.BuildArgv(spec, runner.OsFS, exec.LookPath) @@ -929,6 +940,15 @@ func scanStringFlag(flag string) string { return "" } +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + // resolveTrustFlake checks if the project has a flake.nix and whether the // user has trusted it. On first encounter, prompts interactively and caches // the answer in cellHome. Returns true if DEVCELL_FLAKE_TRUST=1 should be diff --git a/cmd/tart_integration_darwin_test.go b/cmd/tart_integration_darwin_test.go new file mode 100644 index 00000000..6213b444 --- /dev/null +++ b/cmd/tart_integration_darwin_test.go @@ -0,0 +1,303 @@ +//go:build darwin && arm64 + +package main + +// Integration diagnostics for the `cell --os macos` (tart) path. +// +// Run on a Mac (Apple Silicon): +// +// go test -v -run TestTartDarwinIntegration ./cmd/ +// task test:tart +// +// Every subtest writes its raw command output into +// test/results/-TestTartDarwinIntegration// so a failure +// can be diagnosed from the artifacts alone — attach that directory when +// reporting a bug. +// +// The expensive VM stages (tart clone / boot / provision) are NOT run here; +// this covers the host-side stages that gate the auto-build, ending with the +// platform preflight that `cell build --engine=tart` runs first. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/DimmKirr/devcell/internal/runner" + "github.com/DimmKirr/devcell/internal/testutil" + "github.com/DimmKirr/devcell/internal/version" + "github.com/DimmKirr/devcell/internal/vm/tart" +) + +func TestTartDarwinIntegration(t *testing.T) { + flakeRef := runner.ResolveNixhomeRef(version.Version) + + t.Run("environment", func(t *testing.T) { + resultsDir := testutil.TestResultsDir(t, nil) + + env := map[string]string{ + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + "cell_version": version.Version, + "nixhome_flake_ref": flakeRef, + "DEVCELL_NIXHOME": os.Getenv("DEVCELL_NIXHOME"), + "DEVCELL_NIXHOME_PATH": os.Getenv("DEVCELL_NIXHOME_PATH"), + "DEVCELL_CELL_NAME": os.Getenv("DEVCELL_CELL_NAME"), + "nix": captureVersion(t, resultsDir, "nix", "--version"), + "tart": captureVersion(t, resultsDir, "tart", "--version"), + } + writeJSON(t, filepath.Join(resultsDir, "environment.json"), env) + for k, v := range env { + t.Logf("%s = %s", k, v) + } + }) + + t.Run("flake-metadata", func(t *testing.T) { + resultsDir := testutil.TestResultsDir(t, nil) + requireBinary(t, "nix") + + out, err := runLogged(t, resultsDir, "flake-metadata.log", + "nix", "flake", "metadata", flakeRef, "--json") + if err != nil { + t.Fatalf("nix flake metadata %s: %v\n%s", flakeRef, err, out) + } + }) + + // Reproduces the first phase of the tart auto-build + // (build_tart_darwin.go "Platform compatibility check"), which fails + // when a nixhome module pulls in a Linux-only package (e.g. CELL: the + // vm module's virtiofsd broke `cell claude --os macos`). + t.Run("platform-preflight", func(t *testing.T) { + resultsDir := testutil.TestResultsDir(t, nil) + requireBinary(t, "nix") + + // Raw eval first — unlike the CLI (which trims to 5 error lines), + // keep the complete trace for diagnosis. + attr := fmt.Sprintf("%s#platformStrictCheck.aarch64-darwin", flakeRef) + rawOut, rawErr := runLogged(t, resultsDir, "nix-eval-full.log", + "nix", "eval", attr, "--json", "--show-trace") + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) + defer cancel() + preflightErr := runner.PreflightPlatformCheck(ctx, flakeRef, "aarch64-darwin") + + writeJSON(t, filepath.Join(resultsDir, "run.json"), map[string]any{ + "flake_ref": flakeRef, + "attr": attr, + "raw_eval_error": errString(rawErr), + "preflight_error": errString(preflightErr), + }) + + if preflightErr != nil { + t.Fatalf("platform preflight failed (full nix output in %s):\n%v\nraw eval tail:\n%s", + resultsDir, preflightErr, tail(rawOut, 30)) + } + }) + + t.Run("host-nix-detection", func(t *testing.T) { + resultsDir := testutil.TestResultsDir(t, nil) + + hostNix := tart.DetectHostNixStore() + isStore := tart.IsNixStore("/nix") + + result := map[string]any{ + "detected": hostNix, + "is_nix_store": isStore, + } + + if hostNix != "" { + // Validate the store has actual content + entries, err := os.ReadDir(filepath.Join(hostNix, "store")) + if err != nil { + t.Fatalf("detected host nix at %s but can't read store: %v", hostNix, err) + } + result["store_path_count"] = len(entries) + t.Logf("host nix store: %d paths", len(entries)) + + if len(entries) < 10 { + t.Errorf("host nix store has only %d paths — expected a populated store", len(entries)) + } + + // Read content of a few store paths to verify they're accessible + readable := 0 + var samplePaths []string + for i, e := range entries { + if i >= 5 { + break + } + p := filepath.Join(hostNix, "store", e.Name()) + info, err := os.Stat(p) + if err != nil { + t.Errorf("can't stat store path %s: %v", e.Name(), err) + continue + } + if info.IsDir() { + // verify we can list contents inside a store path + inner, err := os.ReadDir(p) + if err != nil { + t.Errorf("can't read store path dir %s: %v", e.Name(), err) + continue + } + samplePaths = append(samplePaths, fmt.Sprintf("%s (%d entries)", e.Name(), len(inner))) + } else { + samplePaths = append(samplePaths, fmt.Sprintf("%s (%d bytes)", e.Name(), info.Size())) + } + readable++ + } + result["sample_paths"] = samplePaths + result["readable_count"] = readable + t.Logf("verified %d store paths are readable", readable) + if readable == 0 { + t.Error("could not read any store path content") + } + + // Find and read a known binary (nix itself should be in the store) + nixBin, _ := exec.LookPath("nix") + if nixBin != "" { + resolved, err := filepath.EvalSymlinks(nixBin) + if err == nil && strings.HasPrefix(resolved, "/nix/store/") { + f, err := os.Open(resolved) + if err != nil { + t.Errorf("can't open nix binary at %s: %v", resolved, err) + } else { + header := make([]byte, 4) + n, _ := f.Read(header) + f.Close() + result["nix_binary"] = resolved + result["nix_binary_header_bytes"] = n + t.Logf("nix binary at %s: read %d header bytes", resolved, n) + if n < 4 { + t.Errorf("could only read %d bytes from nix binary", n) + } + } + } + } + + // Verify DB exists, is non-empty, and has the SQLite magic header + dbPath := filepath.Join(hostNix, "var", "nix", "db", "db.sqlite") + dbInfo, err := os.Stat(dbPath) + if err != nil { + t.Fatalf("detected host nix but db.sqlite missing: %v", err) + } + result["db_size_bytes"] = dbInfo.Size() + t.Logf("nix database: %d bytes", dbInfo.Size()) + + if dbInfo.Size() < 1024 { + t.Errorf("nix database unexpectedly small: %d bytes", dbInfo.Size()) + } + + // Read SQLite header magic to confirm it's a real database + dbFile, err := os.Open(dbPath) + if err != nil { + t.Errorf("can't open db.sqlite: %v", err) + } else { + magic := make([]byte, 16) + n, _ := dbFile.Read(magic) + dbFile.Close() + if n >= 15 && string(magic[:15]) == "SQLite format 3" { + result["db_magic_ok"] = true + t.Log("nix database: valid SQLite header") + } else { + t.Errorf("nix database has unexpected header: %q", string(magic[:n])) + } + } + + // Verify the substituter script references the right paths + script := tart.GenerateHostNixSubstituterScript() + for _, want := range []string{"hostnix", "host-nix-root", "db.sqlite", "local?root="} { + if !strings.Contains(script, want) { + t.Errorf("substituter script missing %q", want) + } + } + result["substituter_script_ok"] = true + + // Verify diskutil shows the Nix Store volume (informational) + diskOut, _ := runLogged(t, resultsDir, "diskutil-list.log", "diskutil", "list") + result["has_nix_store_volume"] = strings.Contains(diskOut, "Nix Store") + } else { + t.Log("no host nix store detected — substituter caching will not be available") + result["note"] = "host nix not installed; build will use 100GB sparse disk" + } + + // Verify disk size constant + if tart.NixVolumeSizeGB != 100 { + t.Errorf("NixVolumeSizeGB = %d, want 100", tart.NixVolumeSizeGB) + } + result["nix_volume_size_gb"] = tart.NixVolumeSizeGB + + writeJSON(t, filepath.Join(resultsDir, "host-nix.json"), result) + }) +} + +// captureVersion records `bin args...` output into -version.log and +// returns its first line ("" / "" instead of failing the +// test — the environment subtest is pure capture). +func captureVersion(t *testing.T, resultsDir, bin string, args ...string) string { + t.Helper() + if _, err := exec.LookPath(bin); err != nil { + return "" + } + out, err := runLogged(t, resultsDir, bin+"-version.log", bin, args...) + if err != nil { + return "" + } + first, _, _ := strings.Cut(strings.TrimSpace(out), "\n") + return first +} + +// runLogged runs a command, tees combined output into resultsDir/logName, +// and returns it. The command line itself is the log's first line. +func runLogged(t *testing.T, resultsDir, logName, bin string, args ...string) (string, error) { + t.Helper() + cmd := exec.Command(bin, args...) + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + content := fmt.Sprintf("$ %s %s\n%s", bin, strings.Join(args, " "), buf.String()) + if werr := os.WriteFile(filepath.Join(resultsDir, logName), []byte(content), 0o644); werr != nil { + t.Fatalf("write %s: %v", logName, werr) + } + return buf.String(), err +} + +func requireBinary(t *testing.T, bin string) { + t.Helper() + if _, err := exec.LookPath(bin); err != nil { + t.Skipf("%s not in PATH", bin) + } +} + +func writeJSON(t *testing.T, path string, v any) { + t.Helper() + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatalf("marshal %s: %v", path, err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func errString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func tail(s string, n int) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} diff --git a/cmd/tart_runner.go b/cmd/tart_runner.go index 831fbbdc..0d1284a9 100644 --- a/cmd/tart_runner.go +++ b/cmd/tart_runner.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "runtime" "strings" "time" @@ -118,7 +117,7 @@ func runTartAgent( SSHTimeout: 120 * time.Second, InitFunc: func() error { logf("auto-build: VM not found — running build with stack=%q", stack) - return runBuildTart(cellName, hostHome, baseDir, stack, nil, false, false, false, cellCfg.Cell.ResolvedTartOCIImage()) + return runBuildTart(cellName, hostHome, baseDir, stack, nil, false, false, false, cellCfg.Cell.ResolvedTartOCIImage(), "full") }, } acquireIn.ApplyDefaults() @@ -192,10 +191,157 @@ func runTartAgent( } } - // Mount project directory inside the VM - projectBasename := filepath.Base(baseDir) - mountScript := tart.GenerateProjectMountScript("project", sessionUser, projectBasename) - logf("mounting project dir: tag=project user=%s basename=%s", sessionUser, projectBasename) + // --- Repair /nix shadow mount (pre-fix templates) --- + // The installer's darwin-store daemon can mount its tiny APFS volume + // over the JHFS+ nix disk at clone boot, hiding the nix-darwin system + // profile and s6. Detect and repair before anything touches /nix. + { + repairScript := tart.GenerateNixShadowRepairScript() + var repOut, repErr strings.Builder + repCmd := exec.CommandContext(context.Background(), "tart", "exec", instanceName, "bash", "-l", "-c", repairScript) + repCmd.Stdout = &repOut + repCmd.Stderr = &repErr + if err := repCmd.Run(); err != nil { + logf("nix shadow repair failed: %v\nstdout: %s\nstderr: %s", err, strings.TrimSpace(repOut.String()), strings.TrimSpace(repErr.String())) + } else { + logf("nix shadow: %s", strings.TrimSpace(repOut.String())) + } + } + + // --- Activate s6 session services --- + // Runs s6 oneshot services (shell-rc, claude-config, etc.) that set up + // the session user's environment. + { + s6Script := tart.GenerateS6SessionActivateScript(sessionUser) + logf("activating s6 session services for %s (envDir=%s svcDir=%s)", sessionUser, tart.S6EnvDir, tart.S6ServicesDir) + logf("s6 script:\n%s", s6Script) + var s6Out, s6Err strings.Builder + s6Cmd := exec.CommandContext(context.Background(), "tart", "exec", instanceName, "bash", "-l", "-c", s6Script) + s6Cmd.Stdout = &s6Out + s6Cmd.Stderr = &s6Err + if err := s6Cmd.Run(); err != nil { + logf("s6 session activation failed: %v\nstdout: %s\nstderr: %s", err, strings.TrimSpace(s6Out.String()), strings.TrimSpace(s6Err.String())) + } else { + logf("s6 session: %s", strings.TrimSpace(s6Out.String())) + } + } + + // --- Re-activate nix-darwin if /run/current-system is missing --- + // nix-darwin's boot activation (org.nixos.activate-system) may not + // have run yet, or may fail if the nix disk wasn't mounted in time. + // The system profile at /nix/var/nix/profiles/system/activate is the + // canonical way to re-trigger activation. + { + checkScript := `echo "nix_mounted=$(mount | grep /nix | head -1 || echo NONE)" +echo "system_profile=$(ls -la /nix/var/nix/profiles/system 2>/dev/null || echo MISSING)" +echo "activate_bin=$(test -x /nix/var/nix/profiles/system/activate && echo EXISTS || echo MISSING)" +test -e /run/current-system && echo "RESULT=OK" || echo "RESULT=MISSING"` + var checkOut strings.Builder + checkCmd := exec.CommandContext(context.Background(), "tart", "exec", instanceName, "bash", "-l", "-c", checkScript) + checkCmd.Stdout = &checkOut + checkErr := checkCmd.Run() + if checkErr == nil { + logf("nix-darwin preflight: %s", strings.TrimSpace(checkOut.String())) + } + if checkErr == nil && strings.Contains(checkOut.String(), "RESULT=MISSING") { + logf("nix-darwin: /run/current-system missing — re-activating system profile") + activateScript := `set -e +if [ -x /nix/var/nix/profiles/system/activate ]; then + sudo /nix/var/nix/profiles/system/activate 2>&1 + echo "ACTIVATED" + ls -la /run/current-system 2>&1 || echo "still missing after activate" + ls /etc/profiles/per-user/devcell/bin/ 2>/dev/null | head -5 || echo "per-user bin still missing" +else + echo "NO_PROFILE" + ls -la /nix/var/nix/profiles/ 2>&1 +fi` + var actOut, actErr strings.Builder + actCmd := exec.CommandContext(context.Background(), "tart", "exec", instanceName, "bash", "-l", "-c", activateScript) + actCmd.Stdout = &actOut + actCmd.Stderr = &actErr + if err := actCmd.Run(); err != nil { + logf("nix-darwin activation failed: %v\nstdout: %s\nstderr: %s", err, strings.TrimSpace(actOut.String()), strings.TrimSpace(actErr.String())) + } else { + logf("nix-darwin activation: %s", strings.TrimSpace(actOut.String())) + } + } else if checkErr == nil { + logf("nix-darwin: /run/current-system present — system already activated") + } + } + + // --- Diagnostic: verify nix-darwin and home-manager paths --- + { + diagScript := fmt.Sprintf(`echo "=== VM PATH DIAGNOSTICS ===" + +echo "--- nix-daemon profile ---" +ls /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>&1 + +echo "--- /run/current-system ---" +ls -la /run/current-system 2>&1 || echo "MISSING" + +echo "--- nix-darwin system profile ---" +ls -la /nix/var/nix/profiles/system 2>&1 || echo "MISSING" + +echo "--- /etc/profiles/per-user/%[1]s/bin (first 20) ---" +ls /etc/profiles/per-user/%[1]s/bin/ 2>/dev/null | head -20 +test -d /etc/profiles/per-user/%[1]s/bin || echo "DIR NOT FOUND" + +echo "--- which claude ---" +. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null +export PATH="/etc/profiles/per-user/%[1]s/bin:/run/current-system/sw/bin:$PATH" +which claude 2>&1 || echo "NOT FOUND" + +echo "--- which node ---" +which node 2>&1 || echo "NOT FOUND" + +echo "--- nix-darwin generation ---" +ls -la /run/current-system 2>/dev/null || echo "NOT FOUND" + +echo "--- home-manager generations for %[1]s ---" +ls -la /nix/var/nix/profiles/per-user/%[1]s/ 2>/dev/null || echo "NO PROFILES" + +echo "--- session user home ---" +ls -la /Users/%[2]s/ 2>/dev/null | head -30 + +echo "--- .zshenv content ---" +cat /Users/%[2]s/.zshenv 2>/dev/null || echo "NO .zshenv" + +echo "--- .zshenv target check ---" +if [ -L /Users/%[2]s/.zshenv ]; then + echo "symlink -> $(readlink /Users/%[2]s/.zshenv)" + echo "target exists: $(test -e /Users/%[2]s/.zshenv && echo YES || echo NO)" +fi + +echo "--- nix /nix mount ---" +mount | grep /nix || echo "NOT MOUNTED" + +echo "--- nix-daemon status ---" +sudo launchctl print system/org.nixos.nix-daemon 2>&1 | head -3 || echo "NOT LOADED" + +echo "--- exec PATH (as session user) ---" +echo "$PATH" + +echo "=== END DIAGNOSTICS ==="`, + tart.DarwinVMUser, + sessionUser) + var diagOut, diagErr strings.Builder + diagCmd := exec.CommandContext(context.Background(), "tart", "exec", instanceName, "bash", "-l", "-c", diagScript) + diagCmd.Stdout = &diagOut + diagCmd.Stderr = &diagErr + if err := diagCmd.Run(); err != nil { + logf("VM diagnostics failed: %v\nstderr: %s", err, strings.TrimSpace(diagErr.String())) + } else { + logf("VM diagnostics:\n%s", diagOut.String()) + } + } + + // Mount project directory inside the VM at the mirrored host path + // (e.g. /Users/dmitry/dev/acme/proj on the host appears at the same + // path in the VM), falling back to ~/ for projects outside + // the host home. + projectPathVM := tart.ProjectPathInVM(hostHome, baseDir, sessionUser) + mountScript := tart.GenerateProjectMountScript("project", sessionUser, projectPathVM) + logf("mounting project dir: tag=project user=%s target=%s", sessionUser, projectPathVM) var mountStderr strings.Builder mountCmd := exec.CommandContext(context.Background(), "tart", "exec", instanceName, "bash", "-l", "-c", mountScript) mountCmd.Stderr = &mountStderr @@ -203,7 +349,7 @@ func runTartAgent( logf("project mount failed: %v (stderr: %s)", err, strings.TrimSpace(mountStderr.String())) return fmt.Errorf("mounting project directory in VM: %w (stderr: %s)", err, strings.TrimSpace(mountStderr.String())) } - logf("project directory mounted at /Users/%s/%s", sessionUser, projectBasename) + logf("project directory mounted at %s", projectPathVM) } // --- lifecycle: simulated VM start (mock only) --- @@ -226,6 +372,7 @@ func runTartAgent( UserArgs: userArgs, EnvVars: envVars, ProjectDir: baseDir, + WorkDir: tart.ProjectPathInVM(hostHome, baseDir, sessionUser), RunAsUser: runAsUser, }) logf("exec command: %s", execCmd) diff --git a/flake.nix b/flake.nix index 6e83e080..84ad69ec 100644 --- a/flake.nix +++ b/flake.nix @@ -55,7 +55,7 @@ version = nixpkgs.lib.removePrefix "v" cellVersion; src = cellSrc; - vendorHash = "sha256-G3kp9kXXi9wcO+cz+R4hFo1XDKPABxAZ+pesRe38gvE="; + vendorHash = "sha256-lwHQ3fQD6c73+a/1iZNW2m//EsHahWhQx3WJTRdZmlc="; subPackages = ["cmd"]; diff --git a/go.mod b/go.mod index 8fc66a1e..736fe13b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/BurntSushi/toml v1.4.0 - github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 + github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/huh v1.0.0 github.com/charmbracelet/lipgloss v1.1.0 @@ -18,7 +18,7 @@ require ( github.com/google/go-containerregistry v0.22.0 github.com/google/uuid v1.6.0 github.com/hydrz/wireguard v0.0.1 - github.com/mattn/go-isatty v0.0.20 + github.com/mattn/go-isatty v0.0.24 github.com/muesli/termenv v0.16.0 github.com/ollama/ollama v0.17.6 github.com/openai/openai-go v1.12.0 @@ -83,7 +83,7 @@ require ( github.com/devcell-sh/go-nixoci v0.1.0 github.com/devcell-sh/go-regedit v0.1.0 github.com/devcell-sh/go-wimlib v0.1.0 - github.com/devcell-sh/go-winkit v0.2.0 + github.com/devcell-sh/go-winkit v0.2.1-0.20260907050558-a2d304a6c6ef github.com/distribution/reference v0.6.0 // indirect github.com/djherbis/times v1.6.0 // indirect github.com/docker/cli v29.7.2+incompatible // indirect @@ -91,7 +91,7 @@ require ( github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.8.4 // indirect + github.com/ebitengine/purego v0.11.0-alpha.6 // indirect github.com/elliotwutingfeng/asciiset v0.0.0-20260129054604-cfde2086bc57 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect diff --git a/go.sum b/go.sum index 3b0bf383..a889e677 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 h1:JFgG/xnwFfbezlUnFMJy0nusZvytYysV4SCS2cYbvws= -github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7/go.mod h1:ISC1gtLcVilLOf23wvTfoQuYbW2q0JevFxPfUzZ9Ybw= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= github.com/charmbracelet/colorprofile v0.4.2 h1:BdSNuMjRbotnxHSfxy+PCSa4xAmz7szw70ktAtWRYrY= @@ -108,8 +108,8 @@ github.com/devcell-sh/go-regedit v0.1.0 h1:+eT+eLZQZtDpKhzjlGBP2DdyNuUhFTX+8354j github.com/devcell-sh/go-regedit v0.1.0/go.mod h1:pWEerGIoAAVu00kuyFaXCmIzIWghcJYwiK8xGha1L5E= github.com/devcell-sh/go-wimlib v0.1.0 h1:pg1WxyqfMm/9Anmp9amfr+nMKwzAc4D5ra4n8qkZkMg= github.com/devcell-sh/go-wimlib v0.1.0/go.mod h1:BFGCDzvSqoVtHxQ8j5GhXHmO8c3w/kLnHDJaOogPstE= -github.com/devcell-sh/go-winkit v0.2.0 h1:qx4udUGbd33q3mE1Cg2nNzN5bpGMymD2VI49sPCC4PU= -github.com/devcell-sh/go-winkit v0.2.0/go.mod h1:knVXG2/GhrkkHZIvie9+NeZLH93F0NbISzeXtMROTUk= +github.com/devcell-sh/go-winkit v0.2.1-0.20260907050558-a2d304a6c6ef h1:xEM/DUsE4tVi5T9YGkWBtB/kYZETjeB9iSKOiSKhr2s= +github.com/devcell-sh/go-winkit v0.2.1-0.20260907050558-a2d304a6c6ef/go.mod h1:BvIWCv/xVDAYJCVaspbZ9ceumdHa8kF9oBimT/guSQo= github.com/digitalocean/go-libvirt v0.0.0-20260609165003-6254771e63a8 h1:R4zqGCPowg1bfJXFxsS122mzkivaMz+wPLxBsF9qsiY= github.com/digitalocean/go-libvirt v0.0.0-20260609165003-6254771e63a8/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4= github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc= @@ -130,8 +130,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= -github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.11.0-alpha.6 h1:xZ7KkRHWH0O/DskwUkFfd6Y4X9vtth85b4iEmNDeIME= +github.com/ebitengine/purego v0.11.0-alpha.6/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/elliotwutingfeng/asciiset v0.0.0-20260129054604-cfde2086bc57 h1:x5yxNrq8XffV/OoNUeFPM6hxHVi5OTspSTBxr/9pemg= github.com/elliotwutingfeng/asciiset v0.0.0-20260129054604-cfde2086bc57/go.mod h1:GLo/8fDswSAniFG+BFIaiSPcK610jyzgEhWYPQwuQdw= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= @@ -226,8 +226,8 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= @@ -411,7 +411,6 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= diff --git a/internal/op/gate.go b/internal/op/gate.go index 03901edc..6743706b 100644 --- a/internal/op/gate.go +++ b/internal/op/gate.go @@ -3,10 +3,11 @@ package op import "strings" // ShouldResolve reports whether the caller should invoke `op item get` for the -// configured 1Password documents. Pure for testability — the caller threads in -// the boolean flag (`--no-1password`), the env var value (`DEVCELL_NO_1PASSWORD`), -// and the resolved document list. Skip when there are no documents, when the -// user explicitly opted out via flag, or via a truthy env value (CELL-42). +// configured 1Password documents. Pure for testability: the caller threads in +// the boolean flag (--no-secrets / --no-1password), the env var value +// (DEVCELL_NO_SECRETS / DEVCELL_NO_1PASSWORD), and the resolved document list. +// Skip when there are no documents, when the user explicitly opted out via +// flag, or via a truthy env value. func ShouldResolve(disableFlag bool, envValue string, docs []string) bool { if len(docs) == 0 { return false diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 233eecce..711ae14f 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -227,6 +227,7 @@ type RunSpec struct { BootDir string // CELL-264: host-side boot dir for fsnotify sentinels; empty disables the bind-mount TTY bool // allocate a pseudo-TTY (-it); set from isatty check on stdin Detach bool // run container in detached mode (-d); set by `cell start` + NoSecrets bool // skip `op run --` prefix and all secrets injection } func (s RunSpec) getenv(key string) string { @@ -243,9 +244,11 @@ func BuildArgv(spec RunSpec, fs FS, lookPath func(string) (string, error)) []str var argv []string - // 1Password passthrough - if opPath, err := lookPath("op"); err == nil && opPath != "" { - argv = append(argv, "op", "run", "--") + // 1Password passthrough (suppressed by --no-secrets) + if !spec.NoSecrets { + if opPath, err := lookPath("op"); err == nil && opPath != "" { + argv = append(argv, "op", "run", "--") + } } dockerRunFlags := []string{"--rm", "--shm-size=" + spec.CellCfg.Docker.ResolvedShmSize(), "--device=/dev/fuse"} diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 5ad1c38b..d30ae633 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -417,6 +417,20 @@ func TestArgv_NoOpPrefixWhenOpMissing(t *testing.T) { } } +func TestArgv_NoSecretsSuppressesOpPrefix(t *testing.T) { + spec := runner.RunSpec{ + Config: baseConfig(), + CellCfg: cfg.CellConfig{}, + Binary: "claude", + DefaultFlags: []string{"--dangerously-skip-permissions"}, + NoSecrets: true, + } + argv := runner.BuildArgv(spec, noopFS(), opLookPath) + if argv[0] == "op" { + t.Error("op run -- prefix must be suppressed when NoSecrets is true") + } +} + // --- cfg env and volumes --- func TestArgv_CfgEnvVarsInArgv(t *testing.T) { diff --git a/internal/runner/thin_build.go b/internal/runner/thin_build.go index 6d236450..fe88c6b0 100644 --- a/internal/runner/thin_build.go +++ b/internal/runner/thin_build.go @@ -2,6 +2,7 @@ package runner import ( "context" + "errors" "fmt" "os" "os/exec" @@ -639,6 +640,9 @@ echo "Done — thin image: %s"`, args := []string{ "docker", "run", "--rm", "--privileged", "--name", containerName, + "--label", "devcell.role=thin-builder", + "--label", "devcell.project=" + projectName, + "--label", "devcell.stack=" + stack, "--platform", platform, "--user", "0", "-v", volumeName + ":/nix", @@ -700,3 +704,53 @@ func isFlakeRef(s string) bool { } return false } + +// ── Builder container identity + slot reclaim ──────────────────────────────── + +// ThinBuilderContainerName names the thin builder for one app +// (`-`, the same key as `cell--run`). One fixed name for +// every project meant a build starting in project B force-removed — SIGKILLed, +// exit 137 with no OOM anywhere — the build project A had in flight. +func ThinBuilderContainerName(appName string) string { + return "devcell-builder-" + appName +} + +// ErrBuilderRunning is returned when the app's builder slot is occupied by a +// live container. Callers must never remove it; wait or let it finish. +var ErrBuilderRunning = errors.New("builder already running") + +// ReclaimBuilderSlot decides what to do with a pre-existing builder container +// of the same name before starting a new build: nothing when absent, remove +// when it exited (a crashed or interrupted run left it behind), and refuse — +// never kill — when it is running. +func ReclaimBuilderSlot(name string, exists, running bool) (remove bool, err error) { + switch { + case !exists: + return false, nil + case running: + return false, fmt.Errorf("%w: %s — another build for this project is in flight; "+ + "follow it with `docker logs -f %s` or remove it with `docker rm -f %s` if it is stale", + ErrBuilderRunning, name, name, name) + default: + return true, nil + } +} + +// BuilderContainerState reports whether a container with the given name +// exists and whether it is currently running. +func BuilderContainerState(ctx context.Context, name string) (exists, running bool) { + out, err := exec.CommandContext(ctx, "docker", "container", "inspect", + "--format", "{{.State.Running}}", name).Output() + if err != nil { + return false, false + } + return true, strings.TrimSpace(string(out)) == "true" +} + +// BuilderOrphanedByCancel is true when the `docker run` client died because +// the caller's context was cancelled (Ctrl-C / SIGTERM). The client is +// SIGKILLed but the container keeps running — an orphan holding the shared +// /nix volume — so the caller must remove its own builder in that case. +func BuilderOrphanedByCancel(runErr, ctxErr error) bool { + return runErr != nil && ctxErr != nil +} diff --git a/internal/runner/thin_build_test.go b/internal/runner/thin_build_test.go index bda59f7a..498cd6ef 100644 --- a/internal/runner/thin_build_test.go +++ b/internal/runner/thin_build_test.go @@ -1,6 +1,8 @@ package runner import ( + "context" + "errors" "runtime" "strconv" "strings" @@ -9,7 +11,7 @@ import ( const ( testCoreImage = "ghcr.io/test/devcell:v0.0.0-core" - testContainer = "devcell-thin-builder" + testContainer = "devcell-builder-test-1" testVolume = "devcell-nix-store" testNixhome = "/home/bob/nixhome" testThinTag = "devcell-user:base-thin" @@ -1186,3 +1188,85 @@ func TestParseMemoryLimit(t *testing.T) { } } } + +// ── Builder container identity + slot reclaim ──────────────────────────────── +// +// One fixed builder name meant every `cell build` (explicit or auto-triggered +// from `cell` in another project) began with `docker rm -f devcell-thin-builder` +// and SIGKILLed whatever build was in flight — exit 137 with no OOM anywhere. +// The builder is now named per app, and a *running* builder is never removed. + +func TestThinBuildArgv_BuilderLabels(t *testing.T) { + argv := ThinBuildArgvFull(testCoreImage, testContainer, testVolume, testNixhome, testThinTag, "local", "aarch64", "base", "go,node", "myproject") + labels := map[string]bool{} + for i, a := range argv { + if a == "--label" && i+1 < len(argv) { + labels[argv[i+1]] = true + } + } + for _, want := range []string{"devcell.role=thin-builder", "devcell.project=myproject", "devcell.stack=base"} { + if !labels[want] { + t.Errorf("missing label %q in builder argv", want) + } + } +} + +func TestThinBuilderContainerName_PerApp(t *testing.T) { + got := ThinBuilderContainerName("nmd.gg-20") + if got != "devcell-builder-nmd.gg-20" { + t.Fatalf("ThinBuilderContainerName = %q, want devcell-builder-nmd.gg-20", got) + } +} + +func TestReclaimBuilderSlot_AbsentDoesNothing(t *testing.T) { + remove, err := ReclaimBuilderSlot("devcell-builder-x", false, false) + if err != nil || remove { + t.Fatalf("absent builder: remove=%v err=%v, want false/nil", remove, err) + } +} + +func TestReclaimBuilderSlot_ExitedIsRemoved(t *testing.T) { + remove, err := ReclaimBuilderSlot("devcell-builder-x", true, false) + if err != nil || !remove { + t.Fatalf("exited builder: remove=%v err=%v, want true/nil", remove, err) + } +} + +func TestReclaimBuilderSlot_RunningIsNeverKilled(t *testing.T) { + remove, err := ReclaimBuilderSlot("devcell-builder-nmd.gg-20", true, true) + if remove { + t.Fatal("running builder must not be scheduled for removal") + } + if err == nil { + t.Fatal("running builder must surface an error to the caller") + } + if !errors.Is(err, ErrBuilderRunning) { + t.Fatalf("err = %v, want ErrBuilderRunning", err) + } + for _, want := range []string{"devcell-builder-nmd.gg-20", "docker logs -f"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q should mention %q", err.Error(), want) + } + } +} + +// Ctrl-C cancels the exec context, which SIGKILLs the `docker run` client but +// leaves the container running as an orphan that holds the shared /nix volume. +// The caller must remove its own builder in exactly that case. +func TestBuilderOrphanedByCancel(t *testing.T) { + cases := []struct { + name string + runErr error + ctxErr error + want bool + }{ + {"clean exit", nil, nil, false}, + {"build failed on its own", errors.New("exit status 1"), nil, false}, + {"cancelled mid-build", errors.New("signal: killed"), context.Canceled, true}, + } + for _, c := range cases { + if got := BuilderOrphanedByCancel(c.runErr, c.ctxErr); got != c.want { + t.Errorf("%s: BuilderOrphanedByCancel = %v, want %v", c.name, got, c.want) + } + } +} diff --git a/internal/vm/tart/image.go b/internal/vm/tart/image.go index fd36313e..a74ed77b 100644 --- a/internal/vm/tart/image.go +++ b/internal/vm/tart/image.go @@ -27,6 +27,10 @@ func ImageName(stack string, modules []string) string { return fmt.Sprintf("disk-%s.img", StackTag(stack, modules)) } +// BaseTemplateName is the tart VM name for a stack-agnostic base template +// (nix installed, store on external disk, but no nix-darwin activation). +const BaseTemplateName = "devcell-tart-base" + // TemplateVMName returns the tart VM name for a built template image. func TemplateVMName(stack string, modules []string) string { return "devcell-tart-" + StackTag(stack, modules) diff --git a/internal/vm/tart/init.go b/internal/vm/tart/init.go index 190b0015..2b2173f1 100644 --- a/internal/vm/tart/init.go +++ b/internal/vm/tart/init.go @@ -58,7 +58,8 @@ type InitConfig struct { CPUs uint MemoryGB uint64 DiskGB uint64 - SSHPort uint16 + SSHPort uint16 + HasHostNix bool } // ApplyDefaults fills zero-value fields with sensible defaults. diff --git a/internal/vm/tart/nix_volume.go b/internal/vm/tart/nix_volume.go index a09c959a..7e19aaea 100644 --- a/internal/vm/tart/nix_volume.go +++ b/internal/vm/tart/nix_volume.go @@ -4,14 +4,38 @@ import ( "fmt" "os" "path/filepath" + + "github.com/DimmKirr/devcell/internal/ux" ) const ( NixVolumeFileName = "nix.img" - NixVolumeSizeGB = 30 + NixVolumeSizeGB = 100 NixVolumeLabel = "DevcellNix" ) +// IsNixStore checks whether the given path contains a functional Nix store +// (has store/ directory and var/nix/db/db.sqlite database). +func IsNixStore(nixPath string) bool { + storePath := filepath.Join(nixPath, "store") + dbPath := filepath.Join(nixPath, "var", "nix", "db", "db.sqlite") + info, err := os.Stat(storePath) + if err != nil || !info.IsDir() { + return false + } + _, err = os.Stat(dbPath) + return err == nil +} + +// DetectHostNixStore returns "/nix" if the host has a functional Nix store, +// empty string otherwise. +func DetectHostNixStore() string { + if IsNixStore("/nix") { + return "/nix" + } + return "" +} + // NixVolumePath returns the path to the global nix store disk image. // Layout: ~/.devcell/darwin/nix-store.img // Shared across all cells — mirrors Docker's single devcell-nix-store volume. @@ -21,9 +45,20 @@ func NixVolumePath(home string) string { // EnsureNixVolume creates a sparse nix store disk image if it doesn't exist. // Returns the path to the (existing or newly created) image. +// Callers should log the returned path and whether it was created or reused. func EnsureNixVolume(home string) (string, error) { imgPath := NixVolumePath(home) - if _, err := os.Stat(imgPath); err == nil { + wantBytes := int64(NixVolumeSizeGB) * 1024 * 1024 * 1024 + if info, err := os.Stat(imgPath); err == nil { + currentGB := info.Size() / (1024 * 1024 * 1024) + ux.Debugf("nix volume exists: %s (%d GB logical)", imgPath, currentGB) + if info.Size() < wantBytes { + ux.Debugf("nix volume undersized (%d GB < %d GB) — growing sparse image", currentGB, NixVolumeSizeGB) + if err := os.Truncate(imgPath, wantBytes); err != nil { + return "", fmt.Errorf("resizing nix volume from %dGB to %dGB: %w", currentGB, NixVolumeSizeGB, err) + } + ux.Debugf("nix volume resized to %d GB (sparse — no extra host disk used)", NixVolumeSizeGB) + } return imgPath, nil } dir := filepath.Dir(imgPath) @@ -40,6 +75,7 @@ func EnsureNixVolume(home string) (string, error) { os.Remove(imgPath) return "", fmt.Errorf("setting nix volume size: %w", err) } + ux.Debugf("nix volume created: %s (sparse %d GB)", imgPath, NixVolumeSizeGB) return imgPath, nil } diff --git a/internal/vm/tart/nix_volume_test.go b/internal/vm/tart/nix_volume_test.go index d2494847..268df49d 100644 --- a/internal/vm/tart/nix_volume_test.go +++ b/internal/vm/tart/nix_volume_test.go @@ -83,6 +83,84 @@ func TestGenerateNixVolumeMountScript_ContainsKeyElements(t *testing.T) { } } +func TestNixVolumeSizeGB(t *testing.T) { + if NixVolumeSizeGB != 100 { + t.Errorf("NixVolumeSizeGB = %d, want 100", NixVolumeSizeGB) + } +} + +func TestIsNixStore_ValidStore(t *testing.T) { + root := t.TempDir() + nixPath := filepath.Join(root, "nix") + os.MkdirAll(filepath.Join(nixPath, "store"), 0o755) + os.MkdirAll(filepath.Join(nixPath, "var", "nix", "db"), 0o755) + os.WriteFile(filepath.Join(nixPath, "var", "nix", "db", "db.sqlite"), []byte("x"), 0o644) + + if !IsNixStore(nixPath) { + t.Error("expected IsNixStore to return true for a valid store layout") + } +} + +func TestIsNixStore_MissingDB(t *testing.T) { + root := t.TempDir() + nixPath := filepath.Join(root, "nix") + os.MkdirAll(filepath.Join(nixPath, "store"), 0o755) + + if IsNixStore(nixPath) { + t.Error("expected IsNixStore to return false when db.sqlite is missing") + } +} + +func TestIsNixStore_MissingStore(t *testing.T) { + root := t.TempDir() + nixPath := filepath.Join(root, "nix") + os.MkdirAll(nixPath, 0o755) + + if IsNixStore(nixPath) { + t.Error("expected IsNixStore to return false when store dir is missing") + } +} + +func TestEnsureNixVolume_ResizesUndersized(t *testing.T) { + home := t.TempDir() + + // Create a 30GB sparse image (old default). + imgPath := NixVolumePath(home) + os.MkdirAll(filepath.Dir(imgPath), 0o755) + f, err := os.Create(imgPath) + if err != nil { + t.Fatal(err) + } + oldSize := int64(30) * 1024 * 1024 * 1024 + f.Truncate(oldSize) + f.Close() + + info, _ := os.Stat(imgPath) + if info.Size() != oldSize { + t.Fatalf("setup: expected %d, got %d", oldSize, info.Size()) + } + + // EnsureNixVolume should grow it to NixVolumeSizeGB. + path, err := EnsureNixVolume(home) + if err != nil { + t.Fatalf("EnsureNixVolume failed: %v", err) + } + + info, _ = os.Stat(path) + wantSize := int64(NixVolumeSizeGB) * 1024 * 1024 * 1024 + if info.Size() != wantSize { + t.Errorf("after resize: size = %d GB, want %d GB", info.Size()/(1024*1024*1024), NixVolumeSizeGB) + } + + // Verify it's still sparse. + if sys, ok := info.Sys().(*syscall.Stat_t); ok { + allocBytes := sys.Blocks * 512 + if allocBytes > 1024*1024 { + t.Errorf("expected sparse after resize (< 1MB on disk), got %d bytes", allocBytes) + } + } +} + func TestNixVolumeMetadataFormat(t *testing.T) { meta := map[string]any{ "type": "nix-store", diff --git a/internal/vm/tart/observer.go b/internal/vm/tart/observer.go index fc3a658b..e5b5763d 100644 --- a/internal/vm/tart/observer.go +++ b/internal/vm/tart/observer.go @@ -22,6 +22,34 @@ type ProvisionStep struct { NeedsPassword bool // true = use password auth (before SSH key is injected) } +// BaseProvisionSteps returns the stack-agnostic provisioning steps: SSH setup, +// sudo, home mount, nix disk prep, nix install, and store swap. These produce +// a base template that any stack can build on. +func BaseProvisionSteps(cfg InitConfig, pubKey string) []ProvisionStep { + steps := []ProvisionStep{ + {Name: "Enable SSH", Command: GenerateSSHEnablementScript(), NeedsPassword: true}, + {Name: "Inject SSH key", Command: GenerateSSHKeyScript(pubKey), NeedsPassword: true}, + {Name: "Configure passwordless sudo", Command: GenerateSudoersScript(cfg.Username)}, + {Name: "Mount home volume", Command: GenerateHomeMountScript(cfg.CellName, cfg.Username)}, + {Name: "Prepare nix disk", Command: GenerateNixDiskPrepScript(cfg.CellName)}, + {Name: "Install Nix", Command: GenerateNixInstallScript()}, + {Name: "Swap nix to external disk", Command: GenerateNixStoreSwapScript(cfg.CellName)}, + } + if cfg.HasHostNix { + steps = append(steps, ProvisionStep{Name: "Configure host nix substituter", Command: GenerateHostNixSubstituterScript()}) + } + return steps +} + +// StackProvisionSteps returns the stack-specific provisioning steps: mount +// nixhome and activate nix-darwin for the configured stack. +func StackProvisionSteps(cfg InitConfig) []ProvisionStep { + return []ProvisionStep{ + {Name: "Mount nixhome", Command: GenerateVirtioFSMountScript("nixhome", "/Volumes/nixhome")}, + {Name: "Activate nix-darwin (" + cfg.Stack + ")", Command: GenerateNixDarwinActivateScript(cfg.Stack, "/Volumes/nixhome")}, + } +} + // ProvisionSteps returns the SSH provisioning commands with human-readable names. // Steps before key injection use NeedsPassword=true since the generated SSH key // isn't in authorized_keys yet. @@ -31,26 +59,19 @@ type ProvisionStep struct { // by ApplyDiskPatch. Only Nix install and home-manager activation remain. func ProvisionSteps(cfg InitConfig, pubKey string, offlineProvisioned bool) []ProvisionStep { if offlineProvisioned { - return []ProvisionStep{ + steps := []ProvisionStep{ {Name: "Preflight diagnostics", Command: GeneratePreflightDiagScript()}, {Name: "Verify sshd FDA grant", Command: GenerateVerifySSHdFDAScript()}, {Name: "Mount home volume", Command: GenerateHomeMountScript(cfg.CellName, cfg.Username)}, {Name: "Prepare nix disk", Command: GenerateNixDiskPrepScript(cfg.CellName)}, {Name: "Install Nix", Command: GenerateNixInstallScript()}, {Name: "Swap nix to external disk", Command: GenerateNixStoreSwapScript(cfg.CellName)}, - {Name: "Mount nixhome", Command: GenerateVirtioFSMountScript("nixhome", "/Volumes/nixhome")}, - {Name: "Activate nix-darwin (" + cfg.Stack + ")", Command: GenerateNixDarwinActivateScript(cfg.Stack, "/Volumes/nixhome")}, } + if cfg.HasHostNix { + steps = append(steps, ProvisionStep{Name: "Configure host nix substituter", Command: GenerateHostNixSubstituterScript()}) + } + steps = append(steps, StackProvisionSteps(cfg)...) + return steps } - return []ProvisionStep{ - {Name: "Enable SSH", Command: GenerateSSHEnablementScript(), NeedsPassword: true}, - {Name: "Inject SSH key", Command: GenerateSSHKeyScript(pubKey), NeedsPassword: true}, - {Name: "Configure passwordless sudo", Command: GenerateSudoersScript(cfg.Username)}, - {Name: "Mount home volume", Command: GenerateHomeMountScript(cfg.CellName, cfg.Username)}, - {Name: "Prepare nix disk", Command: GenerateNixDiskPrepScript(cfg.CellName)}, - {Name: "Install Nix", Command: GenerateNixInstallScript()}, - {Name: "Swap nix to external disk", Command: GenerateNixStoreSwapScript(cfg.CellName)}, - {Name: "Mount nixhome", Command: GenerateVirtioFSMountScript("nixhome", "/Volumes/nixhome")}, - {Name: "Activate nix-darwin (" + cfg.Stack + ")", Command: GenerateNixDarwinActivateScript(cfg.Stack, "/Volumes/nixhome")}, - } + return append(BaseProvisionSteps(cfg, pubKey), StackProvisionSteps(cfg)...) } diff --git a/internal/vm/tart/provision.go b/internal/vm/tart/provision.go index a8c103d9..84055811 100644 --- a/internal/vm/tart/provision.go +++ b/internal/vm/tart/provision.go @@ -1,6 +1,10 @@ package tart -import "fmt" +import ( + "fmt" + "path/filepath" + "strings" +) // GenerateSSHEnablementScript returns a shell script to enable SSH on macOS. func GenerateSSHEnablementScript() string { @@ -193,8 +197,10 @@ fi sudo diskutil mount -mountPoint /nix "$LABEL" # --- Update fstab: remove installer's APFS entry, add our JHFS+ --- +# The installer writes "UUID= /nix apfs rw,noauto,..." — match on the +# mountpoint+fstype, not the volume name (which never appears in the entry). echo "updating fstab..." -sudo sed -i '' '/Nix.Store/d' /etc/fstab 2>/dev/null || true +sudo sed -i '' -E '/\/nix[[:space:]]*apfs/d' /etc/fstab 2>/dev/null || true grep -q "$LABEL" /etc/fstab 2>/dev/null || \ echo "LABEL=$LABEL /nix hfs rw,nobrowse" | sudo tee -a /etc/fstab > /dev/null @@ -222,8 +228,12 @@ sudo tee /Library/LaunchDaemons/com.devcell.mount-nix.plist > /dev/null <<'BOOTP BOOTPLIST sudo launchctl bootstrap system /Library/LaunchDaemons/com.devcell.mount-nix.plist 2>/dev/null || true -# --- Disable installer's APFS mount daemon (fails on clone anyway) --- +# --- Remove installer's APFS mount daemon --- +# bootout alone doesn't survive reboot: the plist would reload on every clone +# boot and mount the tiny installer APFS volume OVER the JHFS+ disk, +# shadowing the real store (observed as missing system profile + dead s6). sudo launchctl bootout system/org.nixos.darwin-store 2>/dev/null || true +sudo rm -f /Library/LaunchDaemons/org.nixos.darwin-store.plist # --- Restart nix daemon on the new mount --- echo "restarting nix-daemon..." @@ -238,6 +248,41 @@ echo "=== Nix store swap complete ==="`, ) } +// GenerateNixShadowRepairScript returns a best-effort script that detects and +// repairs the installer-APFS-over-JHFS+ shadow mount on /nix. +// +// Templates built before the swap-script fix keep the installer's +// org.nixos.darwin-store LaunchDaemon, which re-mounts the tiny APFS +// "Nix Store" volume (initial install only, ~71 store paths) over the JHFS+ +// DevcellNix disk at every clone boot. That hides the nix-darwin system +// profile, s6 binaries, and per-user profiles. This runs at session start, +// before the s6/nix-darwin preflight. Deliberately not `set -e`: on a healthy +// VM every step is a no-op and the session must proceed. +func GenerateNixShadowRepairScript() string { + return `DEV=$(df /nix 2>/dev/null | awk 'NR==2{print $1}') +if [ -n "$DEV" ] && diskutil info "$DEV" 2>/dev/null | grep -q "Volume Name:.*Nix Store"; then + echo "nix-shadow: installer APFS $DEV is shadowing /nix — unmounting" + sudo diskutil unmount "$DEV" 2>/dev/null || sudo diskutil unmount force "$DEV" 2>/dev/null || true + + # Kill the persistence vectors so the shadow doesn't return next boot + sudo launchctl bootout system/org.nixos.darwin-store 2>/dev/null || true + sudo rm -f /Library/LaunchDaemons/org.nixos.darwin-store.plist + sudo sed -i '' -E '/\/nix[[:space:]]*apfs/d' /etc/fstab 2>/dev/null || true + + # If nothing serves /nix now, mount the JHFS+ disk + if ! /sbin/mount | grep -q " /nix "; then + sudo diskutil mount -mountPoint /nix DevcellNix 2>/dev/null || true + fi + + # Restart the nix-daemon against the real store + sudo launchctl kickstart -k system/org.nixos.nix-daemon 2>/dev/null || true + + echo "nix-shadow: /nix now served by $(df /nix 2>/dev/null | awk 'NR==2{print $1}')" +else + echo "nix-shadow: no shadow detected ($DEV serves /nix)" +fi` +} + // GenerateNixDarwinActivateScript returns the nix-darwin activation command. // nix-darwin manages system-level config (LaunchDaemons, /etc/) and user // packages — a superset of home-manager. The flakeDir must contain a flake @@ -256,13 +301,12 @@ echo "nix-darwin: flake dir listing=$(ls %s/flake.nix %s/flake.lock 2>&1)" echo "nix-darwin: /etc/nix/nix.conf=$(cat /etc/nix/nix.conf 2>/dev/null || echo MISSING)" echo "nix-darwin: existing users=$(dscl . -list /Users UniqueID 2>/dev/null | grep -E 'devcell|_nixbld' | head -10)" -# nix-darwin manages /etc/{bashrc,zshrc} — activation aborts if they contain -# unrecognized content. The official Nix installer modifies both files (appends -# nix-daemon.sh sourcing). Renaming to .before-nix-darwin is nix-darwin's -# idiomatic handoff: it signals "I consent, you own this file now." +# nix-darwin manages several /etc files — activation aborts if they contain +# unrecognized content. The Nix installer and macOS defaults write files that +# conflict. Renaming to .before-nix-darwin is nix-darwin's idiomatic handoff. # Idempotent: skip if .before-nix-darwin already exists (prior activation). echo "nix-darwin: checking /etc files for nix-darwin handoff..." -for f in /etc/bashrc /etc/zshrc; do +for f in /etc/bashrc /etc/zshrc /etc/zshenv /etc/zprofile /etc/nix/nix.conf /etc/shells; do if [ -f "$f" ] && [ ! -f "$f.before-nix-darwin" ]; then sudo mv "$f" "$f.before-nix-darwin" echo " backed up $f -> $f.before-nix-darwin (nix-darwin will manage it)" @@ -271,11 +315,73 @@ for f in /etc/bashrc /etc/zshrc; do fi done +# nix's libgit2 fetcher runs as root (HOME=/var/root) but the flake repo sits +# on a VirtioFS share owned by the automount user — libgit2 aborts with +# "repository path ... is not owned by current user". Mark all paths safe in +# root's global gitconfig (libgit2 reads $HOME/.gitconfig; written directly so +# no git binary is needed). Idempotent via the grep guard. +sudo sh -c 'grep -qs "directory = \*" /var/root/.gitconfig 2>/dev/null || printf "[safe]\n\tdirectory = *\n" >> /var/root/.gitconfig' +echo "nix-darwin: root gitconfig=$(sudo cat /var/root/.gitconfig 2>/dev/null || echo MISSING)" + sudo PATH="$PATH" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" HOME=/var/root nix \ --extra-experimental-features 'nix-command flakes' \ run nix-darwin -- switch --flake %s#%s --show-trace --print-build-logs 2>&1`, flakeDir, stack, flakeDir, flakeDir, flakeDir, stack) } +// GenerateHostNixSubstituterScript returns a guest-side script that configures +// the host's shared /nix store as a local substituter. The host's /nix is +// shared read-only via VirtioFS at /Volumes/My Shared Files/hostnix. +// +// To avoid SQLite-over-VirtioFS locking issues, the script copies the nix +// database locally and symlinks the store paths from the VirtioFS mount. +func GenerateHostNixSubstituterScript() string { + return `set -e +echo "=== Configure host nix store as local substituter ===" + +HOST_NIX="/Volumes/My Shared Files/hostnix" +BRIDGE="/tmp/host-nix-root" + +if [ ! -d "$HOST_NIX/store" ]; then + echo "host nix store not available at $HOST_NIX/store — skipping" + exit 0 +fi + +STORE_COUNT=$(ls "$HOST_NIX/store" 2>/dev/null | wc -l | tr -d ' ') +echo "found host nix store: $STORE_COUNT paths" + +# Build a hybrid bridge: local DB copy + symlinked store +sudo mkdir -p "$BRIDGE/nix/var/nix/db" + +# Copy the database locally so SQLite can open it without VirtioFS locking +if [ -f "$HOST_NIX/var/nix/db/db.sqlite" ]; then + echo "copying host nix database..." + sudo cp "$HOST_NIX/var/nix/db/db.sqlite" "$BRIDGE/nix/var/nix/db/db.sqlite" + echo "database copied ($(du -sh "$BRIDGE/nix/var/nix/db/db.sqlite" | cut -f1))" +else + echo "WARNING: host nix database not found — skipping" + exit 0 +fi + +# Symlink the store paths (content-addressable, immutable, safe for reads) +sudo ln -sfn "$HOST_NIX/store" "$BRIDGE/nix/store" + +ls "$BRIDGE/nix/store" > /dev/null 2>&1 || { echo "WARNING: store symlink broken — skipping"; exit 0; } + +# Add as extra substituter in nix.conf (effective for builds before nix-darwin activation overwrites it) +if ! grep -q "host-nix-root" /etc/nix/nix.conf 2>/dev/null; then + echo "extra-substituters = local?root=$BRIDGE" | sudo tee -a /etc/nix/nix.conf + echo "trusted-substituters = local?root=$BRIDGE" | sudo tee -a /etc/nix/nix.conf + sudo launchctl kickstart -k system/org.nixos.nix-daemon 2>/dev/null || true + sleep 2 + echo "host nix substituter configured and daemon restarted" +else + echo "host nix substituter already configured" +fi + +echo "=== Host nix substituter ready ===" +` +} + // GenerateGrantSSHdFDAScript returns a boot-time script that grants Full Disk // Access to sshd in the macOS TCC database. Must run as a LaunchDaemon under // launchd (not over SSH) — launchd has the TCC context to modify the database. @@ -408,17 +514,31 @@ echo "=== VirtioFS mount done ==="`, mountPoint) } -// GenerateProjectMountScript returns a script to mount the project VirtioFS -// share into the user's home directory, mirroring Docker's bind-mount behavior. -func GenerateProjectMountScript(tag, username, projectBasename string) string { - mountPoint := fmt.Sprintf("/Users/%s/%s", username, projectBasename) +// ProjectPathInVM maps the host project path to its in-VM location. +// When the project lives under the host home, the relative path is mirrored +// into the session user's VM home — so /Users/dmitry/dev/acme/proj on the +// host appears at the same path inside the VM (session user == host $USER). +// Projects outside the host home fall back to ~/. +func ProjectPathInVM(hostHome, baseDir, sessionUser string) string { + rel, err := filepath.Rel(hostHome, baseDir) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return fmt.Sprintf("/Users/%s/%s", sessionUser, filepath.Base(baseDir)) + } + return fmt.Sprintf("/Users/%s/%s", sessionUser, rel) +} + +// GenerateProjectMountScript returns a script to symlink the project VirtioFS +// share at mountPoint (an absolute in-VM path from ProjectPathInVM), +// mirroring Docker's bind-mount behavior. Parent directories are created as +// the session user so they stay writable. +func GenerateProjectMountScript(tag, username, mountPoint string) string { return fmt.Sprintf(`set -e echo "=== project mount: tag=%s target=%s ===" AUTOMOUNT_PATH="/Volumes/My Shared Files/%s" if [ -d "$AUTOMOUNT_PATH" ]; then echo "found automount share at $AUTOMOUNT_PATH" - sudo mkdir -p "$(dirname %s)" + sudo -u %s mkdir -p "$(dirname %s)" sudo ln -sfn "$AUTOMOUNT_PATH" %s sudo chown -h %s %s echo "symlinked %s -> $AUTOMOUNT_PATH" @@ -435,7 +555,8 @@ ls %s/ | head -5 || echo "WARNING: mount point is empty" echo "=== project mount done ==="`, tag, mountPoint, tag, - mountPoint, mountPoint, + username, mountPoint, + mountPoint, username, mountPoint, mountPoint, tag, mountPoint, @@ -485,18 +606,85 @@ if [ -d "$AUTOMOUNT_PATH" ]; then for item in "$AUTOMOUNT_PATH"/.[!.]* "$AUTOMOUNT_PATH"/*; do [ -e "$item" ] || continue name=$(basename "$item") + # Shell rc files are platform-specific: the shared copies carry Linux + # paths (/home/, /opt/devcell). The s6 shell-rc service generates + # macOS-correct ones locally instead. + case "$name" in + .zshenv|.zshrc|.profile|.bashrc) continue ;; + esac target="/Users/$USERNAME/$name" if [ ! -e "$target" ] && [ ! -L "$target" ]; then sudo ln -sfn "$item" "$target" sudo chown -h "$USERNAME":staff "$target" fi done + # Drop rc symlinks created by earlier runs of this script — they point at + # the shared Linux-flavored files and would shadow shell-rc's output. + for rc in .zshenv .zshrc .profile .bashrc; do + if [ -L "/Users/$USERNAME/$rc" ]; then + sudo rm -f "/Users/$USERNAME/$rc" + fi + done echo "CellHome contents linked into /Users/$USERNAME" else echo "no CellHome VirtioFS share at $AUTOMOUNT_PATH — skipping" fi`, username) } +// S6EnvDir is where session env vars are written for s6 service scripts. +// /etc on macOS is a symlink to /private/etc (Data volume, writable by root). +// nix-darwin already manages /etc/s6/ via the s6-darwin-renderer activation script. +const S6EnvDir = "/etc/s6/env" + +// S6ServicesDir is where s6 service definitions live (installed by nix-darwin's +// s6-darwin-renderer.nix activation script; read by com.devcell.s6-svscan). +const S6ServicesDir = "/etc/s6/services" + +// GenerateS6SessionActivateScript returns a script that activates s6-rc +// session services for a newly created session user. +// +// The script sets env vars that s6 service scripts read (HOST_USER, SESSION_HOME, +// DEVCELL_HOME) and then runs s6-rc oneshot services if s6 is installed. +// Paths use /etc/s6/ to match nix-darwin's s6-darwin-renderer output. +func GenerateS6SessionActivateScript(sessionUser string) string { + return fmt.Sprintf(`set -e +export HOST_USER="%s" +export SESSION_HOME="/Users/%s" +export DEVCELL_HOME="/Users/%s" + +# Source nix-daemon profile so s6/s6-rc are on PATH (tart exec runs as admin +# whose default PATH doesn't include /run/current-system/sw/bin). +. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true +export PATH="/run/current-system/sw/bin${PATH:+:}${PATH}" + +S6_ENV="%s" +S6_SVC="%s" + +if command -v s6-rc >/dev/null 2>&1 && [ -d "$S6_SVC" ]; then + sudo mkdir -p "$S6_ENV" + echo "$HOST_USER" | sudo tee "$S6_ENV/HOST_USER" > /dev/null + echo "$SESSION_HOME" | sudo tee "$S6_ENV/SESSION_HOME" > /dev/null + echo "$DEVCELL_HOME" | sudo tee "$S6_ENV/DEVCELL_HOME" > /dev/null + echo "s6: activating session services for $HOST_USER" + for svc in shell-rc claude-config codex-config gemini-config opencode-config homedir gcroot mise; do + if [ -d "$S6_SVC/$svc" ]; then + if [ -f "$S6_SVC/$svc/type" ] && [ "$(cat "$S6_SVC/$svc/type")" = "oneshot" ] && [ -x "$S6_SVC/$svc/up" ]; then + echo "s6: running $svc" + sudo -E "$S6_SVC/$svc/up" 2>&1 || echo "s6: $svc failed (non-fatal)" + fi + fi + done + echo "s6: session services activated" +else + echo "s6: s6-rc not available — skipping session activation" +fi`, + sessionUser, + sessionUser, + DarwinVMUser, + S6EnvDir, + S6ServicesDir) +} + // ProvisionedMarkerPath is on the boot disk's writable Data volume. // /private/var persists across tart clone. We use /private/var (not /var) // to avoid any symlink indirection during early boot. diff --git a/internal/vm/tart/provision_test.go b/internal/vm/tart/provision_test.go index 64183570..ebba14b3 100644 --- a/internal/vm/tart/provision_test.go +++ b/internal/vm/tart/provision_test.go @@ -68,6 +68,31 @@ func TestNixDarwinActivateScript(t *testing.T) { } } +func TestNixDarwinActivateScript_HandsOffEtcFiles(t *testing.T) { + script := GenerateNixDarwinActivateScript("ultimate", "/Volumes/nixhome") + for _, f := range []string{"/etc/bashrc", "/etc/zshrc", "/etc/zshenv", "/etc/zprofile", "/etc/nix/nix.conf", "/etc/shells"} { + if !strings.Contains(script, f) { + t.Errorf("activate script must hand off %s to nix-darwin", f) + } + } +} + +func TestNixDarwinActivateScript_GitSafeDirectory(t *testing.T) { + // nix's libgit2 fetcher (running as root, HOME=/var/root) refuses the + // VirtioFS-mounted flake repo: "repository path ... is not owned by + // current user". The script must mark it safe in root's gitconfig + // BEFORE the nix run line. + script := GenerateNixDarwinActivateScript("ultimate", "/Volumes/nixhome") + safeIdx := strings.Index(script, "safe") + if safeIdx < 0 || !strings.Contains(script, "/var/root/.gitconfig") { + t.Fatalf("expected git safe.directory setup in /var/root/.gitconfig, got %q", script) + } + nixRunIdx := strings.Index(script, "nix run nix-darwin") + if nixRunIdx >= 0 && safeIdx > nixRunIdx { + t.Fatal("safe.directory setup must come before the nix run line") + } +} + func TestGrantSSHdFDAScript(t *testing.T) { script := GenerateGrantSSHdFDAScript() @@ -192,6 +217,60 @@ func TestProvisionStepsOnline(t *testing.T) { } } +func TestProvisionStepsWithHostNix(t *testing.T) { + cfg := InitConfig{CellName: "main", Stack: "ultimate", Username: "admin", HasHostNix: true} + steps := ProvisionSteps(cfg, "ssh-ed25519 AAAA", false) + + if len(steps) != 10 { + t.Fatalf("expected 10 online provisioning steps with host nix, got %d", len(steps)) + } + + names := make([]string, len(steps)) + for i, s := range steps { + names[i] = s.Name + } + + var swapIdx, substIdx, nixhomeIdx int + for i, name := range names { + if name == "Swap nix to external disk" { + swapIdx = i + } + if name == "Configure host nix substituter" { + substIdx = i + } + if name == "Mount nixhome" { + nixhomeIdx = i + } + } + if substIdx == 0 { + t.Fatalf("expected 'Configure host nix substituter' step, got %v", names) + } + if substIdx <= swapIdx { + t.Fatalf("host nix substituter (idx %d) must come after swap (idx %d)", substIdx, swapIdx) + } + if substIdx >= nixhomeIdx { + t.Fatalf("host nix substituter (idx %d) must come before nixhome (idx %d)", substIdx, nixhomeIdx) + } +} + +func TestHostNixSubstituterScript(t *testing.T) { + script := GenerateHostNixSubstituterScript() + + for _, want := range []string{ + "hostnix", + "host-nix-root", + "db.sqlite", + "extra-substituters", + "local?root=", + "set -e", + "nix-daemon", + } { + if !strings.Contains(script, want) { + t.Errorf("host nix substituter script should contain %q", want) + } + } +} + func TestProvisionStepsOnlineNixStepUsesOfficial(t *testing.T) { cfg := InitConfig{Stack: "ultimate", Username: "admin"} steps := ProvisionSteps(cfg, "ssh-ed25519 AAAA", false) @@ -278,6 +357,13 @@ func TestNixStoreSwapScript(t *testing.T) { "set -e", "com.devcell.mount-nix", "fstab", + // The installer's boot-time APFS mount daemon must be REMOVED, not just + // booted out — bootout doesn't survive reboot, so cloned VMs would get + // the tiny APFS "Nix Store" volume mounted over the JHFS+ DevcellNix. + "rm -f /Library/LaunchDaemons/org.nixos.darwin-store.plist", + // The installer's fstab entry is "UUID=... /nix apfs ..." — it contains + // no "Nix Store" text, so the old sed pattern never matched it. + "/nix[[:space:]]*apfs", } { if !strings.Contains(script, want) { t.Errorf("swap script should contain %q", want) @@ -292,6 +378,32 @@ func TestNixStoreSwapScript(t *testing.T) { } } +// TestGenerateNixShadowRepairScript covers the runtime repair for templates +// built before the swap-script fix: the installer's darwin-store daemon +// re-mounts its APFS volume over /nix at clone boot, shadowing the JHFS+ +// store that holds nix-darwin profiles and s6. +func TestGenerateNixShadowRepairScript(t *testing.T) { + script := GenerateNixShadowRepairScript() + + for _, want := range []string{ + "df /nix", // detect which device serves /nix + "Nix Store", // identify the installer's APFS volume by name + "diskutil unmount", // remove the shadow + "org.nixos.darwin-store", // kill the persistence vector + "rm -f /Library/LaunchDaemons/org.nixos.darwin-store.plist", + "/nix[[:space:]]*apfs", // purge the installer fstab entry + "DevcellNix", // ensure the JHFS+ disk ends up serving /nix + "nix-daemon", // daemon must be kicked after the swap + } { + if !strings.Contains(script, want) { + t.Errorf("shadow repair script should contain %q", want) + } + } + if strings.Contains(script, "set -e") { + t.Error("repair script must be best-effort — set -e would abort the session on a healthy VM") + } +} + func TestVirtioFSMountScript(t *testing.T) { script := GenerateVirtioFSMountScript("myshare", "/Volumes/myshare") if !strings.Contains(script, "myshare") { @@ -315,12 +427,12 @@ func TestVirtioFSMountScript(t *testing.T) { } func TestProjectMountScript(t *testing.T) { - script := GenerateProjectMountScript("project", "admin", "devcell") + script := GenerateProjectMountScript("project", "admin", "/Users/admin/dev/acme/devcell") if !strings.Contains(script, "set -e") { t.Fatal("expected script to use set -e") } - if !strings.Contains(script, "/Users/admin/devcell") { - t.Fatalf("expected script to mount at /Users/admin/devcell, got %q", script) + if !strings.Contains(script, "/Users/admin/dev/acme/devcell") { + t.Fatalf("expected script to mount at the full mirrored path, got %q", script) } if !strings.Contains(script, "mount_virtiofs") { t.Fatal("expected script to try mount_virtiofs as fallback") @@ -331,6 +443,48 @@ func TestProjectMountScript(t *testing.T) { if !strings.Contains(script, `My Shared Files/project`) { t.Fatal("expected script to check Apple automount path") } + // Parent dirs must be created as the session user, not root — otherwise + // the user can't create siblings under ~/dev/... later. + if !strings.Contains(script, "sudo -u admin mkdir -p") { + t.Fatal("expected parent dirs created as the session user") + } +} + +func TestSetupSessionHomeScriptSkipsShellRcFiles(t *testing.T) { + // Shell rc files in CellHome are generated by the LINUX container's + // shell-rc service and carry Linux paths (/home/, /opt/devcell). + // Symlinking them into the macOS VM home breaks every shell (e.g. + // HISTFILE=/home/dmitry/.zsh_history does not exist on macOS). The s6 + // shell-rc service generates platform-correct ones instead. + script := GenerateSetupSessionHomeScript("dmitry") + for _, rc := range []string{".zshenv", ".zshrc", ".profile", ".bashrc"} { + if !strings.Contains(script, rc) { + t.Errorf("setup home script should explicitly skip %s (platform-specific, owned by shell-rc)", rc) + } + } + if !strings.Contains(script, "continue") { + t.Error("expected a skip (continue) branch for platform-specific rc files") + } +} + +func TestProjectPathInVM(t *testing.T) { + // Project under host home → mirror the relative path into the VM home. + got := ProjectPathInVM("/Users/dmitry", "/Users/dmitry/dev/devcell-sh/devcell", "dmitry") + if got != "/Users/dmitry/dev/devcell-sh/devcell" { + t.Errorf("under-home project: got %q, want mirrored host path", got) + } + + // Different session user still lands under that user's VM home. + got = ProjectPathInVM("/Users/alice", "/Users/alice/work/proj", "bob") + if got != "/Users/bob/work/proj" { + t.Errorf("cross-user mapping: got %q, want /Users/bob/work/proj", got) + } + + // Project outside host home → fall back to ~/. + got = ProjectPathInVM("/Users/dmitry", "/opt/checkouts/thing", "dmitry") + if got != "/Users/dmitry/thing" { + t.Errorf("outside-home project: got %q, want /Users/dmitry/thing", got) + } } func TestProvisionedMarkerScript(t *testing.T) { @@ -346,3 +500,148 @@ func TestCheckProvisionedScript(t *testing.T) { t.Fatal("expected script to check /private/var/devcell-provisioned marker (boot disk, not home)") } } + +func TestBaseProvisionSteps(t *testing.T) { + t.Run("without host nix", func(t *testing.T) { + cfg := InitConfig{CellName: "main", Stack: "ultimate", Username: "admin"} + steps := BaseProvisionSteps(cfg, "ssh-ed25519 AAAA") + + if len(steps) != 7 { + t.Fatalf("expected 7 base steps without host nix, got %d", len(steps)) + } + + wantNames := []string{ + "Enable SSH", + "Inject SSH key", + "Configure passwordless sudo", + "Mount home volume", + "Prepare nix disk", + "Install Nix", + "Swap nix to external disk", + } + for i, want := range wantNames { + if steps[i].Name != want { + t.Errorf("step[%d] = %q, want %q", i, steps[i].Name, want) + } + } + }) + + t.Run("with host nix", func(t *testing.T) { + cfg := InitConfig{CellName: "main", Stack: "ultimate", Username: "admin", HasHostNix: true} + steps := BaseProvisionSteps(cfg, "ssh-ed25519 AAAA") + + if len(steps) != 8 { + t.Fatalf("expected 8 base steps with host nix, got %d", len(steps)) + } + + if steps[7].Name != "Configure host nix substituter" { + t.Errorf("last base step = %q, want 'Configure host nix substituter'", steps[7].Name) + } + }) + + t.Run("password auth on first two steps", func(t *testing.T) { + cfg := InitConfig{CellName: "main", Stack: "ultimate", Username: "admin"} + steps := BaseProvisionSteps(cfg, "ssh-ed25519 AAAA") + + if !steps[0].NeedsPassword { + t.Error("Enable SSH should need password auth") + } + if !steps[1].NeedsPassword { + t.Error("Inject SSH key should need password auth") + } + for _, s := range steps[2:] { + if s.NeedsPassword { + t.Errorf("step %q should not need password auth", s.Name) + } + } + }) +} + +func TestStackProvisionSteps(t *testing.T) { + cfg := InitConfig{CellName: "main", Stack: "ultimate", Username: "admin"} + steps := StackProvisionSteps(cfg) + + if len(steps) != 2 { + t.Fatalf("expected 2 stack steps, got %d", len(steps)) + } + + if steps[0].Name != "Mount nixhome" { + t.Errorf("step[0] = %q, want 'Mount nixhome'", steps[0].Name) + } + if !strings.Contains(steps[1].Name, "Activate nix-darwin") { + t.Errorf("step[1] = %q, want it to contain 'Activate nix-darwin'", steps[1].Name) + } + if !strings.Contains(steps[1].Name, "ultimate") { + t.Errorf("step[1] = %q, want it to contain stack name 'ultimate'", steps[1].Name) + } +} + +func TestProvisionSteps_IsBaseAndStackCombined(t *testing.T) { + cfg := InitConfig{CellName: "main", Stack: "ultimate", Username: "admin", HasHostNix: true} + pubKey := "ssh-ed25519 AAAA" + + all := ProvisionSteps(cfg, pubKey, false) + base := BaseProvisionSteps(cfg, pubKey) + stack := StackProvisionSteps(cfg) + combined := append(base, stack...) + + if len(all) != len(combined) { + t.Fatalf("ProvisionSteps returned %d steps, BaseProvisionSteps+StackProvisionSteps returned %d", + len(all), len(combined)) + } + for i := range all { + if all[i].Name != combined[i].Name { + t.Errorf("step[%d]: ProvisionSteps=%q, combined=%q", i, all[i].Name, combined[i].Name) + } + } +} + +func TestBaseTemplateName(t *testing.T) { + if BaseTemplateName != "devcell-tart-base" { + t.Errorf("BaseTemplateName = %q, want %q", BaseTemplateName, "devcell-tart-base") + } +} + +func TestGenerateS6SessionActivateScript(t *testing.T) { + script := GenerateS6SessionActivateScript("dmitry") + + if !strings.Contains(script, "s6-rc") { + t.Fatal("expected script to use s6-rc for session service activation") + } + if !strings.Contains(script, "dmitry") { + t.Fatal("expected script to reference the session user") + } + if !strings.Contains(script, "shell-rc") { + t.Fatal("expected script to activate shell-rc service") + } + if !strings.Contains(script, "claude-config") { + t.Fatal("expected script to activate claude-config service") + } + if !strings.Contains(script, "HOST_USER") { + t.Fatal("expected script to set HOST_USER env var for service scripts") + } + if !strings.Contains(script, "SESSION_HOME") { + t.Fatal("expected script to set SESSION_HOME env var for service scripts") + } + if !strings.Contains(script, "DEVCELL_HOME") { + t.Fatal("expected script to set DEVCELL_HOME for darwin paths") + } + if !strings.Contains(script, "/Users/devcell") { + t.Fatal("expected DEVCELL_HOME to point at /Users/devcell on darwin") + } + if !strings.Contains(script, S6EnvDir) { + t.Fatalf("expected script to use S6EnvDir (%s)", S6EnvDir) + } + if !strings.Contains(script, S6ServicesDir) { + t.Fatalf("expected script to use S6ServicesDir (%s)", S6ServicesDir) + } + if strings.Contains(script, "mkdir -p") && !strings.Contains(script, "sudo mkdir") { + t.Fatal("mkdir under /etc/s6/ requires sudo") + } + // tart exec runs as admin (uid 501) — service up scripts write to the + // session user's home and /nix/var, so they must run as root with the + // exported env (HOST_USER, SESSION_HOME, DEVCELL_HOME) preserved. + if !strings.Contains(script, `sudo -E "$S6_SVC/$svc/up"`) { + t.Fatal("service up scripts must run via sudo -E (admin can't write session-user home)") + } +} diff --git a/internal/vm/tart/ssh.go b/internal/vm/tart/ssh.go index 889eb8af..1178fe7e 100644 --- a/internal/vm/tart/ssh.go +++ b/internal/vm/tart/ssh.go @@ -6,13 +6,25 @@ import ( "strings" ) +// DarwinVMUser is the fixed nix-darwin/home-manager user inside the macOS VM +// (community-home's darwinVMUser). Agent binaries are installed into its +// per-user profile, regardless of which session user runs the cell. +const DarwinVMUser = "devcell" + +// nixProfileSource makes nix and the home-manager-installed agent binaries +// available in the exec shell. Sourcing nix-daemon.sh only yields nix itself; +// the session user (host $USER) is not DarwinVMUser, so its per-user profile +// and the nix-darwin system profile must be bridged onto PATH explicitly. +const nixProfileSource = `. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || . "$HOME/.nix-profile/etc/profile.d/nix.sh" 2>/dev/null || true; export PATH="/etc/profiles/per-user/` + DarwinVMUser + `/bin:/run/current-system/sw/bin:$PATH"` + // ExecSpec describes a command to run inside a tart VM via `tart exec`. type ExecSpec struct { Binary string // binary to run (e.g. "zsh", "claude") Flags []string // default flags for the binary UserArgs []string // user-provided args EnvVars []string // KEY=VAL pairs to set in the environment - ProjectDir string // host project path — basename is used for cd ~/basename + ProjectDir string // host project path — basename fallback for cd ~/basename + WorkDir string // absolute in-VM project path (ProjectPathInVM); wins over ProjectDir RunAsUser string // if set, wrap command with sudo -u -i } @@ -36,15 +48,17 @@ func BuildExecCommand(spec ExecSpec) string { agentCmd := shellJoinTokens(tokens) var cmd string - if spec.ProjectDir != "" { + switch { + case spec.WorkDir != "": + cmd = "cd " + shellQuoteToken(spec.WorkDir) + " && " + agentCmd + case spec.ProjectDir != "": basename := filepath.Base(spec.ProjectDir) cmd = "cd ~/" + shellQuoteToken(basename) + " && " + agentCmd - } else { + default: cmd = agentCmd } - const nixSource = `. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || . "$HOME/.nix-profile/etc/profile.d/nix.sh" 2>/dev/null || true` - innerCmd := nixSource + "; " + cmd + innerCmd := nixProfileSource + "; " + cmd if spec.RunAsUser != "" { return "sudo -u " + shellQuoteToken(spec.RunAsUser) + " -i bash -l -c " + shellQuoteToken(innerCmd) @@ -87,10 +101,7 @@ func BuildSSHArgv(spec Spec, host string) []string { remoteCmd = agentCmd } - // Explicitly source the nix daemon profile before running the agent binary. - // Determinate installer puts the profile here; fall back to home-manager path. - const nixSource = `. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || . "$HOME/.nix-profile/etc/profile.d/nix.sh" 2>/dev/null || true` - remoteCmd = nixSource + "; " + remoteCmd + remoteCmd = nixProfileSource + "; " + remoteCmd // Build the SSH argv. userHost := spec.SSHUser + "@" + host diff --git a/internal/vm/tart/ssh_test.go b/internal/vm/tart/ssh_test.go index 72f9e27f..968118bc 100644 --- a/internal/vm/tart/ssh_test.go +++ b/internal/vm/tart/ssh_test.go @@ -219,6 +219,28 @@ func TestBuildExecCommand_NixSource(t *testing.T) { } } +func TestBuildExecCommand_DarwinHMProfileOnPath(t *testing.T) { + // home-manager installs agent binaries for the fixed nix-darwin VM user + // (devcell), but the session runs as the host's $USER — the exec command + // must bridge that user's profile bin dir onto PATH or `claude` is not + // found (CELL: --os macos dropped into "command not found"). + cmd := tart.BuildExecCommand(tart.ExecSpec{Binary: "claude", RunAsUser: "dmitry"}) + if !strings.Contains(cmd, "/etc/profiles/per-user/devcell/bin") { + t.Errorf("expected devcell per-user profile bin on PATH, got: %q", cmd) + } + if !strings.Contains(cmd, "/run/current-system/sw/bin") { + t.Errorf("expected nix-darwin system profile bin on PATH, got: %q", cmd) + } +} + +func TestBuildSSHArgv_DarwinHMProfileOnPath(t *testing.T) { + argv := tart.BuildSSHArgv(tart.Spec{Binary: "claude", SSHUser: "dmitry", SSHPort: 22}, "192.168.64.2") + remoteCmd := argv[len(argv)-1] + if !strings.Contains(remoteCmd, "/etc/profiles/per-user/devcell/bin") { + t.Errorf("expected devcell per-user profile bin on PATH, got: %q", remoteCmd) + } +} + func TestBuildExecCommand_ProjectDirCd(t *testing.T) { cmd := tart.BuildExecCommand(tart.ExecSpec{ Binary: "claude", @@ -229,6 +251,22 @@ func TestBuildExecCommand_ProjectDirCd(t *testing.T) { } } +func TestBuildExecCommand_WorkDirOverridesProjectDir(t *testing.T) { + // WorkDir carries the mirrored in-VM path (host path reproduced inside + // the VM); when set it must win over the ~/basename fallback. + cmd := tart.BuildExecCommand(tart.ExecSpec{ + Binary: "claude", + ProjectDir: "/Users/dmitry/dev/devcell-sh/devcell", + WorkDir: "/Users/dmitry/dev/devcell-sh/devcell", + }) + if !strings.Contains(cmd, "cd /Users/dmitry/dev/devcell-sh/devcell") { + t.Errorf("expected cd to mirrored WorkDir, got: %q", cmd) + } + if strings.Contains(cmd, "cd ~/devcell") { + t.Errorf("basename fallback must not be used when WorkDir is set: %q", cmd) + } +} + func TestBuildExecCommand_EnvVars(t *testing.T) { cmd := tart.BuildExecCommand(tart.ExecSpec{ Binary: "claude", diff --git a/internal/vm/tart/vnc_ocr_nocgo.go b/internal/vm/tart/vnc_ocr_nocgo.go new file mode 100644 index 00000000..d7789780 --- /dev/null +++ b/internal/vm/tart/vnc_ocr_nocgo.go @@ -0,0 +1,12 @@ +//go:build darwin && arm64 && !cgo + +package tart + +import "image" + +// FindTextOnScreen without cgo cannot reach the Apple Vision framework +// (vnc_ocr_darwin.go imports "C" and is excluded when CGO_ENABLED=0, e.g. +// cross-compiles from Linux). Reports "not found" like the non-Darwin stub. +func FindTextOnScreen(_ *image.RGBA, _ string) (image.Rectangle, bool) { + return image.Rectangle{}, false +} diff --git a/web/package-lock.json b/web/package-lock.json index a676412b..3a191b41 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -8,7 +8,7 @@ "name": "tmp-astro-temp", "version": "0.0.1", "dependencies": { - "@astrojs/cloudflare": "^12.6.13", + "@astrojs/cloudflare": "12", "astro": "^5.17.1" }, "devDependencies": { diff --git a/web/src/components/FeatureCards.astro b/web/src/components/FeatureCards.astro index 114728d2..b4c8e1b6 100644 --- a/web/src/components/FeatureCards.astro +++ b/web/src/components/FeatureCards.astro @@ -1,48 +1,68 @@ --- --- -
-

Why it matters

-

Auto-approve, safely.

+
+

The payoff

+

What the agent gains inside.

-

"Run this in a container, not your actual machine."

- — Anthropic, Claude Code documentation +

Built-in sandboxes subtract tools to make agents safe. devcell adds tools and stays safe.

- -

Blast radius bounded

-

SSH keys, other repos, host APIs: unreachable. The agent edits freely inside your project. Your host system stays untouched.

-
-
- -

One command, any project

-

cd my-project && cell claude. Working directory mounted automatically, no per-project config needed. Works with Codex and OpenCode too.

+ +

Tools behind the MCP servers

+

Marketplaces install the config. devcell installs the app underneath. OpenTofu actually runs tofu plan, Inkscape actually edits the SVG — display server included. 12 servers today, each with its backing software in the image.

- -

Version-locked toolchain

-

Go, Node.js, Python, Terraform, and more. Nix-pinned at build time. No download URLs that go stale, no version drift between machines.

+ +

A desktop it can see

+

VNC and RDP built in. The agent opens GUI apps, clicks through what it built, and you watch live — or take over at a login screen. Linux by default; VM engines for macOS.

-

Secrets never touch your disk

-

1Password secrets are resolved on the host, injected into the container as env vars, and written to a RAM-only tmpfs at /run/secrets/. When the container stops, they're gone. The LLM never sees actual credential values -- MCP tools resolve placeholder names server-side.

+

Your sessions, not your browser

+

Run cell login <site> on your host: a real browser opens, you log in, it closes. Only that site's session crosses into the cell. The agent never sees your password — and never touches your real browser's history, cache, or other logins.

+
+
+ +
+

The boundary

+

What it can't touch.

+
+

"Run this in a container, not your actual machine."

+ — Anthropic, Claude Code documentation +
+
- -

MCP servers with real tools behind them

-

Not just config stubs. KiCad, Inkscape, and OpenTofu ship in the image alongside their MCP servers, so the agent can actually run tofu plan, analyze PCBs, or edit SVGs. 12 servers today, more with each release.

+ +

Absent, not hidden

+

SSH keys, other repos, host credentials — the agent can't reach them because they aren't there. It can trash its whole machine and lose nothing but the cell: rebuild in minutes. Your project is the one live mount, same as any agent setup.

- -

Stealth Chromium, zero passwords

-

Run cell login on your host to log into any site — a clean browser opens (no CDP, no bot-detection triggers), you log in, press Enter. Cookies and localStorage sync to the container automatically. The agent never sees your password. Anti-fingerprint Playwright replays sessions that pass Cloudflare and Kasada.

+ +

Secrets in RAM only

+

1Password secrets resolve on the host and land on a RAM-only tmpfs at /run/secrets/. Container stops, they're gone. The LLM never sees credential values — MCP tools resolve placeholder names server-side.

- -

Docker or VM — your choice

-

Default: Docker container, zero setup. Add --macos and devcell provisions a Debian ARM64 VM via Vagrant + UTM instead — same nixhome toolchain, same commands. Works for teams that can't use Docker Desktop or need native Linux on Apple Silicon.

+ +

One cell per client

+

Named cells keep engagements apart: Acme's cell has Acme's keys, shell history, and agent memory; BigCorp's has BigCorp's. Same toolchain everywhere — add a project, inherit the setup, duplicate nothing.

+
+
+
+
+ Project + The agent sees the project dir it was launched in — the sole mount. Other repos aren't hidden, they're absent. +
+
+ Cell + Its own home: dotfiles, shell history, agent memory. Shared across that cell's projects, invisible to other cells. +
+
+ Host + Nothing outside the mount and the cell home. ~/.ssh, ~/.aws, your real home: unreachable.
+

Docker by default. When your threat model demands a real kernel boundary, --engine=vagrant runs the same cell in a VM.

diff --git a/web/src/components/Hero.astro b/web/src/components/Hero.astro index 4c858825..38132845 100644 --- a/web/src/components/Hero.astro +++ b/web/src/components/Hero.astro @@ -9,14 +9,14 @@ const { stableVersion, logoGrid } = Astro.props;
-

AI Agent Sandbox · Open Source · Apache 2.0 · ★ Star on GitHub GitHub stars{stableVersion ? · {stableVersion} : · Releases}

-

Agentic Coding,
Without the
Blast Radius.

+

Agent Workspace · Open Source · Apache 2.0 · ★ Star on GitHub GitHub stars{stableVersion ? · {stableVersion} : · Releases}

+

Your Agent
Gets Its Own
Computer.

- Your AI agent can rm -rf / and you're fine. - Auto-approve on your bare machine means the agent sees your SSH keys, other repos, - every credential on disk.

- devcell puts a container between your project and everything else. - Your code goes in. Nothing else comes along. + Auto-approve everything. The agent works inside a cell — a container or VM + with its own home, tools, and desktop. Your SSH keys, other repos, and + credentials aren't hidden from it; they're absent.

+ And inside, it has more than your machine ever gave it: real tools behind + its MCP servers, a browser with your sessions, a screen it can see.

View on GitHub → @@ -33,6 +33,7 @@ const { stableVersion, logoGrid } = Astro.props;
~/myproject $ cell claude
Opening Cell myproject …
✔ mounted /home/alex/myproject
+
✔ ~/.ssh, ~/.aws: not mounted
───────────────────────────────────────