From 3c1fa0b0df2ce6d7af01b5175484f78b5dff1b7e Mon Sep 17 00:00:00 2001 From: James Reilly Date: Mon, 16 Nov 2026 00:48:04 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20janitor:=20prune=20idle=20Claude?= =?UTF-8?q?=20CLI=20session=20scratch=20under=20the=20temp=20directory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Claude CLI keeps per-session scratch in /claude-// / and never removes it. It lives in the container's writable layer, not on /data, so no existing janitor class saw it. On one Docker host two spokes reached 17 GiB and 22 GiB and filled the root disk. PruneClaudeTmp removes each session directory whose newest write anywhere inside is older than 24 hours, reusing the home sweep's symlink-safe newest-write rule, and runs on the janitor's existing 6-hour tick. Co-Authored-By: Claude Opus 5.5 Signed-off-by: James Reilly --- changelog.d/fixed-janitor-claude-tmp.md | 1 + src/cmd/hive/main.go | 3 + src/cmd/hive/sessionprunewire.go | 24 ++++++ src/docs/diskpressure-recovery-runbook.md | 4 + src/pkg/sessionprune/tmpprune.go | 78 ++++++++++++++++++++ src/pkg/sessionprune/tmpprune_test.go | 89 +++++++++++++++++++++++ 6 files changed, 199 insertions(+) create mode 100644 changelog.d/fixed-janitor-claude-tmp.md create mode 100644 src/pkg/sessionprune/tmpprune.go create mode 100644 src/pkg/sessionprune/tmpprune_test.go diff --git a/changelog.d/fixed-janitor-claude-tmp.md b/changelog.d/fixed-janitor-claude-tmp.md new file mode 100644 index 0000000000..4a7b0c3e64 --- /dev/null +++ b/changelog.d/fixed-janitor-claude-tmp.md @@ -0,0 +1 @@ +- **The session janitor now prunes idle Claude CLI scratch under the temp directory.** The Claude CLI keeps per-session scratch in `/claude-///`: task output, scratchpads, and the repositories agents clone or build there. Nothing removed it, and it lives in the container's writable layer rather than on `/data`, so it fills the node's root disk instead of the data volume. On one Docker host, two spokes held 17 GiB and 22 GiB of it after two weeks and filled the disk, which broke every other service on that host. The janitor's 6-hour pass now removes each session directory whose newest write anywhere inside is older than 24 hours (`sessionprune.DefaultTmpRetentionHours`). Like the home sweep, it never follows symlinks and touches nothing outside `claude-*//`. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 0c2235fde6..33feb53358 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1490,6 +1490,7 @@ func (b *boot) wireBootClosures() { maxAge := time.Duration(retentionDays) * 24 * time.Hour homeMaxAge := time.Duration(sessionprune.DefaultHomeRetentionDays) * 24 * time.Hour + tmpMaxAge := time.Duration(sessionprune.DefaultTmpRetentionHours) * time.Hour stop := make(chan struct{}) go func() { @@ -1498,6 +1499,7 @@ func (b *boot) wireBootClosures() { runSessionPrune(b.logger, dir, maxAge) runAgentSessionTreePrune(b.logger, agentSessionTreesRoot, maxAge) runAgentHomePrune(b.logger, agentHomesRoot, homeMaxAge) + runClaudeTmpPrune(b.logger, claudeTmpRoot, tmpMaxAge) ticker := time.NewTicker(sessionPruneInterval) defer ticker.Stop() @@ -1509,6 +1511,7 @@ func (b *boot) wireBootClosures() { runSessionPrune(b.logger, dir, maxAge) runAgentSessionTreePrune(b.logger, agentSessionTreesRoot, maxAge) runAgentHomePrune(b.logger, agentHomesRoot, homeMaxAge) + runClaudeTmpPrune(b.logger, claudeTmpRoot, tmpMaxAge) } } }() diff --git a/src/cmd/hive/sessionprunewire.go b/src/cmd/hive/sessionprunewire.go index ca8858f4cc..ff59c22983 100644 --- a/src/cmd/hive/sessionprunewire.go +++ b/src/cmd/hive/sessionprunewire.go @@ -2,6 +2,7 @@ package main import ( "log/slog" + "os" "time" "github.com/hivecommons/hive/pkg/sessionprune" @@ -22,6 +23,11 @@ const agentHomesRoot = "/data/agents" // (hivecommons/hive#13488). const agentSessionTreesRoot = "/data/home/agents" +// claudeTmpRoot is where the Claude CLI keeps per-session scratch +// (claude-//): the container's temp directory, in its +// writable layer rather than on /data. +var claudeTmpRoot = os.TempDir() + func runSessionPrune(logger *slog.Logger, dir string, maxAge time.Duration) { res, err := sessionprune.Prune(dir, maxAge, time.Now(), logger) if err != nil { @@ -79,3 +85,21 @@ func runAgentSessionTreePrune(logger *slog.Logger, root string, maxAge time.Dura "retention_days", int(maxAge.Hours()/24)) } } + +// runClaudeTmpPrune reclaims idle Claude CLI session scratch under the temp +// directory, which nothing else bounds and which fills the host's root disk. +func runClaudeTmpPrune(logger *slog.Logger, root string, maxAge time.Duration) { + res, err := sessionprune.PruneClaudeTmp(root, maxAge, time.Now(), logger) + if err != nil { + logger.Warn("claude tmp prune failed", "root", root, "error", err) + return + } + if res.Removed > 0 || res.Failed > 0 { + logger.Info("claude tmp prune complete", + "root", root, + "scanned", res.Scanned, + "removed", res.Removed, + "failed", res.Failed, + "retention_hours", int(maxAge.Hours())) + } +} diff --git a/src/docs/diskpressure-recovery-runbook.md b/src/docs/diskpressure-recovery-runbook.md index 6e6296bb9b..d2683ab414 100644 --- a/src/docs/diskpressure-recovery-runbook.md +++ b/src/docs/diskpressure-recovery-runbook.md @@ -97,6 +97,10 @@ to a network-attached class. agent homes (`agent home prune complete`, 14-day retention — `PruneAgentHomes`, added for Ask 3 of #9869). It never follows symlinks and never touches credentials, session-state, beads, or repo checkouts. + - idle **Claude CLI session scratch** under the temp directory + (`/claude-//`, in the container's writable + layer, not `/data`): each session directory whose newest write is older + than 24 hours (`claude tmp prune complete` — `PruneClaudeTmp`). Related: [#2349](https://github.com/hivecommons/hive/issues/2349) (small disks / OKE DiskPressure, same symptom on a different cloud). diff --git a/src/pkg/sessionprune/tmpprune.go b/src/pkg/sessionprune/tmpprune.go new file mode 100644 index 0000000000..54bbef4cfb --- /dev/null +++ b/src/pkg/sessionprune/tmpprune.go @@ -0,0 +1,78 @@ +package sessionprune + +// This file extends the janitor to the per-session scratch the Claude CLI +// keeps outside /data: /claude-///, which holds +// task output, scratchpads and whatever the agent cloned or built there. +// Nothing removes it when a session ends, and it lives in the container's +// writable layer, not on the data volume, so it fills the node's root disk +// instead. On one Docker host two spokes reached 17 GiB and 22 GiB in a +// fortnight and filled the disk, which took the host's other services down. + +import ( + "errors" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "time" +) + +// DefaultTmpRetentionHours bounds how long a Claude CLI session scratch +// directory is kept after its last write. A day is longer than any agent +// session runs and short enough that a busy spoke cannot fill a disk between +// passes; a resumed session only loses its scratch, never its transcript, +// which lives under the agent home. +const DefaultTmpRetentionHours = 24 + +// claudeTmpPrefix is the per-user directory the Claude CLI creates in the +// temp directory: claude-. +const claudeTmpPrefix = "claude-" + +// PruneClaudeTmp removes Claude CLI session scratch directories, +// /claude-*//, whose newest write anywhere inside +// is older than maxAge. Only that exact depth is considered, and a project or +// user directory is left in place even when it is empty. Symlinks are never +// followed, a missing root is not an error, and maxAge <= 0 disables the sweep. +func PruneClaudeTmp(tmpRoot string, maxAge time.Duration, now time.Time, logger *slog.Logger) (Result, error) { + var res Result + if tmpRoot == "" || maxAge <= 0 { + return res, nil + } + users, err := os.ReadDir(tmpRoot) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return res, nil + } + return res, err + } + cutoff := now.Add(-maxAge) + for _, user := range users { + if !user.IsDir() || !strings.HasPrefix(user.Name(), claudeTmpPrefix) { + continue + } + projects, err := os.ReadDir(filepath.Join(tmpRoot, user.Name())) + if err != nil { + logger.Debug("claude tmp prune: cannot read user dir, skipping", "dir", user.Name(), "err", err) + continue + } + for _, project := range projects { + if !project.IsDir() { + continue + } + projectDir := filepath.Join(tmpRoot, user.Name(), project.Name()) + sessions, err := os.ReadDir(projectDir) + if err != nil { + logger.Debug("claude tmp prune: cannot read project dir, skipping", "dir", projectDir, "err", err) + continue + } + for _, session := range sessions { + if !session.IsDir() { + continue + } + pruneHomeTarget(filepath.Join(projectDir, session.Name()), cutoff, &res, logger) + } + } + } + return res, nil +} diff --git a/src/pkg/sessionprune/tmpprune_test.go b/src/pkg/sessionprune/tmpprune_test.go new file mode 100644 index 0000000000..b4b5975a91 --- /dev/null +++ b/src/pkg/sessionprune/tmpprune_test.go @@ -0,0 +1,89 @@ +package sessionprune + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestPruneClaudeTmpRemovesOnlyIdleSessions(t *testing.T) { + root := t.TempDir() + now := time.Date(2026, 11, 16, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + recent := now.Add(-time.Hour) + + project := filepath.Join(root, "claude-2003", "-data-agents-scanner") + idle := filepath.Join(project, "f0a329e7-1341-4cff-82a0-3276b8fbc52c") + live := filepath.Join(project, "9c242e27-ac82-48c9-b1f2-9885910b15af") + mkTree(t, idle, old) + mkTree(t, live, old) + // A live session writes deep inside an otherwise old tree: the newest + // write anywhere inside decides, not the directory's own mtime. + mkTree(t, filepath.Join(live, "scratchpad", "repo"), recent) + + // Outside claude-*//: never touched, however old. + other := filepath.Join(root, "gos-r") + mkTree(t, other, old) + shallow := filepath.Join(root, "claude-2003", "loose-file-dir") + mkTree(t, shallow, old) + + res, err := PruneClaudeTmp(root, 24*time.Hour, now, quietLogger()) + if err != nil { + t.Fatalf("PruneClaudeTmp: %v", err) + } + if res.Removed != 1 || res.Failed != 0 { + t.Fatalf("result = %+v, want 1 removed", res) + } + if _, err := os.Stat(idle); !os.IsNotExist(err) { + t.Errorf("idle session still present: %v", err) + } + for _, keep := range []string{live, other, shallow, project} { + if _, err := os.Stat(keep); err != nil { + t.Errorf("%s removed: %v", keep, err) + } + } +} + +func TestPruneClaudeTmpNeverFollowsSymlinks(t *testing.T) { + root := t.TempDir() + now := time.Date(2026, 11, 16, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + + target := filepath.Join(t.TempDir(), "precious") + mkTree(t, target, old) + project := filepath.Join(root, "claude-1000", "-data-agents-quality") + if err := os.MkdirAll(project, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(project, "link")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(project, filepath.Join(root, "claude-9999")); err != nil { + t.Fatal(err) + } + + if _, err := PruneClaudeTmp(root, 24*time.Hour, now, quietLogger()); err != nil { + t.Fatalf("PruneClaudeTmp: %v", err) + } + if _, err := os.Stat(filepath.Join(target, "data")); err != nil { + t.Fatalf("symlink target was pruned: %v", err) + } +} + +func TestPruneClaudeTmpNoOps(t *testing.T) { + now := time.Now() + for _, tc := range []struct { + root string + maxAge time.Duration + }{ + {"", time.Hour}, + {filepath.Join(t.TempDir(), "missing"), time.Hour}, + {t.TempDir(), 0}, + } { + res, err := PruneClaudeTmp(tc.root, tc.maxAge, now, quietLogger()) + if err != nil || res != (Result{}) { + t.Errorf("PruneClaudeTmp(%q, %v) = %+v, %v; want no-op", tc.root, tc.maxAge, res, err) + } + } +}