From 61996b84630eb6f8fa7bb3283da86889c4876a08 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 7 Nov 2026 14:38:06 -0500 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20agents:=20drain=20CLI=20memory?= =?UTF-8?q?=20into=20draft=20knowledge=20entries=20before=20clearing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds agents.cli_memory.drain_to_knowledge (default false, per-agent override). Before every non-resume kick, and before clear_on_kick wipes the store, the manager lists the agent's own CLI memory stores (Claude ~/.claude/projects/*/memory, Codex $CODEX_HOME/memories*, Gemini ~/.gemini/GEMINI.md) and the hive converts each bullet or paragraph into a state: draft entry in the bead-synth vault the Primer already reads, with source: cli-memory, agent:, cli:, repo: (from the Claude project slug when it matches one of the agent's repos) and a SHA-256 memhash that makes repeated drains idempotent across states and file names. Drafts are never promoted automatically; the approved-only Primer default keeps them out of kicks until a human approves them. The parser and vault writer live in pkg/knowledge/climemory (root module); the Crux manager only collects sources and calls the drainer installed via crux.CLIMemoryDrainHost, so the Crux module imports no knowledge package. One log line per drain: cli-memory: drained N facts from into (dupes K). Proposed by @gregoryhunt at the 2026-11-05 community meeting. Fixes #17033 Signed-off-by: Andrew Anderson --- changelog.d/added-cli-memory-drain.md | 1 + src/cmd/hive/boot_cli_memory_drain.go | 87 +++++ src/cmd/hive/boot_knowledge_primer.go | 7 +- src/cmd/hive/main.go | 7 +- src/crux/agent/cli_memory_drain.go | 104 +++++ src/crux/agent/cli_memory_drain_test.go | 113 ++++++ src/crux/agent/interactive_home.go | 2 +- src/crux/agent/manager.go | 6 +- src/crux/agent/manager_kick.go | 9 +- src/crux/cli_memory_drain.go | 45 +++ src/crux/config/cli_memory.go | 16 +- src/docs/agent-configuration.md | 10 +- src/docs/agent-knowledge-and-memory.md | 47 ++- src/docs/operator-reference/README.md | 2 +- .../operator-reference/runtime-and-quota.md | 36 ++ src/internal/settingsparity/settingsparity.go | 1 + .../settingsparity/settingsparity_test.go | 1 + .../tools/settingsparity/main_test.go | 9 +- .../dashboard/settings_parity_allowlist.txt | 2 + src/pkg/knowledge/climemory/drain.go | 358 ++++++++++++++++++ src/pkg/knowledge/climemory/drain_test.go | 202 ++++++++++ src/pkg/knowledge/climemory/parse.go | 220 +++++++++++ src/pkg/knowledge/climemory/parse_test.go | 119 ++++++ .../climemory/testdata/claude-MEMORY.md | 14 + .../climemory/testdata/claude-topic.md | 9 + .../testdata/codex-memory_summary.md | 8 + .../climemory/testdata/gemini-GEMINI.md | 8 + 27 files changed, 1413 insertions(+), 30 deletions(-) create mode 100644 changelog.d/added-cli-memory-drain.md create mode 100644 src/cmd/hive/boot_cli_memory_drain.go create mode 100644 src/crux/agent/cli_memory_drain.go create mode 100644 src/crux/agent/cli_memory_drain_test.go create mode 100644 src/crux/cli_memory_drain.go create mode 100644 src/pkg/knowledge/climemory/drain.go create mode 100644 src/pkg/knowledge/climemory/drain_test.go create mode 100644 src/pkg/knowledge/climemory/parse.go create mode 100644 src/pkg/knowledge/climemory/parse_test.go create mode 100644 src/pkg/knowledge/climemory/testdata/claude-MEMORY.md create mode 100644 src/pkg/knowledge/climemory/testdata/claude-topic.md create mode 100644 src/pkg/knowledge/climemory/testdata/codex-memory_summary.md create mode 100644 src/pkg/knowledge/climemory/testdata/gemini-GEMINI.md diff --git a/changelog.d/added-cli-memory-drain.md b/changelog.d/added-cli-memory-drain.md new file mode 100644 index 0000000000..0abd573102 --- /dev/null +++ b/changelog.d/added-cli-memory-drain.md @@ -0,0 +1 @@ +- `agents.cli_memory.drain_to_knowledge` (default off) converts each agent's CLI memory (Claude `MEMORY.md` + topic files, Codex `memories*`, Gemini `GEMINI.md`) into `state: draft` knowledge entries in the bead-synth vault before every kick and before `clear_on_kick` wipes the store — tagged `cli-memory`, `agent:`, `cli:`, `repo:` when derivable, deduplicated by content hash, never auto-approved; one `cli-memory: drained N facts …` log line per drain. Proposed by @gregoryhunt (#17033). diff --git a/src/cmd/hive/boot_cli_memory_drain.go b/src/cmd/hive/boot_cli_memory_drain.go new file mode 100644 index 0000000000..c56db0fb8e --- /dev/null +++ b/src/cmd/hive/boot_cli_memory_drain.go @@ -0,0 +1,87 @@ +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/knowledge/climemory" +) + +// beadSynthVaultName is the store name the bead-synth vault is connected +// and registered under; CLI-memory drafts land in the same store. +const beadSynthVaultName = "bead-synth-wiki" + +// cliMemoryDrainEnabled reports whether any enabled agent drains its CLI +// memory into the knowledge store (agents.cli_memory.drain_to_knowledge, +// fleet-wide or per agent). +func (b *boot) cliMemoryDrainEnabled() bool { + if b.cfg == nil { + return false + } + if b.cfg.AgentsCLIMemory.DrainToKnowledge { + return true + } + for _, ac := range b.cfg.EnabledAgents() { + if ac.CLIMemory.DrainMemoryToKnowledge() { + return true + } + } + return false +} + +// installCLIMemoryDrainer connects the bead-synth vault when no bead store +// did and installs the drain hook on the in-process runtime +// (hivecommons/hive#17033). Drained facts are written as `state: draft` +// entries into that vault, so the Primer (approved-only by default) never +// primes them until an operator approves each one. It runs before +// registerKnowledgeStores so the vault is primed like any other store. +func (b *boot) installCLIMemoryDrainer() { + if !b.cliMemoryDrainEnabled() { + return + } + host, ok := b.agentMgr.(crux.CLIMemoryDrainHost) + if !ok { + b.logger.Warn("agents.cli_memory.drain_to_knowledge needs the in-process crux runtime; the sidecar runtime cannot drain CLI memory yet", + "runtime", fmt.Sprintf("%T", b.agentMgr)) + return + } + if b.knowledgeAPI == nil { + b.logger.Warn("agents.cli_memory.drain_to_knowledge is set but the knowledge API is unavailable; CLI memory will not be drained") + return + } + vaultPath := b.beadSynthVaultPath() + if err := os.MkdirAll(vaultPath, 0o755); err != nil { + b.logger.Warn("failed to create vault for CLI memory drain", "path", vaultPath, "error", err) + return + } + if b.knowledgeAPI.GetVaultStore(vaultPath) == nil { + if err := b.knowledgeAPI.ConnectVault(vaultPath, beadSynthVaultName); err != nil { + b.logger.Warn("failed to connect vault for CLI memory drain", "path", vaultPath, "error", err) + return + } + } + layer := knowledge.LayerType(b.cfg.Knowledge.BeadSynthesizer.TargetLayer) + if layer == "" { + layer = knowledge.LayerPersonal + } + drainer := &climemory.Drainer{Dir: vaultPath, Store: b.knowledgeAPI.GetVaultStore(vaultPath), Layer: layer} + cfg := b.cfg + host.SetCLIMemoryDrainer(func(req crux.CLIMemoryDrainRequest) crux.CLIMemoryDrainResult { + sources := make([]climemory.Source, 0, len(req.Sources)) + for _, s := range req.Sources { + sources = append(sources, climemory.Source{CLI: s.CLI, Path: s.Path}) + } + res := drainer.Drain(climemory.Request{Agent: req.Agent, Repos: cfg.ReposForAgent(req.Agent), Sources: sources}) + return crux.CLIMemoryDrainResult{ + Store: beadSynthVaultName, + Drained: res.Drained, + Dupes: res.Dupes, + Skipped: res.Skipped, + Err: errors.Join(res.Errors...), + } + }) + b.logger.Info("CLI memory drain installed", "store", beadSynthVaultName, "path", vaultPath, "layer", layer) +} diff --git a/src/cmd/hive/boot_knowledge_primer.go b/src/cmd/hive/boot_knowledge_primer.go index 5c2890a40f..c8434fc5db 100644 --- a/src/cmd/hive/boot_knowledge_primer.go +++ b/src/cmd/hive/boot_knowledge_primer.go @@ -39,13 +39,16 @@ func (b *boot) registerKnowledgeStores(p *knowledge.Primer) { p.AddFileStore(gsc.Name, store, knowledge.LayerType(gsc.Layer)) } } - if len(b.beadStores) > 0 { + // The bead-synth vault also receives CLI-memory drafts (#17033), so it is + // primed whenever either writer is on; the Primer's approved-only default + // keeps the drafts out of kicks. + if len(b.beadStores) > 0 || b.cliMemoryDrainEnabled() { if store := api.GetVaultStore(b.beadSynthVaultPath()); store != nil { layer := knowledge.LayerType(b.cfg.Knowledge.BeadSynthesizer.TargetLayer) if layer == "" { layer = knowledge.LayerPersonal } - p.AddFileStore("bead-synth-wiki", store, layer) + p.AddFileStore(beadSynthVaultName, store, layer) b.logger.Info("bead-synth vault registered with primer", "layer", layer) } } diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 9f3d24df71..f9155ad016 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3081,7 +3081,7 @@ func (b *boot) bootKnowledgeWith(deps bootKnowledgeDeps) { b.logger.Warn("failed to create bead-synth vault dir", "path", synthVaultPath, "error", err) } if b.knowledgeAPI != nil { - if connErr := b.knowledgeAPI.ConnectVault(synthVaultPath, "bead-synth-wiki"); connErr != nil { + if connErr := b.knowledgeAPI.ConnectVault(synthVaultPath, beadSynthVaultName); connErr != nil { b.logger.Warn("failed to auto-connect bead-synth vault", "path", synthVaultPath, "error", connErr) } else { b.logger.Info("auto-connected bead-synth vault", "path", synthVaultPath) @@ -3138,6 +3138,11 @@ func (b *boot) bootKnowledgeWith(deps bootKnowledgeDeps) { } } + // CLI memory drain (hivecommons/hive#17033); opt-in, makes sure the + // bead-synth vault it writes drafts into is connected before the primer + // registers stores below. + b.installCLIMemoryDrainer() + // Generated repository code maps (#11106); opt-in, connects its vault // before stores are registered with the primer below. b.startCodeMaps() diff --git a/src/crux/agent/cli_memory_drain.go b/src/crux/agent/cli_memory_drain.go new file mode 100644 index 0000000000..55658cfc85 --- /dev/null +++ b/src/crux/agent/cli_memory_drain.go @@ -0,0 +1,104 @@ +package agent + +import ( + "fmt" + "path/filepath" + "sync" + "sync/atomic" + + "github.com/hivecommons/hive/crux" +) + +// CLI memory drain (hivecommons/hive#17033). What a CLI wrote to its private +// memory store is converted into draft knowledge entries before the kick +// (and before clear_on_kick wipes the store), so a one-off inference the CLI +// kept for itself becomes a reviewable fact in the knowledge store instead +// of a hidden notebook. The conversion itself lives in the root module +// (pkg/knowledge/climemory) because the Crux module does not import the +// knowledge packages; the manager only collects the stores and hands them to +// the drainer the hive installs with SetCLIMemoryDrainer +// (crux.CLIMemoryDrainHost). + +// Aliases of the crux host types so the manager and cmd/hive name the same +// hook without cmd/hive importing this package. +type ( + CLIMemorySource = crux.CLIMemorySource + CLIMemoryDrainRequest = crux.CLIMemoryDrainRequest + CLIMemoryDrainResult = crux.CLIMemoryDrainResult + CLIMemoryDrainer = crux.CLIMemoryDrainer +) + +// cliMemoryState groups the manager's CLI-memory bookkeeping so the Manager +// struct carries one field for it. +type cliMemoryState struct { + warnOnce sync.Once // see warnSharedCLIMemoryOnce (hivecommons/hive#17031) + drainer atomic.Pointer[CLIMemoryDrainer] +} + +// CLI names used in CLIMemorySource; they match the climemory constants. +const ( + cliMemoryClaude = "claude" + cliMemoryCodex = "codex" + cliMemoryGemini = "gemini" +) + +// SetCLIMemoryDrainer installs (or with nil, removes) the drain hook. Safe +// to leave unset: agents with drain_to_knowledge then log once per kick that +// no drainer is available and the wipe proceeds as configured. +func (m *Manager) SetCLIMemoryDrainer(fn CLIMemoryDrainer) { + if fn == nil { + m.cliMemory.drainer.Store(nil) + return + } + m.cliMemory.drainer.Store(&fn) +} + +// cliMemoryDrainSources lists the agent's memory stores with the CLI that +// owns each. Same per-agent-home-only rule as cliMemoryStorePaths: a shared +// or bridged store is never drained under one agent's name. +func cliMemoryDrainSources(agent *AgentProcess, backend string) []CLIMemorySource { + var sources []CLIMemorySource + for _, p := range cliMemoryStorePaths(agent, backend) { + cli := cliMemoryCodex + switch { + case filepath.Base(p) == geminiMemoryFileName: + cli = cliMemoryGemini + case filepath.Base(p) == "memory": + cli = cliMemoryClaude + } + sources = append(sources, CLIMemorySource{CLI: cli, Path: p}) + } + return sources +} + +// drainCLIMemoryForKick runs the installed drainer over the agent's memory +// stores when agents.cli_memory.drain_to_knowledge is set. It is called +// before clearCLIMemoryForKick so the facts survive the wipe, and it is +// best-effort: a failed drain is logged and the kick proceeds. The one log +// line per drain is +// +// cli-memory: drained N facts from into (dupes K) +func (m *Manager) drainCLIMemoryForKick(agent *AgentProcess, backend string) { + if !agent.Config.CLIMemory.DrainMemoryToKnowledge() { + return + } + sources := cliMemoryDrainSources(agent, backend) + if len(sources) == 0 { + return + } + fn := m.cliMemory.drainer.Load() + if fn == nil { + m.logger.Warn("cli-memory: drain_to_knowledge is set but no knowledge store is available to drain into", + "agent", agent.Name, "sources", len(sources)) + return + } + res := (*fn)(CLIMemoryDrainRequest{Agent: agent.Name, Backend: backend, Sources: sources}) + if res.Err != nil { + m.logger.Warn("cli-memory: drain failed before kick", + "agent", agent.Name, "store", res.Store, "drained", res.Drained, "error", res.Err) + } + m.logger.Info(fmt.Sprintf("cli-memory: drained %d facts from %s into %s (dupes %d)", + res.Drained, agent.Name, res.Store, res.Dupes), + "agent", agent.Name, "store", res.Store, "drained", res.Drained, + "dupes", res.Dupes, "skipped", res.Skipped, "sources", len(sources)) +} diff --git a/src/crux/agent/cli_memory_drain_test.go b/src/crux/agent/cli_memory_drain_test.go new file mode 100644 index 0000000000..1492db175d --- /dev/null +++ b/src/crux/agent/cli_memory_drain_test.go @@ -0,0 +1,113 @@ +package agent + +import ( + "errors" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +func TestCLIMemoryDrainSources_ClassifiesStoresByCLI(t *testing.T) { + withSharedAgentHome(t) + prev := codexHomePrefix + codexHomePrefix = filepath.Join(t.TempDir(), ".codex-") + t.Cleanup(func() { codexHomePrefix = prev }) + + home := interactiveHomePath("scanner") + for _, p := range []string{ + filepath.Join(home, ".claude", "projects", "-data-agents-scanner", "memory", "MEMORY.md"), + filepath.Join(home, ".gemini", geminiMemoryFileName), + filepath.Join(codexHomePath("scanner"), "memories", "memory_summary.md"), + filepath.Join(codexHomePath("scanner"), "memories_v2", "index.json"), + } { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + ap := &AgentProcess{Name: "scanner", UID: 1001, Config: config.AgentConfig{Backend: "claude"}} + got := cliMemoryDrainSources(ap, "claude") + byCLI := map[string][]string{} + for _, s := range got { + byCLI[s.CLI] = append(byCLI[s.CLI], filepath.Base(s.Path)) + sort.Strings(byCLI[s.CLI]) + } + if len(got) != 4 || + strings.Join(byCLI[cliMemoryClaude], ",") != "memory" || + strings.Join(byCLI[cliMemoryGemini], ",") != geminiMemoryFileName || + strings.Join(byCLI[cliMemoryCodex], ",") != "memories,memories_v2" { + t.Fatalf("sources = %+v", got) + } +} + +func TestDrainCLIMemoryForKick_RunsDrainerBeforeWipeOnlyWhenEnabled(t *testing.T) { + withSharedAgentHome(t) + prev := codexHomePrefix + codexHomePrefix = filepath.Join(t.TempDir(), ".codex-") + t.Cleanup(func() { codexHomePrefix = prev }) + + m := interactiveHomeTestManager(t) + home := interactiveHomePath("scanner") + mem := filepath.Join(home, ".claude", "projects", "-data-agents-scanner", "memory", "MEMORY.md") + if err := os.MkdirAll(filepath.Dir(mem), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mem, []byte("- a fact\n"), 0o644); err != nil { + t.Fatal(err) + } + + var calls []CLIMemoryDrainRequest + var sawFileAtDrain bool + m.SetCLIMemoryDrainer(func(req CLIMemoryDrainRequest) CLIMemoryDrainResult { + calls = append(calls, req) + _, err := os.Stat(mem) + sawFileAtDrain = err == nil + return CLIMemoryDrainResult{Store: "/vault", Drained: 1} + }) + + off := &AgentProcess{Name: "scanner", UID: 1001, Config: config.AgentConfig{ + Backend: "claude", CLIMemory: &config.CLIMemoryConfig{ClearOnKick: true}, + }} + m.drainCLIMemoryForKick(off, "claude") + if len(calls) != 0 { + t.Fatalf("drainer ran with drain_to_knowledge off: %+v", calls) + } + + on := &AgentProcess{Name: "scanner", UID: 1001, Config: config.AgentConfig{ + Backend: "claude", CLIMemory: &config.CLIMemoryConfig{ClearOnKick: true, DrainToKnowledge: true}, + }} + // Same order as deliverKickLocked: drain, then wipe. + m.drainCLIMemoryForKick(on, "claude") + m.clearCLIMemoryForKick(on, "claude") + if len(calls) != 1 || calls[0].Agent != "scanner" || calls[0].Backend != "claude" || len(calls[0].Sources) != 1 { + t.Fatalf("calls = %+v", calls) + } + if calls[0].Sources[0].CLI != cliMemoryClaude || filepath.Base(calls[0].Sources[0].Path) != "memory" { + t.Fatalf("source = %+v", calls[0].Sources[0]) + } + if !sawFileAtDrain { + t.Fatal("memory file was already gone when the drainer ran") + } + if _, err := os.Stat(mem); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("memory not wiped after drain: %v", err) + } + + // Without an installed drainer the kick path must still be safe. + m.SetCLIMemoryDrainer(nil) + if err := os.MkdirAll(filepath.Dir(mem), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mem, []byte("- again\n"), 0o644); err != nil { + t.Fatal(err) + } + m.drainCLIMemoryForKick(on, "claude") + if len(calls) != 1 { + t.Fatalf("removed drainer still invoked: %d calls", len(calls)) + } +} diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index 4c4a15c84a..02e5974e1e 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -460,7 +460,7 @@ func (m *Manager) warnSharedCLIMemoryOnce(spec perAgentCLIHome, shared string) { if err != nil || len(matches) == 0 { continue } - m.sharedCLIMemoryWarnOnce.Do(func() { + m.cliMemory.warnOnce.Do(func() { m.logger.Warn("shared CLI home still holds a persistent memory store that was shared by every agent; it is no longer read by per-agent homes and was left in place for review", "dir", shared, "memory", matches, "see", "hivecommons/hive#17031") }) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index f5d5043405..ff4bc4d705 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -547,9 +547,9 @@ type Manager struct { // rateLimitRelaunchMu guards lastRateLimitRelaunch, the fleet-wide clock // that spaces rate-limit relaunches (rateLimitRelaunchFleetSpacing). Its // own mutex: the pane pollers that consult it must not take m.mu. - rateLimitRelaunchMu sync.Mutex - lastRateLimitRelaunch time.Time - sharedCLIMemoryWarnOnce sync.Once // see warnSharedCLIMemoryOnce (hivecommons/hive#17031) + rateLimitRelaunchMu sync.Mutex + lastRateLimitRelaunch time.Time + cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go // thrashMu guards thrash — its own mutex, NEVER m.mu: the breaker runs on // the output-capture goroutines, and taking m.mu there risks startup diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index ef44072c2e..e213d8ea16 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -293,10 +293,13 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string time.Sleep(staleCheckDelay) } - // Persistent CLI memory (hivecommons/hive#17032) is wiped before the - // conversation /clear below, and skipped on the same resume kicks: a - // resumed session is continuing prior work, not starting a fresh turn. + // Persistent CLI memory is drained into draft knowledge entries + // (hivecommons/hive#17033) and then wiped (hivecommons/hive#17032) before + // the conversation /clear below, and both are skipped on the same resume + // kicks: a resumed session is continuing prior work, not starting a + // fresh turn. if !skipClear { + m.drainCLIMemoryForKick(agent, backend) m.clearCLIMemoryForKick(agent, backend) } if agent.Config.ClearOnKick && !skipClear { diff --git a/src/crux/cli_memory_drain.go b/src/crux/cli_memory_drain.go new file mode 100644 index 0000000000..00d69fb5a4 --- /dev/null +++ b/src/crux/cli_memory_drain.go @@ -0,0 +1,45 @@ +package crux + +// CLI memory drain (hivecommons/hive#17033). The runtime hands the host the +// per-agent CLI memory stores before a kick wipes them; the host converts +// them into draft knowledge entries. Defined here, like the other Host +// capabilities, so cmd/hive can install the hook without importing +// crux/agent; crux/agent aliases these types. + +// CLIMemorySource is one memory store to drain: the CLI that wrote it +// ("claude", "codex", "gemini") and the file or directory it lives in. +type CLIMemorySource struct { + CLI string + Path string +} + +// CLIMemoryDrainRequest names the agent whose stores are being drained. +type CLIMemoryDrainRequest struct { + Agent string + Backend string + Sources []CLIMemorySource +} + +// CLIMemoryDrainResult is what the drainer reports back for the runtime's +// one log line per drain. +type CLIMemoryDrainResult struct { + // Store is the knowledge store (vault path or name) the drafts landed in. + Store string + Drained int + Dupes int + Skipped int + Err error +} + +// CLIMemoryDrainer converts the stores in req into draft knowledge entries. +// It runs synchronously on the kick path, before the memory wipe, so it +// must be bounded. +type CLIMemoryDrainer func(req CLIMemoryDrainRequest) CLIMemoryDrainResult + +// CLIMemoryDrainHost is the optional Host capability the in-process runtime +// implements. It is not part of Host: a sidecar runtime cannot call a Go +// closure synchronously on its kick path, so the hive installs the drainer +// only when the runtime it built satisfies this interface. +type CLIMemoryDrainHost interface { + SetCLIMemoryDrainer(fn CLIMemoryDrainer) +} diff --git a/src/crux/config/cli_memory.go b/src/crux/config/cli_memory.go index 79ff1547ee..1a67264747 100644 --- a/src/crux/config/cli_memory.go +++ b/src/crux/config/cli_memory.go @@ -24,12 +24,19 @@ type CLIMemoryConfig struct { // kick, alongside the /clear that resets the conversation, so each turn // starts from the kick template and the shared knowledge system only. ClearOnKick bool `yaml:"clear_on_kick,omitempty" json:"clear_on_kick,omitempty"` + // DrainToKnowledge converts what the CLI wrote to its memory store into + // draft knowledge entries before each kick (and before ClearOnKick wipes + // the store), so an inference the CLI kept for itself becomes a + // reviewable, scoped fact instead of a hidden per-agent notebook + // (hivecommons/hive#17033). Entries are deduplicated by content hash and + // never promoted automatically. + DrainToKnowledge bool `yaml:"drain_to_knowledge,omitempty" json:"drain_to_knowledge,omitempty"` } // IsZero reports whether no field was set, so an absent block encodes as // nothing and a per-agent nil pointer inherits the fleet value. func (c CLIMemoryConfig) IsZero() bool { - return c.AutoMemory == nil && !c.ClearOnKick + return c.AutoMemory == nil && !c.ClearOnKick && !c.DrainToKnowledge } // AutoMemoryEnabled reports whether the CLI may keep persistent memory. The @@ -44,3 +51,10 @@ func (c *CLIMemoryConfig) AutoMemoryEnabled() bool { func (c *CLIMemoryConfig) ClearMemoryOnKick() bool { return c != nil && c.ClearOnKick } + +// DrainMemoryToKnowledge reports whether the memory store is drained into +// draft knowledge entries before each kick. The receiver may be nil: the +// default is off. +func (c *CLIMemoryConfig) DrainMemoryToKnowledge() bool { + return c != nil && c.DrainToKnowledge +} diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 758d19a3b9..f6e5596837 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -320,6 +320,7 @@ spec omits `launch_cmd`, Hive still builds the backend command normally. cli_memory: # optional override of the fleet agents.cli_memory auto_memory: false # policy: CLI auto-memory (Claude MEMORY.md, Codex clear_on_kick: false # memories, Copilot memory) off; wipe before each kick + drain_to_knowledge: false # first turn each memory fact into a draft knowledge entry restart_strategy: immediate # how to bring a dead session back beads_dir: /data/beads/scanner # work-record (bead) storage; default per-agent replicas: 3 # materialize scanner, scanner-2, scanner-3 (max 5) @@ -638,8 +639,9 @@ Subscription CLIs keep their own long-lived "memory" next to the transcripts: Cl ```yaml agents: cli_memory: - auto_memory: false # default false: pin every CLI's auto-memory feature off - clear_on_kick: false # default false: true wipes this agent's memory stores before each kick + auto_memory: false # default false: pin every CLI's auto-memory feature off + clear_on_kick: false # default false: true wipes this agent's memory stores before each kick + drain_to_knowledge: false # default false: true drains each memory fact into a draft knowledge entry first ``` | Key | Default | Effect | @@ -647,7 +649,9 @@ agents: | `auto_memory` | `false` | When false hive pins the vendor switch off at launch: Claude Code gets `CLAUDE_CODE_DISABLE_AUTO_MEMORY=1` in its environment ([docs](https://code.claude.com/docs/en/memory)); Codex gets `[features] memories = false` merged into `$CODEX_HOME/config.toml` ([docs](https://developers.openai.com/codex/memories)); Copilot CLI gets `"memory": false` in its per-agent `~/.copilot/settings.json` (`copilot help config`). `true` writes the opposite pin so an operator's own setting is never silently shadowed. Gemini CLI has no equivalent switch. | | `clear_on_kick` | `false` | Before every non-resume kick (next to the `/clear` sent by `clear_on_kick`), remove this agent's `~/.claude/projects/*/memory/`, `$CODEX_HOME/memories*/`, and `~/.gemini/GEMINI.md`. Only the per-agent home under `/data/home/agents/` is touched; nothing under the shared `/data/home` is removed, and agents running with `HIVE_SHARED_AGENT_HOME=1` or as uid 0 are skipped. Copilot memory is server-side and has no local store. | -Set `auto_memory: true` on a single agent to let it keep a private notebook while the rest of the fleet stays stateless between kicks. The two knobs are independent: `auto_memory: true` with `clear_on_kick: true` gives a memory that lives for exactly one kick. +| `drain_to_knowledge` | `false` | Before every non-resume kick, and before `clear_on_kick` wipes anything, read this agent's Claude `~/.claude/projects/*/memory/` (`MEMORY.md` plus topic files), Codex `$CODEX_HOME/memories*/` and the `## Gemini Added Memories` section of `~/.gemini/GEMINI.md`, and write each bullet or paragraph as a **draft** entry in the `bead-synth-wiki` vault (`knowledge.bead_synthesizer.vault_path`, default `/data/vaults/bead-synth-wiki`) with `source: cli-memory`, `agent: `, `cli: ` and, when the Claude project slug ends in one of the agent's repos, `repo: `. Entries are deduplicated by a SHA-256 of the normalised text (`memhash`), so repeated drains and the same fact from several agents produce one entry; an entry an operator approved, deprecated or renamed is still a duplicate. Drafts never reach a kick until a human approves them ([#17033](https://github.com/hivecommons/hive/issues/17033)). Needs the in-process crux runtime. | + +Set `auto_memory: true` on a single agent to let it keep a private notebook while the rest of the fleet stays stateless between kicks. The knobs are independent: `auto_memory: true` with `clear_on_kick: true` gives a memory that lives for exactly one kick, and adding `drain_to_knowledge: true` is the "memory → beads" flow: the CLI may take notes during a turn, every note becomes a reviewable draft at the next kick, and the private store is then emptied. The log line per drain is `cli-memory: drained N facts from into bead-synth-wiki (dupes K)`. Every Model Gateway (and the bob backend) also accepts an optional `key_name` — a human-chosen LABEL for the configured key, e.g. `key_name: openrouter-prod-key`. It is safe-to-show metadata, not a secret: the dashboard's gateway row displays it as "Using key: ``", or "(unnamed)" when no label is set, so operators can tell keys apart without ever seeing the value. For gateway keys, the row and settings/status APIs also expose `keySHA256`, the lowercase SHA-256 of the exact effective key string Hive will inject through the inference proxy; this lets operators compare Hive with LiteLLM/OpenRouter key-hash displays without revealing the secret. Saving a replacement key refreshes live inference proxy routes for running agents, so the next kick uses the new key without an agent restart. See [`inference-backends.md`](../../docs/inference-backends.md) for a full YAML example. diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md index 7528d9d3f7..eddf4494d8 100644 --- a/src/docs/agent-knowledge-and-memory.md +++ b/src/docs/agent-knowledge-and-memory.md @@ -175,10 +175,33 @@ in `$CODEX_HOME/config.toml`, Copilot CLI gets `"memory": false` in its per-agent `~/.copilot/settings.json`; Gemini has no equivalent switch. `clear_on_kick` (default false) removes this agent's `~/.claude/projects/*/memory/`, `$CODEX_HOME/memories*/` and -`~/.gemini/GEMINI.md` before each non-resume kick, next to the `/clear`. Both -keys can be overridden per agent. With both off, what each CLI remembers it -remembers across kicks and pod restarts — but, since #17031, not across -agents. See [agent-configuration.md](agent-configuration.md#cli-auto-memory-agentscli_memory). +`~/.gemini/GEMINI.md` before each non-resume kick, next to the `/clear`. All +keys can be overridden per agent. With everything off, what each CLI +remembers it remembers across kicks and pod restarts — but, since #17031, not +across agents. See [agent-configuration.md](agent-configuration.md#cli-auto-memory-agentscli_memory). + +**Memory → knowledge drain** (`agents.cli_memory.drain_to_knowledge`, +`src/crux/agent/cli_memory_drain.go`, `src/pkg/knowledge/climemory`, +[#17033](https://github.com/hivecommons/hive/issues/17033)). Default false. +When on, `deliverKickLocked` runs the drain before the wipe on every +non-resume kick: the manager lists the agent's own stores (never a shared or +symlinked one) and the hive converts them — Claude `MEMORY.md` plus its topic +files, every `*.md` under Codex `memories*/`, the `## Gemini Added Memories` +section of `GEMINI.md` — one bullet or paragraph per entry, into the +`bead-synth-wiki` vault that the Primer already reads +(`registerKnowledgeStores`, `src/cmd/hive/boot_knowledge_primer.go`). Each +entry is written with `state: draft`, `source: cli-memory`, `agent: `, +`cli: `, `repo: ` when the Claude project slug +ends in one of the agent's repos, and a `memhash` (SHA-256 of the normalised +text) that makes the drain idempotent: a fact already in the vault, in any +state and under any file name, counts as a duplicate. Because the Primer is +approved-only by default (section 1), a drained fact is **not** injected into +anyone's kick until an operator approves it through the normal lifecycle — +that review step is the point: the CLI's one-off inference becomes a +candidate, not a rule. One log line per drain: +`cli-memory: drained N facts from into bead-synth-wiki (dupes K)`. +The drain needs the in-process crux runtime; the sidecar runtime logs a +warning and only wipes. ### What persists across what @@ -190,10 +213,10 @@ not visible. |---|---|---|---|---| | **Knowledge** (`# Relevant Knowledge`, AGENTS.md, skills) | re-selected for *this* kick's issues | n/a — re-injected next kick | ✓ vaults on `/data` | ✓ same stores, filtered by its own `agent_scopes` | | **Conversation** — all CLIs | ✗ (`/clear` first, unless `clear_on_kick: false` or a resume kick) | ✗ | ✗ (CLI relaunched) | ✗ | -| **Claude Code memory** (`~/.claude/projects/-data-agents-/memory/`, `~/.claude/CLAUDE.md`, `history.jsonl`) | ✓ unless `cli_memory.clear_on_kick` (and off by default: `auto_memory: false`) | ✓ | ✓ (`/data/home/agents//.claude` is persistent) | ✗ per-agent dir (#17031); only `.credentials.json` / `settings.json` shared. Before #17031 the whole tree was one bridged copy any agent could read or rewrite | -| **Codex memory** (`$CODEX_HOME`) | ✓ unless `cli_memory.clear_on_kick` (off by default) | ✓ | ✓ (`/data/home/.codex-`) | ✗ per-agent dir; only `auth.json` shared | +| **Claude Code memory** (`~/.claude/projects/-data-agents-/memory/`, `~/.claude/CLAUDE.md`, `history.jsonl`) | ✓ unless `cli_memory.clear_on_kick` (and off by default: `auto_memory: false`); with `drain_to_knowledge` each fact lands as a draft knowledge entry first | ✓ | ✓ (`/data/home/agents//.claude` is persistent) | ✗ per-agent dir (#17031); only `.credentials.json` / `settings.json` shared. Before #17031 the whole tree was one bridged copy any agent could read or rewrite | +| **Codex memory** (`$CODEX_HOME`) | ✓ unless `cli_memory.clear_on_kick` (off by default); drained first with `drain_to_knowledge` | ✓ | ✓ (`/data/home/.codex-`) | ✗ per-agent dir; only `auth.json` shared | | **Copilot CLI memory/sessions** (`~/.copilot`) | ✓ | ✓ | ✓ | ✗ per-agent dir (#9444); only `config.json` shared | -| **Gemini memory** (`~/.gemini/GEMINI.md`, state) | ✓ unless `cli_memory.clear_on_kick` | ✓ | ✓ | ✗ per-agent dir (#17031); only sign-in files, `settings.json`, `antigravity-cli/` shared | +| **Gemini memory** (`~/.gemini/GEMINI.md`, state) | ✓ unless `cli_memory.clear_on_kick`; drained first with `drain_to_knowledge` | ✓ | ✓ | ✗ per-agent dir (#17031); only sign-in files, `settings.json`, `antigravity-cli/` shared | | **Bob memory/settings** (`~/.bob`) | ✓ | ✓ | ✓ | ✗ per-agent dir (#17031); only `settings.json` / `settings/` shared | The dashboard's cost accounting reads `data.claude_sessions_dir` (default @@ -216,10 +239,12 @@ Do not treat the following as current behaviour; follow the links. - [#17032](https://github.com/hivecommons/hive/issues/17032) — done: `agents.cli_memory.auto_memory` (default off) and `agents.cli_memory.clear_on_kick`, described above. -- [#17033](https://github.com/hivecommons/hive/issues/17033) — drain CLI - auto-memory into beads/knowledge at session end instead of a hidden - `MEMORY.md` (revives open question 5 of - [design/knowledge-system.md](design/knowledge-system.md)). +- [#17033](https://github.com/hivecommons/hive/issues/17033) — done for the + kick-time drain: `agents.cli_memory.drain_to_knowledge`, described above. + Still open: a Claude Code `SessionEnd` hook that drains without waiting + for the next kick, and a one-shot drain of a pre-#17031 shared store + (sub-issues [#17057](https://github.com/hivecommons/hive/issues/17057), + [#17058](https://github.com/hivecommons/hive/issues/17058)). - Repo `AGENTS.md` injection still requires `project.checkouts_dir`; nearest- file (`ParseNearest`) targeting is deferred ([agents-md.md](agents-md.md)). diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 2e7b230f71..440d3a40d3 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -229,7 +229,7 @@ The keys the agent runtime owns — the part of Hive being carved out as **Crux* | Struct | Embedded in | YAML keys (unchanged position) | |---|---|---| -| `AgentRuntimeConfig` (`pkg/config/agent_config.go`) | `Config` | `agents` (including the `agents.github_api_hourly_cap` scalar and the `agents.session_recycle` block), `backends`, `providers`, `tool_approval`, `agent_sandbox`, `turn` | +| `AgentRuntimeConfig` (`pkg/config/agent_config.go`) | `Config` | `agents` (including the `agents.github_api_hourly_cap` scalar and the `agents.session_recycle` and `agents.cli_memory` blocks), `backends`, `providers`, `tool_approval`, `agent_sandbox`, `turn` | | `AgentRuntimeGovernorConfig` (`pkg/config/agent_config.go`) | `GovernorConfig` | `governor.watchdog`, `governor.litellm`, `governor.vllm`, `governor.llm-d`, `governor.bob`, `governor.trajectory`, `governor.kick_limits`, `governor.gateways`, `governor.rotation`, `governor.provider_budget` | Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through field promotion; only composite literals name the embedded struct (`config.Config{AgentRuntimeConfig: config.AgentRuntimeConfig{Agents: …}}`). `pkg/config/testdata/agent_runtime_golden.{yaml,json}` pins the parse of a representative file so the boundary cannot drift the schema. diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index fb7109f510..06dd942a1d 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -79,6 +79,42 @@ The sampled value is shown as **Session RSS** (with the session age and a `↻N` recycle count) in the agent detail panel. The log line is `agent session recycled` with `rss_mb`, `age` and `reason`. +## Agent CLI memory + +Each subscription CLI keeps a private cross-session memory in the agent's +home (Claude Code `~/.claude/projects/*/memory/`, Codex `$CODEX_HOME/memories*/`, +Gemini `~/.gemini/GEMINI.md`; Copilot memory is server-side). The +`agents.cli_memory` block, overridable per agent, decides what happens to it; +the full table is in +[agent-configuration.md](../agent-configuration.md#cli-auto-memory-agentscli_memory). + +```yaml +agents: + cli_memory: # fleet default; each key may be overridden per agent + auto_memory: false # pin the vendor auto-memory switch off (default) + clear_on_kick: false # wipe the agent's memory stores before every non-resume kick + drain_to_knowledge: false # first convert each memory fact into a draft knowledge entry + scanner: + cli_memory: + auto_memory: true # this agent may take notes… + drain_to_knowledge: true # …which become drafts in bead-synth-wiki at the next kick + clear_on_kick: true # …and the private store is then emptied +``` + +| Key | Default | Effect | +|---|---|---| +| `auto_memory` | `false` | Pins Claude Code (`CLAUDE_CODE_DISABLE_AUTO_MEMORY=1`), Codex (`[features] memories = false`) and Copilot CLI (`"memory": false`) auto-memory off at launch; `true` pins it on. | +| `clear_on_kick` | `false` | Removes the agent's own memory stores before each non-resume kick; shared homes (`HIVE_SHARED_AGENT_HOME=1`, uid 0) are never touched. | +| `drain_to_knowledge` | `false` | Before the wipe, writes every memory bullet or paragraph as a `state: draft` entry into the `bead-synth-wiki` vault (`knowledge.bead_synthesizer.vault_path`), tagged `cli-memory`, `agent:`, `cli:` and `repo:` when derivable, deduplicated by content hash. Drafts are excluded from kicks until approved. | + +The drain logs one line per kick that found a store: +`cli-memory: drained N facts from into bead-synth-wiki (dupes K)` +with `drained`, `dupes`, `skipped` (over the 200-facts-per-drain cap; the +rest drain on the next kick) and `sources`. Review drafts in the dashboard's +knowledge view or the vault directory and approve them through the normal +lifecycle (`PUT /api/knowledge/entry/{id}/state`); nothing promotes them +automatically. + ## GitHub API quota Agents reach `api.github.com` through the Hive GitHub proxy, which injects the diff --git a/src/internal/settingsparity/settingsparity.go b/src/internal/settingsparity/settingsparity.go index b148e2361b..47ea4441d4 100644 --- a/src/internal/settingsparity/settingsparity.go +++ b/src/internal/settingsparity/settingsparity.go @@ -151,6 +151,7 @@ func ConfigLeaves(repoRoot string) ([]Leaf, error) { Leaf{Path: "agents.session_recycle.min_idle", Type: "time.Duration"}, Leaf{Path: "agents.cli_memory.auto_memory", Type: "*bool"}, Leaf{Path: "agents.cli_memory.clear_on_kick", Type: "bool"}, + Leaf{Path: "agents.cli_memory.drain_to_knowledge", Type: "bool"}, ) byPath := map[string]Leaf{} for _, leaf := range leaves { diff --git a/src/internal/settingsparity/settingsparity_test.go b/src/internal/settingsparity/settingsparity_test.go index d496470b85..798dba51fa 100644 --- a/src/internal/settingsparity/settingsparity_test.go +++ b/src/internal/settingsparity/settingsparity_test.go @@ -99,6 +99,7 @@ func TestConfigLeaves(t *testing.T) { {Path: "agents..model", Type: "string"}, {Path: "agents.cli_memory.auto_memory", Type: "*bool"}, {Path: "agents.cli_memory.clear_on_kick", Type: "bool"}, + {Path: "agents.cli_memory.drain_to_knowledge", Type: "bool"}, {Path: "agents.github_api_hourly_cap", Type: "int"}, {Path: "agents.session_recycle.enabled", Type: "*bool"}, {Path: "agents.session_recycle.max_age", Type: "time.Duration"}, diff --git a/src/internal/tools/settingsparity/main_test.go b/src/internal/tools/settingsparity/main_test.go index 56a41ed805..c182f9ebe7 100644 --- a/src/internal/tools/settingsparity/main_test.go +++ b/src/internal/tools/settingsparity/main_test.go @@ -69,6 +69,7 @@ func TestRunMarkdown(t *testing.T) { "| --- | --- | --- | --- | --- | --- |", "| `agents.cli_memory.auto_memory` | `*bool` | | none | | |", "| `agents.cli_memory.clear_on_kick` | `bool` | | none | | |", + "| `agents.cli_memory.drain_to_knowledge` | `bool` | | none | | |", "| `agents.github_api_hourly_cap` | `int` | | none | | may require restart or reconnect |", "| `agents.session_recycle.enabled` | `*bool` | | none | | |", "| `agents.session_recycle.max_age` | `time.Duration` | | none | | |", @@ -95,12 +96,12 @@ func TestRunJSON(t *testing.T) { if err := json.Unmarshal(stdout.Bytes(), &rows); err != nil { t.Fatalf("decode JSON output: %v\n%s", err, stdout.String()) } - if len(rows) != 11 { - t.Fatalf("got %d rows, want 11: %+v", len(rows), rows) + if len(rows) != 12 { + t.Fatalf("got %d rows, want 12: %+v", len(rows), rows) } want := row{Key: "hub.url", Type: "string", Default: "https://hub.example", UI: "read/write", Where: "Settings dialog reference", Notes: "may require restart or reconnect"} - if rows[8] != want { - t.Fatalf("rows[8] = %+v, want %+v", rows[8], want) + if rows[9] != want { + t.Fatalf("rows[9] = %+v, want %+v", rows[9], want) } } diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index ed9f7dfae7..c9042441a4 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -121,6 +121,7 @@ agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings agents..sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..cli_memory.auto_memory # TODO #11854 agents..cli_memory.clear_on_kick # TODO #11854 +agents..cli_memory.drain_to_knowledge # TODO #11854 agents..session_recycle.enabled # TODO #11854 agents..session_recycle.max_age # TODO #11854 agents..session_recycle.max_rss_mb # TODO #11854 @@ -151,6 +152,7 @@ agents..variables..url # TODO #11854 agents..variables..value # TODO #11854 agents.cli_memory.auto_memory # TODO #11854 agents.cli_memory.clear_on_kick # TODO #11854 +agents.cli_memory.drain_to_knowledge # TODO #11854 agents.github_api_hourly_cap # TODO #11854 agents.session_recycle.enabled # TODO #11854 agents.session_recycle.max_age # TODO #11854 diff --git a/src/pkg/knowledge/climemory/drain.go b/src/pkg/knowledge/climemory/drain.go new file mode 100644 index 0000000000..4fb9245d6e --- /dev/null +++ b/src/pkg/knowledge/climemory/drain.go @@ -0,0 +1,358 @@ +package climemory + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/knowledge" +) + +const ( + // SourceTag marks every drained entry; `source: cli-memory` is also + // written as frontmatter so the dashboard filter and the tag agree. + SourceTag = "cli-memory" + // HashKey is the frontmatter key (and tag prefix) carrying the SHA-256 of + // the normalised fact body; it is what makes repeated drains idempotent. + HashKey = "memhash" + // filePrefix + the first hashFilenameHex hex digits of the hash names the + // entry file, so the same fact drained twice targets the same path. + filePrefix = "cli-memory-" + hashFilenameHex = 16 + // maxMemoryFileBytes skips runaway memory files; a CLI note that large is + // a transcript, not a fact. + maxMemoryFileBytes = 512 * 1024 + // maxFactsPerDrain caps one drain so a pathological memory store cannot + // flood the vault; the rest is drained on the next kick (the memory file + // is not touched by this package). + maxFactsPerDrain = 200 + // frontmatterScanBytes bounds how much of an existing entry is read when + // collecting the hashes already in the vault. + frontmatterScanBytes = 4 * 1024 +) + +// Source is one memory store to drain: a file (GEMINI.md, MEMORY.md) or a +// directory walked for *.md (a Claude memory dir, a Codex memories* dir). +type Source struct { + CLI string + Path string +} + +// Request describes one drain. Repos are the "org/repo" names the agent +// serves; a Claude project slug that ends in one of them stamps that repo on +// the entries drained from it. +type Request struct { + Agent string + Repos []string + Sources []Source +} + +// Result reports what the drain did. Drained counts entries written, Dupes +// the facts whose hash was already in the vault (or repeated within this +// drain), Skipped the facts over the per-drain cap. +type Result struct { + Store string + Drained int + Dupes int + Skipped int + Errors []error +} + +// Drainer writes draft entries into Dir. Store, when set, is reindexed after +// a drain that wrote something so the Primer and the dashboard see the new +// drafts without waiting for the index refresh. Layer defaults to personal. +type Drainer struct { + Dir string + Store *knowledge.FileStore + Layer knowledge.LayerType + Now func() time.Time +} + +// Drain reads every source, converts its facts to draft entries and writes +// the ones whose hash is not yet in the vault. Entries are always written +// with `state: draft`; nothing in this package can produce another state. +func (d *Drainer) Drain(req Request) Result { + res := Result{Store: d.Dir} + if d == nil || d.Dir == "" { + res.Errors = append(res.Errors, errors.New("cli-memory drain: no target directory")) + return res + } + if err := os.MkdirAll(d.Dir, 0o755); err != nil { + res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: creating %s: %w", d.Dir, err)) + return res + } + seen, err := existingHashes(d.Dir) + if err != nil { + res.Errors = append(res.Errors, err) + return res + } + now := time.Now + if d.Now != nil { + now = d.Now + } + layer := d.Layer + if layer == "" { + layer = knowledge.LayerPersonal + } + for _, src := range req.Sources { + files, err := memoryFiles(src.Path) + if err != nil { + res.Errors = append(res.Errors, err) + continue + } + repo := repoFromClaudeSlug(src.Path, req.Repos) + for _, file := range files { + content, err := os.ReadFile(file) + if err != nil { + res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: reading %s: %w", file, err)) + continue + } + for _, fact := range ParseFor(src.CLI, string(content)) { + h := Hash(fact.Body) + if seen[h] { + res.Dupes++ + continue + } + if res.Drained >= maxFactsPerDrain { + res.Skipped++ + continue + } + entry := renderEntry(fact, h, entryMeta{ + agent: req.Agent, cli: src.CLI, repo: repo, path: file, layer: layer, at: now().UTC(), + }) + if err := writeAtomic(filepath.Join(d.Dir, EntryFilename(h)), entry); err != nil { + res.Errors = append(res.Errors, err) + continue + } + seen[h] = true + res.Drained++ + } + } + } + if res.Drained > 0 && d.Store != nil { + d.Store.Reindex() + } + return res +} + +// Hash is the SHA-256 of the normalised fact body, hex encoded. +func Hash(body string) string { + sum := sha256.Sum256([]byte(normalise(body))) + return hex.EncodeToString(sum[:]) +} + +// EntryFilename is the vault file name for a fact with the given hash. +func EntryFilename(hash string) string { + if len(hash) > hashFilenameHex { + hash = hash[:hashFilenameHex] + } + return filePrefix + hash + ".md" +} + +type entryMeta struct { + agent, cli, repo, path string + layer knowledge.LayerType + at time.Time +} + +// renderEntry writes the same hand-rolled frontmatter the vault writers use +// (FileStore.WriteFacts, BeadSynthesizer.writeFactToVault) so the entry +// indexes like any other fact, plus the drain provenance keys. +func renderEntry(f Fact, hash string, m entryMeta) string { + tags := []string{SourceTag, "cli:" + m.cli, HashKey + ":" + hash} + if m.agent != "" { + tags = append(tags, "agent:"+m.agent) + } + if m.repo != "" { + tags = append(tags, "repo:"+m.repo) + } + var b strings.Builder + b.WriteString("---\n") + fmt.Fprintf(&b, "title: %s\n", scalar(f.Title())) + fmt.Fprintf(&b, "type: %s\n", knowledge.FactReference) + fmt.Fprintf(&b, "layer: %s\n", scalar(string(m.layer))) + fmt.Fprintf(&b, "state: %s\n", knowledge.StateDraft) + fmt.Fprintf(&b, "source: %s\n", SourceTag) + fmt.Fprintf(&b, "cli: %s\n", scalar(m.cli)) + if m.agent != "" { + fmt.Fprintf(&b, "agent: %s\n", scalar(m.agent)) + } + if m.repo != "" { + fmt.Fprintf(&b, "repo: %s\n", scalar(m.repo)) + } + fmt.Fprintf(&b, "source_path: %s\n", scalar(m.path)) + fmt.Fprintf(&b, "%s: %s\n", HashKey, hash) + fmt.Fprintf(&b, "tags: [%s]\n", list(tags)) + fmt.Fprintf(&b, "drained: %s\n", m.at.Format(time.RFC3339)) + b.WriteString("---\n\n") + b.WriteString(f.Body) + b.WriteString("\n") + return b.String() +} + +// scalar makes s safe as a one-line frontmatter value: the vault parser +// ends the block at the first "\n---" and matches keys by line prefix, so a +// newline inside a value could forge keys (#7688). +func scalar(s string) string { + s = strings.ReplaceAll(s, "\r", "") + s = strings.ReplaceAll(s, "\n", " ") + return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) +} + +func list(items []string) string { + out := make([]string, 0, len(items)) + for _, it := range items { + if v := strings.TrimSpace(strings.ReplaceAll(scalar(it), ",", " ")); v != "" { + out = append(out, v) + } + } + return strings.Join(out, ", ") +} + +// existingHashes collects every memhash already recorded in dir, whatever +// the entry's current state or file name, so an entry an operator approved, +// deprecated or renamed is still a duplicate and is never re-drafted. +func existingHashes(dir string) (map[string]bool, error) { + seen := map[string]bool{} + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("cli-memory drain: listing %s: %w", dir, err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") { + continue + } + if h := hashFromFrontmatter(filepath.Join(dir, e.Name())); h != "" { + seen[h] = true + } + } + return seen, nil +} + +func hashFromFrontmatter(path string) string { + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + sc := bufio.NewScanner(bufio.NewReaderSize(f, frontmatterScanBytes)) + first := true + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if first { + first = false + if line != "---" { + return "" + } + continue + } + if line == "---" { + return "" + } + if v, ok := strings.CutPrefix(line, HashKey+":"); ok { + return strings.Trim(strings.TrimSpace(v), "\"'") + } + } + return "" +} + +// memoryFiles lists the markdown files under path (or path itself when it +// is a file), skipping temp files, hidden entries, symlinks and oversized +// files. A missing path is not an error: the CLI simply wrote nothing. +func memoryFiles(path string) ([]string, error) { + info, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("cli-memory drain: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + // A symlinked store points outside the agent's own home (a shared + // or bridged tree); draining it would attribute another agent's + // memory to this one. + return nil, nil + } + if !info.IsDir() { + if info.Size() > maxMemoryFileBytes || !strings.HasSuffix(path, ".md") { + return nil, nil + } + return []string{path}, nil + } + var files []string + walkErr := filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + name := d.Name() + if p != path && strings.HasPrefix(name, ".") { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() || !strings.HasSuffix(name, ".md") || strings.HasSuffix(name, ".tmp.md") { + return nil + } + fi, err := d.Info() + if err != nil || fi.Mode()&os.ModeSymlink != 0 || fi.Size() > maxMemoryFileBytes { + return nil + } + files = append(files, p) + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("cli-memory drain: walking %s: %w", path, walkErr) + } + sort.Strings(files) + return files, nil +} + +// repoFromClaudeSlug derives "org/repo" from a Claude project slug. Claude +// keys auto-memory by cwd with path separators turned into "-", so a memory +// dir ~/.claude/projects/-data-agents-scanner-hivecommons-hive/memory maps to +// hivecommons/hive when that repo is one the agent serves. Other layouts +// (Codex, Gemini) carry no project and yield "". +func repoFromClaudeSlug(path string, repos []string) string { + parts := strings.Split(filepath.ToSlash(path), "/") + slug := "" + for i, p := range parts { + if p == "projects" && i+1 < len(parts) { + slug = parts[i+1] + break + } + } + if slug == "" { + return "" + } + slug = strings.ToLower(slug) + for _, repo := range repos { + if repo == "" { + continue + } + suffix := strings.ToLower(strings.NewReplacer("/", "-", ".", "-", "_", "-").Replace(repo)) + if strings.HasSuffix(slug, "-"+suffix) || slug == suffix { + return repo + } + } + return "" +} + +func writeAtomic(path, content string) error { + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil { + return fmt.Errorf("cli-memory drain: writing %s: %w", path, err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("cli-memory drain: renaming %s: %w", path, err) + } + return nil +} diff --git a/src/pkg/knowledge/climemory/drain_test.go b/src/pkg/knowledge/climemory/drain_test.go new file mode 100644 index 0000000000..cfcab3f1c2 --- /dev/null +++ b/src/pkg/knowledge/climemory/drain_test.go @@ -0,0 +1,202 @@ +package climemory + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/knowledge" +) + +func copyFixture(t *testing.T, name, dst string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dst, []byte(fixture(t, name)), 0o644); err != nil { + t.Fatal(err) + } +} + +func fixedNow() time.Time { return time.Date(2026, 11, 7, 12, 0, 0, 0, time.UTC) } + +func TestDrainWritesDraftsAndDedupes(t *testing.T) { + home := t.TempDir() + memDir := filepath.Join(home, ".claude", "projects", "-data-agents-scanner-hivecommons-hive", "memory") + copyFixture(t, "claude-MEMORY.md", filepath.Join(memDir, "MEMORY.md")) + copyFixture(t, "claude-topic.md", filepath.Join(memDir, "build-pipeline.md")) + codexDir := filepath.Join(home, ".codex-scanner", "memories") + copyFixture(t, "codex-memory_summary.md", filepath.Join(codexDir, "memory_summary.md")) + gemini := filepath.Join(home, ".gemini", "GEMINI.md") + copyFixture(t, "gemini-GEMINI.md", gemini) + + vault := t.TempDir() + d := &Drainer{Dir: vault, Now: fixedNow} + req := Request{ + Agent: "scanner", + Repos: []string{"hivecommons/hive", "hivecommons/crux"}, + Sources: []Source{ + {CLI: CLIClaude, Path: memDir}, + {CLI: CLICodex, Path: codexDir}, + {CLI: CLIGemini, Path: gemini}, + }, + } + + first := d.Drain(req) + if len(first.Errors) != 0 { + t.Fatalf("errors: %v", first.Errors) + } + // 5 (MEMORY.md) + 3 (topic) + 3 (codex) + 2 (gemini); the MEMORY.md + // index bullet for build-pipeline.md carries its own summary and so is + // a distinct fact from the topic file's paragraphs. + if first.Drained != 13 || first.Dupes != 0 { + t.Fatalf("first drain = %+v", first) + } + files, _ := filepath.Glob(filepath.Join(vault, filePrefix+"*.md")) + if len(files) != 13 { + t.Fatalf("vault has %d entries, want 13", len(files)) + } + + second := d.Drain(req) + if second.Drained != 0 || second.Dupes != 13 || len(second.Errors) != 0 { + t.Fatalf("second drain = %+v", second) + } + files, _ = filepath.Glob(filepath.Join(vault, filePrefix+"*.md")) + if len(files) != 13 { + t.Fatalf("second drain changed the vault: %d entries", len(files)) + } + + // Rewrapping the same fact in the memory file is still a duplicate. + if err := os.WriteFile(gemini, []byte("## Gemini Added Memories\n- The hive dashboard\n listens on port 8080.\n"), 0o644); err != nil { + t.Fatal(err) + } + third := d.Drain(Request{Agent: "scanner", Sources: []Source{{CLI: CLIGemini, Path: gemini}}}) + if third.Drained != 0 || third.Dupes != 1 { + t.Fatalf("rewrapped fact drained again: %+v", third) + } +} + +func TestDrainedEntryFrontmatter(t *testing.T) { + home := t.TempDir() + memDir := filepath.Join(home, ".claude", "projects", "-data-agents-scanner-hivecommons-hive", "memory") + copyFixture(t, "claude-topic.md", filepath.Join(memDir, "build-pipeline.md")) + vault := t.TempDir() + d := &Drainer{Dir: vault, Now: fixedNow} + res := d.Drain(Request{Agent: "scanner", Repos: []string{"hivecommons/hive"}, Sources: []Source{{CLI: CLIClaude, Path: memDir}}}) + if res.Drained != 3 || len(res.Errors) != 0 { + t.Fatalf("drain = %+v", res) + } + want := Fact{Heading: "Build pipeline", Body: "Use the failing job log to fix, then push again."} + data, err := os.ReadFile(filepath.Join(vault, EntryFilename(Hash(want.Body)))) + if err != nil { + t.Fatalf("entry not written under its hash name: %v", err) + } + got := string(data) + for _, line := range []string{ + "title: Build pipeline: Use the failing job log to fix, then push again\n", + "type: reference\n", + "layer: personal\n", + "state: draft\n", + "source: cli-memory\n", + "cli: claude\n", + "agent: scanner\n", + "repo: hivecommons/hive\n", + "memhash: " + Hash(want.Body) + "\n", + "tags: [cli-memory, cli:claude, memhash:" + Hash(want.Body) + ", agent:scanner, repo:hivecommons/hive]\n", + "drained: 2026-11-07T12:00:00Z\n", + "---\n\n" + want.Body + "\n", + } { + if !strings.Contains(got, line) { + t.Errorf("entry missing %q:\n%s", line, got) + } + } +} + +// Drained entries are drafts, and only drafts: the vault parser must read +// them back as draft so the Primer's approved-only default never primes +// them, and nothing in the package can emit another state. +func TestDrainedEntriesAreNeverApproved(t *testing.T) { + vault := t.TempDir() + memDir := filepath.Join(t.TempDir(), "memory") + copyFixture(t, "claude-MEMORY.md", filepath.Join(memDir, "MEMORY.md")) + copyFixture(t, "codex-memory_summary.md", filepath.Join(memDir, "codex.md")) + + store, err := knowledge.NewFileStore(vault, "test", slog.Default()) + if err != nil { + t.Fatal(err) + } + d := &Drainer{Dir: vault, Store: store, Now: fixedNow} + res := d.Drain(Request{Agent: "scanner", Sources: []Source{{CLI: CLIClaude, Path: memDir}}}) + if res.Drained == 0 || len(res.Errors) != 0 { + t.Fatalf("drain = %+v", res) + } + all := store.ListPages("") + if len(all) != res.Drained { + t.Fatalf("store indexed %d pages, drained %d", len(all), res.Drained) + } + for _, f := range all { + if f.EffectiveState() != knowledge.StateDraft { + t.Errorf("%s has state %q, want draft", f.Slug, f.EffectiveState()) + } + } + if primable := knowledge.FilterByLifecycle(all, nil); len(primable) != 0 { + t.Fatalf("%d drained entries pass the approved-only filter: %+v", len(primable), primable) + } + + // A human approving one keeps it out of the next drain (hash still + // present) and never flips the others. + if err := store.SetLifecycleState(all[0].Slug, knowledge.StateApproved); err != nil { + t.Fatal(err) + } + again := d.Drain(Request{Agent: "scanner", Sources: []Source{{CLI: CLIClaude, Path: memDir}}}) + if again.Drained != 0 || again.Dupes != res.Drained { + t.Fatalf("drain after approval = %+v", again) + } + approved := 0 + for _, f := range store.ListPages("") { + if f.EffectiveState() == knowledge.StateApproved { + approved++ + } + } + if approved != 1 { + t.Fatalf("%d approved entries after one manual approval", approved) + } +} + +func TestDrainSkipsSymlinkedAndMissingStores(t *testing.T) { + shared := t.TempDir() + copyFixture(t, "gemini-GEMINI.md", filepath.Join(shared, "GEMINI.md")) + home := t.TempDir() + link := filepath.Join(home, "GEMINI.md") + if err := os.Symlink(filepath.Join(shared, "GEMINI.md"), link); err != nil { + t.Skip("symlinks unavailable") + } + d := &Drainer{Dir: t.TempDir(), Now: fixedNow} + res := d.Drain(Request{Agent: "a", Sources: []Source{ + {CLI: CLIGemini, Path: link}, + {CLI: CLICodex, Path: filepath.Join(home, "does-not-exist")}, + }}) + if res.Drained != 0 || res.Dupes != 0 || len(res.Errors) != 0 { + t.Fatalf("symlinked or missing store drained: %+v", res) + } +} + +func TestRepoFromClaudeSlug(t *testing.T) { + repos := []string{"hivecommons/hive", "kubestellar/console.ui"} + tests := []struct { + path, want string + }{ + {"/data/home/agents/s/.claude/projects/-data-agents-s-hivecommons-hive/memory", "hivecommons/hive"}, + {"/data/home/agents/s/.claude/projects/-data-agents-s-kubestellar-console-ui/memory", "kubestellar/console.ui"}, + {"/data/home/agents/s/.claude/projects/-data-agents-s/memory", ""}, + {"/data/home/.codex-s/memories", ""}, + } + for _, tc := range tests { + if got := repoFromClaudeSlug(tc.path, repos); got != tc.want { + t.Errorf("repoFromClaudeSlug(%q) = %q, want %q", tc.path, got, tc.want) + } + } +} diff --git a/src/pkg/knowledge/climemory/parse.go b/src/pkg/knowledge/climemory/parse.go new file mode 100644 index 0000000000..a4619e2c3b --- /dev/null +++ b/src/pkg/knowledge/climemory/parse.go @@ -0,0 +1,220 @@ +// Package climemory turns the persistent memory the agent CLIs keep for +// themselves (Claude Code auto-memory, Codex memories, Gemini's saved +// GEMINI.md) into draft knowledge entries in a Hive vault, so a CLI's +// private inferences become reviewable, scoped facts instead of a hidden +// per-agent notebook (hivecommons/hive#17033). +// +// The package is split in two: Parse* functions that split one memory file +// into facts, and Drainer, which hashes those facts, deduplicates them +// against the vault and writes the new ones as draft entries. +package climemory + +import ( + "regexp" + "strings" + "unicode" +) + +// CLI names a memory format. They double as the `cli:` frontmatter value and +// the `cli:` tag on drained entries. +const ( + CLIClaude = "claude" + CLICodex = "codex" + CLIGemini = "gemini" +) + +// Fact is one unit of CLI memory: a bullet, a paragraph or a short topic +// file. Body is the normalised text; Heading is the nearest markdown heading +// above it, kept for the entry title. +type Fact struct { + Heading string + Body string +} + +// maxTitleRunes bounds the generated entry title; the full text stays in the +// body. +const maxTitleRunes = 80 + +// geminiMemorySection is the heading Gemini CLI's save_memory / `/memory add` +// write under in GEMINI.md. Everything above it is operator-authored +// instruction, not memory, and is not drained. +const geminiMemorySection = "## Gemini Added Memories" + +var ( + bulletRe = regexp.MustCompile(`^(\s*)([-*+]|\d+[.)])\s+(.*)$`) + headingRe = regexp.MustCompile(`^#{1,6}\s+(.*?)\s*#*\s*$`) + indexLinkRe = regexp.MustCompile(`^\[[^\]]*\]\([^)]+\.md\)\s*([-—–:]\s*)?`) + spaceRe = regexp.MustCompile(`\s+`) +) + +// ParseClaudeMemory splits a Claude Code auto-memory file (MEMORY.md or a +// sibling topic file under ~/.claude/projects//memory/) into facts. +// MEMORY.md is an index: bullets that are only a link to a sibling topic +// file are skipped, because the topic file itself is drained; a link with a +// trailing summary keeps the summary. +func ParseClaudeMemory(content string) []Fact { + return parseMarkdown(content, true) +} + +// ParseCodexMemories splits one Codex memory markdown file +// ($CODEX_HOME/memories*/… such as memory_summary.md or a rollout summary) +// into facts. +func ParseCodexMemories(content string) []Fact { + return parseMarkdown(content, false) +} + +// ParseGeminiMemory splits ~/.gemini/GEMINI.md into facts. Only the +// "## Gemini Added Memories" section is memory; when the heading is absent +// the file is treated as memory in full (a GEMINI.md that exists only in a +// per-agent Hive home was written by the CLI, not by an operator). +func ParseGeminiMemory(content string) []Fact { + if idx := strings.Index(content, geminiMemorySection); idx >= 0 { + content = content[idx:] + } + return parseMarkdown(content, false) +} + +// ParseFor dispatches on cli; unknown names use the generic markdown rules. +func ParseFor(cli, content string) []Fact { + switch cli { + case CLIClaude: + return ParseClaudeMemory(content) + case CLIGemini: + return ParseGeminiMemory(content) + default: + return ParseCodexMemories(content) + } +} + +// parseMarkdown is the shared splitter: every top-level list item is a fact +// (indented continuation lines and nested items are folded into it) and +// every paragraph outside a list is a fact. Headings set the context but +// are never facts themselves; fenced code blocks are kept verbatim inside +// the fact they belong to. +func parseMarkdown(content string, skipIndexLinks bool) []Fact { + var ( + facts []Fact + heading string + cur []string + inItem bool + inFence bool + ) + flush := func() { + text := normalise(strings.Join(cur, "\n")) + cur = cur[:0] + inItem = false + if text == "" { + return + } + if skipIndexLinks { + if loc := indexLinkRe.FindStringIndex(text); loc != nil { + text = strings.TrimSpace(text[loc[1]:]) + if text == "" { + return + } + } + } + facts = append(facts, Fact{Heading: heading, Body: text}) + } + for _, line := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") { + inFence = !inFence + cur = append(cur, line) + continue + } + if inFence { + cur = append(cur, line) + continue + } + if trimmed == "" { + flush() + continue + } + if m := headingRe.FindStringSubmatch(trimmed); m != nil { + flush() + heading = strings.TrimSpace(m[1]) + continue + } + if trimmed == "---" || trimmed == "***" { + flush() + continue + } + if m := bulletRe.FindStringSubmatch(line); m != nil { + if m[1] == "" || !inItem { + flush() + inItem = true + cur = append(cur, m[3]) + continue + } + // Nested item: fold into the parent as a clause. + last := strings.TrimRight(cur[len(cur)-1], " :;,") + cur[len(cur)-1] = last + "; " + m[3] + continue + } + cur = append(cur, trimmed) + } + flush() + return facts +} + +// normalise collapses whitespace outside code fences so the same fact +// written with different wrapping hashes identically. +func normalise(text string) string { + if strings.Contains(text, "```") || strings.Contains(text, "~~~") { + return strings.TrimSpace(text) + } + return strings.TrimSpace(spaceRe.ReplaceAllString(text, " ")) +} + +// Title derives an entry title from the fact: the heading (when it is not +// a generic container such as "Memory") followed by the first sentence of +// the body, truncated on a word boundary. +func (f Fact) Title() string { + body := f.Body + if i := strings.IndexByte(body, '\n'); i >= 0 { + body = body[:i] + } + body = strings.TrimLeft(body, "`#*_ ") + if end := sentenceEnd(body); end > 0 { + body = body[:end] + } + body = strings.TrimSuffix(body, ".") + body = truncateWords(body, maxTitleRunes) + if h := strings.TrimSpace(f.Heading); h != "" && !genericHeading(h) && !strings.HasPrefix(strings.ToLower(body), strings.ToLower(h)) { + return h + ": " + body + } + return body +} + +func sentenceEnd(s string) int { + for i, r := range s { + if (r == '.' || r == '!' || r == '?') && i+1 < len(s) && unicode.IsSpace(rune(s[i+1])) { + return i + } + } + return 0 +} + +func truncateWords(s string, max int) string { + runes := []rune(s) + if len(runes) <= max { + return s + } + cut := string(runes[:max]) + if !unicode.IsSpace(runes[max]) { + if i := strings.LastIndex(cut, " "); i > max/2 { + cut = cut[:i] + } + } + return strings.TrimRight(cut, " ,;:-") + "…" +} + +var genericHeadings = map[string]bool{ + "memory": true, "memories": true, "notes": true, "summary": true, + "gemini added memories": true, "memory summary": true, "project memory": true, +} + +func genericHeading(h string) bool { + return genericHeadings[strings.ToLower(strings.TrimSpace(h))] +} diff --git a/src/pkg/knowledge/climemory/parse_test.go b/src/pkg/knowledge/climemory/parse_test.go new file mode 100644 index 0000000000..c48eed25f1 --- /dev/null +++ b/src/pkg/knowledge/climemory/parse_test.go @@ -0,0 +1,119 @@ +package climemory + +import ( + "os" + "path/filepath" + "testing" +) + +func fixture(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("fixture %s: %v", name, err) + } + return string(data) +} + +func TestParseMemoryFormats(t *testing.T) { + tests := []struct { + name string + cli string + fixture string + want []Fact + }{ + { + name: "claude MEMORY.md index skips bare topic links and folds wrapped bullets", + cli: CLIClaude, + fixture: "claude-MEMORY.md", + want: []Fact{ + {Heading: "Memory", Body: "CI is the only verdict; never run go test locally"}, + {Heading: "Memory", Body: "The operator prefers PRs under 400 lines and squash merges."}, + {Heading: "Memory", Body: "User mentioned HIPAA once in passing; do not assume PHI is in scope for this repository."}, + {Heading: "Repo conventions", Body: "Changelog fragments live at the repository root under `changelog.d/`."}, + {Heading: "Repo conventions", Body: "Commit messages are signed with `-s`; DCO trailer is checked in CI; unsigned commits are rejected"}, + }, + }, + { + name: "claude topic file keeps paragraphs and code fences", + cli: CLIClaude, + fixture: "claude-topic.md", + want: []Fact{ + {Heading: "Build pipeline", Body: "CI is the only verdict. Do not run `go test` or `go build` from an agent pane."}, + {Heading: "Build pipeline", Body: "```sh\ngh pr checks 1234\n```"}, + {Heading: "Build pipeline", Body: "Use the failing job log to fix, then push again."}, + }, + }, + { + name: "codex memory summary splits per section bullet", + cli: CLICodex, + fixture: "codex-memory_summary.md", + want: []Fact{ + {Heading: "Preferences", Body: "Prefers table-driven tests with fixtures under testdata/."}, + {Heading: "Preferences", Body: "Avoids time.Sleep in tests."}, + {Heading: "Project facts", Body: "The scheduler primes kicks from the knowledge store before rendering."}, + }, + }, + { + name: "gemini drains only the Added Memories section", + cli: CLIGemini, + fixture: "gemini-GEMINI.md", + want: []Fact{ + {Heading: "Gemini Added Memories", Body: "The hive dashboard listens on port 8080."}, + {Heading: "Gemini Added Memories", Body: "Release lines are cut from the v5 branch."}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := ParseFor(tc.cli, fixture(t, tc.fixture)) + if len(got) != len(tc.want) { + t.Fatalf("got %d facts, want %d:\n%+v", len(got), len(tc.want), got) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("fact %d:\n got %+v\nwant %+v", i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestParseGeminiWithoutSectionDrainsWholeFile(t *testing.T) { + got := ParseGeminiMemory("- remembered thing\n- another\n") + if len(got) != 2 || got[0].Body != "remembered thing" { + t.Fatalf("got %+v", got) + } +} + +func TestFactTitle(t *testing.T) { + tests := []struct { + name string + fact Fact + want string + }{ + {"generic heading dropped, first sentence kept", Fact{Heading: "Memory", Body: "CI is the verdict. Never run tests locally."}, "CI is the verdict"}, + {"specific heading prefixed", Fact{Heading: "Repo conventions", Body: "Sign every commit"}, "Repo conventions: Sign every commit"}, + {"heading not repeated when body starts with it", Fact{Heading: "Build", Body: "Build with just"}, "Build with just"}, + {"code fence uses first line", Fact{Heading: "", Body: "```sh\ngh pr checks\n```"}, "sh"}, + {"long body truncated on a word", Fact{Body: "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen"}, "one two three four five six seven eight nine ten eleven twelve thirteen fourteen…"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := tc.fact.Title(); got != tc.want { + t.Errorf("Title() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestHashIgnoresWrappingDifferences(t *testing.T) { + a := Hash("the operator prefers\n squash merges") + b := Hash("the operator prefers squash merges") + if a != b { + t.Fatalf("hash differs across whitespace: %s vs %s", a, b) + } + if a == Hash("the operator prefers rebase merges") { + t.Fatal("different facts hashed equal") + } +} diff --git a/src/pkg/knowledge/climemory/testdata/claude-MEMORY.md b/src/pkg/knowledge/climemory/testdata/claude-MEMORY.md new file mode 100644 index 0000000000..8728aa309b --- /dev/null +++ b/src/pkg/knowledge/climemory/testdata/claude-MEMORY.md @@ -0,0 +1,14 @@ +# Memory + +- [Build pipeline](build-pipeline.md) — CI is the only verdict; never run go test locally +- [Deploy notes](deploy-notes.md) +- The operator prefers PRs under 400 lines and squash merges. +- User mentioned HIPAA once in passing; do not assume PHI + is in scope for this repository. + +## Repo conventions + +1. Changelog fragments live at the repository root under `changelog.d/`. +2. Commit messages are signed with `-s`: + - DCO trailer is checked in CI + - unsigned commits are rejected diff --git a/src/pkg/knowledge/climemory/testdata/claude-topic.md b/src/pkg/knowledge/climemory/testdata/claude-topic.md new file mode 100644 index 0000000000..d99c5f1cf6 --- /dev/null +++ b/src/pkg/knowledge/climemory/testdata/claude-topic.md @@ -0,0 +1,9 @@ +# Build pipeline + +CI is the only verdict. Do not run `go test` or `go build` from an agent pane. + +```sh +gh pr checks 1234 +``` + +Use the failing job log to fix, then push again. diff --git a/src/pkg/knowledge/climemory/testdata/codex-memory_summary.md b/src/pkg/knowledge/climemory/testdata/codex-memory_summary.md new file mode 100644 index 0000000000..46d885c39d --- /dev/null +++ b/src/pkg/knowledge/climemory/testdata/codex-memory_summary.md @@ -0,0 +1,8 @@ +# Memory summary + +## Preferences +* Prefers table-driven tests with fixtures under testdata/. +* Avoids time.Sleep in tests. + +## Project facts +* The scheduler primes kicks from the knowledge store before rendering. diff --git a/src/pkg/knowledge/climemory/testdata/gemini-GEMINI.md b/src/pkg/knowledge/climemory/testdata/gemini-GEMINI.md new file mode 100644 index 0000000000..d2587e90f3 --- /dev/null +++ b/src/pkg/knowledge/climemory/testdata/gemini-GEMINI.md @@ -0,0 +1,8 @@ +# Operator instructions + +Always answer in English. This block is written by the operator and is +not memory. + +## Gemini Added Memories +- The hive dashboard listens on port 8080. +- Release lines are cut from the v5 branch. From ffe91d0521e6fa1a4d358a6fa02e99d9ee795eb7 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 7 Nov 2026 15:21:04 -0500 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=8C=B1=20climemory:=20edge-case=20tes?= =?UTF-8?q?ts=20for=20drain=20caps,=20unreadable=20stores,=20frontmatter?= =?UTF-8?q?=20hash=20scan=20and=20parser=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .../knowledge/climemory/drain_edge_test.go | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 src/pkg/knowledge/climemory/drain_edge_test.go diff --git a/src/pkg/knowledge/climemory/drain_edge_test.go b/src/pkg/knowledge/climemory/drain_edge_test.go new file mode 100644 index 0000000000..f703b923c6 --- /dev/null +++ b/src/pkg/knowledge/climemory/drain_edge_test.go @@ -0,0 +1,286 @@ +package climemory + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDrainRejectsMissingTarget(t *testing.T) { + var nilDrainer *Drainer + if res := nilDrainer.Drain(Request{}); len(res.Errors) != 1 || res.Drained != 0 { + t.Fatalf("nil drainer: %+v", res) + } + if res := (&Drainer{}).Drain(Request{}); len(res.Errors) != 1 { + t.Fatalf("empty Dir: %+v", res) + } + // Dir cannot be created because a regular file sits on its path. + blocker := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocker, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + res := (&Drainer{Dir: filepath.Join(blocker, "vault")}).Drain(Request{}) + if len(res.Errors) != 1 || !strings.Contains(res.Errors[0].Error(), "creating") { + t.Fatalf("unmkdirable Dir: %+v", res) + } +} + +func TestDrainCapsFactsPerDrain(t *testing.T) { + home := t.TempDir() + var sb strings.Builder + for i := 0; i < maxFactsPerDrain+5; i++ { + fmt.Fprintf(&sb, "- distinct remembered fact number %d\n", i) + } + mem := filepath.Join(home, "memory_summary.md") + if err := os.WriteFile(mem, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } + d := &Drainer{Dir: t.TempDir(), Now: fixedNow} + res := d.Drain(Request{Agent: "a", Sources: []Source{{CLI: CLICodex, Path: home}}}) + if res.Drained != maxFactsPerDrain || res.Skipped != 5 || len(res.Errors) != 0 { + t.Fatalf("cap not applied: drained=%d skipped=%d errs=%v", res.Drained, res.Skipped, res.Errors) + } + // The second drain picks up the five the cap left behind. + res = d.Drain(Request{Agent: "a", Sources: []Source{{CLI: CLICodex, Path: home}}}) + if res.Drained != 5 || res.Dupes != maxFactsPerDrain || res.Skipped != 0 { + t.Fatalf("second pass: drained=%d dupes=%d skipped=%d", res.Drained, res.Dupes, res.Skipped) + } +} + +func TestDrainReportsUnreadableSourceAndWriteFailure(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("permission bits are not enforced for root") + } + home := t.TempDir() + mem := filepath.Join(home, "notes.md") + if err := os.WriteFile(mem, []byte("- a fact\n"), 0o000); err != nil { + t.Fatal(err) + } + d := &Drainer{Dir: t.TempDir(), Now: fixedNow} + res := d.Drain(Request{Sources: []Source{{CLI: CLICodex, Path: mem}}}) + if res.Drained != 0 || len(res.Errors) != 1 || !strings.Contains(res.Errors[0].Error(), "reading") { + t.Fatalf("unreadable source: %+v", res) + } + + // A read-only vault makes the atomic write fail per fact. + ro := filepath.Join(t.TempDir(), "vault") + if err := os.MkdirAll(ro, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, "ok.md"), []byte("- another fact\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(ro, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(ro, 0o755) }) + res = (&Drainer{Dir: ro, Now: fixedNow}).Drain(Request{Sources: []Source{{CLI: CLICodex, Path: filepath.Join(home, "ok.md")}}}) + if res.Drained != 0 || len(res.Errors) != 1 || !strings.Contains(res.Errors[0].Error(), "writing") { + t.Fatalf("read-only vault: %+v", res) + } + + // An unlistable vault fails before any source is read. + if err := os.Chmod(ro, 0o000); err != nil { + t.Fatal(err) + } + res = (&Drainer{Dir: ro}).Drain(Request{}) + if len(res.Errors) != 1 || !strings.Contains(res.Errors[0].Error(), "listing") { + t.Fatalf("unlistable vault: %+v", res) + } +} + +func TestMemoryFilesFilters(t *testing.T) { + root := t.TempDir() + write := func(rel, body string) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("keep.md", "- a\n") + write("sub/nested.md", "- b\n") + write("draft.tmp.md", "- temp\n") + write("notes.txt", "not markdown\n") + write(".hidden/secret.md", "- hidden\n") + write(".dotfile.md", "- hidden file\n") + write("huge.md", strings.Repeat("x", maxMemoryFileBytes+1)) + + files, err := memoryFiles(root) + if err != nil { + t.Fatal(err) + } + want := []string{filepath.Join(root, "keep.md"), filepath.Join(root, "sub", "nested.md")} + if strings.Join(files, ",") != strings.Join(want, ",") { + t.Fatalf("memoryFiles = %v, want %v", files, want) + } + + // Single-file sources: oversized or non-markdown yield nothing. + for _, rel := range []string{"huge.md", "notes.txt"} { + if got, err := memoryFiles(filepath.Join(root, rel)); err != nil || len(got) != 0 { + t.Fatalf("memoryFiles(%s) = %v, %v", rel, got, err) + } + } + if got, err := memoryFiles(filepath.Join(root, "keep.md")); err != nil || len(got) != 1 { + t.Fatalf("memoryFiles(keep.md) = %v, %v", got, err) + } + + // A path whose parent is a regular file is a real stat error, not ENOENT. + if _, err := memoryFiles(filepath.Join(root, "keep.md", "child")); err == nil { + t.Fatal("expected a stat error for a path under a regular file") + } +} + +func TestHashFromFrontmatter(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p + } + tests := []struct { + name, body, want string + }{ + {"quoted", "---\ntitle: x\nmemhash: \"abc\"\n---\nbody", "abc"}, + {"single-quoted", "---\nmemhash: 'def'\n---\n", "def"}, + {"no frontmatter", "# just a page\nmemhash: nope\n", ""}, + {"key after closing fence", "---\ntitle: x\n---\nmemhash: nope\n", ""}, + {"unterminated without key", "---\ntitle: x\n", ""}, + {"empty", "", ""}, + } + for _, tc := range tests { + if got := hashFromFrontmatter(write(tc.name+".md", tc.body)); got != tc.want { + t.Errorf("%s: hashFromFrontmatter = %q, want %q", tc.name, got, tc.want) + } + } + if got := hashFromFrontmatter(filepath.Join(dir, "missing.md")); got != "" { + t.Errorf("missing file: %q", got) + } + + // existingHashes ignores subdirectories and non-markdown files. + if err := os.Mkdir(filepath.Join(dir, "sub"), 0o755); err != nil { + t.Fatal(err) + } + write("stray.txt", "---\nmemhash: zzz\n---\n") + seen, err := existingHashes(dir) + if err != nil { + t.Fatal(err) + } + if !seen["abc"] || !seen["def"] || seen["zzz"] || seen["nope"] { + t.Fatalf("existingHashes = %v", seen) + } +} + +func TestEntryFilenameAndList(t *testing.T) { + long := strings.Repeat("a", 64) + if got := EntryFilename(long); got != filePrefix+strings.Repeat("a", hashFilenameHex)+".md" { + t.Fatalf("EntryFilename(long) = %q", got) + } + if got := EntryFilename("short"); got != filePrefix+"short.md" { + t.Fatalf("EntryFilename(short) = %q", got) + } + if got := list([]string{"a,b", " ", "c\nd", ""}); got != "a b, c d" { + t.Fatalf("list = %q", got) + } + if got := scalar("line one\r\nline two"); got != "line one line two" { + t.Fatalf("scalar = %q", got) + } +} + +func TestRenderEntryOmitsEmptyProvenance(t *testing.T) { + entry := renderEntry(Fact{Body: "a fact"}, "h", entryMeta{cli: CLICodex, path: "/p", layer: "personal", at: fixedNow()}) + for _, forbidden := range []string{"\nagent:", "\nrepo:", "agent:", "repo:"} { + if strings.Contains(entry, forbidden) { + t.Fatalf("entry carries %q without a value:\n%s", forbidden, entry) + } + } + if !strings.Contains(entry, "state: draft\n") || !strings.HasSuffix(entry, "a fact\n") { + t.Fatalf("entry shape:\n%s", entry) + } +} + +func TestParseMarkdownStructure(t *testing.T) { + src := strings.Join([]string{ + "# Project", + "", + "Intro paragraph", + "spanning lines.", + "", + "---", + "", + "- parent item:", + " - nested one", + " - nested two", + " continuation of parent", + "* second item", + "1. numbered item", + "", + "Fenced:", + "```sh", + "", + "echo spaced", + "```", + "", + "[index](topic.md)", + "", + "[summary](other.md) — kept summary", + "", + }, "\n") + facts := ParseClaudeMemory(src) + bodies := make([]string, 0, len(facts)) + for _, f := range facts { + bodies = append(bodies, f.Body) + if f.Heading != "Project" { + t.Errorf("heading = %q for %q", f.Heading, f.Body) + } + } + want := []string{ + "Intro paragraph spanning lines.", + "parent item; nested one; nested two continuation of parent", + "second item", + "numbered item", + "Fenced:\n```sh\n\necho spaced\n```", + "kept summary", + } + if strings.Join(bodies, "|") != strings.Join(want, "|") { + t.Fatalf("facts:\n%s\nwant:\n%s", strings.Join(bodies, "\n"), strings.Join(want, "\n")) + } + // Codex parsing keeps index-style links as ordinary facts. + if got := ParseCodexMemories("[index](topic.md)\n"); len(got) != 1 || got[0].Body != "[index](topic.md)" { + t.Fatalf("codex index link: %+v", got) + } + if got := ParseFor("unknown-cli", "- x\n"); len(got) != 1 { + t.Fatalf("ParseFor fallback: %+v", got) + } +} + +func TestFactTitleEdges(t *testing.T) { + tests := []struct { + name string + fact Fact + want string + }{ + {"question ends sentence", Fact{Body: "Why is CI slow? Because of shards."}, "Why is CI slow"}, + {"exclamation ends sentence", Fact{Body: "Never force push! Ever."}, "Never force push"}, + {"trailing period trimmed", Fact{Body: "Single sentence."}, "Single sentence"}, + {"markdown lead stripped", Fact{Body: "**bold** start"}, "bold** start"}, + {"heading match is case-insensitive", Fact{Heading: "build", Body: "Build with just"}, "Build with just"}, + {"long word boundary fallback", Fact{Body: strings.Repeat("x", maxTitleRunes+10)}, strings.Repeat("x", maxTitleRunes) + "…"}, + {"cut on exact space", Fact{Body: strings.Repeat("y", maxTitleRunes) + " tail"}, strings.Repeat("y", maxTitleRunes) + "…"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := tc.fact.Title(); got != tc.want { + t.Errorf("Title() = %q, want %q", got, tc.want) + } + }) + } +} From 939d2305a556a9012bf0f09dba8f5f45bf5f39b6 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 7 Nov 2026 15:11:57 -0700 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B=20climemory:=20nil=20Drainer?= =?UTF-8?q?=20returned=20no-target=20error=20without=20dereferencing;=20bo?= =?UTF-8?q?ot=20drain=20installer=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drain built Result{Store: d.Dir} before the nil-receiver guard, so the nil-drainer edge test panicked in CI (test rest 3/4). Return the error result first. Add cmd/hive tests for cliMemoryDrainEnabled and installCLIMemoryDrainer: vault connected without bead stores, drainer installed on a hosted runtime and writes a draft entry with repo/agent tags, dedupes on second drain, and the stand-down paths (disabled, sidecar-like runtime, no knowledge API, unmkdirable vault path). Signed-off-by: Andrew Anderson --- src/cmd/hive/boot_cli_memory_drain_test.go | 171 +++++++++++++++++++++ src/pkg/knowledge/climemory/drain.go | 5 +- 2 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 src/cmd/hive/boot_cli_memory_drain_test.go diff --git a/src/cmd/hive/boot_cli_memory_drain_test.go b/src/cmd/hive/boot_cli_memory_drain_test.go new file mode 100644 index 0000000000..9dd39a4e9b --- /dev/null +++ b/src/cmd/hive/boot_cli_memory_drain_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/climemory" +) + +// drainCapturingRuntime is a hosted runtime that records the drainer the +// boot installs so the test can invoke it the way the manager does on a kick. +type drainCapturingRuntime struct { + hostedRuntime + fn crux.CLIMemoryDrainer +} + +func (r *drainCapturingRuntime) SetCLIMemoryDrainer(fn crux.CLIMemoryDrainer) { r.fn = fn } + +// sidecarLikeRuntime deliberately lacks SetCLIMemoryDrainer. +type sidecarLikeRuntime struct{ hostedRuntime } + +func TestCLIMemoryDrainEnabled(t *testing.T) { + b := &boot{} + if b.cliMemoryDrainEnabled() { + t.Fatal("nil config must not enable the drain") + } + cfg := bootKnowledgeConfig(t) + b.cfg = cfg + if b.cliMemoryDrainEnabled() { + t.Fatal("default config must not enable the drain") + } + ac := cfg.Agents["scanner"] + ac.CLIMemory = &config.CLIMemoryConfig{DrainToKnowledge: true} + cfg.Agents["scanner"] = ac + if !b.cliMemoryDrainEnabled() { + t.Fatal("a per-agent drain_to_knowledge must enable the drain") + } + cfg.Agents["scanner"] = config.AgentConfig{Enabled: false, Backend: "copilot", CLIMemory: &config.CLIMemoryConfig{DrainToKnowledge: true}} + if b.cliMemoryDrainEnabled() { + t.Fatal("a disabled agent must not enable the drain") + } + cfg.AgentsCLIMemory.DrainToKnowledge = true + if !b.cliMemoryDrainEnabled() { + t.Fatal("the fleet-wide flag must enable the drain") + } +} + +func TestInstallCLIMemoryDrainerWritesDraftsIntoBeadSynthVault(t *testing.T) { + cfg := bootKnowledgeConfig(t) + cfg.AgentsCLIMemory.DrainToKnowledge = true + cfg.Knowledge.BeadSynthesizer.TargetLayer = "team" + b := newBootKnowledgeBoot(t, cfg) + var log strings.Builder + b.logger = slog.New(slog.NewTextHandler(&log, nil)) + rt := &drainCapturingRuntime{hostedRuntime: b.agentMgr} + b.agentMgr = rt + b.bootKnowledgeWith(newBootKnowledgeFake(t).deps) + if b.knowledgeAPI == nil { + t.Fatal("knowledge API must be auto-enabled") + } + + b.installCLIMemoryDrainer() + + vaultPath := b.beadSynthVaultPath() + if b.knowledgeAPI.GetVaultStore(vaultPath) == nil { + t.Fatal("the bead-synth vault must be connected even with no bead stores") + } + if rt.fn == nil { + t.Fatalf("drainer must be installed on the in-process runtime; log:\n%s", log.String()) + } + if !strings.Contains(log.String(), "CLI memory drain installed") || !strings.Contains(log.String(), "layer=team") { + t.Fatalf("log:\n%s", log.String()) + } + + memDir := filepath.Join(t.TempDir(), ".claude", "projects", "-data-agents-scanner-acme-widgets", "memory") + if err := os.MkdirAll(memDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(memDir, "MEMORY.md"), []byte("- The widgets build needs CGO_ENABLED=1 on CI runners\n"), 0o644); err != nil { + t.Fatal(err) + } + res := rt.fn(crux.CLIMemoryDrainRequest{Agent: "scanner", Backend: "claude", Sources: []crux.CLIMemorySource{{CLI: climemory.CLIClaude, Path: memDir}}}) + if res.Err != nil || res.Store != beadSynthVaultName || res.Drained != 1 { + t.Fatalf("drain result = %+v", res) + } + entries, err := filepath.Glob(filepath.Join(vaultPath, "cli-memory-*.md")) + if err != nil || len(entries) != 1 { + t.Fatalf("vault entries = %v, err=%v", entries, err) + } + data, err := os.ReadFile(entries[0]) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"state: draft", "repo: acme/widgets", "agent: scanner"} { + if !strings.Contains(string(data), want) { + t.Fatalf("entry missing %q:\n%s", want, data) + } + } + if again := rt.fn(crux.CLIMemoryDrainRequest{Agent: "scanner", Backend: "claude", Sources: []crux.CLIMemorySource{{CLI: climemory.CLIClaude, Path: memDir}}}); again.Drained != 0 || again.Dupes != 1 { + t.Fatalf("second drain = %+v, want dedupe", again) + } + + // Calling again keeps the already-connected vault and reinstalls. + rt.fn = nil + b.installCLIMemoryDrainer() + if rt.fn == nil { + t.Fatal("reinstall must set the drainer again") + } +} + +func TestInstallCLIMemoryDrainerStandsDown(t *testing.T) { + t.Run("disabled", func(t *testing.T) { + cfg := bootKnowledgeConfig(t) + b := newBootKnowledgeBoot(t, cfg) + rt := &drainCapturingRuntime{hostedRuntime: b.agentMgr} + b.agentMgr = rt + b.installCLIMemoryDrainer() + if rt.fn != nil { + t.Fatal("drain off must not install a drainer") + } + }) + t.Run("sidecar runtime warns", func(t *testing.T) { + cfg := bootKnowledgeConfig(t) + cfg.AgentsCLIMemory.DrainToKnowledge = true + b := newBootKnowledgeBoot(t, cfg) + var log strings.Builder + b.logger = slog.New(slog.NewTextHandler(&log, nil)) + b.agentMgr = &sidecarLikeRuntime{hostedRuntime: b.agentMgr} + b.installCLIMemoryDrainer() + if !strings.Contains(log.String(), "sidecar runtime cannot drain") { + t.Fatalf("log:\n%s", log.String()) + } + }) + t.Run("no knowledge API warns", func(t *testing.T) { + cfg := bootKnowledgeConfig(t) + cfg.AgentsCLIMemory.DrainToKnowledge = true + b := newBootKnowledgeBoot(t, cfg) + var log strings.Builder + b.logger = slog.New(slog.NewTextHandler(&log, nil)) + rt := &drainCapturingRuntime{hostedRuntime: b.agentMgr} + b.agentMgr = rt + b.installCLIMemoryDrainer() + if rt.fn != nil || !strings.Contains(log.String(), "knowledge API is unavailable") { + t.Fatalf("installed=%v log:\n%s", rt.fn != nil, log.String()) + } + }) + t.Run("vault path blocked by a file warns", func(t *testing.T) { + cfg := bootKnowledgeConfig(t) + cfg.AgentsCLIMemory.DrainToKnowledge = true + blocker := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocker, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + cfg.Knowledge.BeadSynthesizer.VaultPath = filepath.Join(blocker, "vault") + b := newBootKnowledgeBoot(t, cfg) + var log strings.Builder + b.logger = slog.New(slog.NewTextHandler(&log, nil)) + rt := &drainCapturingRuntime{hostedRuntime: b.agentMgr} + b.agentMgr = rt + b.bootKnowledgeWith(newBootKnowledgeFake(t).deps) + b.installCLIMemoryDrainer() + if rt.fn != nil || !strings.Contains(log.String(), "failed to create vault for CLI memory drain") { + t.Fatalf("installed=%v log:\n%s", rt.fn != nil, log.String()) + } + }) +} diff --git a/src/pkg/knowledge/climemory/drain.go b/src/pkg/knowledge/climemory/drain.go index 4fb9245d6e..de8ed2e8ad 100644 --- a/src/pkg/knowledge/climemory/drain.go +++ b/src/pkg/knowledge/climemory/drain.go @@ -80,11 +80,10 @@ type Drainer struct { // the ones whose hash is not yet in the vault. Entries are always written // with `state: draft`; nothing in this package can produce another state. func (d *Drainer) Drain(req Request) Result { - res := Result{Store: d.Dir} if d == nil || d.Dir == "" { - res.Errors = append(res.Errors, errors.New("cli-memory drain: no target directory")) - return res + return Result{Errors: []error{errors.New("cli-memory drain: no target directory")}} } + res := Result{Store: d.Dir} if err := os.MkdirAll(d.Dir, 0o755); err != nil { res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: creating %s: %w", d.Dir, err)) return res