diff --git a/changelog.d/added-17056-cli-memory-drain-spool-draft-create.md b/changelog.d/added-17056-cli-memory-drain-spool-draft-create.md new file mode 100644 index 0000000000..fcc8d6e6ca --- /dev/null +++ b/changelog.d/added-17056-cli-memory-drain-spool-draft-create.md @@ -0,0 +1 @@ +- CLI memory drain, phase 1 completion (#17056): a fact already in the vault now gains the draining agent's `agent:` tag instead of being ignored, and keeps whatever state an operator gave it, so a deprecated memory is never revived. A memory file whose facts did not all reach the vault (vault unwritable, 200-facts-per-drain cap) is copied to `/data/knowledge/auto-memory-spool//` before `clear_on_kick` wipes the store and retried on the agent's next drain; the drain log line now reports `spooled S`. `POST /api/knowledge/create` takes `state` (`approved` default, or `draft`, which is written into the local channel named by `layer` and never overwrites an existing entry) and accepts a `cli-memory`-tagged fact only as a draft carrying the drain's own tags. `bd create --type knowledge [--state] [--tag]… [--layer]` creates a fact through that endpoint. diff --git a/src/cmd/bd/create_knowledge.go b/src/cmd/bd/create_knowledge.go new file mode 100644 index 0000000000..84ac5cf389 --- /dev/null +++ b/src/cmd/bd/create_knowledge.go @@ -0,0 +1,156 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "strings" +) + +// knowledgeCreateType is the `bd create --type` value that creates a +// knowledge fact instead of a bead (#17056). Facts live in the knowledge +// store the curator, the TOC and `hivectl knowledge state` work on, so this +// goes to the dashboard's POST /api/knowledge/create, the same way `bd kb` +// does, never into the local bead store. +const knowledgeCreateType = "knowledge" + +// maxKnowledgeBodyBytes bounds --body-file; a fact is a note, not a document +// (use `bd kb import-file` for those). +const maxKnowledgeBodyBytes = 256 * 1024 + +// knowledgeCreateRequest is the body of POST /api/knowledge/create. Empty +// fields are omitted so the dashboard applies its defaults (layer project, +// type pattern, state approved). +type knowledgeCreateRequest struct { + Title string `json:"title"` + Body string `json:"body"` + Type string `json:"type,omitempty"` + Tags []string `json:"tags,omitempty"` + Layer string `json:"layer,omitempty"` + State string `json:"state,omitempty"` +} + +// stringList is a repeatable string flag. +type stringList []string + +func (s *stringList) String() string { return strings.Join(*s, ",") } + +func (s *stringList) Set(v string) error { + *s = append(*s, v) + return nil +} + +// createTypeArg returns the value of the last --type/-type flag in args +// (Go's flag package keeps the last one), or "" when there is none. +func createTypeArg(args []string) string { + typ := "" + for i := 0; i < len(args); i++ { + a := args[i] + if a == "--" { + break + } + name := strings.TrimLeft(a, "-") + if name == a { + continue + } + if v, ok := strings.CutPrefix(name, "type="); ok { + typ = v + continue + } + if name == "type" && i+1 < len(args) { + typ = args[i+1] + i++ + } + } + return typ +} + +// parseKnowledgeCreate turns `bd create --type knowledge …` arguments into +// the request body. stdin is read for --body-file -. +func parseKnowledgeCreate(args []string, stdin io.Reader) (knowledgeCreateRequest, error) { + fs := flag.NewFlagSet("create", flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.String("type", knowledgeCreateType, "must be knowledge") + title := fs.String("title", "", "Fact title (required)") + body := fs.String("body", "", "Fact body") + bodyFile := fs.String("body-file", "", "Read the fact body from a file (- for stdin)") + state := fs.String("state", "", "Lifecycle state to create the fact in: draft or approved (default approved)") + layer := fs.String("layer", "", "Knowledge layer, or for a draft the local channel to write to (default project)") + factType := fs.String("fact-type", "", "Fact type, e.g. pattern, gotcha, decision (default pattern)") + var tags stringList + fs.Var(&tags, "tag", "Tag to attach (repeatable)") + if err := fs.Parse(args); err != nil { + return knowledgeCreateRequest{}, err + } + if fs.NArg() > 0 { + return knowledgeCreateRequest{}, fmt.Errorf("unexpected argument %q", fs.Arg(0)) + } + if strings.TrimSpace(*title) == "" { + return knowledgeCreateRequest{}, errors.New("--title is required") + } + if (*body == "") == (*bodyFile == "") { + return knowledgeCreateRequest{}, errors.New("exactly one of --body and --body-file is required") + } + text := *body + if *bodyFile != "" { + var r io.Reader = stdin + if *bodyFile != "-" { + f, err := os.Open(*bodyFile) + if err != nil { + return knowledgeCreateRequest{}, err + } + defer func() { _ = f.Close() }() + r = f + } + data, err := io.ReadAll(io.LimitReader(r, maxKnowledgeBodyBytes+1)) + if err != nil { + return knowledgeCreateRequest{}, fmt.Errorf("reading --body-file: %w", err) + } + if len(data) > maxKnowledgeBodyBytes { + return knowledgeCreateRequest{}, fmt.Errorf("--body-file is over %d bytes", maxKnowledgeBodyBytes) + } + text = string(data) + } + if strings.TrimSpace(text) == "" { + return knowledgeCreateRequest{}, errors.New("the fact body is empty") + } + return knowledgeCreateRequest{ + Title: *title, + Body: text, + Type: *factType, + Tags: tags, + Layer: *layer, + State: strings.ToLower(strings.TrimSpace(*state)), + }, nil +} + +func cmdCreateKnowledge(args []string) { + req, err := parseKnowledgeCreate(args, os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "bd create --type knowledge: %v\n", err) + os.Exit(1) + } + payload, err := json.Marshal(req) + if err != nil { + fmt.Fprintf(os.Stderr, "bd create --type knowledge: %v\n", err) + os.Exit(1) + } + body, err := kbPost(dashboardURL()+"/api/knowledge/create", string(payload)) + if err != nil { + fmt.Fprintf(os.Stderr, "bd create --type knowledge: %v\n", err) + os.Exit(1) + } + var resp struct { + Title string `json:"title"` + Layer string `json:"layer"` + State string `json:"state"` + } + if err := json.Unmarshal(body, &resp); err != nil { + fmt.Fprintf(os.Stderr, "bd create --type knowledge: invalid response: %v\n", err) + os.Exit(1) + } + fmt.Printf("Created knowledge fact %q in %s (state: %s)\n", resp.Title, resp.Layer, resp.State) +} diff --git a/src/cmd/bd/create_knowledge_test.go b/src/cmd/bd/create_knowledge_test.go new file mode 100644 index 0000000000..ce03456a9b --- /dev/null +++ b/src/cmd/bd/create_knowledge_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestCreateTypeArg(t *testing.T) { + cases := []struct { + args []string + want string + }{ + {[]string{"--title", "x", "--type", "knowledge"}, "knowledge"}, + {[]string{"-type=knowledge", "--title", "x"}, "knowledge"}, + {[]string{"--type", "task", "--type=knowledge"}, "knowledge"}, + {[]string{"--type=knowledge", "--type", "bug"}, "bug"}, + {[]string{"--title", "x"}, ""}, + {[]string{"--", "--type", "knowledge"}, ""}, + } + for _, tc := range cases { + if got := createTypeArg(tc.args); got != tc.want { + t.Errorf("createTypeArg(%q) = %q, want %q", tc.args, got, tc.want) + } + } +} + +func TestParseKnowledgeCreate(t *testing.T) { + bodyFile := filepath.Join(t.TempDir(), "entry.md") + if err := os.WriteFile(bodyFile, []byte("from a file\n"), 0o644); err != nil { + t.Fatal(err) + } + got, err := parseKnowledgeCreate([]string{ + "--type", "knowledge", "--title", "T", "--body-file", bodyFile, + "--state", "Draft", "--tag", "cli-memory", "--tag", "agent:scanner", "--layer", "drafts", + }, strings.NewReader("")) + if err != nil { + t.Fatal(err) + } + want := knowledgeCreateRequest{Title: "T", Body: "from a file\n", Tags: []string{"cli-memory", "agent:scanner"}, Layer: "drafts", State: "draft"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %+v\nwant %+v", got, want) + } + + if got, err := parseKnowledgeCreate([]string{"--title", "T", "--body-file", "-"}, strings.NewReader("stdin body")); err != nil || got.Body != "stdin body" { + t.Fatalf("stdin body: %+v %v", got, err) + } + + for _, args := range [][]string{ + {"--body", "b"}, // no title + {"--title", "T"}, // no body + {"--title", "T", "--body", "b", "--body-file", bodyFile}, // both bodies + {"--title", "T", "--body", " "}, // blank body + {"--title", "T", "--body", "b", "extra"}, // stray argument + {"--title", "T", "--body", "b", "--priority", "1"}, // bead-only flag + } { + if _, err := parseKnowledgeCreate(args, strings.NewReader("")); err == nil { + t.Errorf("parseKnowledgeCreate(%q) accepted", args) + } + } +} + +// bd create --type knowledge posts to the dashboard, not the bead store. +func TestCmdCreateKnowledgePostsToDashboard(t *testing.T) { + t.Setenv("BD_DIR", t.TempDir()) + var got map[string]any + kbTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/knowledge/create" { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Errorf("decoding request: %v", err) + } + jsonResponse(t, w, map[string]any{"ok": true, "title": "T", "layer": "drafts", "state": "draft"}) + }) + cmdCreate([]string{"--type", "knowledge", "--title", "T", "--body", "b", "--state", "draft", "--tag", "cli-memory", "--layer", "drafts"}) + want := map[string]any{"title": "T", "body": "b", "state": "draft", "tags": []any{"cli-memory"}, "layer": "drafts"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("payload = %v, want %v", got, want) + } + if entries, _ := os.ReadDir(os.Getenv("BD_DIR")); len(entries) != 0 { + t.Fatalf("knowledge create touched the bead store: %d entries", len(entries)) + } +} diff --git a/src/cmd/bd/main.go b/src/cmd/bd/main.go index a347622853..1e1edb4327 100644 --- a/src/cmd/bd/main.go +++ b/src/cmd/bd/main.go @@ -97,6 +97,9 @@ Commands: list [--json] [--status=] [--actor=] List beads ready [--json] [--actor=] List open/unblocked beads create --title "..." --type --priority <0-4> --actor [--external-ref ] + create --type knowledge --title "..." (--body "..." | --body-file ) + [--state draft|approved] [--tag ]... [--layer ] [--fact-type ] + Create a knowledge fact via the dashboard API update --claim Claim a bead (set in_progress) update --status Change status update --set-metadata key=value Set metadata key @@ -177,9 +180,13 @@ func cmdReady(args []string) { // ---------- create ---------- func cmdCreate(args []string) { + if createTypeArg(args) == knowledgeCreateType { + cmdCreateKnowledge(args) + return + } fs := flag.NewFlagSet("create", flag.ExitOnError) title := fs.String("title", "", "Bead title (required)") - beadType := fs.String("type", "task", "Bead type: bug|feature|task|epic|chore|decision|advisory") + beadType := fs.String("type", "task", "Bead type: bug|feature|task|epic|chore|decision|advisory|knowledge") priority := fs.Int("priority", 2, "Priority 0-4 (0=critical, 4=minor)") actor := fs.String("actor", "", "Actor name (required)") extRef := fs.String("external-ref", "", "External reference (e.g. issue URL)") diff --git a/src/cmd/hive/boot_cli_memory_drain.go b/src/cmd/hive/boot_cli_memory_drain.go index c56db0fb8e..99260f8215 100644 --- a/src/cmd/hive/boot_cli_memory_drain.go +++ b/src/cmd/hive/boot_cli_memory_drain.go @@ -14,6 +14,12 @@ import ( // and registered under; CLI-memory drafts land in the same store. const beadSynthVaultName = "bead-synth-wiki" +// cliMemorySpoolDir keeps a copy of memory files whose facts could not all +// be stored, per agent, so clear_on_kick does not lose them; the drainer +// retries them on the agent's next drain (#17056). A var so tests can point +// it at a temp dir. +var cliMemorySpoolDir = "/data/knowledge/auto-memory-spool" + // cliMemoryDrainEnabled reports whether any enabled agent drains its CLI // memory into the knowledge store (agents.cli_memory.drain_to_knowledge, // fleet-wide or per agent). @@ -67,7 +73,7 @@ func (b *boot) installCLIMemoryDrainer() { if layer == "" { layer = knowledge.LayerPersonal } - drainer := &climemory.Drainer{Dir: vaultPath, Store: b.knowledgeAPI.GetVaultStore(vaultPath), Layer: layer} + drainer := &climemory.Drainer{Dir: vaultPath, Store: b.knowledgeAPI.GetVaultStore(vaultPath), Layer: layer, SpoolDir: cliMemorySpoolDir} cfg := b.cfg host.SetCLIMemoryDrainer(func(req crux.CLIMemoryDrainRequest) crux.CLIMemoryDrainResult { sources := make([]climemory.Source, 0, len(req.Sources)) @@ -80,6 +86,7 @@ func (b *boot) installCLIMemoryDrainer() { Drained: res.Drained, Dupes: res.Dupes, Skipped: res.Skipped, + Spooled: res.Spooled, Err: errors.Join(res.Errors...), } }) diff --git a/src/cmd/hive/boot_cli_memory_drain_test.go b/src/cmd/hive/boot_cli_memory_drain_test.go index 9dd39a4e9b..c297b04479 100644 --- a/src/cmd/hive/boot_cli_memory_drain_test.go +++ b/src/cmd/hive/boot_cli_memory_drain_test.go @@ -54,6 +54,9 @@ func TestInstallCLIMemoryDrainerWritesDraftsIntoBeadSynthVault(t *testing.T) { cfg := bootKnowledgeConfig(t) cfg.AgentsCLIMemory.DrainToKnowledge = true cfg.Knowledge.BeadSynthesizer.TargetLayer = "team" + prevSpool := cliMemorySpoolDir + cliMemorySpoolDir = t.TempDir() + t.Cleanup(func() { cliMemorySpoolDir = prevSpool }) b := newBootKnowledgeBoot(t, cfg) var log strings.Builder b.logger = slog.New(slog.NewTextHandler(&log, nil)) diff --git a/src/crux/agent/cli_memory_drain.go b/src/crux/agent/cli_memory_drain.go index 55658cfc85..7609f0b30a 100644 --- a/src/crux/agent/cli_memory_drain.go +++ b/src/crux/agent/cli_memory_drain.go @@ -77,7 +77,7 @@ func cliMemoryDrainSources(agent *AgentProcess, backend string) []CLIMemorySourc // best-effort: a failed drain is logged and the kick proceeds. The one log // line per drain is // -// cli-memory: drained N facts from into (dupes K) +// cli-memory: drained N facts from into (dupes K, spooled S) func (m *Manager) drainCLIMemoryForKick(agent *AgentProcess, backend string) { if !agent.Config.CLIMemory.DrainMemoryToKnowledge() { return @@ -97,8 +97,8 @@ func (m *Manager) drainCLIMemoryForKick(agent *AgentProcess, backend string) { m.logger.Warn("cli-memory: drain failed before kick", "agent", agent.Name, "store", res.Store, "drained", res.Drained, "error", res.Err) } - m.logger.Info(fmt.Sprintf("cli-memory: drained %d facts from %s into %s (dupes %d)", - res.Drained, agent.Name, res.Store, res.Dupes), + m.logger.Info(fmt.Sprintf("cli-memory: drained %d facts from %s into %s (dupes %d, spooled %d)", + res.Drained, agent.Name, res.Store, res.Dupes, res.Spooled), "agent", agent.Name, "store", res.Store, "drained", res.Drained, - "dupes", res.Dupes, "skipped", res.Skipped, "sources", len(sources)) + "dupes", res.Dupes, "skipped", res.Skipped, "spooled", res.Spooled, "sources", len(sources)) } diff --git a/src/crux/cli_memory_drain.go b/src/crux/cli_memory_drain.go index 00d69fb5a4..ba6b6198e5 100644 --- a/src/crux/cli_memory_drain.go +++ b/src/crux/cli_memory_drain.go @@ -21,13 +21,16 @@ type CLIMemoryDrainRequest struct { } // CLIMemoryDrainResult is what the drainer reports back for the runtime's -// one log line per drain. +// one log line per drain. Spooled counts memory files kept in the hive's +// spool because not all of their facts were stored; they are retried on the +// agent's next drain, so the wipe that follows the drain does not lose them. type CLIMemoryDrainResult struct { // Store is the knowledge store (vault path or name) the drafts landed in. Store string Drained int Dupes int Skipped int + Spooled int Err error } diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index f6e5596837..6ebddb3cc5 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -649,9 +649,9 @@ agents: | `auto_memory` | `false` | When false hive pins the vendor switch off at launch: Claude Code gets `CLAUDE_CODE_DISABLE_AUTO_MEMORY=1` in its environment ([docs](https://code.claude.com/docs/en/memory)); Codex gets `[features] memories = false` merged into `$CODEX_HOME/config.toml` ([docs](https://developers.openai.com/codex/memories)); Copilot CLI gets `"memory": false` in its per-agent `~/.copilot/settings.json` (`copilot help config`). `true` writes the opposite pin so an operator's own setting is never silently shadowed. Gemini CLI has no equivalent switch. | | `clear_on_kick` | `false` | Before every non-resume kick (next to the `/clear` sent by `clear_on_kick`), remove this agent's `~/.claude/projects/*/memory/`, `$CODEX_HOME/memories*/`, and `~/.gemini/GEMINI.md`. Only the per-agent home under `/data/home/agents/` is touched; nothing under the shared `/data/home` is removed, and agents running with `HIVE_SHARED_AGENT_HOME=1` or as uid 0 are skipped. Copilot memory is server-side and has no local store. | -| `drain_to_knowledge` | `false` | Before every non-resume kick, and before `clear_on_kick` wipes anything, read this agent's Claude `~/.claude/projects/*/memory/` (`MEMORY.md` plus topic files), Codex `$CODEX_HOME/memories*/` and the `## Gemini Added Memories` section of `~/.gemini/GEMINI.md`, and write each bullet or paragraph as a **draft** entry in the `bead-synth-wiki` vault (`knowledge.bead_synthesizer.vault_path`, default `/data/vaults/bead-synth-wiki`) with `source: cli-memory`, `agent: `, `cli: ` and, when the Claude project slug ends in one of the agent's repos, `repo: `. Entries are deduplicated by a SHA-256 of the normalised text (`memhash`), so repeated drains and the same fact from several agents produce one entry; an entry an operator approved, deprecated or renamed is still a duplicate. Drafts never reach a kick until a human approves them ([#17033](https://github.com/hivecommons/hive/issues/17033)). Needs the in-process crux runtime. | +| `drain_to_knowledge` | `false` | Before every non-resume kick, and before `clear_on_kick` wipes anything, read this agent's Claude `~/.claude/projects/*/memory/` (`MEMORY.md` plus topic files), Codex `$CODEX_HOME/memories*/` and the `## Gemini Added Memories` section of `~/.gemini/GEMINI.md`, and write each bullet or paragraph as a **draft** entry in the `bead-synth-wiki` vault (`knowledge.bead_synthesizer.vault_path`, default `/data/vaults/bead-synth-wiki`) with `source: cli-memory`, `agent: `, `cli: ` and, when the Claude project slug ends in one of the agent's repos, `repo: `. Entries are deduplicated by a SHA-256 of the normalised text (`memhash`), so repeated drains and the same fact from several agents produce one entry, which gains an `agent:` tag per draining agent; an entry an operator approved, deprecated or renamed is still a duplicate and keeps its state. A memory file whose facts did not all reach the vault (vault unwritable, 200-facts-per-drain cap) is copied to `/data/knowledge/auto-memory-spool//` before the wipe and retried on the next drain ([#17056](https://github.com/hivecommons/hive/issues/17056)). Drafts never reach a kick until a human approves them ([#17033](https://github.com/hivecommons/hive/issues/17033)). Needs the in-process crux runtime. | -Set `auto_memory: true` on a single agent to let it keep a private notebook while the rest of the fleet stays stateless between kicks. The knobs are independent: `auto_memory: true` with `clear_on_kick: true` gives a memory that lives for exactly one kick, and adding `drain_to_knowledge: true` is the "memory → beads" flow: the CLI may take notes during a turn, every note becomes a reviewable draft at the next kick, and the private store is then emptied. The log line per drain is `cli-memory: drained N facts from into bead-synth-wiki (dupes K)`. +Set `auto_memory: true` on a single agent to let it keep a private notebook while the rest of the fleet stays stateless between kicks. The knobs are independent: `auto_memory: true` with `clear_on_kick: true` gives a memory that lives for exactly one kick, and adding `drain_to_knowledge: true` is the "memory → beads" flow: the CLI may take notes during a turn, every note becomes a reviewable draft at the next kick, and the private store is then emptied. The log line per drain is `cli-memory: drained N facts from into bead-synth-wiki (dupes K, spooled S)`. Every Model Gateway (and the bob backend) also accepts an optional `key_name` — a human-chosen LABEL for the configured key, e.g. `key_name: openrouter-prod-key`. It is safe-to-show metadata, not a secret: the dashboard's gateway row displays it as "Using key: ``", or "(unnamed)" when no label is set, so operators can tell keys apart without ever seeing the value. For gateway keys, the row and settings/status APIs also expose `keySHA256`, the lowercase SHA-256 of the exact effective key string Hive will inject through the inference proxy; this lets operators compare Hive with LiteLLM/OpenRouter key-hash displays without revealing the secret. Saving a replacement key refreshes live inference proxy routes for running agents, so the next kick uses the new key without an agent restart. See [`inference-backends.md`](../../docs/inference-backends.md) for a full YAML example. diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md index 122418e51a..4ae3dadba4 100644 --- a/src/docs/agent-knowledge-and-memory.md +++ b/src/docs/agent-knowledge-and-memory.md @@ -218,12 +218,22 @@ entry is written with `state: draft`, `source: cli-memory`, `agent: `, `cli: `, `repo: ` when the Claude project slug ends in one of the agent's repos, and a `memhash` (SHA-256 of the normalised text) that makes the drain idempotent: a fact already in the vault, in any -state and under any file name, counts as a duplicate. Because the Primer is +state and under any file name, counts as a duplicate. A duplicate is never +rewritten; it only gains the draining agent's `agent:` tag, so an +inference several agents reached on their own is one entry with one tag per +agent, and a fact an operator deprecated or superseded stays that way however +often a CLI writes it again ([#17056](https://github.com/hivecommons/hive/issues/17056)). +A memory file whose facts do not all reach the vault (the vault is +unwritable, or the 200-facts-per-drain cap was hit) is copied to +`/data/knowledge/auto-memory-spool//` first, so the `clear_on_kick` +wipe that follows loses nothing; the agent's next drain retries the spool +before its live store and deletes each spooled copy once all its facts are +stored. The drain never blocks the kick. Because the Primer is approved-only by default (section 1), a drained fact is **not** injected into anyone's kick until an operator approves it through the normal lifecycle — that review step is the point: the CLI's one-off inference becomes a candidate, not a rule. One log line per drain: -`cli-memory: drained N facts from into bead-synth-wiki (dupes K)`. +`cli-memory: drained N facts from into bead-synth-wiki (dupes K, spooled S)`. The drain needs the in-process crux runtime; the sidecar runtime logs a warning and only wipes. diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 8dfc69b6e6..38937cecb1 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -19,7 +19,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:387` | | `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:388` | | `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:389` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:390` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:`, `cli:`, `memhash:` and `repo:/` tags (400 otherwise) | `pkg/dashboard/api.go:390` | | `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:391` | | `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:397` | | `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:398` | diff --git a/src/docs/beads-cli.md b/src/docs/beads-cli.md index ca236a891e..c9165ba067 100644 --- a/src/docs/beads-cli.md +++ b/src/docs/beads-cli.md @@ -8,7 +8,7 @@ |---|---| | `bd list [--json] [--status ] [--actor ]` | List beads; filter by status or actor. | | `bd ready [--json] [--actor ]` | Show open, unblocked work ready to claim. | -| `bd create --title "..." --type --priority <0-4> --actor [--external-ref ]` | Create work. Types include `bug`, `feature`, `task`, `epic`, `chore`, `decision`, and `advisory`; priority `0` is highest. | +| `bd create --title "..." --type --priority <0-4> --actor [--external-ref ]` | Create work. Types include `bug`, `feature`, `task`, `epic`, `chore`, `decision`, and `advisory`; priority `0` is highest. `--type knowledge` creates a knowledge fact instead; see [Knowledge commands](#knowledge-commands). | | `bd update --claim` | Mark a bead `in_progress`. | | `bd update --status open|in_progress|blocked|done|closed` | Change status directly. | | `bd update --set-metadata key=value` / `--unset-metadata key` | Add or remove custom metadata in the bead ledger. | @@ -62,7 +62,7 @@ bd close bead-123 ## Knowledge commands -`bd kb` talks to the dashboard API (`BD_DASHBOARD_URL`, default `http://localhost:3001`) rather than the local bead store. +`bd kb` and `bd create --type knowledge` talk to the dashboard API (`BD_DASHBOARD_URL`, default `http://localhost:3001`) rather than the local bead store. | Command | Use | |---|---| @@ -73,3 +73,14 @@ bd close bead-123 | `bd kb ctx7-search ` | Search Context7 library IDs. | | `bd kb import-ctx7 [--name ] [--query ]` | Import Context7 docs into the community layer. | | `bd kb list-docs` | List imported documents. | +| `bd create --type knowledge --title "..." (--body "..." \| --body-file ) [--state draft\|approved] [--tag ]... [--layer ] [--fact-type ]` | Create one knowledge fact through `POST /api/knowledge/create`. Defaults: layer `project`, fact type `pattern`, state `approved`. `--body-file -` reads stdin (256 KiB max). | + +`--state draft` writes the fact into the local channel (vault) named by `--layer`, with `state: draft`, so it stays out of every kick until an operator approves it (`hivectl knowledge state approved` or `PUT /api/knowledge/entry/{id}/state`). Remote wiki layers only record a status, so the dashboard refuses a draft for a layer that is not a local channel, and it refuses a draft whose title already names an entry in that channel (HTTP 409) rather than overwrite it. Only `draft` and `approved` are accepted at create time. + +A fact tagged `cli-memory` claims to come from a CLI's memory store ([memory → knowledge drain](agent-knowledge-and-memory.md)). The dashboard accepts one only as `--state draft`, and only with the tags the drain itself writes: `cli-memory`, `agent:`, `cli:`, `memhash:` and `repo:/`. + +```bash +bd create --type knowledge --state draft --layer notes \ + --title "Read the failing job log first" --body-file note.md \ + --tag cli-memory --tag agent:scanner --tag cli:codex +``` diff --git a/src/docs/design/README.md b/src/docs/design/README.md index b2e8d4cea0..25c40add6e 100644 --- a/src/docs/design/README.md +++ b/src/docs/design/README.md @@ -22,7 +22,7 @@ that status is the thing to check before treating a page as current behaviour: - [Crux: the agent runtime boundary](crux.md) — **Stage 1 shipped (v5); Stage 2 blocked on `pkg/config`.** RFC [#13536](https://github.com/hivecommons/hive/issues/13536): names the agent runtime inside Hive (`pkg/agent` and its ring) as Crux, lists what stays in Hive, records the staged plan (in-repo boundary → nested module → `crux serve`) and the naming decision. Inbound and outbound ratchets in `src/internal/testutil/crux_*_ratchet_test.go`, the tmux funnel, `crux.Runtime`, Crux-owned value types, CI path filters and the `crux` CLI are live; the nested module waits on an agent-runtime config leaf (the doc lists the measured outbound edges and prerequisites). -- [Draining CLI auto-memory into draft knowledge](cli-memory-drain.md) — **Partly shipped** (phase 1: manager-side drain; phase 3: review flow, draft exclusion, legacy drain; phase 2, the Claude Code `SessionEnd` hook, is open). #17033: turns each coding CLI's hidden memory store (Claude Code `projects/*/memory`, Codex `memories`, Gemini `GEMINI.md`) into `draft` knowledge facts tagged `cli-memory` and `agent:`. Claude Code drains through a `SessionEnd` hook; the others drain on the manager side at kick time, before `clear_on_kick`. Decides content-hash dedup across agents, an ingest-only drain with the existing curator as the only promoter, and untrusted-content handling (never primed while draft). Implementation is split into three child issues. +- [Draining CLI auto-memory into draft knowledge](cli-memory-drain.md) — **Partly shipped** (phase 1: manager-side drain, dedup tags, spool, draft create path; phase 3: review flow, draft exclusion, legacy drain; phase 2, the Claude Code `SessionEnd` hook, is open). #17033: turns each coding CLI's hidden memory store (Claude Code `projects/*/memory`, Codex `memories`, Gemini `GEMINI.md`) into `draft` knowledge facts tagged `cli-memory` and `agent:`. Claude Code drains through a `SessionEnd` hook; the others drain on the manager side at kick time, before `clear_on_kick`. Decides content-hash dedup across agents, an ingest-only drain with the existing curator as the only promoter, and untrusted-content handling (never primed while draft). Implementation is split into three child issues. - [Crux: generic tool policy for the MITM proxy](crux-tool-policy.md) — **Design only (proposed).** #14228, child of #14138: a shared operation vocabulary with per-forge adapters, a `relay()` verdict, deny-by-default for unrecognised operations, and the proxy's evaluation pipeline written down once. It also covers migrating behind a parity test and treating #14211 as the first non-GitHub adapter. diff --git a/src/docs/design/cli-memory-drain.md b/src/docs/design/cli-memory-drain.md index 5c3899d78c..b218157e60 100644 --- a/src/docs/design/cli-memory-drain.md +++ b/src/docs/design/cli-memory-drain.md @@ -1,8 +1,9 @@ # Draining CLI auto-memory into draft knowledge (#17033) -Status: **design only.** Nothing described here is implemented yet. The three -open questions in #17033 are decided below; the work is cut into three -implementation children, listed in [Implementation plan](#implementation-plan). +Status: **phase 1 shipped, phases 2 and 3 open.** The three open questions in +#17033 are decided below; the work is cut into three implementation children, +listed in [Implementation plan](#implementation-plan). Phase 1 shipped with +the differences recorded in [Phase 1 as shipped](#phase-1-as-shipped). All citations are against `origin/v5` at `7a929356`. The CLI-memory isolation and disable work this design builds on (#17031, #17032) is in open PRs @@ -339,6 +340,31 @@ Child: #17056. not revived, partial failure leaves the file, symlinked or shared stores are skipped, spool and retry, kick never blocked. +#### Phase 1 as shipped + +Phase 1 landed in #17086 and the follow-up for #17056. Where the shipped code +differs from the plan above, the shipped code is authoritative: + +- **Drain library.** The drain is `src/pkg/knowledge/climemory`, run in the + hive process and installed on the in-process runtime through + `crux.CLIMemoryDrainHost`, not an HTTP-only `src/crux/agent/memdrain`. It + writes into the `bead-synth-wiki` vault the Primer already reads, so the + Crux module still imports nothing from `src/pkg`. +- **Names.** The config key is `agents.cli_memory.drain_to_knowledge`, and + the provenance tag is `cli-memory` (with `source: cli-memory` frontmatter) + rather than `auto-memory`. +- **Dedup.** As designed: a duplicate in any state only gains the draining + agent's `agent:` tag; its state is never reset. +- **Store emptying.** The drain does not truncate the store; `clear_on_kick` + does. A file whose facts did not all reach the vault is **copied** to + `/data/knowledge/auto-memory-spool//` before the wipe and retried on + the next drain, which gives the "partial failure loses nothing" property + without a second remover running as the agent. +- **Create path.** `CreateFactRequest.state` and the `cli-memory` tag + allow-list on `POST /api/knowledge/create`, and `bd create --type knowledge`, + shipped as planned; a draft is written into the local channel named by + `layer`. + ### Phase 2 — Claude Code `SessionEnd` and `PostToolUse` hooks Child: #17057. diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index 155b668c2d..0602262e7b 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -105,12 +105,16 @@ agents: |---|---|---| | `auto_memory` | `false` | Pins Claude Code (`CLAUDE_CODE_DISABLE_AUTO_MEMORY=1`), Codex (`[features] memories = false`) and Copilot CLI (`"memory": false`) auto-memory off at launch; `true` pins it on. | | `clear_on_kick` | `false` | Removes the agent's own memory stores before each non-resume kick; shared homes (`HIVE_SHARED_AGENT_HOME=1`, uid 0) are never touched. | -| `drain_to_knowledge` | `false` | Before the wipe, writes every memory bullet or paragraph as a `state: draft` entry into the `bead-synth-wiki` vault (`knowledge.bead_synthesizer.vault_path`), tagged `cli-memory`, `agent:`, `cli:` and `repo:` when derivable, deduplicated by content hash. Drafts are excluded from kicks until approved. | +| `drain_to_knowledge` | `false` | Before the wipe, writes every memory bullet or paragraph as a `state: draft` entry into the `bead-synth-wiki` vault (`knowledge.bead_synthesizer.vault_path`), tagged `cli-memory`, `agent:`, `cli:` and `repo:` when derivable, deduplicated by content hash; a duplicate only gains the draining agent's `agent:` tag and keeps its state. Drafts are excluded from kicks until approved. | The drain logs one line per kick that found a store: -`cli-memory: drained N facts from into bead-synth-wiki (dupes K)` -with `drained`, `dupes`, `skipped` (over the 200-facts-per-drain cap; the -rest drain on the next kick) and `sources`. Review drafts in the dashboard's +`cli-memory: drained N facts from into bead-synth-wiki (dupes K, spooled S)` +with `drained`, `dupes`, `skipped` (over the 200-facts-per-drain cap), +`spooled` and `sources`. A memory file whose facts did not all reach the vault +(cap hit, vault unwritable) is copied to +`/data/knowledge/auto-memory-spool//` before `clear_on_kick` wipes the +store, retried first on the agent's next drain, and deleted from the spool +once every fact in it is stored. Review drafts in the dashboard's knowledge view or the vault directory and approve them through the normal lifecycle (`PUT /api/knowledge/entry/{id}/state`); nothing promotes them automatically. diff --git a/src/pkg/dashboard/api_knowledge.go b/src/pkg/dashboard/api_knowledge.go index 7f8b23094e..42adba4592 100644 --- a/src/pkg/dashboard/api_knowledge.go +++ b/src/pkg/dashboard/api_knowledge.go @@ -2,6 +2,7 @@ package dashboard import ( "crypto/sha256" + "errors" "fmt" "net/http" "os" @@ -13,6 +14,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/knowledge/climemory" ) func (s *Server) handleKnowledgeToggle(w http.ResponseWriter, r *http.Request) { @@ -590,6 +592,20 @@ func (s *Server) handleKnowledgeCreate(w http.ResponseWriter, r *http.Request) { jsonError(w, "title and body are required", http.StatusBadRequest) return } + state, err := knowledge.ParseCreateState(req.State) + if err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + // A fact that claims to be drained from CLI memory must enter as a + // draft carrying only the drain's own tags (#17056). + if err := climemory.ValidateCreate(req.State, req.Tags); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if state == "" { + state = knowledge.StateApproved + } if req.Layer == "" { req.Layer = "project" } @@ -602,11 +618,18 @@ func (s *Server) handleKnowledgeCreate(w http.ResponseWriter, r *http.Request) { } if err := s.deps.Knowledge.CreateFact(s.deps.Ctx, req); err != nil { - jsonError(w, err.Error(), http.StatusInternalServerError) + status := http.StatusInternalServerError + switch { + case errors.Is(err, knowledge.ErrNoDraftChannel): + status = http.StatusBadRequest + case errors.Is(err, knowledge.ErrFactExists): + status = http.StatusConflict + } + jsonError(w, err.Error(), status) return } - s.auditFromRequest(r, "knowledge_create_fact", auditDetail("title", req.Title, "layer", req.Layer), "") - jsonResponse(w, map[string]interface{}{"ok": true, "title": req.Title, "layer": req.Layer}) + s.auditFromRequest(r, "knowledge_create_fact", auditDetail("title", req.Title, "layer", req.Layer, "state", string(state)), "") + jsonResponse(w, map[string]interface{}{"ok": true, "title": req.Title, "layer": req.Layer, "state": state}) } func (s *Server) handleKnowledgeUpdate(w http.ResponseWriter, r *http.Request) { diff --git a/src/pkg/dashboard/api_knowledge_create_state_test.go b/src/pkg/dashboard/api_knowledge_create_state_test.go new file mode 100644 index 0000000000..81f56bc9b0 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_create_state_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/knowledge" +) + +// TestHandleKnowledgeCreate_StateAndDrainProvenance covers the create-time +// lifecycle state and the CLI-memory provenance guard (#17056): a fact +// claiming drain provenance enters only as a draft with the drain's own +// tags, a draft lands in a local channel with `state: draft`, and a draft +// never overwrites an existing entry. +func TestHandleKnowledgeCreate_StateAndDrainProvenance(t *testing.T) { + s, deps, wiki := apiServerWithKnowledge(t) + defer wiki.Close() + vault := t.TempDir() + if err := deps.Knowledge.ConnectVault(vault, "drafts"); err != nil { + t.Fatal(err) + } + drained := []string{"cli-memory", "agent:scanner", "cli:codex", "memhash:0123456789abcdef", "repo:hivecommons/hive"} + + rejected := []struct { + name string + req knowledge.CreateFactRequest + code int + }{ + {"deprecated is not a create state", knowledge.CreateFactRequest{Title: "t", Body: "b", State: "deprecated"}, http.StatusBadRequest}, + {"unknown state", knowledge.CreateFactRequest{Title: "t", Body: "b", State: "live"}, http.StatusBadRequest}, + {"drain tag without state", knowledge.CreateFactRequest{Title: "t", Body: "b", Layer: "drafts", Tags: drained}, http.StatusBadRequest}, + {"drain tag approved", knowledge.CreateFactRequest{Title: "t", Body: "b", Layer: "drafts", Tags: drained, State: "approved"}, http.StatusBadRequest}, + {"drain tag, case folded", knowledge.CreateFactRequest{Title: "t", Body: "b", Layer: "drafts", Tags: []string{"CLI-Memory"}, State: "approved"}, http.StatusBadRequest}, + {"drain tag with a foreign tag", knowledge.CreateFactRequest{Title: "t", Body: "b", Layer: "drafts", Tags: append([]string{"security"}, drained...), State: "draft"}, http.StatusBadRequest}, + {"drain tag with a path-like agent", knowledge.CreateFactRequest{Title: "t", Body: "b", Layer: "drafts", Tags: []string{"cli-memory", "agent:../x"}, State: "draft"}, http.StatusBadRequest}, + {"draft into a layer with no channel", knowledge.CreateFactRequest{Title: "t", Body: "b", Layer: "project", State: "draft"}, http.StatusBadRequest}, + } + for _, tc := range rejected { + if rec := doPost(s, "/api/knowledge/create", tc.req); rec.Code != tc.code { + t.Errorf("%s: status = %d (%s), want %d", tc.name, rec.Code, strings.TrimSpace(rec.Body.String()), tc.code) + } + } + if entries, _ := os.ReadDir(vault); len(entries) != 0 { + t.Fatalf("rejected requests wrote %d files into the channel", len(entries)) + } + + req := knowledge.CreateFactRequest{Title: "Codex memory note", Body: "Run the failing job log first.", Layer: "drafts", Tags: drained, State: "draft"} + rec := doPost(s, "/api/knowledge/create", req) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"state":"draft"`) { + t.Fatalf("draft create: %d %s", rec.Code, rec.Body.String()) + } + files, _ := filepath.Glob(filepath.Join(vault, "*.md")) + if len(files) != 1 { + t.Fatalf("channel has %v, want one entry", files) + } + data, err := os.ReadFile(files[0]) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "\nstate: draft\n") { + t.Fatalf("entry is not a draft:\n%s", data) + } + store := deps.Knowledge.GetVaultStore(vault) + store.Reindex() + if got := knowledge.FilterByLifecycle(store.ListPages(""), nil); len(got) != 0 { + t.Fatalf("a created draft passed the approved-only filter: %+v", got) + } + + // Same title again: never overwrite, whatever the new state. + if rec := doPost(s, "/api/knowledge/create", req); rec.Code != http.StatusConflict { + t.Fatalf("duplicate draft: status = %d, want 409", rec.Code) + } + + // No state keeps the old default path (the remote project layer). + if rec := doPost(s, "/api/knowledge/create", knowledge.CreateFactRequest{Title: "plain", Body: "b"}); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"state":"approved"`) { + t.Fatalf("default create: %d %s", rec.Code, rec.Body.String()) + } +} diff --git a/src/pkg/knowledge/api.go b/src/pkg/knowledge/api.go index 8e6ce4c5c6..d01ebcbda1 100644 --- a/src/pkg/knowledge/api.go +++ b/src/pkg/knowledge/api.go @@ -290,6 +290,13 @@ func (k *KnowledgeAPI) Stats(ctx context.Context) map[string]interface{} { } // CreateFactRequest is the payload for creating a new fact. +// +// State is the lifecycle state the fact is created in: empty or "approved" +// (the default) ingests into the layer as before; "draft" writes the fact +// into the local vault (channel) named Layer with `state: draft`, because +// only a local channel records lifecycle state and lets an operator approve +// or discard the fact later (#17056). Deprecated and superseded are states +// a fact moves into, never ones it is created in. type CreateFactRequest struct { Title string `json:"title"` Body string `json:"body"` @@ -297,16 +304,40 @@ type CreateFactRequest struct { Tags []string `json:"tags"` Layer string `json:"layer"` Confidence float64 `json:"confidence"` + State string `json:"state,omitempty"` +} + +var ( + // ErrNoDraftChannel is returned by CreateFact for a draft fact whose + // Layer names no connected local channel. + ErrNoDraftChannel = errors.New("draft facts are written to a local channel") + // ErrFactExists is returned by CreateFact for a draft fact whose title + // maps to an entry that already exists in the channel. + ErrFactExists = errors.New("a fact with this title already exists") +) + +// ParseCreateState validates the state a fact is created in. Empty means +// the default (approved); only draft and approved are accepted. +func ParseCreateState(s string) (LifecycleState, error) { + if strings.TrimSpace(s) == "" { + return "", nil + } + st, err := ParseLifecycleState(s) + if err != nil { + return "", err + } + if st != StateDraft && st != StateApproved { + return "", fmt.Errorf("a fact is created as %s or %s, not %s", StateDraft, StateApproved, st) + } + return st, nil } // CreateFact ingests a new fact into the specified layer. func (k *KnowledgeAPI) CreateFact(ctx context.Context, req CreateFactRequest) error { - layer := LayerType(req.Layer) - client := k.clientForLayer(layer) - if client == nil { - return fmt.Errorf("layer %s has no configured endpoint", req.Layer) + state, err := ParseCreateState(req.State) + if err != nil { + return err } - fact := ExtractedFact{ Title: req.Title, Body: req.Body, @@ -316,6 +347,15 @@ func (k *KnowledgeAPI) CreateFact(ctx context.Context, req CreateFactRequest) er SourcePR: "manual", SourceDate: time.Now(), } + if state == StateDraft { + return k.createDraftFact(req.Layer, fact) + } + + layer := LayerType(req.Layer) + client := k.clientForLayer(layer) + if client == nil { + return fmt.Errorf("layer %s has no configured endpoint", req.Layer) + } if err := client.IngestFacts(ctx, []ExtractedFact{fact}); err != nil { return fmt.Errorf("ingesting fact: %w", err) @@ -325,6 +365,31 @@ func (k *KnowledgeAPI) CreateFact(ctx context.Context, req CreateFactRequest) er return nil } +// createDraftFact writes fact with `state: draft` into the local channel +// named channel. A remote wiki layer cannot hold a draft: it records only a +// status, and an unknown status loads as approved, so a draft sent there +// would reach primers at once. An existing entry with the same title is +// never overwritten, so a draft cannot replace an approved fact. +func (k *KnowledgeAPI) createDraftFact(channel string, fact ExtractedFact) error { + v := k.vaultByName(channel) + if v == nil { + return fmt.Errorf("%w; no channel named %q", ErrNoDraftChannel, channel) + } + if strings.TrimSpace(fact.Title) == "" { + return errors.New("a draft fact needs a title") + } + if _, err := os.Lstat(v.factPath(fact.Title)); err == nil { + return fmt.Errorf("%w in channel %q: %q", ErrFactExists, channel, fact.Title) + } + fact.State = StateDraft + if err := v.WriteFacts([]ExtractedFact{fact}); err != nil { + return fmt.Errorf("writing draft fact to channel %q: %w", channel, err) + } + k.triggerVaultReindex(v.RootDir()) + k.logger.Info("draft fact created", "title", fact.Title, "channel", channel, "type", fact.Type) + return nil +} + // UpdateFactRequest is the payload for updating an existing fact. // // State and SupersededBy change a fact's lifecycle (#11102). On a local diff --git a/src/pkg/knowledge/climemory/drain.go b/src/pkg/knowledge/climemory/drain.go index 411ce362fd..d0b86b84c5 100644 --- a/src/pkg/knowledge/climemory/drain.go +++ b/src/pkg/knowledge/climemory/drain.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" "github.com/hivecommons/hive/pkg/knowledge" @@ -32,8 +33,8 @@ const ( // a transcript, not a fact. maxMemoryFileBytes = 512 * 1024 // maxFactsPerDrain caps one drain so a pathological memory store cannot - // flood the vault; the rest is drained on the next kick (the memory file - // is not touched by this package). + // flood the vault; a file with facts over the cap is spooled (when a + // spool is configured) and drained on the next kick. maxFactsPerDrain = 200 // frontmatterScanBytes bounds how much of an existing entry is read when // collecting the hashes already in the vault. @@ -58,91 +59,170 @@ type Request struct { // Result reports what the drain did. Drained counts entries written, Dupes // the facts whose hash was already in the vault (or repeated within this -// drain), Skipped the facts over the per-drain cap. +// drain), Skipped the facts over the per-drain cap, Spooled the memory files +// copied to the spool because not all of their facts were stored. type Result struct { Store string Drained int Dupes int Skipped int + Spooled int Errors []error } // Drainer writes draft entries into Dir. Store, when set, is reindexed after -// a drain that wrote something so the Primer and the dashboard see the new +// a drain that changed the vault so the Primer and the dashboard see the new // drafts without waiting for the index refresh. Layer defaults to personal. +// +// SpoolDir, when set, keeps a copy of every memory file whose facts did not +// all reach the vault (the vault is unwritable, or the per-drain cap was +// reached) under SpoolDir//, so clear_on_kick wiping the store right +// after the drain does not lose them. The agent's spool is retried first on +// its next drain and each spooled file is removed once all its facts are +// stored. type Drainer struct { - Dir string - Store *knowledge.FileStore - Layer knowledge.LayerType - Now func() time.Time + Dir string + Store *knowledge.FileStore + Layer knowledge.LayerType + SpoolDir string + Now func() time.Time + + // mu serialises drains: two agents reaching the same fact at once must + // not race on the entry's tag rewrite or its spool directory. + mu sync.Mutex +} + +// memFile is one memory file to drain. path is the file in the agent's +// store (recorded as source_path); spoolDir is set when the content is +// being retried from the spool. +type memFile struct { + cli, path, repo string + spoolDir string +} + +// drainRun is the state of one Drain call. +type drainRun struct { + d *Drainer + req Request + res *Result + seen map[string]string // memhash → vault entry path + changed bool + now func() time.Time + layer knowledge.LayerType } // Drain reads every source, converts its facts to draft entries and writes // the ones whose hash is not yet in the vault. Entries are always written // with `state: draft`; nothing in this package can produce another state. +// A fact already in the vault, in any state, is never rewritten: it only +// gains this agent's agent: tag, so a fact an operator deprecated or +// superseded stays that way however often a CLI writes it again. func (d *Drainer) Drain(req Request) Result { if d == nil || d.Dir == "" { return Result{Errors: []error{errors.New("cli-memory drain: no target directory")}} } + d.mu.Lock() + defer d.mu.Unlock() res := Result{Store: d.Dir} - if err := os.MkdirAll(d.Dir, 0o755); err != nil { - res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: creating %s: %w", d.Dir, err)) - return res - } - seen, err := existingHashes(d.Dir) - if err != nil { - res.Errors = append(res.Errors, err) - return res - } - now := time.Now + r := &drainRun{d: d, req: req, res: &res, now: time.Now, layer: d.Layer} if d.Now != nil { - now = d.Now + r.now = d.Now } - layer := d.Layer - if layer == "" { - layer = knowledge.LayerPersonal + if r.layer == "" { + r.layer = knowledge.LayerPersonal } + var files []memFile for _, src := range req.Sources { - files, err := memoryFiles(src.Path) + paths, err := memoryFiles(src.Path) if err != nil { res.Errors = append(res.Errors, err) continue } repo := repoFromClaudeSlug(src.Path, req.Repos) - for _, file := range files { - content, err := os.ReadFile(file) - if err != nil { - res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: reading %s: %w", file, err)) - continue - } - for _, fact := range ParseFor(src.CLI, string(content)) { - h := Hash(fact.Body) - if seen[h] { - res.Dupes++ - continue - } - if res.Drained >= maxFactsPerDrain { - res.Skipped++ - continue - } - entry := renderEntry(fact, h, entryMeta{ - agent: req.Agent, cli: src.CLI, repo: repo, path: file, layer: layer, at: now().UTC(), - }) - if err := writeAtomic(filepath.Join(d.Dir, EntryFilename(h)), entry); err != nil { - res.Errors = append(res.Errors, err) - continue - } - seen[h] = true - res.Drained++ + for _, p := range paths { + files = append(files, memFile{cli: src.CLI, path: p, repo: repo}) + } + } + seen, err := r.prepareVault() + if err != nil { + res.Errors = append(res.Errors, err) + for _, f := range files { + r.spool(f, nil) + } + return res + } + r.seen = seen + for _, f := range r.spooled() { + content, err := os.ReadFile(filepath.Join(f.spoolDir, spoolContentFile)) + if err != nil { + res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: reading spooled %s: %w", f.spoolDir, err)) + continue + } + if r.drainFile(f, content) { + if err := os.RemoveAll(f.spoolDir); err != nil { + res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: removing spooled %s: %w", f.spoolDir, err)) } } } - if res.Drained > 0 && d.Store != nil { + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + res.Errors = append(res.Errors, fmt.Errorf("cli-memory drain: reading %s: %w", f.path, err)) + continue + } + if !r.drainFile(f, content) { + r.spool(f, content) + } + } + if r.changed && d.Store != nil { d.Store.Reindex() } return res } +func (r *drainRun) prepareVault() (map[string]string, error) { + if err := os.MkdirAll(r.d.Dir, 0o755); err != nil { + return nil, fmt.Errorf("cli-memory drain: creating %s: %w", r.d.Dir, err) + } + return existingHashes(r.d.Dir) +} + +// drainFile stores every fact in content and reports whether all of them +// are now in the vault (written now, or already there). +func (r *drainRun) drainFile(f memFile, content []byte) bool { + complete := true + for _, fact := range ParseFor(f.cli, string(content)) { + h := Hash(fact.Body) + if existing, ok := r.seen[h]; ok { + r.res.Dupes++ + tagged, err := addAgentTag(existing, r.req.Agent) + if err != nil { + r.res.Errors = append(r.res.Errors, err) + } + r.changed = r.changed || tagged + continue + } + if r.res.Drained >= maxFactsPerDrain { + r.res.Skipped++ + complete = false + continue + } + path := filepath.Join(r.d.Dir, EntryFilename(h)) + entry := renderEntry(fact, h, entryMeta{ + agent: r.req.Agent, cli: f.cli, repo: f.repo, path: f.path, layer: r.layer, at: r.now().UTC(), + }) + if err := writeAtomic(path, entry); err != nil { + r.res.Errors = append(r.res.Errors, err) + complete = false + continue + } + r.seen[h] = path + r.res.Drained++ + r.changed = true + } + return complete +} + // Hash is the SHA-256 of the normalised fact body, hex encoded. func Hash(body string) string { sum := sha256.Sum256([]byte(normalise(body))) @@ -217,11 +297,12 @@ func list(items []string) string { return strings.Join(out, ", ") } -// existingHashes collects every memhash already recorded in dir, whatever -// the entry's current state or file name, so an entry an operator approved, -// deprecated or renamed is still a duplicate and is never re-drafted. -func existingHashes(dir string) (map[string]bool, error) { - seen := map[string]bool{} +// existingHashes maps every memhash already recorded in dir to the entry +// that carries it, whatever the entry's current state or file name, so an +// entry an operator approved, deprecated or renamed is still a duplicate and +// is never re-drafted. +func existingHashes(dir string) (map[string]string, error) { + seen := map[string]string{} entries, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("cli-memory drain: listing %s: %w", dir, err) @@ -230,8 +311,9 @@ func existingHashes(dir string) (map[string]bool, error) { if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") { continue } - if h := hashFromFrontmatter(filepath.Join(dir, e.Name())); h != "" { - seen[h] = true + p := filepath.Join(dir, e.Name()) + if h := hashFromFrontmatter(p); h != "" { + seen[h] = p } } return seen, nil diff --git a/src/pkg/knowledge/climemory/drain_edge_test.go b/src/pkg/knowledge/climemory/drain_edge_test.go index f703b923c6..4b6a70eb2d 100644 --- a/src/pkg/knowledge/climemory/drain_edge_test.go +++ b/src/pkg/knowledge/climemory/drain_edge_test.go @@ -174,7 +174,7 @@ func TestHashFromFrontmatter(t *testing.T) { if err != nil { t.Fatal(err) } - if !seen["abc"] || !seen["def"] || seen["zzz"] || seen["nope"] { + if seen["abc"] == "" || seen["def"] == "" || seen["zzz"] != "" || seen["nope"] != "" { t.Fatalf("existingHashes = %v", seen) } } diff --git a/src/pkg/knowledge/climemory/guard.go b/src/pkg/knowledge/climemory/guard.go new file mode 100644 index 0000000000..23e2f2d03e --- /dev/null +++ b/src/pkg/knowledge/climemory/guard.go @@ -0,0 +1,69 @@ +package climemory + +import ( + "fmt" + "regexp" + "strings" + + "github.com/hivecommons/hive/pkg/knowledge" +) + +var ( + // agentNameRe bounds the agent names the drain records in tags and in + // spool paths: no path separators, no leading dot. + agentNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) + repoTagRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + memhashRe = regexp.MustCompile(`^[0-9a-f]{16,64}$`) +) + +// ValidateCreate checks a knowledge create request against the drain's +// provenance rules (#17056). A request without the SourceTag is not a +// drained fact and passes. One that carries it must be created as draft, +// and every tag must be one the drain itself writes: SourceTag, +// agent:, cli:, memhash: and +// repo:/. That keeps a caller from creating an approved fact +// that claims to come from CLI memory, or from smuggling arbitrary tags +// (which scopes and primers select on) onto one. +func ValidateCreate(state string, tags []string) error { + drained := false + for _, t := range tags { + if strings.EqualFold(strings.TrimSpace(t), SourceTag) { + drained = true + break + } + } + if !drained { + return nil + } + st, err := knowledge.ParseLifecycleState(state) + if err != nil || st != knowledge.StateDraft { + return fmt.Errorf("a fact tagged %s must be created with state %s", SourceTag, knowledge.StateDraft) + } + for _, t := range tags { + if !allowedDrainTag(strings.TrimSpace(t)) { + return fmt.Errorf("tag %q is not allowed on a fact tagged %s", t, SourceTag) + } + } + return nil +} + +func allowedDrainTag(tag string) bool { + if strings.EqualFold(tag, SourceTag) { + return true + } + key, val, ok := strings.Cut(tag, ":") + if !ok { + return false + } + switch key { + case "agent": + return agentNameRe.MatchString(val) + case "cli": + return val == CLIClaude || val == CLICodex || val == CLIGemini + case HashKey: + return memhashRe.MatchString(val) + case "repo": + return repoTagRe.MatchString(val) + } + return false +} diff --git a/src/pkg/knowledge/climemory/guard_test.go b/src/pkg/knowledge/climemory/guard_test.go new file mode 100644 index 0000000000..6ceee03712 --- /dev/null +++ b/src/pkg/knowledge/climemory/guard_test.go @@ -0,0 +1,51 @@ +package climemory + +import ( + "strings" + "testing" +) + +// TestValidateCreate pins the create-path provenance guard (#17056): a fact +// tagged cli-memory must be a draft carrying only the tags the drain itself +// writes; anything else passes through untouched. +func TestValidateCreate(t *testing.T) { + hash := strings.Repeat("ab", 32) + tests := []struct { + name string + state string + tags []string + wantErr string + }{ + {"untagged approved fact passes", "approved", []string{"deploy", "agent:scanner"}, ""}, + {"untagged fact with no state passes", "", nil, ""}, + {"full drain tag set as draft", "draft", []string{SourceTag, "agent:scanner", "cli:claude", HashKey + ":" + hash, "repo:hivecommons/hive"}, ""}, + {"source tag matched case-insensitively and trimmed", "DRAFT", []string{" CLI-Memory ", "cli:codex"}, ""}, + {"gemini cli and short memhash", "draft", []string{SourceTag, "cli:gemini", HashKey + ":" + hash[:16]}, ""}, + {"approved drained fact refused", "approved", []string{SourceTag}, "must be created with state draft"}, + {"missing state refused", "", []string{SourceTag}, "must be created with state draft"}, + {"unknown state refused", "pending", []string{SourceTag}, "must be created with state draft"}, + {"foreign plain tag refused", "draft", []string{SourceTag, "deploy"}, `tag "deploy" is not allowed`}, + {"unknown key refused", "draft", []string{SourceTag, "scope:org"}, `tag "scope:org" is not allowed`}, + {"path-like agent refused", "draft", []string{SourceTag, "agent:../etc"}, "not allowed"}, + {"leading-dot agent refused", "draft", []string{SourceTag, "agent:.hidden"}, "not allowed"}, + {"unknown cli refused", "draft", []string{SourceTag, "cli:copilot"}, "not allowed"}, + {"uppercase memhash refused", "draft", []string{SourceTag, HashKey + ":" + strings.ToUpper(hash)}, "not allowed"}, + {"too-short memhash refused", "draft", []string{SourceTag, HashKey + ":abc123"}, "not allowed"}, + {"repo without owner refused", "draft", []string{SourceTag, "repo:hive"}, "not allowed"}, + {"repo with extra segment refused", "draft", []string{SourceTag, "repo:a/b/c"}, "not allowed"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := ValidateCreate(tc.state, tc.tags) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("ValidateCreate(%q, %v) = %v, want nil", tc.state, tc.tags, err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ValidateCreate(%q, %v) = %v, want error containing %q", tc.state, tc.tags, err, tc.wantErr) + } + }) + } +} diff --git a/src/pkg/knowledge/climemory/legacy.go b/src/pkg/knowledge/climemory/legacy.go index 1fb200c220..bbabf5ebc5 100644 --- a/src/pkg/knowledge/climemory/legacy.go +++ b/src/pkg/knowledge/climemory/legacy.go @@ -87,9 +87,9 @@ func (d *Drainer) TruncateDrained(sources []Source) (truncated, kept []string, e return truncated, kept, errors.Join(errs...) } -func allStored(facts []Fact, seen map[string]bool) bool { +func allStored(facts []Fact, seen map[string]string) bool { for _, f := range facts { - if !seen[Hash(f.Body)] { + if _, ok := seen[Hash(f.Body)]; !ok { return false } } diff --git a/src/pkg/knowledge/climemory/spool.go b/src/pkg/knowledge/climemory/spool.go new file mode 100644 index 0000000000..38bc27b369 --- /dev/null +++ b/src/pkg/knowledge/climemory/spool.go @@ -0,0 +1,178 @@ +package climemory + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// Spool layout: SpoolDir///{memory.md, source.json}, where key +// is derived from the CLI and the store path so re-spooling the same file +// overwrites its earlier copy instead of piling up. +const ( + spoolContentFile = "memory.md" + spoolMetaFile = "source.json" + spoolKeyHex = 16 +) + +// spoolMeta records what a spooled copy is needed to drain it like the +// original: the parser to use and the provenance stamped on its entries. +type spoolMeta struct { + CLI string `json:"cli"` + SourcePath string `json:"source_path"` + Repo string `json:"repo,omitempty"` +} + +// agentSpoolDir is SpoolDir/, or "" when no spool is configured or +// the agent name is not safe as a path component. +func (r *drainRun) agentSpoolDir() string { + if r.d.SpoolDir == "" || !agentNameRe.MatchString(r.req.Agent) { + return "" + } + return filepath.Join(r.d.SpoolDir, r.req.Agent) +} + +// spool keeps a copy of f (content, or the file itself when content is nil) +// in the agent's spool. A file already being retried from the spool stays +// where it is. +func (r *drainRun) spool(f memFile, content []byte) { + dir := r.agentSpoolDir() + if dir == "" || f.spoolDir != "" { + return + } + if content == nil { + var err error + if content, err = os.ReadFile(f.path); err != nil { + r.res.Errors = append(r.res.Errors, fmt.Errorf("cli-memory drain: reading %s to spool: %w", f.path, err)) + return + } + } + sum := sha256.Sum256([]byte(f.cli + "\x00" + f.path)) + entryDir := filepath.Join(dir, hex.EncodeToString(sum[:])[:spoolKeyHex]) + meta, err := json.Marshal(spoolMeta{CLI: f.cli, SourcePath: f.path, Repo: f.repo}) + if err == nil { + err = os.MkdirAll(entryDir, 0o700) + } + if err == nil { + err = writeFilePrivate(filepath.Join(entryDir, spoolContentFile), content) + } + if err == nil { + // The metadata goes last: a spool entry without it is never + // retried, so a half-written copy cannot be drained. + err = writeFilePrivate(filepath.Join(entryDir, spoolMetaFile), meta) + } + if err != nil { + r.res.Errors = append(r.res.Errors, fmt.Errorf("cli-memory drain: spooling %s: %w", f.path, err)) + return + } + r.res.Spooled++ +} + +// spooled lists the agent's spooled files, oldest key order first. An entry +// whose metadata is unreadable is reported and left for the operator. +func (r *drainRun) spooled() []memFile { + dir := r.agentSpoolDir() + if dir == "" { + return nil + } + entries, err := os.ReadDir(dir) + if err != nil { + if !os.IsNotExist(err) { + r.res.Errors = append(r.res.Errors, fmt.Errorf("cli-memory drain: listing spool %s: %w", dir, err)) + } + return nil + } + var out []memFile + for _, e := range entries { + if !e.IsDir() { + continue + } + entryDir := filepath.Join(dir, e.Name()) + data, err := os.ReadFile(filepath.Join(entryDir, spoolMetaFile)) + if err != nil { + if !os.IsNotExist(err) { + r.res.Errors = append(r.res.Errors, fmt.Errorf("cli-memory drain: reading spool %s: %w", entryDir, err)) + } + continue + } + var m spoolMeta + if err := json.Unmarshal(data, &m); err != nil || (m.CLI != CLIClaude && m.CLI != CLICodex && m.CLI != CLIGemini) { + r.res.Errors = append(r.res.Errors, fmt.Errorf("cli-memory drain: spool entry %s has invalid metadata", entryDir)) + continue + } + out = append(out, memFile{cli: m.CLI, path: m.SourcePath, repo: m.Repo, spoolDir: entryDir}) + } + sort.Slice(out, func(i, j int) bool { return out[i].spoolDir < out[j].spoolDir }) + return out +} + +func writeFilePrivate(path string, data []byte) error { + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// addAgentTag adds agent: to the tags of the vault entry at path +// and reports whether it changed the file. It edits only the inline `tags:` +// list of the frontmatter (adding one when absent) and never touches the +// state, so a fact under review keeps its state whichever agent writes it. +func addAgentTag(path, agent string) (bool, error) { + if path == "" || !agentNameRe.MatchString(agent) { + return false, nil + } + data, err := os.ReadFile(path) + if err != nil { + return false, fmt.Errorf("cli-memory drain: reading %s: %w", path, err) + } + content := string(data) + const open = "---\n" + if !strings.HasPrefix(content, open) { + return false, nil + } + end := strings.Index(content[len(open):], "\n---") + if end < 0 { + return false, nil + } + fm := content[len(open) : len(open)+end] + rest := content[len(open)+end:] + tag := "agent:" + agent + lines := strings.Split(fm, "\n") + found := false + for i, line := range lines { + v, ok := strings.CutPrefix(line, "tags:") + if !ok { + continue + } + v = strings.TrimSpace(v) + if !strings.HasPrefix(v, "[") || !strings.HasSuffix(v, "]") { + return false, fmt.Errorf("cli-memory drain: %s: tags is not an inline list; %s not added", path, tag) + } + var items []string + for _, it := range strings.Split(v[1:len(v)-1], ",") { + it = strings.Trim(strings.TrimSpace(it), "\"'") + if it == tag { + return false, nil + } + if it != "" { + items = append(items, it) + } + } + lines[i] = "tags: [" + list(append(items, tag)) + "]" + found = true + break + } + if !found { + lines = append(lines, "tags: ["+tag+"]") + } + if err := writeAtomic(path, open+strings.Join(lines, "\n")+rest); err != nil { + return false, err + } + return true, nil +} diff --git a/src/pkg/knowledge/climemory/spool_test.go b/src/pkg/knowledge/climemory/spool_test.go new file mode 100644 index 0000000000..cffdc4d4e1 --- /dev/null +++ b/src/pkg/knowledge/climemory/spool_test.go @@ -0,0 +1,215 @@ +package climemory + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/knowledge" +) + +func writeMemory(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func readEntry(t *testing.T, vault, body string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(vault, EntryFilename(Hash(body)))) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// A second agent draining a fact already in the vault adds its agent: tag +// to the one entry instead of creating another, and never resets the state +// an operator set: a deprecated fact stays deprecated. +func TestDrainDedupAddsAgentTagAndNeverRevives(t *testing.T) { + vault := t.TempDir() + d := &Drainer{Dir: vault, Now: fixedNow} + const fact = "The dashboard listens on port 8080." + scannerMem := filepath.Join(t.TempDir(), "memories", "notes.md") + writeMemory(t, scannerMem, "- "+fact+"\n") + if res := d.Drain(Request{Agent: "scanner", Sources: []Source{{CLI: CLICodex, Path: scannerMem}}}); res.Drained != 1 || len(res.Errors) != 0 { + t.Fatalf("first drain = %+v", res) + } + + // The operator discards it. + store, err := knowledge.NewFileStore(vault, "test", slog.Default()) + if err != nil { + t.Fatal(err) + } + pages := store.ListPages("") + if len(pages) != 1 { + t.Fatalf("vault pages = %d", len(pages)) + } + if err := store.SetLifecycleState(pages[0].Slug, knowledge.StateDeprecated); err != nil { + t.Fatal(err) + } + + geminiMem := filepath.Join(t.TempDir(), ".gemini", "GEMINI.md") + writeMemory(t, geminiMem, "Operator context.\n\n## Gemini Added Memories\n- "+fact+"\n") + for i := range 2 { // the second pass must be a no-op + res := d.Drain(Request{Agent: "reviewer", Sources: []Source{{CLI: CLIGemini, Path: geminiMem}}}) + if res.Drained != 0 || res.Dupes != 1 || len(res.Errors) != 0 { + t.Fatalf("pass %d: %+v", i, res) + } + } + entries, _ := filepath.Glob(filepath.Join(vault, "*.md")) + if len(entries) != 1 { + t.Fatalf("vault has %d entries, want the one fact", len(entries)) + } + entry := readEntry(t, vault, fact) + if strings.Count(entry, "agent:reviewer") != 1 || !strings.Contains(entry, "agent:scanner") { + t.Fatalf("tags not merged once:\n%s", entry) + } + store.Reindex() + got := store.ListPages("") + if len(got) != 1 || got[0].EffectiveState() != knowledge.StateDeprecated { + t.Fatalf("discarded fact revived: %+v", got) + } + if len(knowledge.FilterByLifecycle(got, nil)) != 0 { + t.Fatal("deprecated fact passes the approved-only filter after a re-drain") + } +} + +func TestAddAgentTag(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name, in, want string + changed bool + wantErr bool + }{ + {"appends", "---\ntitle: x\ntags: [cli-memory, agent:a]\n---\n\nbody\n", "---\ntitle: x\ntags: [cli-memory, agent:a, agent:b]\n---\n\nbody\n", true, false}, + {"already tagged", "---\ntags: [agent:b]\n---\nbody\n", "---\ntags: [agent:b]\n---\nbody\n", false, false}, + {"no tags line", "---\ntitle: x\n---\nbody\n", "---\ntitle: x\ntags: [agent:b]\n---\nbody\n", true, false}, + {"no frontmatter", "body only\n", "body only\n", false, false}, + {"block list", "---\ntags:\n - a\n---\nbody\n", "---\ntags:\n - a\n---\nbody\n", false, true}, + } + for _, tc := range cases { + p := filepath.Join(dir, tc.name+".md") + writeMemory(t, p, tc.in) + changed, err := addAgentTag(p, "b") + if changed != tc.changed || (err != nil) != tc.wantErr { + t.Errorf("%s: changed=%v err=%v", tc.name, changed, err) + } + if data, _ := os.ReadFile(p); string(data) != tc.want { + t.Errorf("%s: file =\n%s\nwant\n%s", tc.name, data, tc.want) + } + } + if changed, err := addAgentTag(filepath.Join(dir, "appends.md"), "../x"); changed || err != nil { + t.Errorf("unsafe agent name: changed=%v err=%v", changed, err) + } +} + +// When the vault cannot be written, every memory file is copied into the +// agent's spool, so the clear_on_kick wipe that follows loses nothing; +// failing again replaces the copy rather than adding one. The next drain +// that can write stores the spooled facts with their original provenance +// and empties the spool. +func TestDrainSpoolsOnVaultFailureAndRetries(t *testing.T) { + home := t.TempDir() + memDir := filepath.Join(home, ".claude", "projects", "-data-agents-scanner-hivecommons-hive", "memory") + writeMemory(t, filepath.Join(memDir, "MEMORY.md"), "- Always read the failing job log.\n- Sign every commit.\n") + spool := t.TempDir() + blocker := filepath.Join(t.TempDir(), "file") + writeMemory(t, blocker, "x") + req := Request{Agent: "scanner", Repos: []string{"hivecommons/hive"}, Sources: []Source{{CLI: CLIClaude, Path: memDir}}} + + broken := &Drainer{Dir: filepath.Join(blocker, "vault"), SpoolDir: spool, Now: fixedNow} + for i := range 2 { + res := broken.Drain(req) + if res.Drained != 0 || res.Spooled != 1 || len(res.Errors) != 1 { + t.Fatalf("failed drain %d = %+v", i, res) + } + } + if entries, _ := os.ReadDir(filepath.Join(spool, "scanner")); len(entries) != 1 { + t.Fatalf("spool entries after two failures = %d, want 1", len(entries)) + } + // clear_on_kick wipes the store. + if err := os.RemoveAll(memDir); err != nil { + t.Fatal(err) + } + + vault := t.TempDir() + d := &Drainer{Dir: vault, SpoolDir: spool, Now: fixedNow} + res := d.Drain(req) + if res.Drained != 2 || res.Spooled != 0 || len(res.Errors) != 0 { + t.Fatalf("retry drain = %+v", res) + } + entry := readEntry(t, vault, "Always read the failing job log.") + for _, want := range []string{"state: draft", "repo: hivecommons/hive", "source_path: " + filepath.Join(memDir, "MEMORY.md")} { + if !strings.Contains(entry, want) { + t.Errorf("spooled entry lost %q:\n%s", want, entry) + } + } + if left, _ := os.ReadDir(filepath.Join(spool, "scanner")); len(left) != 0 { + t.Fatalf("spool not emptied after a complete retry: %d entries", len(left)) + } + if res := d.Drain(req); res.Drained != 0 || res.Dupes != 0 || len(res.Errors) != 0 { + t.Fatalf("drain after retry = %+v", res) + } +} + +// A file whose facts only partly reach the vault (here: the per-drain cap) +// is spooled and completed from the spool on the next drain. +func TestDrainSpoolsPartialFile(t *testing.T) { + mem := filepath.Join(t.TempDir(), "memories", "notes.md") + var sb strings.Builder + for i := range maxFactsPerDrain + 3 { + fmt.Fprintf(&sb, "- distinct remembered fact number %d\n", i) + } + writeMemory(t, mem, sb.String()) + spool := t.TempDir() + d := &Drainer{Dir: t.TempDir(), SpoolDir: spool, Now: fixedNow} + req := Request{Agent: "scanner", Sources: []Source{{CLI: CLICodex, Path: mem}}} + + res := d.Drain(req) + if res.Drained != maxFactsPerDrain || res.Skipped != 3 || res.Spooled != 1 { + t.Fatalf("capped drain = %+v", res) + } + entries, _ := os.ReadDir(filepath.Join(spool, "scanner")) + if len(entries) != 1 { + t.Fatalf("spool entries = %d, want 1", len(entries)) + } + + // The store is wiped; the next drain completes the file from the spool. + if err := os.Remove(mem); err != nil { + t.Fatal(err) + } + res = d.Drain(req) + if res.Drained != 3 || res.Dupes != maxFactsPerDrain || res.Spooled != 0 || len(res.Errors) != 0 { + t.Fatalf("spool retry = %+v", res) + } + if entries, _ := os.ReadDir(filepath.Join(spool, "scanner")); len(entries) != 0 { + t.Fatalf("spool not emptied: %d", len(entries)) + } +} + +// A failed drain without a spool, or for an agent whose name is unsafe as a +// path, writes nothing outside the vault. +func TestDrainSpoolNeedsSafeAgentAndDir(t *testing.T) { + mem := filepath.Join(t.TempDir(), "notes.md") + writeMemory(t, mem, "- a fact\n") + blocker := filepath.Join(t.TempDir(), "file") + writeMemory(t, blocker, "x") + spool := t.TempDir() + for _, agent := range []string{"", "../escape", ".hidden"} { + d := &Drainer{Dir: filepath.Join(blocker, "vault"), SpoolDir: spool} + if res := d.Drain(Request{Agent: agent, Sources: []Source{{CLI: CLICodex, Path: mem}}}); res.Spooled != 0 { + t.Errorf("agent %q spooled: %+v", agent, res) + } + } + if entries, _ := os.ReadDir(spool); len(entries) != 0 { + t.Fatalf("spool written for unsafe agents: %d entries", len(entries)) + } +} diff --git a/src/pkg/knowledge/filestore.go b/src/pkg/knowledge/filestore.go index 7b2a1caf0b..0988c253d3 100644 --- a/src/pkg/knowledge/filestore.go +++ b/src/pkg/knowledge/filestore.go @@ -117,9 +117,7 @@ func (s *FileStore) WriteFacts(facts []ExtractedFact) error { // clobber a shared "untitled.md". continue } - slug := slugify(f.Title) - filename := strings.ReplaceAll(slug+".md", "/", "_") - path := filepath.Join(s.rootDir, filename) + path := s.factPath(f.Title) var buf strings.Builder buf.WriteString("---\n") @@ -133,6 +131,9 @@ func (s *FileStore) WriteFacts(facts []ExtractedFact) error { if len(f.Tags) > 0 { fmt.Fprintf(&buf, "tags: [%s]\n", sanitizeFrontmatterList(f.Tags)) } + if f.State != "" { + fmt.Fprintf(&buf, "state: %s\n", sanitizeFrontmatterValue(string(f.State))) + } if len(f.Related) > 0 { fmt.Fprintf(&buf, "related: [%s]\n", sanitizeFrontmatterList(f.Related)) } @@ -151,6 +152,11 @@ func (s *FileStore) WriteFacts(facts []ExtractedFact) error { return nil } +// factPath is the file WriteFacts writes a fact with this title to. +func (s *FileStore) factPath(title string) string { + return filepath.Join(s.rootDir, strings.ReplaceAll(slugify(title)+".md", "/", "_")) +} + func (s *FileStore) reindex() { pages := make(map[string]filePage) diff --git a/src/pkg/knowledge/types.go b/src/pkg/knowledge/types.go index 8ba97a9408..a40e3daf2c 100644 --- a/src/pkg/knowledge/types.go +++ b/src/pkg/knowledge/types.go @@ -149,6 +149,9 @@ type ExtractedFact struct { // Sensitive marks a fact that must stay inside the hive; repository // publishing skips it (see IsSensitiveFact). Sensitive bool `json:"sensitive,omitempty"` + // State is the lifecycle state recorded when the fact is written to a + // local vault. Empty writes no `state:` key, which loads as approved. + State LifecycleState `json:"state,omitempty"` } func containsAny(s string, needles ...string) bool {