diff --git a/README.md b/README.md
index a7630d7..1dd043f 100644
--- a/README.md
+++ b/README.md
@@ -568,6 +568,51 @@ agents:
# LOG_FORMAT=json switches to JSON lines.
log_level: error
+# Optional: conversation compaction. When the context fills up, nib
+# summarizes the older turns and keeps the recent ones verbatim.
+# summary_max_tokens caps the output the summary request reserves (default
+# 16384; the model's own output cap wins when it is smaller). The summary
+# prompt is fitted into the window minus that reservation. If the backend
+# still rejects it as too large, nib summarizes fewer of the older turns and
+# keeps the rest verbatim, instead of cutting the text it summarizes.
+#
+# nib reads a backend's "request too large" error to decide what to do:
+# compact when the prompt itself does not fit (kind "context"), lower the
+# requested output when only prompt + output does not fit ("budget"), or
+# lower the output cap when the output alone is above the model's maximum
+# ("output_cap"), and then retry the turn once. When the summary alone does
+# not make the history fit, nib stubs old tool outputs, then drops older turns
+# (saved as an artifact:// the model can read), and as a last resort compacts
+# without the model. If the retry still does not fit, the turn fails, but the
+# compaction is kept, so the next turn does not overflow again. When the tool
+# schemas and system prompt alone fill the window, compaction cannot help:
+# nib keeps the conversation as it was, does not compact (neither on
+# overflow nor automatically), and names the MCP servers that take the most
+# room, so you can disable them or filter their tools. When a tool call is
+# cut off because the reply ran out of room in the window (for example a
+# whole file in one write), nib retries the turn once with a note that asks
+# the model to split the call, after compacting when the prompt is large.
+# When the reasoning filled the window, the retry also uses one lower
+# reasoning effort. nib never caps the output for this. A stream that ends
+# before the reply is finished is retried at most 3 times, and a tool call
+# with invalid JSON arguments once. It knows the
+# wording of llama.cpp/LocalAI, vLLM, OpenAI,
+# Anthropic and Gemini. overflow_patterns teaches it another backend without a
+# rebuild: the rows are tried before the built-in ones, first match wins. A
+# regex states its figures with the named groups window, total, input and
+# output. A row with an invalid regex or kind is logged and skipped. A
+# backend-specific wording (built-in or from this list) counts at any HTTP
+# status, because LocalAI reports an overflow as HTTP 500; a generic phrase
+# such as "context window" counts only with status 400, 413 or none. An error
+# with HTTP status 400 or 413 that no pattern recognises is logged at debug
+# level, so you can copy its text from the log.
+compaction:
+ summary_max_tokens: 16384
+ overflow_patterns:
+ - name: my-backend
+ kind: context
+ regex: 'prompt has (?P\d+) tokens, limit is (?P\d+)'
+
# Optional: external MCP servers
mcp_servers:
filesystem:
diff --git a/chat/basic_compact.go b/chat/basic_compact.go
new file mode 100644
index 0000000..6101e68
--- /dev/null
+++ b/chat/basic_compact.go
@@ -0,0 +1,154 @@
+package chat
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+
+ openai "github.com/sashabaranov/go-openai"
+)
+
+// basicCompactMarker opens the user-role message that stands in for the head
+// basicCompact drops.
+const basicCompactMarker = "Conversation compacted (basic fallback mode). Previous tool outputs and assistant responses were elided to fit the context window."
+
+const (
+ basicAssistantChars = 200 // assistant text kept per line
+ basicArgsChars = 80 // tool arguments kept per line when there is no path
+)
+
+// basicLine is one rendered line of the dropped head. User lines are the
+// user's own words and go last when the result must shrink.
+type basicLine struct {
+ text string
+ user bool
+}
+
+// basicCompact compacts the fragment without an LLM call. It is the last
+// resort when the summarizer is unavailable or iterativeTrim ran out of steps.
+//
+// The whole head is saved as an artifact first. The head is then replaced by
+// one user-role message: the user messages verbatim, one line per assistant
+// text (cut to 200 characters) and one stub line per tool call. The results
+// are rendered as text, never as tool messages, so no tool result is left
+// without its call. The tail, split by splitForCompaction, keeps its pairing.
+//
+// When the result does not fit, activity lines go first, oldest first, and
+// then user lines, oldest first; they all stay in the artifact. When even the
+// tail does not fit, the fragment is left alone and the validation error is
+// returned.
+func (s *Session) basicCompact(ctx context.Context) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ cfg := s.compactionConfig()
+ budget := s.trimBudget(cfg)
+ state := s.fragmentSnapshot()
+ msgs := state.frag.Messages
+
+ head, tail := splitForCompaction(msgs, max(cfg.KeepRecent, 1))
+ if len(head) == 0 {
+ return fmt.Errorf("%w: nothing to compact", ErrCompactionNoOp)
+ }
+
+ marker := basicCompactMarker
+ if uri := s.spillCompactionHead(renderMessages(head)); uri != "" {
+ marker += " Full history available at " + uri + "."
+ }
+ marker += " User messages are preserved verbatim."
+
+ var system []openai.ChatCompletionMessage
+ for _, m := range head {
+ if m.Role == "system" {
+ system = append(system, m)
+ }
+ }
+ lines := basicRender(head)
+
+ build := func() []openai.ChatCompletionMessage {
+ var b strings.Builder
+ b.WriteString(marker)
+ for _, l := range lines {
+ b.WriteString("\n")
+ b.WriteString(l.text)
+ }
+ out := append([]openai.ChatCompletionMessage(nil), system...)
+ out = append(out, openai.ChatCompletionMessage{Role: "user", Content: b.String()})
+ return append(out, tail...)
+ }
+
+ candidate := build()
+ for _, dropUser := range []bool{false, true} {
+ for estimateTokens(candidate) > budget {
+ i := oldestLine(lines, dropUser)
+ if i < 0 {
+ break
+ }
+ lines = append(lines[:i], lines[i+1:]...)
+ candidate = build()
+ }
+ }
+ if err := validateCompaction(msgs, candidate, budget); err != nil {
+ return fmt.Errorf("basic compact: %w", err)
+ }
+ s.installFragment(candidate, compactedDisplay(state.messages, tail))
+ return nil
+}
+
+// oldestLine is the index of the first line of the kind asked for, or -1.
+func oldestLine(lines []basicLine, user bool) int {
+ for i, l := range lines {
+ if l.user == user {
+ return i
+ }
+ }
+ return -1
+}
+
+// basicRender renders the head as text lines: user messages verbatim, the
+// first 200 characters of assistant text, and one stub per tool call. System
+// messages are kept as messages by the caller, and tool results are covered by
+// their call's stub.
+func basicRender(head []openai.ChatCompletionMessage) []basicLine {
+ var lines []basicLine
+ for _, m := range head {
+ switch m.Role {
+ case "user":
+ if m.Content != "" {
+ lines = append(lines, basicLine{text: "User: " + m.Content, user: true})
+ }
+ case "assistant":
+ if text := strings.TrimSpace(m.Content); text != "" {
+ lines = append(lines, basicLine{text: "Assistant: " + cutRunes(text, basicAssistantChars)})
+ }
+ for _, tc := range m.ToolCalls {
+ lines = append(lines, basicLine{text: fmt.Sprintf("[tool %s(%s) ran, result elided]", tc.Function.Name, basicToolTarget(tc.Function.Arguments))})
+ }
+ }
+ }
+ return lines
+}
+
+// basicToolTarget is what a tool stub names: the path argument when there is
+// one, or else the first 80 characters of the arguments.
+func basicToolTarget(args string) string {
+ var parsed map[string]any
+ if json.Unmarshal([]byte(args), &parsed) == nil {
+ for _, key := range []string{"path", "file_path"} {
+ if p, ok := parsed[key].(string); ok && p != "" {
+ return p
+ }
+ }
+ }
+ return cutRunes(args, basicArgsChars)
+}
+
+// cutRunes returns the first n characters of s.
+func cutRunes(s string, n int) string {
+ r := []rune(s)
+ if len(r) <= n {
+ return s
+ }
+ return string(r[:n])
+}
diff --git a/chat/basic_compact_test.go b/chat/basic_compact_test.go
new file mode 100644
index 0000000..d9f0d5b
--- /dev/null
+++ b/chat/basic_compact_test.go
@@ -0,0 +1,184 @@
+package chat
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/mudler/nib/mcp"
+ openai "github.com/sashabaranov/go-openai"
+)
+
+// The test session has a 1000-token window, so the budget is 500 byte/4
+// tokens (2000 bytes).
+
+func basicToolCall(id, name, args string) openai.ToolCall {
+ return openai.ToolCall{ID: id, Type: openai.ToolTypeFunction, Function: openai.FunctionCall{Name: name, Arguments: args}}
+}
+
+func TestBasicCompactRendersHeadAndKeepsTail(t *testing.T) {
+ bigResult := "RESULT-BODY" + strings.Repeat("r", 3000)
+ longAssistant := "a1" + strings.Repeat("z", 400)
+ frag := []openai.ChatCompletionMessage{
+ {Role: "system", Content: "sys"},
+ {Role: "user", Content: "u1 please do X\n exactly like this"},
+ {Role: "assistant", ToolCalls: []openai.ToolCall{basicToolCall("c1", "read", `{"path":"/a/b.go"}`)}},
+ {Role: "tool", ToolCallID: "c1", Content: bigResult},
+ {Role: "assistant", Content: longAssistant},
+ {Role: "user", Content: "u2 now run ls"},
+ {Role: "assistant", ToolCalls: []openai.ToolCall{basicToolCall("c2", "bash", `{"command":"ls"}`)}},
+ {Role: "tool", ToolCallID: "c2", Content: "ok"},
+ {Role: "assistant", Content: "done"},
+ }
+ llm := &fakeSummaryLLM{reply: "SUMMARY-TEXT"}
+ s := newCompactTestSession(llm, 3, frag, frag)
+ s.artifacts = mcp.NewArtifactStore()
+
+ if err := s.basicCompact(context.Background()); err != nil {
+ t.Fatalf("basicCompact: %v", err)
+ }
+ if llm.calls != 0 {
+ t.Fatalf("basicCompact must not call the LLM, got %d calls", llm.calls)
+ }
+ got := trimFragment(s)
+ if len(got) != 5 {
+ t.Fatalf("want [system, marker, 3 tail], got %d: %+v", len(got), got)
+ }
+ if got[0].Role != "system" || got[0].Content != "sys" {
+ t.Fatalf("head system message must be kept first, got %s %q", got[0].Role, got[0].Content)
+ }
+ body := got[1].Content
+ if got[1].Role != "user" || !strings.Contains(body, "basic fallback mode") || !strings.Contains(body, "artifact://1") {
+ t.Fatalf("want a user-role marker naming artifact://1, got %s %q", got[1].Role, body)
+ }
+ for _, want := range []string{
+ "u1 please do X\n exactly like this",
+ "u2 now run ls",
+ "[tool read(/a/b.go) ran, result elided]",
+ "Assistant: " + longAssistant[:200],
+ } {
+ if !strings.Contains(body, want) {
+ t.Fatalf("marker missing %q:\n%s", want, body)
+ }
+ }
+ if strings.Contains(body, longAssistant[:201]) || strings.Contains(body, "RESULT-BODY") {
+ t.Fatalf("assistant text must be cut to 200 chars and tool output elided:\n%s", body)
+ }
+ for i, m := range got[2:] {
+ orig := frag[6+i]
+ if m.Role != orig.Role || m.Content != orig.Content || m.ToolCallID != orig.ToolCallID || len(m.ToolCalls) != len(orig.ToolCalls) {
+ t.Fatalf("tail message %d not kept verbatim: %+v", i, m)
+ }
+ }
+ for _, m := range got {
+ if m.Role == "tool" && m.ToolCallID == "c1" {
+ t.Fatalf("head tool result must not survive as a tool message")
+ }
+ }
+ assertPaired(t, got)
+ if estimateTokens(got) > s.trimBudget(s.compactionConfig()) {
+ t.Fatalf("result over budget: %d", estimateTokens(got))
+ }
+ art := s.artifacts.Get(1)
+ if art == nil || !strings.Contains(art.Content, "RESULT-BODY") || !strings.Contains(art.Content, "u1 please do X") {
+ t.Fatalf("artifact must hold the full untruncated head, got %+v", art)
+ }
+ if len(s.messages) == 0 || s.messages[0].Role != "assistant" || s.messages[len(s.messages)-1].Content != "done" {
+ t.Fatalf("display copy not rebuilt: %+v", s.messages)
+ }
+}
+
+func TestBasicCompactDropsActivityLinesWhenTight(t *testing.T) {
+ var frag []openai.ChatCompletionMessage
+ for i := range 10 {
+ frag = append(frag,
+ openai.ChatCompletionMessage{Role: "user", Content: fmt.Sprintf("user-%d", i)},
+ openai.ChatCompletionMessage{Role: "assistant", Content: fmt.Sprintf("asst-%d-", i) + strings.Repeat("q", 250)},
+ )
+ }
+ frag = append(frag,
+ openai.ChatCompletionMessage{Role: "user", Content: "last-user" + strings.Repeat("t", 400)},
+ openai.ChatCompletionMessage{Role: "assistant", Content: "last-asst" + strings.Repeat("t", 400)},
+ )
+ s := newCompactTestSession(&fakeSummaryLLM{}, 2, frag, frag)
+ s.artifacts = mcp.NewArtifactStore()
+
+ if err := s.basicCompact(context.Background()); err != nil {
+ t.Fatalf("basicCompact: %v", err)
+ }
+ got := trimFragment(s)
+ if estimateTokens(got) > s.trimBudget(s.compactionConfig()) {
+ t.Fatalf("result over budget: %d", estimateTokens(got))
+ }
+ body := got[0].Content
+ for i := range 10 {
+ if !strings.Contains(body, fmt.Sprintf("user-%d\n", i)) {
+ t.Fatalf("user-%d must be preserved while activity lines can go:\n%s", i, body)
+ }
+ }
+ if strings.Contains(body, "asst-0-") {
+ t.Fatalf("the oldest activity line must be dropped first:\n%s", body)
+ }
+ if !strings.Contains(body, "asst-9-") {
+ t.Fatalf("the newest activity line should survive:\n%s", body)
+ }
+ if got[len(got)-1].Content != frag[len(frag)-1].Content {
+ t.Fatalf("tail must be kept verbatim")
+ }
+}
+
+func TestBasicCompactDropsOldUsersAsLastResort(t *testing.T) {
+ var frag []openai.ChatCompletionMessage
+ for i := range 8 {
+ frag = append(frag,
+ openai.ChatCompletionMessage{Role: "user", Content: fmt.Sprintf("user-%d-", i) + strings.Repeat("w", 400)},
+ openai.ChatCompletionMessage{Role: "assistant", Content: fmt.Sprintf("asst-%d", i)},
+ )
+ }
+ frag = append(frag,
+ openai.ChatCompletionMessage{Role: "user", Content: "last-user"},
+ openai.ChatCompletionMessage{Role: "assistant", Content: "last-asst"},
+ )
+ s := newCompactTestSession(&fakeSummaryLLM{}, 2, frag, frag)
+ s.artifacts = mcp.NewArtifactStore()
+
+ if err := s.basicCompact(context.Background()); err != nil {
+ t.Fatalf("basicCompact: %v", err)
+ }
+ got := trimFragment(s)
+ if estimateTokens(got) > s.trimBudget(s.compactionConfig()) {
+ t.Fatalf("result over budget: %d", estimateTokens(got))
+ }
+ body := got[0].Content
+ if strings.Contains(body, "Assistant: ") {
+ t.Fatalf("activity lines must all go before any user line:\n%s", body)
+ }
+ if strings.Contains(body, "user-0-") {
+ t.Fatalf("the oldest user line must be dropped when nothing else fits:\n%s", body)
+ }
+ if !strings.Contains(body, "user-7-"+strings.Repeat("w", 400)) {
+ t.Fatalf("the newest user line must survive verbatim:\n%s", body)
+ }
+ if art := s.artifacts.Get(1); art == nil || !strings.Contains(art.Content, "user-0-") {
+ t.Fatalf("dropped user lines must stay in the artifact")
+ }
+}
+
+func TestBasicCompactFailsWhenTailAloneOverBudget(t *testing.T) {
+ frag := []openai.ChatCompletionMessage{
+ {Role: "user", Content: "u1" + strings.Repeat("x", 800)},
+ {Role: "assistant", Content: "a1"},
+ {Role: "user", Content: strings.Repeat("y", 4000)},
+ }
+ s := newCompactTestSession(&fakeSummaryLLM{}, 1, frag, frag)
+
+ err := s.basicCompact(context.Background())
+ if !errors.Is(err, ErrCompactionOverBudget) {
+ t.Fatalf("want ErrCompactionOverBudget, got %v", err)
+ }
+ if got := trimFragment(s); len(got) != len(frag) {
+ t.Fatalf("a failed basic compact must leave the fragment alone, got %d messages", len(got))
+ }
+}
diff --git a/chat/compact.go b/chat/compact.go
index e37872c..cbeabc4 100644
--- a/chat/compact.go
+++ b/chat/compact.go
@@ -2,9 +2,7 @@ package chat
import (
"context"
- "errors"
"fmt"
- "strconv"
"strings"
"github.com/mudler/nib/types"
@@ -97,8 +95,8 @@ func shouldAutoCompact(cfg types.CompactionConfig, window, promptTokens int) boo
// 4096-context model is exactly nib's audience, so the floor sits below any
// real served model and above any plausible misparse of an output-token count
// (the largest default max_tokens backends print in overflow errors is in the
-// hundreds — vLLM's "512 output tokens" is the known example, and it is only
-// skipped today because tokenCountRe's \s* cannot span the word "output").
+// hundreds — vLLM's "512 output tokens" is the known example; the overflow
+// catalog's named groups keep it out of the window, see overflow_patterns.go).
//
// The ceiling exists because strconv.Atoi clamps an absurdly long digit run to
// MaxInt instead of erroring, so garbage lands far ABOVE any floor rather than
@@ -116,6 +114,15 @@ const (
// Threshold default already is).
const defaultReserveTokens = 4096
+// defaultSummaryMaxTokens is the fallback for an unset CompactionConfig.
+// SummaryMaxTokens, repeated from config.Load for the same reason.
+const defaultSummaryMaxTokens = 16384
+
+// maxSummaryAttempts bounds how many summary requests summarizeFitting sends
+// for one compaction. Each overflow moves the boundary so the head shrinks by
+// the backend's stated overshoot, so a fit normally takes one or two retries.
+const maxSummaryAttempts = 6
+
// rememberWindow records a context window a backend stated for a specific
// model. Values outside the plausibility band are discarded rather than
// stored, because an implausible figure is a parse artefact and storing one
@@ -210,6 +217,16 @@ func canRecoverFromOverflow(turnCtx context.Context, err error) bool {
return turnCtx.Err() == nil && isContextOverflow(err)
}
+// canShrinkSummary reports whether a rejected summary request is worth
+// retrying with a smaller prompt: a context overflow, or a budget overflow
+// (the prompt fits but prompt + reserved output does not), the user did not
+// cancel. The summary's output reservation is already capped, so shrinking
+// its prompt is what fits a budget overflow here; this is not compaction of
+// the conversation, which a budget overflow must never trigger.
+func canShrinkSummary(ctx context.Context, err error) bool {
+ return ctx.Err() == nil && isWindowOverflow(err)
+}
+
// overflowRetries reports how many context-overflow recoveries the current turn
// performed. Read by tests; the cap itself is enforced in SendMessage.
func (s *Session) overflowRetries() int {
@@ -517,35 +534,37 @@ func fitSummaryInput(pieces []summaryPiece, maxTokens int) string {
// summaryRetryTarget returns the prompt size to retry a summary at after the
// backend rejected a prompt of sent (byte/4) tokens.
//
-// byte/4 is only an estimate, and a backend can count the same text as more
-// tokens than that. The rejection states both figures in the backend's own
-// count, so their ratio corrects the estimate. The 10% margin covers the
-// estimate varying across the text. With no figures to go on, it halves.
+// The rejection states the request size and the window in the backend's own
+// count. Their difference is by how many tokens the request overshot, and that
+// is a fixed number, not a ratio: the output the request reserves does not
+// shrink with the prompt. Scaling the prompt by window/request left the
+// reservation whole, so a large reservation overflowed again on every retry.
+// The 10% margin covers the byte/4 estimate varying across the text. With no
+// figures to go on, it halves. A result of zero or less means the reservation
+// alone does not fit, and no prompt will.
func summaryRetryTarget(err error, sent int) int {
needs, allows, ok := overflowFigures(err)
if !ok {
return sent / 2
}
- return int(float64(sent) * float64(allows) / float64(needs) * 0.9)
+ return int(float64(sent-(needs-allows)) * 0.9)
}
// overflowFigures reads the request size and the window from an overflow
-// error, applying the same rule as learnedWindowFrom: the larger figure is the
-// request and the smaller is the limit, whatever order the backend used.
+// error for the subtractive summary retry: (Total, Window) from the catalog
+// row, with Input + Output (or Input alone) standing in for a missing total.
+// A context or a budget overflow qualifies; a generic wording states no
+// figures and reports ok == false, so the caller halves instead.
func overflowFigures(err error) (needs, allows int, ok bool) {
- for e := err; e != nil; e = errors.Unwrap(e) {
- m := tokenCountRe.FindAllStringSubmatch(e.Error(), -1)
- if !hasOverflowMarker(e.Error()) || len(m) < 2 {
- continue
- }
- a, _ := strconv.Atoi(m[0][1])
- b, _ := strconv.Atoi(m[1][1])
- needs, allows = max(a, b), min(a, b)
- if allows > 0 {
- return needs, allows, true
- }
+ info := classifyOverflow(err)
+ if info.Kind != KindContext && info.Kind != KindBudget {
+ return 0, 0, false
+ }
+ needs, allows = info.overflowNeeds(), info.Window
+ if needs <= 0 || allows <= 0 {
+ return 0, 0, false
}
- return 0, 0, false
+ return needs, allows, true
}
// CompactHistory summarizes the older portion of the conversation via the LLM
@@ -565,37 +584,40 @@ func (s *Session) CompactHistory() (before, after int, err error) {
// passed-in ctx governs the summarization LLM call, allowing callers (e.g.
// auto-compaction) to make it cancellable via a per-turn context.
func (s *Session) compactHistory(ctx context.Context) (before, after int, err error) {
+ return s.compactHistoryKeep(ctx, s.compactionConfig().KeepRecent)
+}
+
+// compactHistoryKeep is compactHistory with the tail length passed in rather
+// than read from the config. Overflow recovery uses it to retry with a
+// smaller tail without writing the shared CompactionConfig.
+func (s *Session) compactHistoryKeep(ctx context.Context, keep int) (before, after int, err error) {
msgs := s.fragment.Messages
before = estimateTokens(msgs)
- cfg := s.compactionConfig()
- head, tail := splitForCompaction(msgs, cfg.KeepRecent)
s.prunedMu.Lock()
- head = stubbedView(head, s.prunedIDs)
- s.prunedMu.Unlock()
- pieces := renderMessages(head)
- if len(pieces) == 0 {
- return before, before, nil // nothing to compact
+ pruned := make(map[string]string, len(s.prunedIDs))
+ for k, v := range s.prunedIDs {
+ pruned[k] = v
}
- // Save the full untruncated head as an artifact so nothing is lost
- // when the lossy summary truncates or drops messages. The model can
- // page through it via the read tool (artifact://N) or search it with
- // search_artifacts.
- var artifactURI string
- if !cfg.DisableArtifactSpill && s.artifacts != nil {
- var b strings.Builder
- for _, p := range pieces {
- b.WriteString(p.text)
- }
- if b.Len() > 0 {
- artifactURI = s.artifacts.Save("compaction", b.String())
+ compressed := s.copyCompressedLocked()
+ s.prunedMu.Unlock()
+ // The summary sees what the requests saw: stubs, then the progressive
+ // compression levels (see progressivePrune).
+ view := func(m []openai.ChatCompletionMessage) []openai.ChatCompletionMessage {
+ if len(compressed) == 0 {
+ return stubbedView(m, pruned)
}
+ return requestView(m, pruned, compressed)
}
- summary, err := s.summarize(ctx, pieces)
+ summary, head, tail, err := s.summarizeFitting(ctx, msgs, keep, view)
if err != nil {
return before, before, err
}
+ if len(head) == 0 {
+ return before, before, nil // nothing to compact
+ }
+ artifactURI := s.spillCompactionHead(renderMessages(view(head)))
// Build the new state up front; swap only after success (atomic).
newFragMsgs := append([]openai.ChatCompletionMessage{summaryMessage(summary, artifactURI)}, tail...)
@@ -627,58 +649,282 @@ func (s *Session) compactHistory(ctx context.Context) (before, after int, err er
return before, after, nil
}
-// summarize asks the model for the compaction summary of pieces, fitting the
-// prompt into the window. It counts the spend of every call it makes.
-func (s *Session) summarize(ctx context.Context, pieces []summaryPiece) (string, error) {
+// summaryPrefix is the instruction the rendered conversation follows in the
+// summary prompt.
+const summaryPrefix = compactInstruction + "\n\n--- CONVERSATION ---\n"
+
+// summaryOutputTokens is the output the summary request reserves: the smaller
+// of the model's output cap and cfg.SummaryMaxTokens, and never more than half
+// of a known window, so the prompt keeps room beside it.
+//
+// It is set on the request explicitly. Left to the client, the request
+// reserves the model's whole output cap, and a backend that checks prompt +
+// reservation against the window rejects every summary prompt larger than the
+// window minus that cap, however short the summary would have been.
+func (s *Session) summaryOutputTokens() int {
cfg := s.compactionConfig()
- // The summary prompt has to fit the same window as any other request, and
- // the head it summarizes is by nature most of a window that just filled up.
- const prefix = compactInstruction + "\n\n--- CONVERSATION ---\n"
- limit := 0
- if window := s.contextWindow(); window > 0 {
- limit = max(ContextBudget(cfg, window)-tokensOf(prefix), 1)
+ out := cfg.SummaryMaxTokens
+ if out <= 0 {
+ out = defaultSummaryMaxTokens
}
- prompt := prefix + fitSummaryInput(pieces, limit)
- llm, _ := s.currentLLM()
- res, aerr := llm.Ask(ctx, cogito.NewFragment().AddMessage(cogito.UserMessageRole, prompt))
- // Exactly once: the backend's rejection states by how much the estimate was
- // off, so a single corrected retry fits, and a second failure means the
- // backend cannot take even the minimal prompt.
- if aerr != nil && canRecoverFromOverflow(ctx, aerr) {
- if res.Status != nil {
- s.addUsage(res.Status.LastUsage)
+ cap, window := s.requestLimits()
+ if cap > 0 {
+ out = min(out, cap)
+ }
+ if window > 0 {
+ out = max(min(out, window/2), 1)
+ }
+ return out
+}
+
+// summaryPromptLimit is the budget, in byte/4 tokens, for the rendered
+// conversation in the first summary request: the window less the summary's
+// output reservation (and never more than ContextBudget), less the
+// instruction. 0 means no window is known, so no limit.
+func (s *Session) summaryPromptLimit(output int) int {
+ window := s.contextWindow()
+ if window <= 0 {
+ return 0
+ }
+ budget := min(ContextBudget(s.compactionConfig(), window), window-output)
+ return max(budget-tokensOf(summaryPrefix), 1)
+}
+
+// summarizeFitting summarizes the head of msgs that splitForCompaction(msgs,
+// keep) leaves, and returns the summary with the head it covers and the tail
+// to keep verbatim. view maps messages to what the summary prompt should see
+// (the pruned view of them). An empty head, with a nil error, means there was
+// nothing to compact.
+//
+// When the window is known and the rendered head is larger than the summary
+// prompt budget (summaryPromptLimit), the head is summarized in chunks fitted
+// to that budget from the start (see rollingCover): fitting it into one
+// request would cut and drop messages on purpose. Otherwise the whole head
+// goes out in one request first, and when the backend rejects it as too
+// large, the head is summarized in chunks instead, each fitted to the retry
+// target summaryRetryTarget derives from the rejection. When the chunks cover the whole head, the result is the summary
+// and the original tail. When they stop at maxRollingChunks, the head ends
+// where they stopped and the rest joins the tail, verbatim: nothing is lost,
+// and the next compaction can summarize it. The loop stops on a non-overflow
+// error, a cancelled context, a target of zero or less, or after
+// maxSummaryAttempts overflows in a row, and returns the last error. It
+// changes no session state other than the usage it counts.
+func (s *Session) summarizeFitting(ctx context.Context, msgs []openai.ChatCompletionMessage, keep int, view func([]openai.ChatCompletionMessage) []openai.ChatCompletionMessage) (summary string, head, tail []openai.ChatCompletionMessage, err error) {
+ head, tail = splitForCompaction(msgs, keep)
+ if len(head) == 0 || len(renderMessages(view(head))) == 0 {
+ return "", nil, nil, nil
+ }
+ limit := s.summaryPromptLimit(s.summaryOutputTokens())
+ pieces := renderMessages(view(head))
+ if limit > 0 && piecesTokens(pieces) > limit {
+ // The head is known not to fit: fitSummaryInput would cut it down
+ // on purpose, so it goes out in chunks from the start instead. The
+ // target is the whole prompt, instruction included.
+ return s.rollingHead(ctx, msgs, head, view, limit+tokensOf(summaryPrefix), 0)
+ }
+ summary, sent, serr := s.summarize(ctx, pieces, limit)
+ if serr == nil {
+ return summary, head, tail, nil
+ }
+ if maxSummaryAttempts <= 1 || !canShrinkSummary(ctx, serr) {
+ return "", nil, nil, serr
+ }
+ target := summaryRetryTarget(serr, sent)
+ if target <= 0 {
+ return "", nil, nil, serr
+ }
+ return s.rollingHead(ctx, msgs, head, view, target, 1)
+}
+
+// rollingHead summarizes head in chunks fitted to target (see rollingCover)
+// and splits msgs where the chunks stopped. failures is passed through to
+// rollingCover.
+func (s *Session) rollingHead(ctx context.Context, msgs, head []openai.ChatCompletionMessage, view func([]openai.ChatCompletionMessage) []openai.ChatCompletionMessage, target, failures int) (summary string, h, tail []openai.ChatCompletionMessage, err error) {
+ summary, covered, err := s.rollingCover(ctx, head, view, target, failures)
+ if err != nil {
+ return "", nil, nil, err
+ }
+ // head is msgs[:len(head)], so what the chunks did not cover and the
+ // original tail are one contiguous run of msgs.
+ return summary, msgs[:covered], msgs[covered:], nil
+}
+
+// piecesTokens is the byte/4 size of pieces joined.
+func piecesTokens(pieces []summaryPiece) int {
+ n := 0
+ for _, p := range pieces {
+ n += len(p.text)
+ }
+ return n / 4
+}
+
+// maxRollingChunks bounds how many chunks rollingCover summarizes for one
+// compaction. What is left after that stays verbatim.
+const maxRollingChunks = 8
+
+// rollingInstruction is the instruction for every chunk after the first: the
+// running summary comes first in the prompt, the chunk's messages after it.
+const rollingInstruction = compactInstruction + "\n\n" +
+ "A summary of the earlier part of this conversation is given first. " +
+ "Merge it with the new messages into one summary in the same format; " +
+ "keep every file path, identifier and decision from both."
+
+// rollingPrefix is the prompt a later chunk's rendered messages follow.
+func rollingPrefix(running string) string {
+ return rollingInstruction + "\n\n--- EARLIER SUMMARY ---\n" + running + "\n\n--- NEW MESSAGES ---\n"
+}
+
+// rollingSummary summarizes head in consecutive chunks, each fitted to a
+// prompt of target byte/4 tokens, and returns the one summary that stands
+// for all of head. It fails when maxRollingChunks chunks do not cover head;
+// summarizeFitting, which keeps the uncovered rest verbatim instead, uses
+// rollingCover directly.
+func (s *Session) rollingSummary(ctx context.Context, head []openai.ChatCompletionMessage, view func([]openai.ChatCompletionMessage) []openai.ChatCompletionMessage, target int) (summary string, err error) {
+ summary, covered, err := s.rollingCover(ctx, head, view, target, 0)
+ if err != nil {
+ return "", err
+ }
+ if covered < len(head) {
+ return "", fmt.Errorf("compaction summary covered %d of %d messages in %d chunks", covered, len(head), maxRollingChunks)
+ }
+ return summary, nil
+}
+
+// rollingCover summarizes head from oldest to newest in chunks, and returns
+// the running summary and how many of head's messages it stands for.
+//
+// Each chunk is the longest run that fits target: the first after the
+// compaction instruction, each later one after rollingInstruction and the
+// running summary, whose reply then replaces the running summary. A chunk
+// ends only where splitForCompaction could, so it never separates a tool
+// call from its results. A run too large for target on its own still forms
+// a chunk, and fitSummaryInput cuts it down. When the backend rejects a chunk
+// anyway, target drops by the stated overshoot (summaryRetryTarget) and the
+// chunk is rebuilt from the same start, so the rest moves on to the next
+// chunk. failures is how many overflows in a row the caller already had;
+// maxSummaryAttempts of them in a row stop it. It stops after
+// maxRollingChunks chunks, and on any error it returns nothing covered.
+func (s *Session) rollingCover(ctx context.Context, head []openai.ChatCompletionMessage, view func([]openai.ChatCompletionMessage) []openai.ChatCompletionMessage, target, failures int) (summary string, covered int, err error) {
+ viewed := view(head)
+ if len(viewed) != len(head) {
+ return "", 0, fmt.Errorf("compaction view changed the message count")
+ }
+ sizes := make([]int, len(viewed))
+ for i := range viewed {
+ for _, p := range renderMessages(viewed[i : i+1]) {
+ sizes[i] += len(p.text)
}
- limit = max(summaryRetryTarget(aerr, tokensOf(prompt))-tokensOf(prefix), 1)
- prompt = prefix + fitSummaryInput(pieces, limit)
- res, aerr = llm.Ask(ctx, cogito.NewFragment().AddMessage(cogito.UserMessageRole, prompt))
}
+ boundary := func(i int) bool {
+ return i == len(head) || (head[i].Role != "tool" && len(head[i-1].ToolCalls) == 0)
+ }
+
+ running := ""
+ start, chunks := 0, 0
+ for start < len(head) && chunks < maxRollingChunks {
+ prefix := summaryPrefix
+ if chunks > 0 {
+ prefix = rollingPrefix(running)
+ }
+ budget := target - tokensOf(prefix)
+ end, acc := 0, 0
+ for e := start + 1; e <= len(head); e++ {
+ acc += sizes[e-1]
+ if !boundary(e) {
+ continue
+ }
+ if end == 0 || acc/4 <= budget {
+ end = e
+ }
+ if acc/4 > budget {
+ break
+ }
+ }
+ pieces := renderMessages(viewed[start:end])
+ if len(pieces) == 0 {
+ start = end // nothing in it for a summary to cover
+ continue
+ }
+ reply, sent, serr := s.summarizeWith(ctx, prefix, pieces, max(budget, 1))
+ if serr == nil {
+ running, start, failures = reply, end, 0
+ chunks++
+ continue
+ }
+ failures++
+ if failures >= maxSummaryAttempts || !canShrinkSummary(ctx, serr) {
+ return "", 0, serr
+ }
+ if target = summaryRetryTarget(serr, sent); target <= 0 {
+ return "", 0, serr
+ }
+ }
+ if chunks == 0 {
+ return "", 0, fmt.Errorf("compaction produced an empty summary")
+ }
+ return running, start, nil
+}
+
+// spillCompactionHead saves the rendered head a summary stands for as an
+// artifact, so nothing is lost to the lossy summary, and returns its URI ("" when
+// spilling is off or there is no store). The model can page through it with
+// the read tool (artifact://N) or search it with search_artifacts.
+func (s *Session) spillCompactionHead(pieces []summaryPiece) string {
+ if s.compactionConfig().DisableArtifactSpill || s.artifacts == nil {
+ return ""
+ }
+ var b strings.Builder
+ for _, p := range pieces {
+ b.WriteString(p.text)
+ }
+ if b.Len() == 0 {
+ return ""
+ }
+ return s.artifacts.Save("compaction", b.String())
+}
+
+// summarize sends one summary request for pieces, fitted into limit byte/4
+// tokens (0 means no limit) after the instruction, and returns the summary and
+// the byte/4 size of the prompt it sent. It counts the spend of the call.
+//
+// The request goes out through CreateChatCompletion with MaxTokens set to
+// summaryOutputTokens. Ask would leave the reservation to the client, which
+// reserves the model's whole output cap.
+func (s *Session) summarize(ctx context.Context, pieces []summaryPiece, limit int) (summary string, sentTokens int, err error) {
+ return s.summarizeWith(ctx, summaryPrefix, pieces, limit)
+}
+
+// summarizeWith is summarize with the instruction prefix the rendered pieces
+// follow in the prompt.
+func (s *Session) summarizeWith(ctx context.Context, prefix string, pieces []summaryPiece, limit int) (summary string, sentTokens int, err error) {
+ prompt := prefix + fitSummaryInput(pieces, limit)
+ sentTokens = tokensOf(prompt)
+ llm, _ := s.currentLLM()
+ reply, usage, aerr := llm.CreateChatCompletion(ctx, openai.ChatCompletionRequest{
+ Messages: []openai.ChatCompletionMessage{{Role: cogito.UserMessageRole.String(), Content: prompt}},
+ MaxTokens: s.summaryOutputTokens(),
+ })
// Compaction is not free, and it fires exactly when a session has already
// grown expensive — so leaving it out would understate the runs that cost
// the most.
//
- // LastUsage, not CumulativeUsage: Ask is a single call on a throwaway
- // fragment that no ExecuteTools run ever stamped, so CumulativeUsage is
- // zero here and reading it would count nothing at all.
- //
// Counted before BOTH exits below on purpose. What the backend served it
// billed, whether the summary then came back empty or the call came back an
// error, and a rejected summary must not also erase the spend — the same
- // rule the interrupted-turn path in SendMessage follows. Placing this after
- // either early return would make compaction the one path where paid-for
- // tokens vanish, precisely on the sessions that spend the most. Clients
- // that discard the fragment on error simply leave Status nil and add
- // nothing, so the guard is what keeps this honest rather than optimistic.
- if res.Status != nil {
- s.addUsage(res.Status.LastUsage)
- }
+ // rule the interrupted-turn path in SendMessage follows. A client that
+ // reports nothing on error simply adds zero.
+ s.addUsage(usage)
if aerr != nil {
- return "", fmt.Errorf("compaction summary failed: %w", aerr)
+ return "", sentTokens, fmt.Errorf("compaction summary failed: %w", aerr)
+ }
+ var content string
+ if ch := reply.ChatCompletionResponse.Choices; len(ch) > 0 {
+ content = ch[0].Message.Content
}
- last := res.LastMessage()
- if last == nil || strings.TrimSpace(last.Content) == "" {
- return "", fmt.Errorf("compaction produced an empty summary")
+ if strings.TrimSpace(content) == "" {
+ return "", sentTokens, fmt.Errorf("compaction produced an empty summary")
}
- return last.Content, nil
+ return content, sentTokens, nil
}
// summaryMessage wraps a compaction summary as the message that stands in for
diff --git a/chat/compact_fit_test.go b/chat/compact_fit_test.go
index 62980bf..dc29d56 100644
--- a/chat/compact_fit_test.go
+++ b/chat/compact_fit_test.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "reflect"
"strings"
"testing"
@@ -20,16 +21,18 @@ type promptCapturingLLM struct {
}
func (f *promptCapturingLLM) Ask(ctx context.Context, fr cogito.Fragment) (cogito.Fragment, error) {
- prompt := fr.Messages[len(fr.Messages)-1].Content
- f.prompts = append(f.prompts, prompt)
- if f.limit > 0 && tokensOf(prompt) > f.limit {
- return cogito.Fragment{}, fmt.Errorf("rpc error: code = Internal desc = request (%d tokens) exceeds the available context size (%d tokens), try increasing it", tokensOf(prompt), f.limit)
- }
- return fr.AddMessage(cogito.AssistantMessageRole, "SUMMARY"), nil
+ return cogito.Fragment{}, errors.New("the summary must be sent with CreateChatCompletion, not Ask")
}
func (f *promptCapturingLLM) CreateChatCompletion(ctx context.Context, req openai.ChatCompletionRequest) (cogito.LLMReply, cogito.LLMUsage, error) {
- return cogito.LLMReply{}, cogito.LLMUsage{}, nil
+ prompt := req.Messages[len(req.Messages)-1].Content
+ f.prompts = append(f.prompts, prompt)
+ if f.limit > 0 && tokensOf(prompt) > f.limit {
+ return cogito.LLMReply{}, cogito.LLMUsage{}, fmt.Errorf("rpc error: code = Internal desc = request (%d tokens) exceeds the available context size (%d tokens), try increasing it", tokensOf(prompt), f.limit)
+ }
+ return cogito.LLMReply{ChatCompletionResponse: openai.ChatCompletionResponse{
+ Choices: []openai.ChatCompletionChoice{{Message: openai.ChatCompletionMessage{Role: "assistant", Content: "SUMMARY"}}},
+ }}, cogito.LLMUsage{}, nil
}
// toolTurn is one assistant tool call plus its result.
@@ -90,20 +93,24 @@ func TestCompactHistoryFitsTheSummaryPromptInTheBudget(t *testing.T) {
if _, _, err := s.CompactHistory(); err != nil {
t.Fatalf("CompactHistory: %v", err)
}
- if got := tokensOf(llm.prompts[0]); got > budget {
- t.Fatalf("summary prompt is ~%d tokens, budget is %d", got, budget)
+ // The head is known not to fit, so it goes out in chunks from the start.
+ for i, p := range llm.prompts {
+ if got := tokensOf(p); got > budget {
+ t.Fatalf("summary prompt %d is ~%d tokens, budget is %d", i, got, budget)
+ }
}
if !strings.Contains(llm.prompts[0], "goal: fix the parser") {
t.Fatal("fitting the prompt dropped the user's goal")
}
- if !strings.Contains(llm.prompts[0], "f5.go") {
+ if !strings.Contains(strings.Join(llm.prompts, ""), "f5.go") {
t.Fatal("fitting the prompt dropped the record of which tools ran")
}
}
func TestCompactHistoryRetriesSmallerWhenTheSummaryOverflows(t *testing.T) {
// The backend tokenizes denser than byte/4, so a prompt the estimate says
- // fits can still overflow. The backend's own figures say by how much.
+ // fits can still overflow. The backend's own figures say by how much, and
+ // the head is then summarized in chunks that fit.
frag := []openai.ChatCompletionMessage{{Role: "user", Content: "goal"}}
for i := range 6 {
frag = append(frag, toolTurn(fmt.Sprintf("c%d", i), fmt.Sprintf("f%d.go", i), strings.Repeat("x", 8000))...)
@@ -112,6 +119,7 @@ func TestCompactHistoryRetriesSmallerWhenTheSummaryOverflows(t *testing.T) {
openai.ChatCompletionMessage{Role: "user", Content: "u2"},
openai.ChatCompletionMessage{Role: "assistant", Content: "a2"},
)
+ orig := cloneMessages(frag)
llm := &promptCapturingLLM{limit: 1500}
s := newCompactTestSession(llm, 2, frag, frag)
s.compaction.MaxContextTokens = 8000
@@ -119,21 +127,26 @@ func TestCompactHistoryRetriesSmallerWhenTheSummaryOverflows(t *testing.T) {
if _, _, err := s.CompactHistory(); err != nil {
t.Fatalf("CompactHistory: %v", err)
}
- if len(llm.prompts) != 2 {
- t.Fatalf("summary calls = %d, want 2 (one overflow, one smaller retry)", len(llm.prompts))
+ if n := len(llm.prompts); n < 2 || n > 1+maxRollingChunks {
+ t.Fatalf("summary calls = %d, want an overflow and then smaller chunks", n)
}
- if !strings.Contains(s.fragment.Messages[0].Content, "SUMMARY") {
- t.Fatal("the retried summary was not installed")
+ if got := tokensOf(llm.prompts[len(llm.prompts)-1]); got > 1500 {
+ t.Fatalf("the accepted prompt is ~%d tokens, over the backend's 1500", got)
}
+ // The messages no chunk covered are kept verbatim instead.
+ checkVerbatimTail(t, orig, s.fragment.Messages)
}
-func TestCompactHistoryRetriesTheSummaryOnlyOnce(t *testing.T) {
+func TestCompactHistoryGivesUpWhenNoSummaryFits(t *testing.T) {
+ // Not even the instruction fits, and the rejection carries no usable
+ // figures: the loop halves until it runs out of attempts.
frag := []openai.ChatCompletionMessage{
{Role: "user", Content: strings.Repeat("u", 4000)},
{Role: "assistant", Content: "a1"},
{Role: "user", Content: "u2"},
{Role: "assistant", Content: "a2"},
}
+ orig := cloneMessages(frag)
llm := &promptCapturingLLM{limit: 10}
s := newCompactTestSession(llm, 1, frag, frag)
s.compaction.MaxContextTokens = 1 << 20
@@ -142,7 +155,10 @@ func TestCompactHistoryRetriesTheSummaryOnlyOnce(t *testing.T) {
if !isContextOverflow(errors.Unwrap(err)) {
t.Fatalf("want the summarizer's overflow, got %v", err)
}
- if len(llm.prompts) != 2 {
- t.Fatalf("summary calls = %d, want exactly 2", len(llm.prompts))
+ if len(llm.prompts) != maxSummaryAttempts {
+ t.Fatalf("summary calls = %d, want exactly %d", len(llm.prompts), maxSummaryAttempts)
+ }
+ if !reflect.DeepEqual(s.fragment.Messages, orig) {
+ t.Fatal("a failed compaction changed the fragment")
}
}
diff --git a/chat/compact_internal_test.go b/chat/compact_internal_test.go
index 96a9271..c10baa2 100644
--- a/chat/compact_internal_test.go
+++ b/chat/compact_internal_test.go
@@ -96,11 +96,14 @@ func TestEstimateAndHumanTokens(t *testing.T) {
}
}
-// fakeSummaryLLM is a minimal cogito.LLM that returns a canned summary from Ask.
+// fakeSummaryLLM is a minimal cogito.LLM that returns a canned summary from
+// Ask and from CreateChatCompletion, which is what compaction sends. lastReq is
+// the last CreateChatCompletion request.
type fakeSummaryLLM struct {
- reply string
- err error
- calls int
+ reply string
+ err error
+ calls int
+ lastReq openai.ChatCompletionRequest
}
func (f *fakeSummaryLLM) Ask(ctx context.Context, fr cogito.Fragment) (cogito.Fragment, error) {
@@ -116,7 +119,19 @@ func (f *fakeSummaryLLM) Ask(ctx context.Context, fr cogito.Fragment) (cogito.Fr
}
func (f *fakeSummaryLLM) CreateChatCompletion(ctx context.Context, req openai.ChatCompletionRequest) (cogito.LLMReply, cogito.LLMUsage, error) {
- return cogito.LLMReply{}, cogito.LLMUsage{}, nil
+ f.calls++
+ f.lastReq = req
+ if f.err != nil {
+ return cogito.LLMReply{}, cogito.LLMUsage{}, f.err
+ }
+ return replyWith(f.reply), cogito.LLMUsage{TotalTokens: 1}, nil
+}
+
+// replyWith is a completion whose single choice says content.
+func replyWith(content string) cogito.LLMReply {
+ return cogito.LLMReply{ChatCompletionResponse: openai.ChatCompletionResponse{
+ Choices: []openai.ChatCompletionChoice{{Message: openai.ChatCompletionMessage{Role: "assistant", Content: content}}},
+ }}
}
func newCompactTestSession(llm cogito.LLM, keepRecent int, frag []openai.ChatCompletionMessage, disp []openai.ChatCompletionMessage) *Session {
diff --git a/chat/compact_validate.go b/chat/compact_validate.go
new file mode 100644
index 0000000..67f83f5
--- /dev/null
+++ b/chat/compact_validate.go
@@ -0,0 +1,69 @@
+package chat
+
+import (
+ "errors"
+ "fmt"
+
+ openai "github.com/sashabaranov/go-openai"
+)
+
+// Sentinel errors returned by validateCompaction. They are exported so the
+// overflow-recovery caller can switch on them with errors.Is.
+var (
+ ErrCompactionNoOp = errors.New("compaction was a no-op")
+ ErrCompactionGrew = errors.New("compaction result is larger than the original")
+ ErrCompactionOverBudget = errors.New("compaction result exceeds the context budget")
+ ErrCompactionEmpty = errors.New("compaction result is empty")
+)
+
+// validateCompaction gates a compaction result before it is sent. It returns
+// the first failing check: empty result, growth, no-op, over budget, then
+// broken tool-call pairing. Growth is checked before no-op, because a result
+// that grew is never equal in size and the growth check must be reachable.
+// budget is decided by the caller (the context budget, minus any
+// non-compactable floor it knows about).
+func validateCompaction(before, after []openai.ChatCompletionMessage, budget int) error {
+ if len(after) == 0 {
+ return ErrCompactionEmpty
+ }
+ was, now := estimateTokens(before), estimateTokens(after)
+ if now > was {
+ return fmt.Errorf("%w: %d > %d tokens", ErrCompactionGrew, now, was)
+ }
+ if now == was {
+ return fmt.Errorf("%w: still %d tokens", ErrCompactionNoOp, now)
+ }
+ if now > budget {
+ return fmt.Errorf("%w: %d > %d tokens", ErrCompactionOverBudget, now, budget)
+ }
+ return validateToolPairing(after)
+}
+
+// validateToolPairing reports a tool result whose ToolCallID has no earlier
+// assistant tool call, and an assistant tool call with no tool result after
+// it. Either one makes the backend reject the request.
+func validateToolPairing(msgs []openai.ChatCompletionMessage) error {
+ pending := map[string]bool{} // call ID -> seen a result yet
+ for i, m := range msgs {
+ for _, tc := range m.ToolCalls {
+ pending[tc.ID] = false
+ }
+ if m.Role == openai.ChatMessageRoleTool {
+ answered, ok := pending[m.ToolCallID]
+ if !ok {
+ return fmt.Errorf("tool result %q at message %d has no matching tool call", m.ToolCallID, i)
+ }
+ if !answered {
+ pending[m.ToolCallID] = true
+ }
+ }
+ }
+ for i, m := range msgs {
+ for _, tc := range m.ToolCalls {
+ if !pending[tc.ID] {
+ return fmt.Errorf("tool call %q at message %d has no tool result", tc.ID, i)
+ }
+ }
+ }
+ return nil
+}
diff --git a/chat/compact_validate_test.go b/chat/compact_validate_test.go
new file mode 100644
index 0000000..c8ae330
--- /dev/null
+++ b/chat/compact_validate_test.go
@@ -0,0 +1,124 @@
+package chat
+
+import (
+ "errors"
+ "strings"
+ "testing"
+
+ openai "github.com/sashabaranov/go-openai"
+)
+
+func vmsg(role string, n int) openai.ChatCompletionMessage {
+ return openai.ChatCompletionMessage{Role: role, Content: strings.Repeat("x", n*4)}
+}
+
+func vcall(id string) openai.ChatCompletionMessage {
+ return openai.ChatCompletionMessage{
+ Role: openai.ChatMessageRoleAssistant,
+ ToolCalls: []openai.ToolCall{{
+ ID: id, Type: openai.ToolTypeFunction,
+ Function: openai.FunctionCall{Name: "read", Arguments: "{}"},
+ }},
+ }
+}
+
+func vresult(id string) openai.ChatCompletionMessage {
+ return openai.ChatCompletionMessage{Role: openai.ChatMessageRoleTool, ToolCallID: id, Content: "ok"}
+}
+
+func TestValidateCompactionAcceptsAValidResult(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 100), vcall("a"), vresult("a"), vmsg("assistant", 100)}
+ after := []openai.ChatCompletionMessage{vmsg("user", 10), vcall("a"), vresult("a")}
+ if err := validateCompaction(before, after, 1000); err != nil {
+ t.Fatalf("valid compaction rejected: %v", err)
+ }
+}
+
+func TestValidateCompactionRejectsEmpty(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 100)}
+ if err := validateCompaction(before, nil, 1000); !errors.Is(err, ErrCompactionEmpty) {
+ t.Fatalf("err = %v, want ErrCompactionEmpty", err)
+ }
+}
+
+func TestValidateCompactionRejectsGrowth(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 10)}
+ after := []openai.ChatCompletionMessage{vmsg("user", 20)}
+ if err := validateCompaction(before, after, 1000); !errors.Is(err, ErrCompactionGrew) {
+ t.Fatalf("err = %v, want ErrCompactionGrew", err)
+ }
+}
+
+func TestValidateCompactionRejectsNoOp(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 50), vmsg("assistant", 50)}
+ after := []openai.ChatCompletionMessage{vmsg("user", 100)}
+ if err := validateCompaction(before, after, 1000); !errors.Is(err, ErrCompactionNoOp) {
+ t.Fatalf("err = %v, want ErrCompactionNoOp", err)
+ }
+}
+
+func TestValidateCompactionRejectsNoOpFromSplit(t *testing.T) {
+ // A split whose head comes back unchanged reassembles into the original
+ // history, the same size: the known no-op cause.
+ before := []openai.ChatCompletionMessage{vmsg("user", 50), vmsg("assistant", 50)}
+ head, tail := splitForCompaction(before, 1)
+ after := append(append([]openai.ChatCompletionMessage{}, head...), tail...)
+ if err := validateCompaction(before, after, 1000); !errors.Is(err, ErrCompactionNoOp) {
+ t.Fatalf("err = %v, want ErrCompactionNoOp", err)
+ }
+}
+
+func TestValidateCompactionRejectsOverBudget(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 500)}
+ after := []openai.ChatCompletionMessage{vmsg("user", 200)}
+ if err := validateCompaction(before, after, 100); !errors.Is(err, ErrCompactionOverBudget) {
+ t.Fatalf("err = %v, want ErrCompactionOverBudget", err)
+ }
+}
+
+func TestValidateCompactionGrowthBeatsNoOpAndBudget(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 10)}
+ after := []openai.ChatCompletionMessage{vmsg("user", 500)}
+ if err := validateCompaction(before, after, 100); !errors.Is(err, ErrCompactionGrew) {
+ t.Fatalf("err = %v, want ErrCompactionGrew", err)
+ }
+}
+
+func TestValidateCompactionRejectsOrphanToolResult(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 100), vcall("a"), vresult("a")}
+ after := []openai.ChatCompletionMessage{vmsg("user", 10), vresult("a")}
+ err := validateCompaction(before, after, 1000)
+ if err == nil || errors.Is(err, ErrCompactionEmpty) || errors.Is(err, ErrCompactionGrew) ||
+ errors.Is(err, ErrCompactionNoOp) || errors.Is(err, ErrCompactionOverBudget) {
+ t.Fatalf("err = %v, want a tool-pairing error", err)
+ }
+}
+
+func TestValidateCompactionRejectsOrphanToolCall(t *testing.T) {
+ before := []openai.ChatCompletionMessage{vmsg("user", 100), vcall("a"), vresult("a")}
+ after := []openai.ChatCompletionMessage{vmsg("user", 10), vcall("a")}
+ if err := validateCompaction(before, after, 1000); err == nil {
+ t.Fatal("orphan tool call accepted")
+ }
+}
+
+func TestValidateCompactionToolPairing(t *testing.T) {
+ cases := []struct {
+ name string
+ msgs []openai.ChatCompletionMessage
+ ok bool
+ }{
+ {"paired", []openai.ChatCompletionMessage{vcall("a"), vresult("a")}, true},
+ {"no tools", []openai.ChatCompletionMessage{vmsg("user", 1)}, true},
+ {"orphan result", []openai.ChatCompletionMessage{vresult("a")}, false},
+ {"result before call", []openai.ChatCompletionMessage{vresult("a"), vcall("a")}, false},
+ {"orphan call", []openai.ChatCompletionMessage{vcall("a"), vmsg("user", 1)}, false},
+ {"mismatched id", []openai.ChatCompletionMessage{vcall("a"), vresult("b")}, false},
+ }
+ for _, c := range cases {
+ err := validateToolPairing(c.msgs)
+ if (err == nil) != c.ok {
+ t.Errorf("%s: err = %v, want ok=%v", c.name, err, c.ok)
+ }
+ }
+}
diff --git a/chat/contextlive.go b/chat/contextlive.go
index 749bc9d..d8be410 100644
--- a/chat/contextlive.go
+++ b/chat/contextlive.go
@@ -2,9 +2,13 @@ package chat
import (
"context"
+ "encoding/json"
+ "math"
+ "slices"
"sync/atomic"
"github.com/mudler/cogito"
+ "github.com/mudler/xlog"
"github.com/sashabaranov/go-openai"
)
@@ -29,9 +33,100 @@ import (
// active distinguishes "no turn is running" from "a turn is running and has
// not been billed yet". Without it a zero would be indistinguishable from an
// idle session and ContextTokens could not tell which source to trust.
+//
+// floor is different: it is the fixed per-request overhead (tool schemas and
+// system messages) the wrapper measured, byte/4, on the last request that
+// carried tools. ratio is how many tokens the backend counts per byte/4
+// estimated token, learned from the reported usage; the floor is reported
+// scaled by it (see schemaFloor). Neither is reset between turns, because they
+// describe the session's configuration and backend, not the conversation. See
+// SchemaBudget.
type liveUsage struct {
prompt atomic.Int64
active atomic.Bool
+
+ floor atomic.Int64
+ measured atomic.Bool
+ ratio atomic.Uint64 // math.Float64bits; 0 means not calibrated yet
+
+ // clamped records whether clampOutputTokens lowered the last request's
+ // max_tokens below the cap. A tool call truncated on such a request ran
+ // out of window, not of cap (see truncationCause).
+ clamped atomic.Bool
+ // note is a one-turn user-role message the next request carries on the
+ // wire only, then drops: it never reaches the fragment (see
+ // trackedLLM.prepare).
+ note atomic.Pointer[string]
+}
+
+// setNote arms the note the next request carries; "" disarms it.
+func (l *liveUsage) setNote(n string) {
+ if n == "" {
+ l.note.Store(nil)
+ return
+ }
+ l.note.Store(&n)
+}
+
+// takeNote returns the armed note and disarms it.
+func (l *liveUsage) takeNote() string {
+ if p := l.note.Swap(nil); p != nil {
+ return *p
+ }
+ return ""
+}
+
+// lastClamped reports whether the last request's output reservation was
+// lowered below the cap by clampOutputTokens.
+func (l *liveUsage) lastClamped() bool {
+ return l.clamped.Load()
+}
+
+// maxTokenizerRatio caps how many tokens per byte/4 estimated token the floor
+// calibration believes a backend counts.
+const maxTokenizerRatio = 4.0
+
+// recordFloor stores the measured fixed overhead (byte/4) of the latest
+// request.
+func (l *liveUsage) recordFloor(n int) {
+ l.floor.Store(int64(n))
+ l.measured.Store(true)
+}
+
+// setRatio stores the tokenizer ratio, clamped to [1, maxTokenizerRatio].
+func (l *liveUsage) setRatio(r float64) {
+ switch {
+ case r < 1:
+ r = 1
+ case r > maxTokenizerRatio:
+ r = maxTokenizerRatio
+ }
+ l.ratio.Store(math.Float64bits(r))
+}
+
+// tokenizerRatio returns the calibrated ratio, or 1 before any calibration.
+func (l *liveUsage) tokenizerRatio() float64 {
+ if b := l.ratio.Load(); b != 0 {
+ return math.Float64frombits(b)
+ }
+ return 1
+}
+
+// schemaFloor returns the latest measured floor scaled by the tokenizer
+// ratio, and whether any request was measured yet.
+func (l *liveUsage) schemaFloor() (int, bool) {
+ return int(float64(l.floor.Load()) * l.tokenizerRatio()), l.measured.Load()
+}
+
+// promptOverhead is how many more tokens the backend reported for a request
+// than the byte/4 estimate of the messages it was measured against: the
+// tool schemas, anything else the estimate left out, and tokenizer skew.
+// Zero when the report is not larger.
+func promptOverhead(reported, estimated int) int {
+ if reported <= estimated {
+ return 0
+ }
+ return reported - estimated
}
// begin arms the tracker for a new turn, discarding the previous turn's
@@ -39,6 +134,8 @@ type liveUsage struct {
func (l *liveUsage) begin() {
l.prompt.Store(0)
l.active.Store(true)
+ l.clamped.Store(false)
+ l.note.Store(nil)
}
// end disarms it, handing authority back to s.fragment — which by then holds
@@ -46,6 +143,7 @@ func (l *liveUsage) begin() {
func (l *liveUsage) end() {
l.active.Store(false)
l.prompt.Store(0)
+ l.note.Store(nil)
}
// reset drops the in-flight figure without disarming: compaction replaced the
@@ -76,12 +174,88 @@ func (l *liveUsage) promptTokens() int {
// trackedLLM is the LLM the turn loop actually calls: the session's client,
// with every request's reported usage recorded on the way back.
+//
+// It also sizes each request's output reservation (see clampOutputTokens).
+// limits reports the session's output cap and context window; nil leaves every
+// request as the caller built it.
type trackedLLM struct {
cogito.LLM
- live *liveUsage
+ live *liveUsage
+ limits func() (cap, window int)
+}
+
+// prepare measures the request's fixed overhead and applies clampOutputTokens
+// with the session's current limits. The tool schemas are marshalled once for
+// both. It returns the byte/4 estimate of the whole request (messages, tools
+// and system messages), which calibrate compares with the backend's report,
+// or -1 when the request must not calibrate.
+//
+// Only a request that carries tools is measured: a tool-less request (a
+// summary, say) says nothing about the overhead of a turn's requests. A
+// request with an image part is measured but does not calibrate: the backend
+// counts the image, the byte/4 estimate does not, and the ratio would absorb
+// it.
+func (t *trackedLLM) prepare(request openai.ChatCompletionRequest) (openai.ChatCompletionRequest, int) {
+ // A one-turn note goes on this request only. The fragment is cogito's,
+ // built from its own messages, so nothing here is stored.
+ if note := t.live.takeNote(); note != "" {
+ request.Messages = append(slices.Clip(request.Messages),
+ openai.ChatCompletionMessage{Role: openai.ChatMessageRoleUser, Content: note})
+ }
+ toolBytes := toolSchemaBytes(request.Tools)
+ whole := -1
+ if len(request.Tools) > 0 {
+ sysBytes := 0
+ image := false
+ var other []openai.ChatCompletionMessage
+ for _, m := range request.Messages {
+ if m.Role == openai.ChatMessageRoleSystem {
+ sysBytes += len(m.Content)
+ } else {
+ other = append(other, m)
+ }
+ for _, p := range m.MultiContent {
+ if p.Type == openai.ChatMessagePartTypeImageURL {
+ image = true
+ }
+ }
+ }
+ fixed := (toolBytes + sysBytes) / 4
+ t.live.recordFloor(fixed)
+ if !image {
+ whole = estimateTokens(other) + fixed
+ }
+ }
+ explicit := request.MaxTokens != 0 || request.MaxCompletionTokens != 0
+ if t.limits == nil {
+ t.live.clamped.Store(false)
+ return request, whole
+ }
+ cap, window := t.limits()
+ out := clampOutputTokensSized(request, cap, window, toolBytes/4)
+ // An explicit value is cogito's own retry of the previous request with
+ // a raised cap: it keeps that request's record.
+ if !explicit {
+ t.live.clamped.Store(cap > 0 && window > 0 && out.MaxTokens < cap)
+ }
+ return out, whole
+}
+
+// calibrate sets the tokenizer ratio from the backend's report for a request
+// prepare measured (whole >= 0): promptTokens / whole, clamped to
+// [1, maxTokenizerRatio]. The floor is the measured overhead times this ratio,
+// never the raw residual promptTokens - estimate: that residual also holds the
+// skew on the whole conversation, and would make the floor grow with the
+// history.
+func (t *trackedLLM) calibrate(promptTokens, whole int) {
+ if whole <= 0 || promptTokens <= 0 {
+ return
+ }
+ t.live.setRatio(float64(promptTokens) / float64(whole))
}
func (t *trackedLLM) CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest) (cogito.LLMReply, cogito.LLMUsage, error) {
+ request, whole := t.prepare(request)
var usage cogito.LLMUsage
reply, err := retryRequest(ctx, func() (cogito.LLMReply, error) {
r, u, e := t.LLM.CreateChatCompletion(ctx, request)
@@ -89,6 +263,7 @@ func (t *trackedLLM) CreateChatCompletion(ctx context.Context, request openai.Ch
return r, e
})
t.live.record(usage.PromptTokens)
+ t.calibrate(usage.PromptTokens, whole)
return reply, usage, err
}
@@ -120,6 +295,7 @@ type trackedStreamingLLM struct {
// The relay goroutine is bounded by the source channel: it ends when the
// provider closes it, which cogito's own consumer loop already depends on.
func (t *trackedStreamingLLM) CreateChatCompletionStream(ctx context.Context, request openai.ChatCompletionRequest) (<-chan cogito.StreamEvent, error) {
+ request, whole := t.prepare(request)
src, err := retryRequest(ctx, func() (<-chan cogito.StreamEvent, error) {
return t.stream.CreateChatCompletionStream(ctx, request)
})
@@ -132,6 +308,7 @@ func (t *trackedStreamingLLM) CreateChatCompletionStream(ctx context.Context, re
for ev := range src {
if ev.Type == cogito.StreamEventDone {
t.live.record(ev.Usage.PromptTokens)
+ t.calibrate(ev.Usage.PromptTokens, whole)
}
out <- ev
}
@@ -139,11 +316,71 @@ func (t *trackedStreamingLLM) CreateChatCompletionStream(ctx context.Context, re
return out, nil
}
-// trackUsage wraps llm so the turn's requests report their size to live,
+// trackUsage wraps llm so the turn's requests report their size to live and
+// reserve only the output room the window has left (limits may be nil),
// preserving streaming support when the client has it.
-func trackUsage(llm cogito.LLM, live *liveUsage) cogito.LLM {
+func trackUsage(llm cogito.LLM, live *liveUsage, limits func() (cap, window int)) cogito.LLM {
+ t := trackedLLM{LLM: llm, live: live, limits: limits}
if s, ok := llm.(cogito.StreamingLLM); ok {
- return &trackedStreamingLLM{trackedLLM: trackedLLM{LLM: llm, live: live}, stream: s}
+ return &trackedStreamingLLM{trackedLLM: t, stream: s}
+ }
+ return &t
+}
+
+const (
+ // minOutputTokens is the smallest output reservation a clamped request
+ // carries. A prompt that already fills the window still gets this much,
+ // and the overflow path deals with the rejection.
+ minOutputTokens = 1024
+ // outputSafetyMargin absorbs the error of the bytes/4 prompt estimate.
+ outputSafetyMargin = 512
+)
+
+// clampOutputTokens sizes a request's output reservation to what the window
+// has left after its prompt.
+//
+// The client sends its output cap as max_tokens on every request. A backend
+// such as vLLM rejects a request when prompt + max_tokens exceeds the window,
+// whatever the model actually generates. With a cap that is a large fraction
+// of the window (regolo: 96000 of 210000), every prompt above window − cap
+// failed, compaction summaries included. cogito applies the client's cap only
+// when the request carries none, so setting it here decides the reservation.
+//
+// The result is min(cap, window − prompt − outputSafetyMargin), floored at
+// minOutputTokens (and never above cap). The prompt estimate is bytes/4 over
+// the messages and the JSON of the tool schemas, which are real prompt tokens.
+// A request with an explicit value, or unknown limits (cap or window 0), is
+// returned unchanged.
+func clampOutputTokens(request openai.ChatCompletionRequest, cap, window int) openai.ChatCompletionRequest {
+ return clampOutputTokensSized(request, cap, window, toolSchemaBytes(request.Tools)/4)
+}
+
+// toolSchemaBytes is the size of the JSON of a request's tools.
+func toolSchemaBytes(tools []openai.Tool) int {
+ if len(tools) == 0 {
+ return 0
+ }
+ b, err := json.Marshal(tools)
+ if err != nil {
+ return 0
+ }
+ return len(b)
+}
+
+// clampOutputTokensSized is clampOutputTokens with the tool schemas already
+// measured (toolTokens), so the wrapper does not marshal them twice.
+func clampOutputTokensSized(request openai.ChatCompletionRequest, cap, window, toolTokens int) openai.ChatCompletionRequest {
+ if cap <= 0 || window <= 0 || request.MaxTokens != 0 || request.MaxCompletionTokens != 0 {
+ return request
+ }
+ prompt := estimateTokens(request.Messages) + toolTokens
+ v := min(cap, window-prompt-outputSafetyMargin)
+ if v < minOutputTokens {
+ v = min(minOutputTokens, cap)
+ }
+ if v < cap {
+ xlog.Debug("output reservation clamped", "from", cap, "to", v, "prompt_estimate", prompt, "window", window)
}
- return &trackedLLM{LLM: llm, live: live}
+ request.MaxTokens = v
+ return request
}
diff --git a/chat/errors.go b/chat/errors.go
index 43f16ee..9af1b9f 100644
--- a/chat/errors.go
+++ b/chat/errors.go
@@ -3,41 +3,10 @@ package chat
import (
"errors"
"fmt"
- "regexp"
- "strconv"
"strings"
-)
-// contextOverflowMarkers are substrings different backends emit when a request
-// exceeds the model's context window. Only a fragment of each message is
-// matched here, because the wording around it varies. The full messages, which
-// learnedWindowFrom parses for the model's stated limit, are:
-//
-// - llama.cpp / LocalAI (tools/server/server-context.cpp):
-// "request (9739 tokens) exceeds the available context size (8192 tokens), try increasing it"
-// - OpenAI:
-// "This model's maximum context length is 8192 tokens. However, your messages resulted in 9739 tokens. Please reduce the length of the messages."
-// - vLLM, current (vllm/renderers/params.py):
-// "This model's maximum context length is 8192 tokens. However, you requested 512 output tokens and your prompt contains 9739 input tokens, for a total of 10251 tokens. Please reduce the length of the input prompt or the number of requested output tokens."
-// - vLLM, older OpenAI-compatible server:
-// "This model's maximum context length is 8192 tokens. However, you requested 10251 tokens (9739 in the messages, 512 in the completion). Please reduce the length of the messages or completion."
-//
-// This comment used to abbreviate vLLM's message to "maximum context length is
-// 8192 tokens". That fragment is enough to trip a marker but carries only ONE
-// figure, so it teaches learnedWindowFrom nothing — an abbreviation that reads
-// as a complete message invites tests asserting coverage nib does not have.
-// Record the whole thing.
-var contextOverflowMarkers = []string{
- "exceeds the available context size",
- "maximum context length",
- "context length",
- "context window",
- "context size",
-}
-
-// tokenCountRe pulls token counts (e.g. "8192 tokens") out of a backend error
-// so we can report how far over the limit the request was.
-var tokenCountRe = regexp.MustCompile(`(\d{3,})\s*tokens`)
+ "github.com/mudler/cogito"
+)
// FriendlyError wraps a noisy backend error with a short, actionable message
// while preserving the original via Unwrap (so errors.Is/As still work).
@@ -56,7 +25,7 @@ func humanizeError(err error) error {
if err == nil {
return nil
}
- if isContextOverflow(err) {
+ if isWindowOverflow(err) {
return &FriendlyError{err: err, msg: contextOverflowMessage(err.Error())}
}
if isRateLimitError(err) {
@@ -65,9 +34,82 @@ func humanizeError(err error) error {
if isEmptyReply(err) {
return &FriendlyError{err: err, msg: emptyReplyMessage}
}
+ // cogito's sentinels. The turn builds these messages with the figures
+ // only it knows; this is the fallback for an error it did not rewrite.
+ if _, done := err.(*FriendlyError); !done {
+ var te *cogito.ToolArgumentsTruncatedError
+ switch {
+ case errors.Is(err, cogito.ErrStreamInterrupted):
+ return &FriendlyError{err: err, msg: streamInterruptedMessage(0)}
+ case errors.Is(err, cogito.ErrToolArgumentsInvalid):
+ return &FriendlyError{err: err, msg: toolArgsInvalidMessage}
+ case errors.As(err, &te):
+ return &FriendlyError{err: err, msg: toolArgsTruncatedMessage(te, truncCap)}
+ }
+ }
return err
}
+// streamInterruptedMessage is the text for a turn whose stream kept ending
+// before the backend finished it. tries is how many times the turn was sent,
+// or 0 when unknown.
+func streamInterruptedMessage(tries int) string {
+ n := ""
+ if tries > 0 {
+ n = fmt.Sprintf(" (tried %d times)", tries)
+ }
+ return "the connection to the model ended before the reply finished" + n +
+ ". The backend or a proxy in front of it may have timed out on a long reply."
+}
+
+const toolArgsInvalidMessage = "the model produced a tool call with invalid arguments several times. " +
+ "Try rephrasing, or ask it to split the change into smaller steps."
+
+// truncationCause is why a tool call was cut by finish_reason=length.
+type truncationCause int
+
+const (
+ // truncCap: the output cap was reached with room left in the window.
+ truncCap truncationCause = iota
+ // truncToolCall: the window ran out while the model wrote a long call.
+ truncToolCall
+ // truncReasoning: the window ran out while the model was reasoning.
+ truncReasoning
+)
+
+// toolArgsTruncatedMessage is the text for a tool call cut by the output
+// limit, by cause.
+func toolArgsTruncatedMessage(te *cogito.ToolArgumentsTruncatedError, cause truncationCause) string {
+ switch cause {
+ case truncToolCall:
+ return fmt.Sprintf("the model's call to %s did not fit in the context window (about %d tokens of arguments on a prompt of %d). "+
+ "Ask it to make the change in smaller steps.", te.ToolName, te.ArgumentsBytes/4, te.PromptTokens)
+ case truncReasoning:
+ return fmt.Sprintf("the model's reasoning filled the context window (about %d tokens on a prompt of %d). "+
+ "Ask it to split the task, or lower the reasoning effort.", te.ReasoningBytes/4, te.PromptTokens)
+ }
+ limit := "the output limit"
+ if te.MaxTokens > 0 {
+ limit = fmt.Sprintf("the output limit (max_tokens %d)", te.MaxTokens)
+ }
+ return "the model's tool call was longer than " + limit +
+ ". Raise the model's max_tokens, or ask for smaller edits."
+}
+
+// truncationNote is the one-turn user-role note sent with the retry after a
+// tool call was cut because the window ran out. It is never stored in the
+// history.
+func truncationNote(te *cogito.ToolArgumentsTruncatedError, cause truncationCause) string {
+ if cause == truncReasoning {
+ return fmt.Sprintf("Your previous reply was cut off: its reasoning (about %d tokens) ran out of room in the context window before the call to %s was complete. "+
+ "Keep the plan shorter and work in smaller steps (for example write a file in parts, or use edit for targeted changes).",
+ te.ReasoningBytes/4, te.ToolName)
+ }
+ return fmt.Sprintf("Your previous call to %s was cut off after about %d tokens because the reply ran out of room in the context window. "+
+ "Split it into smaller calls (for example write the file in parts, or use edit for targeted changes).",
+ te.ToolName, te.ArgumentsBytes/4)
+}
+
// emptyReplyMarkers are the texts cogito uses when every decision attempt came
// back with no text and no tool call: "streaming decision produced no content
// (finish_reason=...)" on the streaming path, "no choices: 0" on the other.
@@ -106,31 +148,28 @@ const emptyReplyMessage = "the model returned an empty reply (no text and no too
// A bool parameter would leave one function whose every caller passes a
// constant, and both branches would still have to be tested separately.
func humanizeTurnError(err error, compacted bool) error {
- if compacted && isContextOverflow(err) {
+ if compacted && isWindowOverflow(err) {
return &FriendlyError{err: err, msg: contextOverflowRetriedMessage(err.Error())}
}
return humanizeError(err)
}
// isContextOverflow reports whether err is a backend complaining that the
-// request did not fit the model's context window. Factored out of
-// humanizeError so the recovery path and the message path cannot drift apart
-// by consulting different marker lists.
+// prompt itself did not fit the model's context window: the one rejection
+// compaction can fix. A budget overflow (prompt + requested output) and an
+// output-cap error are not context overflows and must not trigger compaction.
+// Every overflow question in the package reads classifyOverflow, so the
+// recovery path and the message path cannot drift apart.
func isContextOverflow(err error) bool {
- return err != nil && hasOverflowMarker(err.Error())
+ return classifyOverflow(err).Kind == KindContext
}
-// hasOverflowMarker holds the single walk over contextOverflowMarkers. Both
-// isContextOverflow and learnedWindowFrom go through it, so there is exactly
-// one marker list and one matching rule in the package.
-func hasOverflowMarker(msg string) bool {
- low := strings.ToLower(msg)
- for _, marker := range contextOverflowMarkers {
- if strings.Contains(low, marker) {
- return true
- }
- }
- return false
+// isWindowOverflow reports a context or a budget overflow: the request, as
+// sent, did not fit the window. It decides the user-facing message, which is
+// true for both; recovery must still tell them apart.
+func isWindowOverflow(err error) bool {
+ k := classifyOverflow(err).Kind
+ return k == KindContext || k == KindBudget
}
// learnedWindowFrom extracts the model's real context window from an overflow
@@ -139,67 +178,52 @@ func hasOverflowMarker(msg string) bool {
// so inconsistently, so this error is nib's only trustworthy source; inferring
// a window from a model name would be a guess presented as a fact.
//
-// Two figures are required. tokenCountRe can match a single number, and with
-// one number there is no way to tell whether it is the limit or the request
-// size — treating a request size as the window would raise the compaction
-// trigger above the real limit and suppress compaction exactly when it is most
-// needed. Fewer than two figures teaches nothing.
-//
-// The window is the SMALLER figure regardless of the order the backend printed
-// them, matching contextOverflowMessage.
+// A window is learned only from a catalog row that names it (the "window"
+// group): a generic wording, or a row that states only a request size, teaches
+// nothing, because mistaking a request size for the window would raise the
+// compaction trigger above the real limit. Context and budget overflows both
+// state the real window; an output-cap error states the maximum OUTPUT, which
+// is not the window.
//
-// The whole unwrap chain is tried, not just err.Error(). humanizeError wraps
-// the backend error in a FriendlyError whose own text drops the second figure
-// — contextOverflowMessage prints "model allows 262144)" with no "tokens" for
-// tokenCountRe to anchor on — so reading only the outermost message would
-// learn nothing from a humanized error. Recovery runs before humanizeError
-// today, but that ordering is one edit away from silently never learning a
-// window again, and no test would catch it. Parsing every level makes the
-// order irrelevant.
+// classifyOverflow walks the whole unwrap chain, so a humanized error (whose
+// own text drops the figures) still teaches the window of the error it wraps.
func learnedWindowFrom(err error) (int, bool) {
- for e := err; e != nil; e = errors.Unwrap(e) {
- if n, ok := windowFromMessage(e.Error()); ok {
- return n, true
- }
+ info := classifyOverflow(err)
+ if (info.Kind == KindContext || info.Kind == KindBudget) && info.Window > 0 {
+ return info.Window, true
}
return 0, false
}
-// windowFromMessage applies the two-figure rule to one error message.
-func windowFromMessage(msg string) (int, bool) {
- if !hasOverflowMarker(msg) {
- return 0, false
- }
- m := tokenCountRe.FindAllStringSubmatch(msg, -1)
- if len(m) < 2 {
- return 0, false
- }
- a, _ := strconv.Atoi(m[0][1])
- b, _ := strconv.Atoi(m[1][1])
- if b < a {
- a = b
- }
- if a <= 0 {
- return 0, false
+// overflowNeeds is the request size an overflow states: the total, else input
+// plus output, else the input alone (llama.cpp and OpenAI state only the
+// prompt, and reserve no output in the count).
+func (o overflowInfo) overflowNeeds() int {
+ switch {
+ case o.Total > 0:
+ return o.Total
+ case o.Input > 0 && o.Output > 0:
+ return o.Input + o.Output
+ default:
+ return o.Input
}
- return a, true
}
// overflowDetail renders the parenthesised token figures for an overflow
-// message, or "" when the backend did not report two of them. The larger count
-// is the request size and the smaller is the model's limit, regardless of the
-// order the backend printed them — the same rule learnedWindowFrom applies.
+// message, or "" when the backend stated none. Which figure is the request
+// size and which the limit comes from the catalog row, not number order.
func overflowDetail(raw string) string {
- m := tokenCountRe.FindAllStringSubmatch(raw, -1)
- if len(m) < 2 {
- return ""
+ info := classifyOverflow(errors.New(raw))
+ needs := info.overflowNeeds()
+ switch {
+ case needs > 0 && info.Window > 0:
+ return fmt.Sprintf(" (needs ~%d tokens, model allows %d)", needs, info.Window)
+ case info.Window > 0:
+ return fmt.Sprintf(" (model allows %d)", info.Window)
+ case needs > 0:
+ return fmt.Sprintf(" (needs ~%d tokens)", needs)
}
- needs, _ := strconv.Atoi(m[0][1])
- allows, _ := strconv.Atoi(m[1][1])
- if allows > needs {
- needs, allows = allows, needs
- }
- return fmt.Sprintf(" (needs ~%d tokens, model allows %d)", needs, allows)
+ return ""
}
// contextOverflowMessage builds the user-facing text for the FIRST context-
@@ -225,3 +249,16 @@ func contextOverflowRetriedMessage(raw string) string {
return "the request is still larger than the model's context window" + overflowDetail(raw) +
" after compacting the conversation and retrying. Compacting again will not help: increase the backend's context size, or reduce the enabled tools/MCP servers."
}
+
+// errSchemaFloor marks a turn that failed because the tool schemas and the
+// system prompt alone leave no room for a request: no compaction can help, so
+// nib kept the conversation instead of compacting it away.
+var errSchemaFloor = errors.New("tool schemas and system prompt do not fit the context window")
+
+// schemaFloorMessage is the user-facing text for errSchemaFloor. It reads
+// like the schema-budget notice (sizes, the largest servers, what to do), and
+// says the conversation was kept.
+func schemaFloorMessage(sb SchemaBudget, window int) string {
+ return schemaBudgetNotice(sb, window) +
+ ", or increase the backend's context size. The conversation was kept: compacting it cannot make the request fit."
+}
diff --git a/chat/errors_window_test.go b/chat/errors_window_test.go
index 1d1bb72..5b9082c 100644
--- a/chat/errors_window_test.go
+++ b/chat/errors_window_test.go
@@ -33,11 +33,9 @@ func TestLearnedWindowFromRealBackendPhrasings(t *testing.T) {
want: 8192,
},
{
- // vLLM current, vllm/renderers/params.py. Note the near-miss: the
- // "512 output tokens" and "9739 input tokens" figures do NOT match
- // tokenCountRe, because the intervening word breaks `\s*tokens`.
- // The two that do match are the limit and the total, so the
- // smaller is still the window.
+ // vLLM current, vllm/renderers/params.py. The catalog row names
+ // each figure (window, output, input, total), so the window is
+ // read by name, not guessed from number order.
name: "vLLM, current",
msg: "This model's maximum context length is 8192 tokens. However, you requested 512 output tokens and your prompt contains 9739 input tokens, for a total of 10251 tokens. Please reduce the length of the input prompt or the number of requested output tokens.",
want: 8192,
diff --git a/chat/midturn.go b/chat/midturn.go
index 9976366..6beabe5 100644
--- a/chat/midturn.go
+++ b/chat/midturn.go
@@ -2,7 +2,6 @@ package chat
import (
"context"
- "strings"
"github.com/mudler/cogito"
"github.com/mudler/xlog"
@@ -81,8 +80,9 @@ func sameMessage(a, b openai.ChatCompletionMessage) bool {
}
// manipulate is the cogito.WithMessagesManipulator body for a turn: it puts the
-// summary in place of the history it covers, prunes tool output, and compacts
-// when the result crosses the auto-compaction trigger.
+// summary in place of the history it covers, prunes and progressively
+// compresses tool output (progressivePrune), and compacts when the result
+// crosses the auto-compaction trigger.
func (c *turnCompactor) manipulate(msgs []openai.ChatCompletionMessage) []openai.ChatCompletionMessage {
base, repl := 0, 0
view := msgs
@@ -91,10 +91,10 @@ func (c *turnCompactor) manipulate(msgs []openai.ChatCompletionMessage) []openai
view = make([]openai.ChatCompletionMessage, 0, repl+len(msgs)-base)
view = append(append(view, c.replacement...), msgs[base:]...)
}
- out := c.s.pruneMessages(view)
+ out := c.s.progressivePrune(view)
overhead := c.overhead()
- if !c.failed && c.s.shouldCompactNow(estimateTokens(out)+overhead) {
+ if !c.failed && c.s.shouldCompactNow(estimateTokens(out)+overhead) && !c.s.autoCompactBlocked(lastUserTokens(out)) {
out = c.compact(msgs, out, base, repl, overhead)
}
c.sent = estimateTokens(out)
@@ -112,10 +112,10 @@ func (c *turnCompactor) manipulate(msgs []openai.ChatCompletionMessage) []openai
// report measured both, so its excess over the last estimate is added back.
func (c *turnCompactor) overhead() int {
reported := c.s.live.promptTokens()
- if c.sent <= 0 || reported <= c.sent {
+ if c.sent <= 0 {
return 0
}
- return reported - c.sent
+ return promptOverhead(reported, c.sent)
}
// compact summarizes the head of out and returns the request with the summary
@@ -126,49 +126,43 @@ func (c *turnCompactor) compact(msgs, out []openai.ChatCompletionMessage, base,
// The kept tail has to fit too, or the summary buys nothing and the next
// step compacts again. Shrink it toward the last tool step when it does
// not; splitForCompaction still never separates a call from its results.
- var head, tail []openai.ChatCompletionMessage
- for keep := max(c.s.compactionConfig().KeepRecent, 1); keep >= 1; keep-- {
- h, t := splitForCompaction(out, keep)
+ keep := 0
+ for k := max(c.s.compactionConfig().KeepRecent, 1); k >= 1; k-- {
+ h, t := splitForCompaction(out, k)
if h == nil {
continue
}
- head, tail = h, t
+ keep = k
if !c.s.shouldCompactNow(estimateTokens(t) + overhead) {
break
}
}
// A head that is only the previous summary has nothing new in it.
- if len(head) <= repl {
+ if keep == 0 {
return out
}
- pieces := renderMessages(head)
- if len(pieces) == 0 {
+ if h, _ := splitForCompaction(out, keep); len(h) <= repl {
return out
}
- // Save the full untruncated head as a compaction artifact, the same
- // as the end-of-turn path, so nothing is lost to the lossy summary.
- var artifactURI string
- cfg := c.s.compactionConfig()
- if !cfg.DisableArtifactSpill && c.s.artifacts != nil {
- var b strings.Builder
- for _, p := range pieces {
- b.WriteString(p.text)
- }
- if b.Len() > 0 {
- artifactURI = c.s.artifacts.Save("compaction", b.String())
- }
- }
-
if c.s.callbacks.OnStatus != nil {
c.s.callbacks.OnStatus("Compacting conversation…")
}
- summary, err := c.s.summarize(c.ctx, pieces)
+ // out is already the pruned request, so the summary sees it as it is.
+ summary, head, tail, err := c.s.summarizeFitting(c.ctx, out, keep, func(m []openai.ChatCompletionMessage) []openai.ChatCompletionMessage { return m })
if err != nil {
xlog.Warn("mid-turn compaction failed", "error", err)
c.failed = true
return out
}
+ // The overflow retries may have moved the boundary back until the head
+ // is only the previous summary, which has nothing new in it.
+ if len(head) <= repl {
+ return out
+ }
+ // Save the head actually summarized as a compaction artifact, the same
+ // as the end-of-turn path, so nothing is lost to the lossy summary.
+ artifactURI := c.s.spillCompactionHead(renderMessages(head))
// renderMessages leaves system messages out of the summary, so they are
// kept as they are: dropping them would lose the system prompt.
diff --git a/chat/midturn_compact_test.go b/chat/midturn_compact_test.go
index 401606e..45ca915 100644
--- a/chat/midturn_compact_test.go
+++ b/chat/midturn_compact_test.go
@@ -223,11 +223,18 @@ func answer(marker string, tokens int) string {
// The real failure: a turn that starts well under the trigger grows past the
// window through its own tool results, because auto-compaction only ran after
-// the turn. Two 9000-token results take the third request to about 18000
-// tokens, over the 15200 trigger; the turn must compact before sending it.
+// the turn. Two 7500-token results and the ~1300-token system prompt take the
+// third request to about 16300 tokens, over the 15200 trigger; the turn must
+// compact before sending it.
+//
+// The results are sized so the request lands between 80% and 90% of the
+// 19000 budget. There progressivePrune leaves a one-line result as it is, so
+// the request still crosses the trigger. Above 90% it elides the older result
+// and the request drops under the trigger without a summary, which is the
+// compression working, not the compaction under test.
func TestMidTurnCompactionKeepsEveryRequestUnderTheTrigger(t *testing.T) {
_, backend, events := runMidTurn(t, []string{
- answer("FIRST_ANSWER", 9000), answer("SECOND_ANSWER", 9000), answer("THIRD_ANSWER", 1000),
+ answer("FIRST_ANSWER", 7500), answer("SECOND_ANSWER", 7500), answer("THIRD_ANSWER", 1000),
})
turns := backend.turns()
@@ -273,7 +280,7 @@ func TestMidTurnCompactionKeepsEveryRequestUnderTheTrigger(t *testing.T) {
// the compaction carry the same one.
func TestMidTurnCompactionReusesTheSummary(t *testing.T) {
_, backend, _ := runMidTurn(t, []string{
- answer("FIRST_ANSWER", 9000), answer("SECOND_ANSWER", 9000), answer("THIRD_ANSWER", 1000),
+ answer("FIRST_ANSWER", 7500), answer("SECOND_ANSWER", 7500), answer("THIRD_ANSWER", 1000),
})
if n := backend.summaryCount(); n != 1 {
@@ -301,7 +308,7 @@ func TestMidTurnCompactionReusesTheSummary(t *testing.T) {
// and every tool result follows the assistant message that called it.
func TestMidTurnCompactionKeepsSystemPromptAndToolPairs(t *testing.T) {
_, backend, _ := runMidTurn(t, []string{
- answer("FIRST_ANSWER", 9000), answer("SECOND_ANSWER", 9000), answer("THIRD_ANSWER", 1000),
+ answer("FIRST_ANSWER", 7500), answer("SECOND_ANSWER", 7500), answer("THIRD_ANSWER", 1000),
})
turns := backend.turns()
@@ -341,7 +348,7 @@ func TestMidTurnCompactionKeepsSystemPromptAndToolPairs(t *testing.T) {
// copy records the compaction the way compactHistory does.
func TestMidTurnCompactionLeavesTheFragmentCompacted(t *testing.T) {
s, _, _ := runMidTurn(t, []string{
- answer("FIRST_ANSWER", 9000), answer("SECOND_ANSWER", 9000), answer("THIRD_ANSWER", 1000),
+ answer("FIRST_ANSWER", 7500), answer("SECOND_ANSWER", 7500), answer("THIRD_ANSWER", 1000),
})
s.historyMu.Lock()
@@ -411,8 +418,8 @@ func TestNoMidTurnCompactionUnderTheTrigger(t *testing.T) {
// off by the length of the first replacement, it would drop or repeat a step.
func TestMidTurnCompactionTwiceInOneTurn(t *testing.T) {
s, backend, _ := runMidTurn(t, []string{
- answer("FIRST_ANSWER", 9000), answer("SECOND_ANSWER", 9000),
- answer("THIRD_ANSWER", 9000), answer("FOURTH_ANSWER", 1000),
+ answer("FIRST_ANSWER", 7500), answer("SECOND_ANSWER", 7500),
+ answer("THIRD_ANSWER", 7500), answer("FOURTH_ANSWER", 1000),
})
if n := backend.summaryCount(); n != 2 {
diff --git a/chat/modellimits.go b/chat/modellimits.go
index 9acffb3..8c648d8 100644
--- a/chat/modellimits.go
+++ b/chat/modellimits.go
@@ -3,8 +3,11 @@ package chat
import (
"context"
+ "github.com/mudler/cogito"
+ "github.com/mudler/nib/auth"
"github.com/mudler/nib/llmprovider"
"github.com/mudler/nib/llmprovider/catalog"
+ "github.com/mudler/nib/types"
)
// A model has two limits nib can only learn by asking the endpoint: the
@@ -53,6 +56,11 @@ func (s *Session) ensureModelLimits(ctx context.Context) {
cancel()
if !res.Omit && res.MaxTokens > 0 {
setter.SetMaxTokens(res.MaxTokens)
+ s.modelMu.Lock()
+ if s.llmModel == model {
+ s.outputCap = res.MaxTokens
+ }
+ s.modelMu.Unlock()
}
}
@@ -67,3 +75,85 @@ func (s *Session) ensureModelLimits(ctx context.Context) {
type maxTokensSetter interface {
SetMaxTokens(int)
}
+
+// factoryOutputCap is the output cap the LLM factory gave llm when it built it
+// for provider: the same resolution (config, then catalog, no network), so the
+// session knows what each request reserves without asking the client. A
+// client that carries no cap (no SetMaxTokens) reports 0, which leaves its
+// requests unclamped.
+func factoryOutputCap(llm cogito.LLM, provider types.ModelProviderConfig, store *auth.Store) int {
+ if _, ok := llm.(maxTokensSetter); !ok {
+ return 0
+ }
+ baseURL, apiKey, _ := llmprovider.ModelsEndpoint(provider, store)
+ res := catalog.ResolveMaxTokens(nil, provider, baseURL, apiKey)
+ if res.Omit || res.MaxTokens <= 0 {
+ return 0
+ }
+ return res.MaxTokens
+}
+
+// requestLimits reports the output cap and context window the turn's requests
+// are clamped against (see clampOutputTokens), read under one lock so a model
+// switch cannot pair one model's cap with another's window. A one-turn
+// override (turnOutputCap) lowers the cap, and never raises it.
+func (s *Session) requestLimits() (cap, window int) {
+ s.modelMu.RLock()
+ defer s.modelMu.RUnlock()
+ cap = s.outputCap
+ if o := s.turnOutputCap; o > 0 && (cap <= 0 || o < cap) {
+ cap = o
+ }
+ return cap, s.windowLocked()
+}
+
+// setTurnOutputCap sets (or, with 0, clears) the current turn's output cap
+// override.
+func (s *Session) setTurnOutputCap(v int) {
+ s.modelMu.Lock()
+ s.turnOutputCap = v
+ s.modelMu.Unlock()
+}
+
+// lowerOutputCap lowers the output cap of model to max after the backend
+// said the requested output is above what it allows. It never raises the
+// cap. It reports whether the cap changed, so the caller retries only a
+// request that will differ. The client's own cap is lowered too, because it
+// is what a request carries when the window is unknown and nothing clamps.
+func (s *Session) lowerOutputCap(max int, model string) bool {
+ if max <= 0 {
+ return false
+ }
+ s.modelMu.Lock()
+ if s.llmModel != model || (s.outputCap > 0 && s.outputCap <= max) {
+ s.modelMu.Unlock()
+ return false
+ }
+ s.outputCap = max
+ llm := s.llm
+ s.modelMu.Unlock()
+ if setter, ok := llm.(maxTokensSetter); ok {
+ setter.SetMaxTokens(max)
+ }
+ return true
+}
+
+// budgetRetryOutput is the output reservation that fits a budget overflow:
+// the window less the input the backend counted, less outputSafetyMargin. The
+// backend's count is exact, which nib's estimate is not. It reports false when
+// the figures are missing or the result is below minOutputTokens: then the
+// prompt itself leaves no useful room, and only a smaller prompt helps.
+func budgetRetryOutput(info overflowInfo) (int, bool) {
+ input := info.Input
+ if input <= 0 && info.Total > 0 && info.Output > 0 {
+ input = info.Total - info.Output
+ }
+ if info.Window <= 0 || input <= 0 {
+ return 0, false
+ }
+ out := info.Window - input - outputSafetyMargin
+ if out < minOutputTokens {
+ return 0, false
+ }
+ return out, true
+}
diff --git a/chat/output_clamp_test.go b/chat/output_clamp_test.go
new file mode 100644
index 0000000..9c72660
--- /dev/null
+++ b/chat/output_clamp_test.go
@@ -0,0 +1,172 @@
+package chat
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/mudler/cogito"
+ "github.com/mudler/nib/types"
+ "github.com/sashabaranov/go-openai"
+)
+
+// promptOf builds a request whose estimated prompt is about tokens tokens
+// (estimateTokens counts bytes/4).
+func promptOf(tokens int) openai.ChatCompletionRequest {
+ return openai.ChatCompletionRequest{
+ Messages: []openai.ChatCompletionMessage{{Role: "user", Content: strings.Repeat("a", tokens*4)}},
+ }
+}
+
+func TestClampOutputLeavesRoomForThePrompt(t *testing.T) {
+ got := clampOutputTokens(promptOf(70000), 96000, 96000)
+ want := 96000 - 70000 - outputSafetyMargin
+ if got.MaxTokens != want {
+ t.Fatalf("MaxTokens = %d, want %d", got.MaxTokens, want)
+ }
+}
+
+func TestClampOutputNeverAboveCap(t *testing.T) {
+ got := clampOutputTokens(promptOf(100), 8000, 96000)
+ if got.MaxTokens != 8000 {
+ t.Fatalf("MaxTokens = %d, want the cap 8000", got.MaxTokens)
+ }
+}
+
+func TestClampOutputFloorWhenPromptFillsWindow(t *testing.T) {
+ got := clampOutputTokens(promptOf(120000), 96000, 96000)
+ if got.MaxTokens != minOutputTokens {
+ t.Fatalf("MaxTokens = %d, want minOutputTokens %d", got.MaxTokens, minOutputTokens)
+ }
+}
+
+func TestClampOutputLeavesExplicitValueAlone(t *testing.T) {
+ req := promptOf(70000)
+ req.MaxTokens = 50000
+ if got := clampOutputTokens(req, 96000, 96000); got.MaxTokens != 50000 || got.MaxCompletionTokens != 0 {
+ t.Fatalf("explicit MaxTokens changed: %+v", got.MaxTokens)
+ }
+ req = promptOf(70000)
+ req.MaxCompletionTokens = 50000
+ if got := clampOutputTokens(req, 96000, 96000); got.MaxTokens != 0 || got.MaxCompletionTokens != 50000 {
+ t.Fatalf("explicit MaxCompletionTokens changed: %d/%d", got.MaxTokens, got.MaxCompletionTokens)
+ }
+}
+
+func TestClampOutputUnknownLimitsLeaveRequestAlone(t *testing.T) {
+ if got := clampOutputTokens(promptOf(70000), 0, 96000); got.MaxTokens != 0 {
+ t.Fatalf("cap 0: MaxTokens = %d, want 0", got.MaxTokens)
+ }
+ if got := clampOutputTokens(promptOf(70000), 96000, 0); got.MaxTokens != 0 {
+ t.Fatalf("window 0: MaxTokens = %d, want 0", got.MaxTokens)
+ }
+}
+
+func TestClampOutputCountsToolSchemas(t *testing.T) {
+ req := promptOf(10000)
+ without := clampOutputTokens(req, 96000, 96000).MaxTokens
+
+ req.Tools = []openai.Tool{{
+ Type: openai.ToolTypeFunction,
+ Function: &openai.FunctionDefinition{
+ Name: "big",
+ Description: strings.Repeat("d", 40000), // ~10k tokens of schema
+ },
+ }}
+ with := clampOutputTokens(req, 96000, 96000).MaxTokens
+ if without-with < 9000 {
+ t.Fatalf("tool schemas not counted: without=%d with=%d", without, with)
+ }
+}
+
+// captureLLMReq records the request it receives, on both paths.
+type captureLLMReq struct {
+ got openai.ChatCompletionRequest
+ stream openai.ChatCompletionRequest
+}
+
+func (c *captureLLMReq) Ask(ctx context.Context, f cogito.Fragment) (cogito.Fragment, error) {
+ return f, nil
+}
+
+func (c *captureLLMReq) CreateChatCompletion(ctx context.Context, req openai.ChatCompletionRequest) (cogito.LLMReply, cogito.LLMUsage, error) {
+ c.got = req
+ return cogito.LLMReply{}, cogito.LLMUsage{}, nil
+}
+
+func (c *captureLLMReq) CreateChatCompletionStream(ctx context.Context, req openai.ChatCompletionRequest) (<-chan cogito.StreamEvent, error) {
+ c.stream = req
+ ch := make(chan cogito.StreamEvent)
+ close(ch)
+ return ch, nil
+}
+
+func TestClampOutputAppliedByTrackedLLM(t *testing.T) {
+ inner := &captureLLMReq{}
+ var live liveUsage
+ llm := trackUsage(inner, &live, func() (int, int) { return 96000, 96000 })
+
+ if _, _, err := llm.CreateChatCompletion(context.Background(), promptOf(70000)); err != nil {
+ t.Fatal(err)
+ }
+ want := 96000 - 70000 - outputSafetyMargin
+ if inner.got.MaxTokens != want {
+ t.Fatalf("blocking MaxTokens = %d, want %d", inner.got.MaxTokens, want)
+ }
+
+ sl, ok := llm.(cogito.StreamingLLM)
+ if !ok {
+ t.Fatal("wrapper lost streaming support")
+ }
+ ch, err := sl.CreateChatCompletionStream(context.Background(), promptOf(70000))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for range ch {
+ }
+ if inner.stream.MaxTokens != want {
+ t.Fatalf("streaming MaxTokens = %d, want %d", inner.stream.MaxTokens, want)
+ }
+}
+
+func TestClampOutputNilLimitsIsNoop(t *testing.T) {
+ inner := &captureLLMReq{}
+ var live liveUsage
+ llm := trackUsage(inner, &live, nil)
+ if _, _, err := llm.CreateChatCompletion(context.Background(), promptOf(70000)); err != nil {
+ t.Fatal(err)
+ }
+ if inner.got.MaxTokens != 0 {
+ t.Fatalf("MaxTokens = %d, want untouched 0", inner.got.MaxTokens)
+ }
+}
+
+func TestClampOutputSessionLimits(t *testing.T) {
+ s := &Session{
+ llmModel: "m",
+ outputCap: 96000,
+ compaction: types.CompactionConfig{MaxContextTokens: 96000},
+ }
+ if c, w := s.requestLimits(); c != 96000 || w != 96000 {
+ t.Fatalf("requestLimits = %d,%d", c, w)
+ }
+ s.learnedWindow, s.learnedWindowModel = 210000, "m"
+ if _, w := s.requestLimits(); w != 210000 {
+ t.Fatalf("learned window not used: %d", w)
+ }
+}
+
+// capSetterLLM is a client that carries an output cap, like LocalAIClient.
+type capSetterLLM struct{ captureLLMReq }
+
+func (c *capSetterLLM) SetMaxTokens(int) {}
+
+func TestClampOutputFactoryCapRecorded(t *testing.T) {
+ p := types.ModelProviderConfig{Model: "some-model", MaxTokens: 96000, BaseURL: "http://localhost:1/v1"}
+ if got := factoryOutputCap(&capSetterLLM{}, p, nil); got != 96000 {
+ t.Fatalf("factoryOutputCap = %d, want 96000", got)
+ }
+ if got := factoryOutputCap(&captureLLMReq{}, p, nil); got != 0 {
+ t.Fatalf("adapter without a cap: factoryOutputCap = %d, want 0", got)
+ }
+}
diff --git a/chat/overflow_e2e_test.go b/chat/overflow_e2e_test.go
index cae229c..69f0550 100644
--- a/chat/overflow_e2e_test.go
+++ b/chat/overflow_e2e_test.go
@@ -84,8 +84,8 @@ func TestOverflowRecoveryOverHTTP(t *testing.T) {
// Seed enough history that compaction has a head to summarise.
s.fragment = s.fragment.
- AddMessage("user", "u1").AddMessage("assistant", "a1").
- AddMessage("user", "u2").AddMessage("assistant", "a2")
+ AddMessage("user", "u1").AddMessage("assistant", "a1 "+overflowFiller).
+ AddMessage("user", "u2").AddMessage("assistant", "a2 "+overflowFiller)
got, err := s.SendMessage("what changed?")
if err != nil {
@@ -249,8 +249,8 @@ func TestOverflowRetryKeepsTheSystemPrompt(t *testing.T) {
// FIRST (the first turn's own add), then history accumulates after it.
s.ensureSystemPrompt()
s.fragment = s.fragment.
- AddMessage("user", "u1").AddMessage("assistant", "a1").
- AddMessage("user", "u2").AddMessage("assistant", "a2")
+ AddMessage("user", "u1").AddMessage("assistant", "a1 "+overflowFiller).
+ AddMessage("user", "u2").AddMessage("assistant", "a2 "+overflowFiller)
if got := s.fragment.Messages[0].Role; got != "system" {
t.Fatalf("seeded fragment starts with %q, not the system prompt: the bug would be hidden", got)
}
@@ -336,8 +336,8 @@ func TestOverflowRecoveryOverStreamedErrorChunk(t *testing.T) {
defer s.Close()
s.fragment = s.fragment.
- AddMessage("user", "u1").AddMessage("assistant", "a1").
- AddMessage("user", "u2").AddMessage("assistant", "a2")
+ AddMessage("user", "u1").AddMessage("assistant", "a1 "+overflowFiller).
+ AddMessage("user", "u2").AddMessage("assistant", "a2 "+overflowFiller)
got, err := s.SendMessage("what changed?")
if err != nil {
diff --git a/chat/overflow_floor_test.go b/chat/overflow_floor_test.go
new file mode 100644
index 0000000..ba6ffac
--- /dev/null
+++ b/chat/overflow_floor_test.go
@@ -0,0 +1,252 @@
+package chat
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "sync"
+ "testing"
+
+ sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp"
+ "github.com/mudler/cogito"
+ "github.com/mudler/nib/types"
+ openai "github.com/sashabaranov/go-openai"
+)
+
+// A floor above the window: compaction cannot make any request fit, so an
+// overflow must not summarize, must not touch the history, and must say which
+// MCP servers take the room.
+func TestOverflowFloorKeepsConversation(t *testing.T) {
+ llm := &overflowLLM{failures: 99}
+ s := newOverflowSession(t, llm)
+ // The overflow teaches a 262144-token window; github alone is about 300k.
+ s.clients = []*sdkmcp.ClientSession{
+ connectTestMCP(t, "github", 60, 20000),
+ connectTestMCP(t, "jira", 10, 4000),
+ connectTestMCP(t, "linear", 4, 2000),
+ connectTestMCP(t, "slack", 1, 100),
+ }
+ s.ensureSystemPrompt()
+ before := append([]openai.ChatCompletionMessage(nil), s.fragment.Messages...)
+
+ _, err := s.SendMessage("what changed?")
+ if !errors.Is(err, errSchemaFloor) {
+ t.Fatalf("err = %v, want the schema-floor error", err)
+ }
+ msg := err.Error()
+ for _, want := range []string{"github", "jira", "linear", "kept"} {
+ if !strings.Contains(msg, want) {
+ t.Fatalf("message %q does not mention %q", msg, want)
+ }
+ }
+ if strings.Contains(msg, "slack") {
+ t.Fatalf("message %q names more than the three largest servers", msg)
+ }
+ llm.mu.Lock()
+ asks := llm.asks
+ llm.mu.Unlock()
+ if asks != 0 {
+ t.Fatalf("the summarizer ran %d times; the floor is the problem, not the history", asks)
+ }
+ if s.overflowRetries() != 0 {
+ t.Fatalf("overflow retries = %d, want 0", s.overflowRetries())
+ }
+ got := s.fragment.Messages
+ if len(got) != len(before) {
+ t.Fatalf("fragment has %d messages, want the %d from before the turn", len(got), len(before))
+ }
+ for i := range got {
+ if got[i].Role != before[i].Role || got[i].Content != before[i].Content {
+ t.Fatalf("message %d changed: %+v", i, got[i])
+ }
+ }
+ if n := len(s.messages); n == 0 || s.messages[n-1].Role != "user" || s.messages[n-1].Content != "what changed?" {
+ t.Fatalf("the user's message is not the last in the display copy: %+v", s.messages)
+ }
+}
+
+// A floor well below the window: recovery runs as before.
+func TestOverflowFloorBelowWindowRecovers(t *testing.T) {
+ llm := &overflowLLM{failures: 1}
+ s := newOverflowSession(t, llm)
+ if _, err := s.SendMessage("what changed?"); err != nil {
+ t.Fatalf("SendMessage: %v", err)
+ }
+ llm.mu.Lock()
+ asks := llm.asks
+ llm.mu.Unlock()
+ if asks == 0 || s.overflowRetries() != 1 {
+ t.Fatalf("asks = %d, retries = %d: recovery did not run", asks, s.overflowRetries())
+ }
+}
+
+// An estimate may undercount, so it blocks only when it alone is over the
+// limit; a measured floor blocks once it leaves no room.
+func TestOverflowFloorEstimateOnly(t *testing.T) {
+ s := newCompactTestSession(&fakeSummaryLLM{}, 2, nil, nil) // window 1000
+ s.systemPrompt = strings.Repeat("s", 3200) // ~800 tokens
+
+ sb, blocked := s.schemaFloorBlocks(1000, 300)
+ if sb.Measured || blocked {
+ t.Fatalf("estimate %d below the window blocked (measured %v)", sb.Floor, sb.Measured)
+ }
+ s.live.recordFloor(sb.Floor)
+ if _, blocked := s.schemaFloorBlocks(1000, 300); !blocked {
+ t.Fatal("a measured floor that leaves no room did not block")
+ }
+
+ s2 := newCompactTestSession(&fakeSummaryLLM{}, 2, nil, nil)
+ s2.systemPrompt = strings.Repeat("s", 4400) // ~1100 tokens
+ if sb, blocked := s2.schemaFloorBlocks(1000, 0); !blocked {
+ t.Fatalf("estimate %d over the window did not block", sb.Floor)
+ }
+}
+
+// floorUsageLLM answers every turn request and reports its byte/4 size plus
+// extra as prompt tokens, standing in for a large tool-schema floor.
+type floorUsageLLM struct {
+ mu sync.Mutex
+ extra int
+ asks int
+}
+
+func (f *floorUsageLLM) summary() string {
+ f.mu.Lock()
+ f.asks++
+ f.mu.Unlock()
+ return "summary of earlier turns"
+}
+
+func (f *floorUsageLLM) CreateChatCompletion(ctx context.Context, req openai.ChatCompletionRequest) (cogito.LLMReply, cogito.LLMUsage, error) {
+ if n := len(req.Messages); n > 0 && strings.HasPrefix(req.Messages[n-1].Content, compactInstruction) {
+ return replyWith(f.summary()), cogito.LLMUsage{PromptTokens: 20, TotalTokens: 22}, nil
+ }
+ p := estimateTokens(req.Messages) + f.extra
+ return replyWith("ok"), cogito.LLMUsage{PromptTokens: p, CompletionTokens: 1, TotalTokens: p + 1}, nil
+}
+
+func (f *floorUsageLLM) Ask(ctx context.Context, fr cogito.Fragment) (cogito.Fragment, error) {
+ out := fr.AddMessage("assistant", f.summary())
+ if out.Status != nil {
+ out.Status.LastUsage = cogito.LLMUsage{PromptTokens: 20, TotalTokens: 22}
+ }
+ return out, nil
+}
+
+// End-of-turn auto-compaction is skipped when the floor alone is over the
+// budget, and runs otherwise.
+func TestOverflowFloorEndOfTurnCompaction(t *testing.T) {
+ run := func(t *testing.T, extra int, sys, history string) (*floorUsageLLM, *Session) {
+ llm := &floorUsageLLM{extra: extra}
+ s := newOverflowSession(t, llm)
+ if sys != "" {
+ s.systemPrompt = sys
+ }
+ // Budget 40000 - 4096 = 35904; trigger 28723.
+ s.compaction.MaxContextTokens = 40000
+ s.fragment = cogito.NewFragment(
+ openai.ChatCompletionMessage{Role: "user", Content: "u1"},
+ openai.ChatCompletionMessage{Role: "assistant", Content: "a1 " + history},
+ openai.ChatCompletionMessage{Role: "user", Content: "u2"},
+ openai.ChatCompletionMessage{Role: "assistant", Content: "a2 " + history},
+ )
+ if _, err := s.SendMessage("next"); err != nil {
+ t.Fatalf("SendMessage: %v", err)
+ }
+ return llm, s
+ }
+
+ t.Run("floor over budget", func(t *testing.T) {
+ // A system prompt of 37000 tokens: the floor is the fixed overhead
+ // itself, not a skew the backend reports on top of the history.
+ llm, s := run(t, 0, strings.Repeat("s", 37000*4), "short")
+ if llm.asks != 0 {
+ t.Fatalf("auto-compaction summarized %d times with the floor (%d) over the budget", llm.asks, s.SchemaBudget().Floor)
+ }
+ if !fragmentHas(s, func(m openai.ChatCompletionMessage) bool { return m.Content == "u1" }) {
+ t.Fatal("the history was compacted away")
+ }
+ })
+ t.Run("floor under budget", func(t *testing.T) {
+ llm, _ := run(t, 5000, "", strings.Repeat("earlier detail ", 3500)) // ~13k tokens each
+ if llm.asks == 0 {
+ t.Fatal("auto-compaction did not run")
+ }
+ })
+}
+
+// Mid-turn compaction is skipped when the floor alone is over the budget, and
+// runs otherwise.
+func TestOverflowFloorMidTurnCompaction(t *testing.T) {
+ msgs := func() []openai.ChatCompletionMessage {
+ long := strings.Repeat("x", 400) // 100 tokens
+ return []openai.ChatCompletionMessage{
+ {Role: "user", Content: "u1 " + long}, {Role: "assistant", Content: "a1 " + long},
+ {Role: "user", Content: "u2 " + long}, {Role: "assistant", Content: "a2 " + long},
+ {Role: "user", Content: "u3 " + long}, {Role: "assistant", Content: "a3 " + long},
+ }
+ }
+ for _, tc := range []struct {
+ name string
+ floor int
+ compact bool
+ }{
+ {"floor over budget", 600, false}, // window 1000, budget 500
+ {"floor under budget", 10, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ llm := &fakeSummaryLLM{reply: "S"}
+ s := newCompactTestSession(llm, 2, nil, nil)
+ s.live.recordFloor(tc.floor)
+ in := msgs()
+ out := s.newTurnCompactor(context.Background()).manipulate(in)
+ if got := llm.calls > 0; got != tc.compact {
+ t.Fatalf("summarized = %v, want %v", got, tc.compact)
+ }
+ if !tc.compact && len(out) != len(in) {
+ t.Fatalf("request changed: %d messages, want %d", len(out), len(in))
+ }
+ })
+ }
+}
+
+// With a large reserve (budget window/2) and a backend counting 1.5x, a long
+// history must not read as a schema floor: the history is the problem, so
+// end-of-turn auto-compaction runs. The old calibration took the whole
+// residual (schemas plus the skew on the history) as the floor and skipped it.
+func TestOverflowFloorSkewDoesNotBlockAutoCompaction(t *testing.T) {
+ s := newCompactTestSession(&fakeSummaryLLM{}, 2, nil, nil)
+ s.compaction = types.CompactionConfig{MaxContextTokens: 96000, ReserveTokens: 48000, Threshold: 0.8, KeepRecent: 2}
+ llm := trackUsage(&skewLLM{ratio: 1.5}, &s.live, s.requestLimits)
+ req, raw := skewRequest(5000, 90000)
+ _, usage, err := llm.CreateChatCompletion(context.Background(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !s.shouldCompactNow(usage.PromptTokens) {
+ t.Fatalf("%d prompt tokens did not cross the trigger", usage.PromptTokens)
+ }
+ if s.autoCompactBlocked(10) {
+ t.Fatalf("auto-compaction blocked by a floor of %d (schemas about %d x 1.5) with budget 48000",
+ s.SchemaBudget().Floor, raw)
+ }
+}
+
+// A tools-heavy request, schemas 80k of a 96k window and no skew, still
+// blocks: there the schemas are the problem, whatever is compacted.
+func TestOverflowFloorToolsHeavyBlocks(t *testing.T) {
+ s := newCompactTestSession(&fakeSummaryLLM{}, 2, nil, nil)
+ s.compaction = types.CompactionConfig{MaxContextTokens: 96000, ReserveTokens: 48000, Threshold: 0.8, KeepRecent: 2}
+ llm := trackUsage(&skewLLM{ratio: 1}, &s.live, s.requestLimits)
+ req, raw := skewRequest(80000, 1000)
+ if _, _, err := llm.CreateChatCompletion(context.Background(), req); err != nil {
+ t.Fatal(err)
+ }
+ sb, blocked := s.schemaFloorBlocks(ContextBudget(s.compactionConfig(), s.contextWindow()), 10)
+ if !blocked || !sb.Measured || sb.Floor != raw {
+ t.Fatalf("floor %d (measured %v, want %d) did not block", sb.Floor, sb.Measured, raw)
+ }
+ if !s.autoCompactBlocked(10) {
+ t.Fatal("auto-compaction not blocked by an 80k floor")
+ }
+}
diff --git a/chat/overflow_patterns.go b/chat/overflow_patterns.go
new file mode 100644
index 0000000..10fbe9e
--- /dev/null
+++ b/chat/overflow_patterns.go
@@ -0,0 +1,371 @@
+package chat
+
+import (
+ "errors"
+ "regexp"
+ "strconv"
+ "strings"
+ "sync/atomic"
+
+ "github.com/mudler/cogito"
+ "github.com/mudler/nib/types"
+ "github.com/mudler/xlog"
+ openai "github.com/sashabaranov/go-openai"
+)
+
+// OverflowKind says what a backend rejection means for recovery.
+type OverflowKind int
+
+const (
+ // KindNone: the error is not about size.
+ KindNone OverflowKind = iota
+ // KindContext: the prompt itself does not fit the window. Compact.
+ KindContext
+ // KindBudget: the prompt fits, but prompt + requested output does not.
+ // Lower max_tokens and retry. Never compact.
+ KindBudget
+ // KindOutputCap: the requested output alone is above the model's maximum.
+ // Lower the output cap. Do not compact.
+ KindOutputCap
+)
+
+// String returns the name fixtures and config use for the kind.
+func (k OverflowKind) String() string {
+ switch k {
+ case KindContext:
+ return "context"
+ case KindBudget:
+ return "budget"
+ case KindOutputCap:
+ return "output_cap"
+ default:
+ return "none"
+ }
+}
+
+func parseOverflowKind(s string) (OverflowKind, bool) {
+ switch s {
+ case "context":
+ return KindContext, true
+ case "budget":
+ return KindBudget, true
+ case "output_cap":
+ return KindOutputCap, true
+ }
+ return KindNone, false
+}
+
+// overflowInfo is what classifyOverflow read from an error. A zero figure
+// means the message did not state it.
+type overflowInfo struct {
+ Kind OverflowKind
+ Pattern string
+ Status int
+ Window int
+ Total int
+ Input int
+ Output int
+}
+
+// hasFigures reports whether the match stated any number at all.
+func (o overflowInfo) hasFigures() bool {
+ return o.Window > 0 || o.Total > 0 || o.Input > 0 || o.Output > 0
+}
+
+// overflowPattern is one row of the catalog. Figures come from the named
+// capture groups window, total, input and output, so a row states what each
+// number means instead of relying on the order the backend printed them.
+type overflowPattern struct {
+ name string
+ kind OverflowKind
+ re *regexp.Regexp
+ strong bool
+}
+
+// Evidence strength of a row, for the status gate (see classifyOverflowWith).
+// A strong row (a structured code or a specific provider wording) classifies
+// at any HTTP status; a weak row (a generic phrase) only with 400, 413 or an
+// unknown status.
+const (
+ strongEvidence = true
+ weakEvidence = false
+)
+
+// builtinOverflowPatterns are tried in order, most specific first; the first
+// match wins. Each row with figures is backed by a fixture in
+// testdata/overflow. The generic phrases at the end detect an overflow in a
+// wording no row names, but state no figures, so no window is learned from
+// them and the summary retry falls back to halving.
+var builtinOverflowPatterns = []overflowPattern{
+ // vLLM, newest (seen through regolo).
+ {"vllm_total", KindContext, regexp.MustCompile(`maximum context length of (?P\d+) tokens\. You requested a total of (?P\d+) tokens: (?P\d+) tokens from the input messages and (?P