fix(chat): stop context overflows from leaving sessions stuck

README.md

@@ -568,6 +568,51 @@ agents:
 # LOG_FORMAT=json switches to JSON lines.
 log_level: error
 
+# Optional: conversation compaction. When the context fills up, nib
+# summarizes the older turns and keeps the recent ones verbatim.
+# summary_max_tokens caps the output the summary request reserves (default
+# 16384; the model's own output cap wins when it is smaller). The summary
+# prompt is fitted into the window minus that reservation. If the backend
+# still rejects it as too large, nib summarizes fewer of the older turns and
+# keeps the rest verbatim, instead of cutting the text it summarizes.
+#
+# nib reads a backend's "request too large" error to decide what to do:
+# compact when the prompt itself does not fit (kind "context"), lower the
+# requested output when only prompt + output does not fit ("budget"), or
+# lower the output cap when the output alone is above the model's maximum
+# ("output_cap"), and then retry the turn once. When the summary alone does
+# not make the history fit, nib stubs old tool outputs, then drops older turns
+# (saved as an artifact:// the model can read), and as a last resort compacts
+# without the model. If the retry still does not fit, the turn fails, but the
+# compaction is kept, so the next turn does not overflow again. When the tool
+# schemas and system prompt alone fill the window, compaction cannot help:
+# nib keeps the conversation as it was, does not compact (neither on
+# overflow nor automatically), and names the MCP servers that take the most
+# room, so you can disable them or filter their tools. When a tool call is
+# cut off because the reply ran out of room in the window (for example a
+# whole file in one write), nib retries the turn once with a note that asks
+# the model to split the call, after compacting when the prompt is large.
+# When the reasoning filled the window, the retry also uses one lower
+# reasoning effort. nib never caps the output for this. A stream that ends
+# before the reply is finished is retried at most 3 times, and a tool call
+# with invalid JSON arguments once. It knows the
+# wording of llama.cpp/LocalAI, vLLM, OpenAI,
+# Anthropic and Gemini. overflow_patterns teaches it another backend without a
+# rebuild: the rows are tried before the built-in ones, first match wins. A
+# regex states its figures with the named groups window, total, input and
+# output. A row with an invalid regex or kind is logged and skipped. A
+# backend-specific wording (built-in or from this list) counts at any HTTP
+# status, because LocalAI reports an overflow as HTTP 500; a generic phrase
+# such as "context window" counts only with status 400, 413 or none. An error
+# with HTTP status 400 or 413 that no pattern recognises is logged at debug
+# level, so you can copy its text from the log.
+compaction:
+  summary_max_tokens: 16384
+  overflow_patterns:
+    - name: my-backend
+      kind: context
+      regex: 'prompt has (?P<input>\d+) tokens, limit is (?P<window>\d+)'
+
 # Optional: external MCP servers
 mcp_servers:
   filesystem:

chat/basic_compact.go

@@ -0,0 +1,154 @@
+package chat
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	openai "github.com/sashabaranov/go-openai"
+)
+
+// basicCompactMarker opens the user-role message that stands in for the head
+// basicCompact drops.
+const basicCompactMarker = "Conversation compacted (basic fallback mode). Previous tool outputs and assistant responses were elided to fit the context window."
+
+const (
+	basicAssistantChars = 200 // assistant text kept per line
+	basicArgsChars      = 80  // tool arguments kept per line when there is no path
+)
+
+// basicLine is one rendered line of the dropped head. User lines are the
+// user's own words and go last when the result must shrink.
+type basicLine struct {
+	text string
+	user bool
+}
+
+// basicCompact compacts the fragment without an LLM call. It is the last
+// resort when the summarizer is unavailable or iterativeTrim ran out of steps.
+//
+// The whole head is saved as an artifact first. The head is then replaced by
+// one user-role message: the user messages verbatim, one line per assistant
+// text (cut to 200 characters) and one stub line per tool call. The results
+// are rendered as text, never as tool messages, so no tool result is left
+// without its call. The tail, split by splitForCompaction, keeps its pairing.
+//
+// When the result does not fit, activity lines go first, oldest first, and
+// then user lines, oldest first; they all stay in the artifact. When even the
+// tail does not fit, the fragment is left alone and the validation error is
+// returned.
+func (s *Session) basicCompact(ctx context.Context) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	cfg := s.compactionConfig()
+	budget := s.trimBudget(cfg)
+	state := s.fragmentSnapshot()
+	msgs := state.frag.Messages
+
+	head, tail := splitForCompaction(msgs, max(cfg.KeepRecent, 1))
+	if len(head) == 0 {
+		return fmt.Errorf("%w: nothing to compact", ErrCompactionNoOp)
+	}
+
+	marker := basicCompactMarker
+	if uri := s.spillCompactionHead(renderMessages(head)); uri != "" {
+		marker += " Full history available at " + uri + "."
+	}
+	marker += " User messages are preserved verbatim."
+
+	var system []openai.ChatCompletionMessage
+	for _, m := range head {
+		if m.Role == "system" {
+			system = append(system, m)
+		}
+	}
+	lines := basicRender(head)
+
+	build := func() []openai.ChatCompletionMessage {
+		var b strings.Builder
+		b.WriteString(marker)
+		for _, l := range lines {
+			b.WriteString("\n")
+			b.WriteString(l.text)
+		}
+		out := append([]openai.ChatCompletionMessage(nil), system...)
+		out = append(out, openai.ChatCompletionMessage{Role: "user", Content: b.String()})
+		return append(out, tail...)
+	}
+
+	candidate := build()
+	for _, dropUser := range []bool{false, true} {
+		for estimateTokens(candidate) > budget {
+			i := oldestLine(lines, dropUser)
+			if i < 0 {
+				break
+			}
+			lines = append(lines[:i], lines[i+1:]...)
+			candidate = build()
+		}
+	}
+	if err := validateCompaction(msgs, candidate, budget); err != nil {
+		return fmt.Errorf("basic compact: %w", err)
+	}
+	s.installFragment(candidate, compactedDisplay(state.messages, tail))
+	return nil
+}
+
+// oldestLine is the index of the first line of the kind asked for, or -1.
+func oldestLine(lines []basicLine, user bool) int {
+	for i, l := range lines {
+		if l.user == user {
+			return i
+		}
+	}
+	return -1
+}
+
+// basicRender renders the head as text lines: user messages verbatim, the
+// first 200 characters of assistant text, and one stub per tool call. System
+// messages are kept as messages by the caller, and tool results are covered by
+// their call's stub.
+func basicRender(head []openai.ChatCompletionMessage) []basicLine {
+	var lines []basicLine
+	for _, m := range head {
+		switch m.Role {
+		case "user":
+			if m.Content != "" {
+				lines = append(lines, basicLine{text: "User: " + m.Content, user: true})
+			}
+		case "assistant":
+			if text := strings.TrimSpace(m.Content); text != "" {
+				lines = append(lines, basicLine{text: "Assistant: " + cutRunes(text, basicAssistantChars)})
+			}
+			for _, tc := range m.ToolCalls {
+				lines = append(lines, basicLine{text: fmt.Sprintf("[tool %s(%s) ran, result elided]", tc.Function.Name, basicToolTarget(tc.Function.Arguments))})
+			}
+		}
+	}
+	return lines
+}
+
+// basicToolTarget is what a tool stub names: the path argument when there is
+// one, or else the first 80 characters of the arguments.
+func basicToolTarget(args string) string {
+	var parsed map[string]any
+	if json.Unmarshal([]byte(args), &parsed) == nil {
+		for _, key := range []string{"path", "file_path"} {
+			if p, ok := parsed[key].(string); ok && p != "" {
+				return p
+			}
+		}
+	}
+	return cutRunes(args, basicArgsChars)
+}
+
+// cutRunes returns the first n characters of s.
+func cutRunes(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	return string(r[:n])
+}

chat/basic_compact_test.go

@@ -0,0 +1,184 @@
+package chat
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/mudler/nib/mcp"
+	openai "github.com/sashabaranov/go-openai"
+)
+
+// The test session has a 1000-token window, so the budget is 500 byte/4
+// tokens (2000 bytes).
+
+func basicToolCall(id, name, args string) openai.ToolCall {
+	return openai.ToolCall{ID: id, Type: openai.ToolTypeFunction, Function: openai.FunctionCall{Name: name, Arguments: args}}
+}
+
+func TestBasicCompactRendersHeadAndKeepsTail(t *testing.T) {
+	bigResult := "RESULT-BODY" + strings.Repeat("r", 3000)
+	longAssistant := "a1" + strings.Repeat("z", 400)
+	frag := []openai.ChatCompletionMessage{
+		{Role: "system", Content: "sys"},
+		{Role: "user", Content: "u1 please do X\n  exactly like this"},
+		{Role: "assistant", ToolCalls: []openai.ToolCall{basicToolCall("c1", "read", `{"path":"/a/b.go"}`)}},
+		{Role: "tool", ToolCallID: "c1", Content: bigResult},
+		{Role: "assistant", Content: longAssistant},
+		{Role: "user", Content: "u2 now run ls"},
+		{Role: "assistant", ToolCalls: []openai.ToolCall{basicToolCall("c2", "bash", `{"command":"ls"}`)}},
+		{Role: "tool", ToolCallID: "c2", Content: "ok"},
+		{Role: "assistant", Content: "done"},
+	}
+	llm := &fakeSummaryLLM{reply: "SUMMARY-TEXT"}
+	s := newCompactTestSession(llm, 3, frag, frag)
+	s.artifacts = mcp.NewArtifactStore()
+
+	if err := s.basicCompact(context.Background()); err != nil {
+		t.Fatalf("basicCompact: %v", err)
+	}
+	if llm.calls != 0 {
+		t.Fatalf("basicCompact must not call the LLM, got %d calls", llm.calls)
+	}
+	got := trimFragment(s)
+	if len(got) != 5 {
+		t.Fatalf("want [system, marker, 3 tail], got %d: %+v", len(got), got)
+	}
+	if got[0].Role != "system" || got[0].Content != "sys" {
+		t.Fatalf("head system message must be kept first, got %s %q", got[0].Role, got[0].Content)
+	}
+	body := got[1].Content
+	if got[1].Role != "user" || !strings.Contains(body, "basic fallback mode") || !strings.Contains(body, "artifact://1") {
+		t.Fatalf("want a user-role marker naming artifact://1, got %s %q", got[1].Role, body)
+	}
+	for _, want := range []string{
+		"u1 please do X\n  exactly like this",
+		"u2 now run ls",
+		"[tool read(/a/b.go) ran, result elided]",
+		"Assistant: " + longAssistant[:200],
+	} {
+		if !strings.Contains(body, want) {
+			t.Fatalf("marker missing %q:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, longAssistant[:201]) || strings.Contains(body, "RESULT-BODY") {
+		t.Fatalf("assistant text must be cut to 200 chars and tool output elided:\n%s", body)
+	}
+	for i, m := range got[2:] {
+		orig := frag[6+i]
+		if m.Role != orig.Role || m.Content != orig.Content || m.ToolCallID != orig.ToolCallID || len(m.ToolCalls) != len(orig.ToolCalls) {
+			t.Fatalf("tail message %d not kept verbatim: %+v", i, m)
+		}
+	}
+	for _, m := range got {
+		if m.Role == "tool" && m.ToolCallID == "c1" {
+			t.Fatalf("head tool result must not survive as a tool message")
+		}
+	}
+	assertPaired(t, got)
+	if estimateTokens(got) > s.trimBudget(s.compactionConfig()) {
+		t.Fatalf("result over budget: %d", estimateTokens(got))
+	}
+	art := s.artifacts.Get(1)
+	if art == nil || !strings.Contains(art.Content, "RESULT-BODY") || !strings.Contains(art.Content, "u1 please do X") {
+		t.Fatalf("artifact must hold the full untruncated head, got %+v", art)
+	}
+	if len(s.messages) == 0 || s.messages[0].Role != "assistant" || s.messages[len(s.messages)-1].Content != "done" {
+		t.Fatalf("display copy not rebuilt: %+v", s.messages)
+	}
+}
+
+func TestBasicCompactDropsActivityLinesWhenTight(t *testing.T) {
+	var frag []openai.ChatCompletionMessage
+	for i := range 10 {
+		frag = append(frag,
+			openai.ChatCompletionMessage{Role: "user", Content: fmt.Sprintf("user-%d", i)},
+			openai.ChatCompletionMessage{Role: "assistant", Content: fmt.Sprintf("asst-%d-", i) + strings.Repeat("q", 250)},
+		)
+	}
+	frag = append(frag,
+		openai.ChatCompletionMessage{Role: "user", Content: "last-user" + strings.Repeat("t", 400)},
+		openai.ChatCompletionMessage{Role: "assistant", Content: "last-asst" + strings.Repeat("t", 400)},
+	)
+	s := newCompactTestSession(&fakeSummaryLLM{}, 2, frag, frag)
+	s.artifacts = mcp.NewArtifactStore()
+
+	if err := s.basicCompact(context.Background()); err != nil {
+		t.Fatalf("basicCompact: %v", err)
+	}
+	got := trimFragment(s)
+	if estimateTokens(got) > s.trimBudget(s.compactionConfig()) {
+		t.Fatalf("result over budget: %d", estimateTokens(got))
+	}
+	body := got[0].Content
+	for i := range 10 {
+		if !strings.Contains(body, fmt.Sprintf("user-%d\n", i)) {
+			t.Fatalf("user-%d must be preserved while activity lines can go:\n%s", i, body)
+		}
+	}
+	if strings.Contains(body, "asst-0-") {
+		t.Fatalf("the oldest activity line must be dropped first:\n%s", body)
+	}
+	if !strings.Contains(body, "asst-9-") {
+		t.Fatalf("the newest activity line should survive:\n%s", body)
+	}
+	if got[len(got)-1].Content != frag[len(frag)-1].Content {
+		t.Fatalf("tail must be kept verbatim")
+	}
+}
+
+func TestBasicCompactDropsOldUsersAsLastResort(t *testing.T) {
+	var frag []openai.ChatCompletionMessage
+	for i := range 8 {
+		frag = append(frag,
+			openai.ChatCompletionMessage{Role: "user", Content: fmt.Sprintf("user-%d-", i) + strings.Repeat("w", 400)},
+			openai.ChatCompletionMessage{Role: "assistant", Content: fmt.Sprintf("asst-%d", i)},
+		)
+	}
+	frag = append(frag,
+		openai.ChatCompletionMessage{Role: "user", Content: "last-user"},
+		openai.ChatCompletionMessage{Role: "assistant", Content: "last-asst"},
+	)
+	s := newCompactTestSession(&fakeSummaryLLM{}, 2, frag, frag)
+	s.artifacts = mcp.NewArtifactStore()
+
+	if err := s.basicCompact(context.Background()); err != nil {
+		t.Fatalf("basicCompact: %v", err)
+	}
+	got := trimFragment(s)
+	if estimateTokens(got) > s.trimBudget(s.compactionConfig()) {
+		t.Fatalf("result over budget: %d", estimateTokens(got))
+	}
+	body := got[0].Content
+	if strings.Contains(body, "Assistant: ") {
+		t.Fatalf("activity lines must all go before any user line:\n%s", body)
+	}
+	if strings.Contains(body, "user-0-") {
+		t.Fatalf("the oldest user line must be dropped when nothing else fits:\n%s", body)
+	}
+	if !strings.Contains(body, "user-7-"+strings.Repeat("w", 400)) {
+		t.Fatalf("the newest user line must survive verbatim:\n%s", body)
+	}
+	if art := s.artifacts.Get(1); art == nil || !strings.Contains(art.Content, "user-0-") {
+		t.Fatalf("dropped user lines must stay in the artifact")
+	}
+}
+
+func TestBasicCompactFailsWhenTailAloneOverBudget(t *testing.T) {
+	frag := []openai.ChatCompletionMessage{
+		{Role: "user", Content: "u1" + strings.Repeat("x", 800)},
+		{Role: "assistant", Content: "a1"},
+		{Role: "user", Content: strings.Repeat("y", 4000)},
+	}
+	s := newCompactTestSession(&fakeSummaryLLM{}, 1, frag, frag)
+
+	err := s.basicCompact(context.Background())
+	if !errors.Is(err, ErrCompactionOverBudget) {
+		t.Fatalf("want ErrCompactionOverBudget, got %v", err)
+	}
+	if got := trimFragment(s); len(got) != len(frag) {
+		t.Fatalf("a failed basic compact must leave the fragment alone, got %d messages", len(got))
+	}
+}