From 211d5e9d2b8d7e475c159f0ce7cf16417135fefc Mon Sep 17 00:00:00 2001 From: Tom Daniel <0xtomdaniel@gmail.com> Date: Thu, 15 Oct 2026 20:57:21 -0600 Subject: [PATCH] Mark the request's end on its last block, so a 1h block TTL does not clash with a top-level 5m mark --- src/cache.ts | 20 ++++++++++++++++---- test/memory.test.ts | 28 ++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/cache.ts b/src/cache.ts index f42bfd21..8c0da2d2 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -30,8 +30,10 @@ function lastBlock(message: Record) { return record(block) ? block : undefined; } -/** Anthropic: marks on the view (see MARKS) plus automatic end-of-request caching, so the next call finds the last one's entry - * and pays only for the lines after it. */ +/** Anthropic: marks on the view (see MARKS) plus a mark at the end of the request, so the next call finds the last one's entry + * and pays only for the lines after it. The end mark sits on the last block itself rather than as top-level automatic caching: + * a proxy or setting that gives block marks a 1h TTL makes Anthropic refuse a top-level mark, whose TTL defaults to 5m, on the + * same block ("Top-level cache_control has ttl='5m' but the target block already has cache_control with ttl='1h'"). */ export function cachePayload(payload: unknown): unknown { if (!record(payload) || !Array.isArray(payload.messages)) return payload; const messages = payload.messages; @@ -60,7 +62,17 @@ export function cachePayload(payload: unknown): unknown { // Pi may end the request with an empty system message carrying settings (effort); it holds nothing to cache. const [before, last] = messages.filter(message => !record(message) || message.role !== 'system').slice(-2); const turn = isAtWork(last) && record(before) && before.role === 'user' ? lastBlock(before) : undefined; - if (turn) turn.cache_control = { type: 'ephemeral' }; - else payload.cache_control = { type: 'ephemeral' }; + const end = turn ?? endBlock(messages); + if (end) end.cache_control = { type: 'ephemeral' }; return payload; } + +/** The last block of the last non-system message that can carry a mark (thinking blocks can't). */ +function endBlock(messages: unknown[]) { + const message = messages.filter(m => !record(m) || m.role !== 'system').at(-1); + if (!record(message)) return undefined; + if (typeof message.content === 'string') return lastBlock(message); + if (!Array.isArray(message.content)) return undefined; + const block: unknown = message.content.findLast((b: unknown) => record(b) && b.type !== 'thinking' && b.type !== 'redacted_thinking'); + return record(block) ? block : undefined; +} diff --git a/test/memory.test.ts b/test/memory.test.ts index b2e39bf3..b932df3b 100644 --- a/test/memory.test.ts +++ b/test/memory.test.ts @@ -254,12 +254,36 @@ test('the view goes in blocks of 4 lines, with marks on its last whole block, 20 assert.equal((JSON.stringify(output).match(/cache_control/g) ?? []).length, 4, quoted ? 'a quoted closing tag keeps the marks' : 'plain view'); assert.ok(output.system.every(b => !('cache_control' in b)) && !('cache_control' in output.tools[0])); const blocks = output.messages[0].content; - assert.deepEqual(blocks.flatMap((b, j) => 'cache_control' in b ? [j] : []), [pieces.length - 42, pieces.length - 22, pieces.length - 2]); - assert.ok(output.cache_control); + assert.deepEqual(blocks.flatMap((b, j) => 'cache_control' in b ? [j] : []), [pieces.length - 42, pieces.length - 22, pieces.length - 2, blocks.length - 1]); + assert.equal(output.cache_control, undefined, 'no top-level automatic mark'); + assert.deepEqual(blocks.at(-1)!.cache_control, { type: 'ephemeral' }, 'the end mark sits on the last block'); assert.equal(blocks.map(b => b.text).join(''), view + 'new question'); } }); +test('a request with no status line marks its last block, not the top level, so a 1h block TTL set by a proxy does not clash', () => { + const view = '\n' + '0+1|summary of a decision\n'.repeat(200) + ''; + // As a compactor call ends: the view, then the instruction, then Pi's empty settings message. + const payload = { messages: [ + { role: 'user', content: [{ type: 'text', text: view }, { type: 'text', text: 'compress this' }] }, + { role: 'assistant', content: [{ type: 'thinking', thinking: 'hm' }, { type: 'text', text: 'line' }] }, + { role: 'user', content: 'again, shorter' }, + { role: 'system', content: [], output_config: { effort: 'low' } }, + ] }; + const output = cachePayload(payload) as { cache_control?: unknown, messages: { content: string | { type: string, cache_control?: unknown }[] }[] }; + assert.equal(output.cache_control, undefined); + assert.deepEqual(output.messages[2].content, [{ type: 'text', text: 'again, shorter', cache_control: { type: 'ephemeral' } }]); + // What a proxy that sets 1h on block marks sends on: every mark carries the same TTL, and nothing top-level defaults to 5m. + const sent = JSON.parse(JSON.stringify(output).replaceAll('{"type":"ephemeral"}', '{"type":"ephemeral","ttl":"1h"}')); + assert.ok(!('cache_control' in sent)); + assert.ok((JSON.stringify(sent).match(/cache_control/g) ?? []).length <= 4); + const thinkingLast = cachePayload({ messages: [ + { role: 'user', content: [{ type: 'text', text: view }] }, + { role: 'assistant', content: [{ type: 'text', text: 'ok' }, { type: 'thinking', thinking: 'later' }] }, + ] }) as { messages: { content: { type: string, cache_control?: unknown }[] }[] }; + assert.deepEqual(thinkingLast.messages[1].content.map(b => 'cache_control' in b), [true, false], 'thinking blocks are skipped'); +}); + test('after a turn adds up to 60 blocks of lines, a mark of the next call still finds one of the last call\'s within 20 blocks', () => { const line = '0+1|summary of a decision\n', marks = (lines: number) => { const payload = { messages: [{ role: 'user', content: [{ type: 'text', text: '\n' + line.repeat(lines) + '' }] }] };