diff --git a/src/serve/openai_common.cpp b/src/serve/openai_common.cpp index 047be69d5b..f553f474fd 100644 --- a/src/serve/openai_common.cpp +++ b/src/serve/openai_common.cpp @@ -144,8 +144,25 @@ void apply_openai_prompt_cache_policy(GenerationRequest& request, OpenAIPromptCa const bool automatic_enabled = policy.automatic != OpenAIPromptCacheAutomatic::Disabled && automatic_target != nullptr; const bool automatic_merges_explicit = automatic_enabled && automatic_target->has_value(); - const std::size_t explicit_write_slots = + // Issue #142: the leading-instruction automatic candidate is a distinct + // marker when the leading run is unmarked, so it consumes one of the four + // frontend marker slots (frontend.cpp rejects more than four). + std::size_t leading_instruction = static_cast(-1); + if (policy.auto_system_shared_prefix) { + for (std::size_t index = 0; index < request.messages.size(); ++index) { + const ChatRole role = request.messages[index].role; + if (role != ChatRole::System && role != ChatRole::Developer) { break; } + leading_instruction = index; + } + } + const bool system_candidate = + policy.auto_system_shared_prefix && automatic_enabled && + leading_instruction != static_cast(-1) && + !request.messages[leading_instruction].cache_boundary_after.has_value(); + const std::size_t base_slots = !automatic_enabled || automatic_merges_explicit ? 4U : 3U; + const std::size_t explicit_write_slots = + base_slots - (system_candidate ? 1U : 0U); const std::size_t first_selected = explicit_boundaries.size() > explicit_write_slots ? explicit_boundaries.size() - explicit_write_slots : 0U; @@ -170,6 +187,18 @@ void apply_openai_prompt_cache_policy(GenerationRequest& request, OpenAIPromptCa } else { *automatic_target = CacheBoundary{.evidence = evidence}; } + // Issue #142: a second automatic candidate at the leading + // system/developer frontier (agent siblings share the long head). + // Mark the end of the contiguous leading instruction run so requests + // sharing both system and developer turns reuse the whole prefix. + if (policy.auto_system_shared_prefix && + leading_instruction != static_cast(-1)) { + ChatTurn& turn = request.messages[leading_instruction]; + if (!turn.cache_boundary_after) { + turn.cache_boundary_after = CacheBoundary{ + .evidence = ninfer::SharedCandidateEvidence::DefaultAutomatic}; + } + } } // OpenAI already defines the automatic/explicit write policy for every request. Existing // exact shared residents are still considered by the Engine independently of this switch. diff --git a/src/serve/openai_common.h b/src/serve/openai_common.h index 3531d3cb45..62c778bd48 100644 --- a/src/serve/openai_common.h +++ b/src/serve/openai_common.h @@ -20,6 +20,11 @@ enum class OpenAIPromptCacheAutomatic : std::uint8_t { struct OpenAIPromptCachePolicy { OpenAIPromptCacheAutomatic automatic = OpenAIPromptCacheAutomatic::Default; + // Issue #142: additionally publish a shared-prefix candidate at the + // leading system/developer frontier so agent sibling sessions share a + // long head even without a client marker. The last-content implicit + // candidate stays untouched. + bool auto_system_shared_prefix = true; }; [[nodiscard]] bool parse_openai_prompt_cache_breakpoint(const nlohmann::json& value, diff --git a/src/serve/openai_responses.h b/src/serve/openai_responses.h index e752612dc1..f32812ca8f 100644 --- a/src/serve/openai_responses.h +++ b/src/serve/openai_responses.h @@ -34,6 +34,10 @@ struct OpenAIResponsesPromptRequest { std::vector input_items; std::optional instructions; std::optional previous_response_id; + // Applied after prompt resolution assembles generation.messages (the + // policy needs the resolved system/developer turns to place the + // leading-instruction candidate). + std::optional cache_policy; }; struct OpenAIResponsesCreateRequest { @@ -72,7 +76,9 @@ struct BuiltOpenAIResponse { }; OpenAIResponsesCreateRequest parse_openai_responses_create_request(const nlohmann::json& body, - const RequestLimits& limits); + const RequestLimits& limits, + bool auto_system_shared_prefix = + true); OpenAIResponsesPromptRequest parse_openai_responses_input_tokens_request(const nlohmann::json& body, diff --git a/src/serve/openai_responses_http.cpp b/src/serve/openai_responses_http.cpp index 8351731025..1a0a16bd74 100644 --- a/src/serve/openai_responses_http.cpp +++ b/src/serve/openai_responses_http.cpp @@ -245,7 +245,8 @@ void HttpServer::handle_responses(const httplib::Request& req, httplib::Response try { RequestLimits limits; limits.default_max_tokens = options_.default_max_tokens; - request = parse_openai_responses_create_request(parse_json_body(req), limits); + request = parse_openai_responses_create_request( + parse_json_body(req), limits, options_.auto_system_shared_prefix); validate_openai_model(request.prompt.model, public_model_id_); resolved = resolve_openai_responses_prompt(request.prompt, openai_responses_store_, id, request.store); diff --git a/src/serve/openai_responses_request.cpp b/src/serve/openai_responses_request.cpp index f460ed921a..7c682779b2 100644 --- a/src/serve/openai_responses_request.cpp +++ b/src/serve/openai_responses_request.cpp @@ -1158,16 +1158,21 @@ void validate_common_top_level(const Json& body, bool create) { } // namespace OpenAIResponsesCreateRequest parse_openai_responses_create_request(const Json& body, - const RequestLimits& limits) { + const RequestLimits& limits, + bool auto_system_shared_prefix) { require_object(body); validate_common_top_level(body, true); reject_unsupported_platform_fields(body); - const OpenAIPromptCachePolicy cache_policy = parse_openai_prompt_cache_policy(body); + OpenAIPromptCachePolicy cache_policy = parse_openai_prompt_cache_policy(body); + cache_policy.auto_system_shared_prefix = auto_system_shared_prefix; ParsedPromptFields parsed = parse_prompt_fields(body, limits); - apply_openai_prompt_cache_policy(parsed.prompt.generation, cache_policy); OpenAIResponsesCreateRequest out; out.prompt = std::move(parsed.prompt); + // The prompt policy is applied after resolution assembles + // generation.messages (resolve_openai_responses_prompt), because at parse + // time the leading instructions/input turns are not yet in messages. + out.prompt.cache_policy = cache_policy; out.tools = std::move(parsed.wire_tools); out.tool_choice = std::move(parsed.wire_tool_choice); out.tool_identities = std::move(parsed.tool_identities); diff --git a/src/serve/openai_responses_state.cpp b/src/serve/openai_responses_state.cpp index f6ec66a4b4..0cedb745e5 100644 --- a/src/serve/openai_responses_state.cpp +++ b/src/serve/openai_responses_state.cpp @@ -1,3 +1,4 @@ +#include "serve/openai_common.h" #include "serve/openai_responses.h" #include "serve/request_validation.h" @@ -179,6 +180,12 @@ resolve_openai_responses_prompt(const OpenAIResponsesPromptRequest& request, std::make_move_iterator(context.begin()), std::make_move_iterator(context.end())); + // The prompt cache policy needs the resolved leading instructions/input + // turns, so it is applied here rather than at parse time. + if (request.cache_policy) { + apply_openai_prompt_cache_policy(resolved.generation, *request.cache_policy); + } + if (response_id) { if (parent_record) { resolved.session_key = parent_record->session_key; diff --git a/src/serve/serve_options.cpp b/src/serve/serve_options.cpp index 66fef22197..ace5271885 100644 --- a/src/serve/serve_options.cpp +++ b/src/serve/serve_options.cpp @@ -79,7 +79,7 @@ std::string serve_usage_text(const char* argv0) { "[--kv-dtype bf16|int8|fp8] [--spec mtp|dflash --draft-tokens N] " "[--default-max-tokens N] [--default-thinking-budget N] " "[--vision] [--no-cuda-graph] [--no-prefix-reuse] " - "[--lm-head-draft] [--no-thinking] [--preserve-thinking] [--cors] " + "[--lm-head-draft] [--no-thinking] [--preserve-thinking] [--no-auto-system-shared-prefix] [--cors] " "[--temperature F] [--top-p F] [--top-k N] [--min-p F] [--presence-penalty F] " "[--frequency-penalty F] [--seed N] [--greedy]\n" " serves OpenAI Responses/Chat Completions and Anthropic Messages endpoints\n" @@ -281,6 +281,8 @@ ServeOptions parse_serve_options(int argc, char** argv) { options.use_cuda_graph = false; } else if (arg == "--no-prefix-reuse") { options.allow_prefix_reuse = false; + } else if (arg == "--no-auto-system-shared-prefix") { + options.auto_system_shared_prefix = false; } else if (arg == "--lm-head-draft") { options.speculative.proposal_head = ProposalHead::Optimized; } else if (arg == "--no-thinking") { diff --git a/src/serve/serve_options.h b/src/serve/serve_options.h index c529fbc84f..628d871860 100644 --- a/src/serve/serve_options.h +++ b/src/serve/serve_options.h @@ -47,6 +47,9 @@ struct ServeOptions { bool enable_vision = false; bool use_cuda_graph = true; bool allow_prefix_reuse = true; + // Issue #142: publish a shared-prefix candidate at the leading + // system/developer frontier (default on for agent workloads). + bool auto_system_shared_prefix = true; bool enable_thinking = true; // default thinking mode for the generation prompt (--no-thinking opts out) bool preserve_thinking = false;