From 7308a070292e08e528b5355a17e090b0e2833c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9ctor=20Ram=C3=B3n=20Jim=C3=A9nez?= Date: Tue, 1 Sep 2026 16:39:53 +0200 Subject: [PATCH] feat(serve): expose llama.cpp-compatible model metadata on /v1/models --- docs/serving.md | 24 +++++++++- include/ninfer/engine.h | 1 + include/ninfer/types.h | 15 +++++++ src/runtime/engine/engine.cpp | 7 +++ src/serve/generation_service.h | 4 ++ src/serve/http_server.cpp | 7 ++- src/serve/http_server.h | 1 + src/serve/openai_common.cpp | 45 +++++++++++++------ src/serve/openai_common.h | 4 +- .../ninfer/targets/qwen3_6_27b/package.h | 3 ++ src/targets/qwen3_6_27b/impl/package.cpp | 11 +++++ .../ninfer/targets/qwen3_6_35b_a3b/package.h | 3 ++ src/targets/qwen3_6_35b_a3b/impl/package.cpp | 10 +++++ src/targets/registry.cpp | 27 +++++++++++ src/targets/registry.h | 1 + tests/test_openai_schema.cpp | 23 +++++++++- 16 files changed, 164 insertions(+), 22 deletions(-) diff --git a/docs/serving.md b/docs/serving.md index 5fb0bde843..0ecfc206bb 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -54,8 +54,8 @@ omitted at startup. | Method and path | Behavior | |---|---| | `GET /health` | Engine readiness | -| `GET /v1/models` | configured OpenAI model alias and effective `max_model_len` | -| `GET /v1/models/{id}` | lookup of the configured alias and effective `max_model_len` | +| `GET /v1/models` | configured OpenAI model alias, effective `max_model_len`, and a llama.cpp-compatible `meta` object | +| `GET /v1/models/{id}` | lookup of the configured alias, `max_model_len`, and `meta` object | | `POST /v1/chat/completions` | OpenAI-style chat generation | | `POST /v1/responses` | OpenAI Responses Core generation, state, typed Items, and SSE | | `POST /v1/responses/input_tokens` | Responses prompt-token count without generation | @@ -81,6 +81,26 @@ request is waiting or prefilling. A peer whose TCP stack remains connected and a cannot be distinguished from a reading application; proxies must close their upstream NInfer connection when the downstream client disappears. +## Models + +`GET /v1/models` and `GET /v1/models/{id}` return the configured public OpenAI model alias +(defaults to the artifact `identity.model_id`, overridable with `--model-id`) together with the +effective `max_model_len` (the `--max-context` ceiling) and a `meta` object in the shape exposed by +`llama.cpp`. The `meta` facts describe the registered artifact behind the alias: + +| Field | Meaning | +|---|---| +| `n_vocab` | tokenizer token domain | +| `n_ctx` | configured per-request context ceiling (equal to `max_model_len`) | +| `n_ctx_train` | model native/training context | +| `n_embd` | model embedding width | +| `n_params` | total logical weight elements across the registered artifact tensors | +| `size` | encoded weight payload bytes of the registered artifact | +| `ftype` | registered weights profile (the NInfer quantization name) | + +`GET /v1/models/{id}` returns the same object for the single configured alias and a `404` for any +other id. + ## OpenAI Chat Completions ```bash diff --git a/include/ninfer/engine.h b/include/ninfer/engine.h index b33ae47b76..4b0aa774cb 100644 --- a/include/ninfer/engine.h +++ b/include/ninfer/engine.h @@ -104,6 +104,7 @@ class Engine { [[nodiscard]] const EngineOptions& options() const; [[nodiscard]] LoadSummary load_summary() const; + [[nodiscard]] ModelMetadata model_metadata() const; [[nodiscard]] MemorySummary memory_summary() const; [[nodiscard]] RuntimeStats runtime_stats() const; [[nodiscard]] MediaCacheSummary media_cache_summary() const; diff --git a/include/ninfer/types.h b/include/ninfer/types.h index d9c8c93c6d..58f38f2f3c 100644 --- a/include/ninfer/types.h +++ b/include/ninfer/types.h @@ -948,6 +948,21 @@ struct ContextCostSummary { std::filesystem::path preset_path; }; +// Static facts about the registered model, independent of the current request context and memory +// layout. The target package owns the dimension facts (vocab_size, embedding_size, +// native_context); the Engine completes the registered identity (model_id, weights_id) and the +// artifact-measured facts (parameters, weight_bytes). Serving renders these into the +// OpenAI-compatible /v1/models model object and its llama.cpp-compatible `meta` field. +struct ModelMetadata { + std::string model_id; // Registered identity (artifact identity.model_id). + std::string weights_id; // Registered weights profile (artifact identity.weights_id). + std::uint64_t vocab_size = 0; // Tokenizer token domain (meta n_vocab). + std::uint64_t embedding_size = 0; // Model embedding width (meta n_embd). + std::uint64_t native_context = 0; // Model native/training context (meta n_ctx_train). + std::uint64_t parameters = 0; // Total logical weight elements (meta n_params). + std::uint64_t weight_bytes = 0; // Encoded weight payload bytes (meta size). +}; + struct LoadSummary { std::string target; std::string model_id; diff --git a/src/runtime/engine/engine.cpp b/src/runtime/engine/engine.cpp index ee15cf8ab8..1c1c359ec9 100644 --- a/src/runtime/engine/engine.cpp +++ b/src/runtime/engine/engine.cpp @@ -230,6 +230,7 @@ class Engine::Impl { auto constructed = targets::construct_target(options, device); active = std::move(constructed.active); load = std::move(constructed.load); + model_metadata = std::move(constructed.model_metadata); sampling_defaults = constructed.sampling_defaults; StartupPhaseScope finalize_phase(options.startup_observer, StartupPhase::EngineFinalize); core = std::visit( @@ -266,6 +267,7 @@ class Engine::Impl { DeviceContext device; targets::ActiveTarget active; LoadSummary load; + ModelMetadata model_metadata; ModelSamplingDefaults sampling_defaults; Core core; }; @@ -482,6 +484,11 @@ LoadSummary Engine::load_summary() const { return impl_->load; } +ModelMetadata Engine::model_metadata() const { + if (impl_ == nullptr) { throw std::logic_error("Engine is moved from"); } + return impl_->model_metadata; +} + MemorySummary Engine::memory_summary() const { if (impl_ == nullptr) { throw std::logic_error("Engine is moved from"); } return std::visit( diff --git a/src/serve/generation_service.h b/src/serve/generation_service.h index 8540d6ddc0..43e072a319 100644 --- a/src/serve/generation_service.h +++ b/src/serve/generation_service.h @@ -104,6 +104,10 @@ class GenerationService { [[nodiscard]] ninfer::LoadSummary load_summary() const { return engine_->load_summary(); } + [[nodiscard]] ninfer::ModelMetadata model_metadata() const { + return engine_->model_metadata(); + } + [[nodiscard]] ninfer::MemorySummary memory_summary() const { return engine_->memory_summary(); } [[nodiscard]] ninfer::RuntimeStats runtime_stats() const { return engine_->runtime_stats(); } diff --git a/src/serve/http_server.cpp b/src/serve/http_server.cpp index e1c12dcd32..a089daff40 100644 --- a/src/serve/http_server.cpp +++ b/src/serve/http_server.cpp @@ -478,7 +478,8 @@ void HttpServer::register_routes() { } void HttpServer::handle_models(const httplib::Request&, httplib::Response& res) const { - res.set_content(make_models_list(public_model_id_, unix_time_now(), options_.max_context), + res.set_content(make_models_list(public_model_id_, unix_time_now(), options_.max_context, + model_metadata_), "application/json"); } @@ -493,7 +494,8 @@ void HttpServer::handle_model(const httplib::Request& req, httplib::Response& re write_openai_error(res, error); return; } - res.set_content(make_model_object(public_model_id_, unix_time_now(), options_.max_context), + res.set_content(make_model_object(public_model_id_, unix_time_now(), options_.max_context, + model_metadata_), "application/json"); } @@ -505,6 +507,7 @@ void HttpServer::attach(GenerationService& service) { } const ninfer::LoadSummary load = service.load_summary(); public_model_id_ = resolve_public_model_id(options_, load.model_id); + model_metadata_ = service.model_metadata(); service_ = &service; request_jsonl_.write_server_start(options_, service.engine_options(), service.sampling_defaults(), public_model_id_, load, diff --git a/src/serve/http_server.h b/src/serve/http_server.h index 24314949fa..0c2811752a 100644 --- a/src/serve/http_server.h +++ b/src/serve/http_server.h @@ -100,6 +100,7 @@ class HttpServer { GenerationService* service_ = nullptr; ServeOptions options_; std::string public_model_id_; + ninfer::ModelMetadata model_metadata_; OpenAIResponsesStore openai_responses_store_; OperationalLog operational_log_; JsonlRequestLog request_jsonl_; diff --git a/src/serve/openai_common.cpp b/src/serve/openai_common.cpp index f16a90f61b..12afd202de 100644 --- a/src/serve/openai_common.cpp +++ b/src/serve/openai_common.cpp @@ -38,6 +38,31 @@ std::string responses_identifier(std::string_view prefix) { using Json = nlohmann::json; +namespace { + +// OpenAI model object carrying the llama.cpp-compatible `meta` field. `id` is the public OpenAI +// alias; the `meta` facts describe the registered artifact behind it. n_ctx is the configured +// per-request context ceiling (equal to max_model_len); n_ctx_train is the model's native +// training context. ftype is the registered weights profile (the NInfer quantization name). +Json make_model_object_json(const std::string& model_id, std::int64_t created, + std::uint32_t max_model_len, const ninfer::ModelMetadata& metadata) { + return Json{{"id", model_id}, + {"object", "model"}, + {"created", created}, + {"owned_by", "ninfer"}, + {"max_model_len", max_model_len}, + {"meta", + Json{{"n_vocab", metadata.vocab_size}, + {"n_ctx", max_model_len}, + {"n_ctx_train", metadata.native_context}, + {"n_embd", metadata.embedding_size}, + {"n_params", metadata.parameters}, + {"size", metadata.weight_bytes}, + {"ftype", metadata.weights_id}}}}; +} + +} // namespace + bool parse_openai_prompt_cache_breakpoint(const RequestJson& value, std::string_view param) { if (!value.contains("prompt_cache_breakpoint") || value.at("prompt_cache_breakpoint").is_null()) { @@ -178,26 +203,18 @@ void apply_openai_prompt_cache_policy(GenerationRequest& request, OpenAIPromptCa } std::string make_models_list(const std::string& model_id, std::int64_t created, - std::uint32_t max_model_len) { + std::uint32_t max_model_len, const ninfer::ModelMetadata& metadata) { // vLLM/llama.cpp-compatible discovery metadata for the configured per-request context limit. const Json payload = {{"object", "list"}, - {"data", Json::array({Json{{"id", model_id}, - {"object", "model"}, - {"created", created}, - {"owned_by", "ninfer"}, - {"max_model_len", max_model_len}}})}}; + {"data", + Json::array({make_model_object_json(model_id, created, max_model_len, + metadata)})}}; return payload.dump(); } std::string make_model_object(const std::string& model_id, std::int64_t created, - std::uint32_t max_model_len) { - // vLLM/llama.cpp-compatible discovery metadata for the configured per-request context limit. - const Json payload = {{"id", model_id}, - {"object", "model"}, - {"created", created}, - {"owned_by", "ninfer"}, - {"max_model_len", max_model_len}}; - return payload.dump(); + std::uint32_t max_model_len, const ninfer::ModelMetadata& metadata) { + return make_model_object_json(model_id, created, max_model_len, metadata).dump(); } std::string make_error_body(const ApiError& error) { diff --git a/src/serve/openai_common.h b/src/serve/openai_common.h index 1935327676..1c582bdf8d 100644 --- a/src/serve/openai_common.h +++ b/src/serve/openai_common.h @@ -29,9 +29,9 @@ struct OpenAIPromptCachePolicy { void apply_openai_prompt_cache_policy(GenerationRequest& request, OpenAIPromptCachePolicy policy); std::string make_models_list(const std::string& model_id, std::int64_t created, - std::uint32_t max_model_len); + std::uint32_t max_model_len, const ninfer::ModelMetadata& metadata); std::string make_model_object(const std::string& model_id, std::int64_t created, - std::uint32_t max_model_len); + std::uint32_t max_model_len, const ninfer::ModelMetadata& metadata); std::string make_error_body(const ApiError& error); std::int64_t unix_time_now(); diff --git a/src/targets/qwen3_6_27b/export/ninfer/targets/qwen3_6_27b/package.h b/src/targets/qwen3_6_27b/export/ninfer/targets/qwen3_6_27b/package.h index e6bdc80061..eee3fb41b9 100644 --- a/src/targets/qwen3_6_27b/export/ninfer/targets/qwen3_6_27b/package.h +++ b/src/targets/qwen3_6_27b/export/ninfer/targets/qwen3_6_27b/package.h @@ -123,6 +123,9 @@ struct Package { using Program = qwen3_6::Program; [[nodiscard]] static ModelSamplingDefaults sampling_defaults(std::string_view model); + // Static dimension facts (vocab_size, embedding_size, native_context). The Engine completes + // model_id, weights_id, parameters, and weight_bytes before the metadata is published. + [[nodiscard]] static ModelMetadata model_metadata(); [[nodiscard]] static WeightsProfile resolve_weights(const artifact::ArtifactIdentity& identity); [[nodiscard]] static LoadPlan plan_load(artifact::Binder& binder, const EngineOptions& options, WeightsProfile weights_profile); diff --git a/src/targets/qwen3_6_27b/impl/package.cpp b/src/targets/qwen3_6_27b/impl/package.cpp index c844d21eda..a078871aa5 100644 --- a/src/targets/qwen3_6_27b/impl/package.cpp +++ b/src/targets/qwen3_6_27b/impl/package.cpp @@ -82,6 +82,17 @@ ModelSamplingDefaults Package::sampling_defaults(std::string_view model) { std::string(target_key) + "'"); } +ModelMetadata Package::model_metadata() { + // Static dimension facts owned by this target. The qwen3.6-27b and qwen3.8-27b identities + // share one architecture, so both register the same facts; the Engine sets model_id/weights_id + // from the loaded identity and measures parameters/weight_bytes from the artifact. + return ModelMetadata{ + .vocab_size = static_cast(qwen3_6::kTokenDomain), + .embedding_size = static_cast(detail::TextConfig::hidden), + .native_context = static_cast(detail::kNativeContext), + }; +} + Package::WeightsProfile Package::resolve_weights(const artifact::ArtifactIdentity& identity) { if (identity.model_id == model_id && identity.weights_id == "groupwise-int") { return WeightsProfile::Qwen36GroupwiseInt; diff --git a/src/targets/qwen3_6_35b_a3b/export/ninfer/targets/qwen3_6_35b_a3b/package.h b/src/targets/qwen3_6_35b_a3b/export/ninfer/targets/qwen3_6_35b_a3b/package.h index 467969453d..928008ff87 100644 --- a/src/targets/qwen3_6_35b_a3b/export/ninfer/targets/qwen3_6_35b_a3b/package.h +++ b/src/targets/qwen3_6_35b_a3b/export/ninfer/targets/qwen3_6_35b_a3b/package.h @@ -118,6 +118,9 @@ struct Package { using Program = qwen3_6::Program; [[nodiscard]] static ModelSamplingDefaults sampling_defaults(std::string_view model); + // Static dimension facts (vocab_size, embedding_size, native_context). The Engine completes + // model_id, weights_id, parameters, and weight_bytes before the metadata is published. + [[nodiscard]] static ModelMetadata model_metadata(); [[nodiscard]] static WeightsProfile resolve_weights(const artifact::ArtifactIdentity& identity); [[nodiscard]] static LoadPlan plan_load(artifact::Binder& binder, const EngineOptions& options, WeightsProfile weights_profile); diff --git a/src/targets/qwen3_6_35b_a3b/impl/package.cpp b/src/targets/qwen3_6_35b_a3b/impl/package.cpp index 15e55730e0..6de5b56a86 100644 --- a/src/targets/qwen3_6_35b_a3b/impl/package.cpp +++ b/src/targets/qwen3_6_35b_a3b/impl/package.cpp @@ -64,6 +64,16 @@ ModelSamplingDefaults Package::sampling_defaults(std::string_view model) { std::string(target_key) + "'"); } +ModelMetadata Package::model_metadata() { + // Static dimension facts owned by this target. The Engine sets model_id/weights_id from the + // loaded identity and measures parameters/weight_bytes from the artifact. + return ModelMetadata{ + .vocab_size = static_cast(qwen3_6::kTokenDomain), + .embedding_size = static_cast(detail::TextConfig::hidden), + .native_context = static_cast(detail::kNativeContext), + }; +} + Package::WeightsProfile Package::resolve_weights(const artifact::ArtifactIdentity& identity) { if (identity.model_id == model_id && identity.weights_id == "groupwise-int") { return WeightsProfile::GroupwiseInt; diff --git a/src/targets/registry.cpp b/src/targets/registry.cpp index 57bc001eae..82cfcdc0b8 100644 --- a/src/targets/registry.cpp +++ b/src/targets/registry.cpp @@ -84,6 +84,24 @@ std::size_t current_free_device_bytes() { return free_bytes; } +// Artifact-measured model facts. n_params is the total logical (dequantized) element count over +// every registered weight tensor; size is the total encoded payload bytes of those tensors. Both +// are pure functions of the artifact's tensor inventory, independent of device memory layout. +void measure_artifact_weights(artifact::Reader& reader, ModelMetadata& metadata) { + std::uint64_t parameters = 0; + std::uint64_t weight_bytes = 0; + for (const auto& object : reader.objects()) { + const auto* tensor = std::get_if(&object); + if (tensor == nullptr) { continue; } // Non-weight resources (tokenizer, templates). + std::uint64_t elements = 1; + for (const auto dimension : tensor->shape) { elements *= dimension; } + parameters += elements; + weight_bytes += artifact::object_bytes(*tensor); + } + metadata.parameters = parameters; + metadata.weight_bytes = weight_bytes; +} + template ConstructedTarget construct_registered(const EngineOptions& options, DeviceContext& device, artifact::Reader& reader, Clock::time_point load_start, @@ -150,8 +168,17 @@ ConstructedTarget construct_registered(const EngineOptions& options, DeviceConte summary.tensor_count = stats.tensor_count; summary.resource_count = stats.resource_count; summary.context_cost = context_cost.summary; + + // The target owns the static dimension facts; the registered identity and the artifact + // measurements complete the model metadata served by /v1/models. + ModelMetadata metadata = Target::model_metadata(); + metadata.model_id = identity.model_id; + metadata.weights_id = identity.weights_id; + measure_artifact_weights(reader, metadata); + return ConstructedTarget{.active = ActiveTarget(std::move(instance)), .load = std::move(summary), + .model_metadata = std::move(metadata), .sampling_defaults = sampling_defaults, .context_cost = std::move(context_cost.model)}; } diff --git a/src/targets/registry.h b/src/targets/registry.h index 16ceb36613..9f4dbb1c19 100644 --- a/src/targets/registry.h +++ b/src/targets/registry.h @@ -83,6 +83,7 @@ using ActiveTarget = struct ConstructedTarget { ActiveTarget active; LoadSummary load; + ModelMetadata model_metadata; ModelSamplingDefaults sampling_defaults; runtime::ContextMachineCostModel context_cost; }; diff --git a/tests/test_openai_schema.cpp b/tests/test_openai_schema.cpp index c4378233f4..22f0f701d0 100644 --- a/tests/test_openai_schema.cpp +++ b/tests/test_openai_schema.cpp @@ -764,13 +764,32 @@ int test_stream_observations() { int test_common_objects() { int failures = 0; - const Json models = Json::parse(make_models_list("qwen", 7, 240000)); + const ninfer::ModelMetadata metadata{ + .model_id = "qwen3.6-27b", + .weights_id = "groupwise-int", + .vocab_size = 248077, + .embedding_size = 5120, + .native_context = 262144, + .parameters = 27000000000ULL, + .weight_bytes = 17000000000ULL, + }; + const Json models = Json::parse(make_models_list("qwen", 7, 240000, metadata)); failures += check(models["data"][0]["id"] == "qwen" && models["data"][0]["max_model_len"] == 240000, "models list advertises the configured context limit"); - const Json model = Json::parse(make_model_object("qwen", 7, 240000)); + const Json list_meta = models["data"][0]["meta"]; + failures += check(list_meta["n_vocab"] == 248077 && list_meta["n_ctx"] == 240000 && + list_meta["n_ctx_train"] == 262144 && list_meta["n_embd"] == 5120 && + list_meta["n_params"] == 27000000000ULL && + list_meta["size"] == 17000000000ULL && + list_meta["ftype"] == "groupwise-int", + "models list exposes the llama.cpp-compatible model meta"); + const Json model = Json::parse(make_model_object("qwen", 7, 240000, metadata)); failures += check(model["max_model_len"] == 240000, "model lookup advertises the configured context limit"); + failures += check(model["meta"]["n_embd"] == 5120 && model["meta"]["n_ctx_train"] == 262144 && + model["meta"]["ftype"] == "groupwise-int", + "model lookup exposes the llama.cpp-compatible model meta"); const Json error = Json::parse(make_error_body( ApiError{.status = 400, .message = "bad", .param = "messages", .code = "invalid"})); failures += check(error["error"]["param"] == "messages" && error["error"]["code"] == "invalid",