diff --git a/CMakeLists.txt b/CMakeLists.txt index d529e08..2bf7915 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,6 +149,7 @@ endif() set(VOICEDETECT_SRC src/voicedetect.cpp src/voicedetect_capi.cpp + src/gguf_check.cpp src/model.cpp src/model_loader.cpp src/backend.cpp diff --git a/README.md b/README.md index 85e06b1..420773f 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ The frozen symbol set: - `voicedetect_capi_abi_version` - integer LocalAI checks for compatibility; bump it on any breaking change to the frozen symbols (additive functions do not require a bump). - `voicedetect_capi_load` / `voicedetect_capi_free` - load a GGUF model, free the context. +- `voicedetect_capi_load_from_memory` / `voicedetect_capi_load_from_memory_prefixed` / `voicedetect_capi_last_load_error` - load a model from a buffer instead of a path, and read why a load failed (see "Loading from memory" below). - `voicedetect_capi_last_error` - human-readable last error on a context. - `voicedetect_capi_embed_path` / `voicedetect_capi_embed_pcm` - L2-normalized embedding from a WAV file or in-memory mono float PCM (PCM is linearly resampled to 16 kHz if needed). - `voicedetect_capi_verify_paths` - cosine distance (`1 - cosine_similarity`) between two clips plus a same-speaker verdict against a threshold. @@ -218,6 +219,25 @@ if (json) { voicedetect_capi_free_string(json); } voicedetect_capi_free(ctx); ``` +### Loading from memory + +A model can be loaded from a GGUF that is already in memory, with no file path and no temporary file. The same code runs on Linux, macOS and Windows. + +```c +voicedetect_ctx* ctx = voicedetect_capi_load_from_memory(bytes, n_bytes); +if (!ctx) fprintf(stderr, "%s\n", voicedetect_capi_last_load_error()); +free(bytes); // allowed: the loader has already copied what it needs +``` + +- **Ownership.** The loader copies the tensor data into its own memory during the call. `data` is only read inside the call, so you may free or overwrite it as soon as the function returns, whether it succeeded or not. While the call runs, the buffer and the model are in memory together. +- **Errors.** A NULL or empty buffer, a truncated buffer, and a corrupt buffer all return `NULL`. `voicedetect_capi_last_load_error()` returns the reason for the last failed load on the calling thread. No byte outside `[data, data + size)` is read. The same function also reports path-load errors. (`voicedetect_capi_load` used to return `NULL` without a reason.) +- **Threads.** Loads are independent. Different threads may load at the same time, from different buffers or from the same read-only buffer. A context is used by one thread at a time, as before. +- **C++.** `vd::Model::load_from_memory(data, size, &err)` and `vd::ModelLoader::load_from_memory(data, size)`. `vd::Model::load(path, &err)` gained the same optional error out-parameter. + +**Models inside a larger GGUF (bundles).** A GGUF can hold several models if every key and every tensor name of a model carries a prefix, for example `voice.`. `voicedetect_capi_load_from_memory_prefixed(data, size, "voice.")` reads such a component straight from the whole bundle. Only the tensors under the prefix are copied, so the caller does not need to build a standalone copy of the component first. Tensor names that are stored as string values in the model's metadata (the block manifests) are not prefixed: the loader uses them as they are. A prefix that matches no tensor is an error. An empty prefix behaves like `voicedetect_capi_load_from_memory`. + +Limits: the buffer must hold the complete GGUF (the loader does not stream). The tensors are copied once, so the peak memory is the buffer plus the model. + --- ## Model coverage @@ -243,6 +263,8 @@ export VOICEDETECT_TEST_AUDIO=tests/fixtures/clip.wav ctest --test-dir build --output-on-failure ``` +`test_load_memory` is model-independent: it builds GGUF files and bundles in memory and checks equivalence, truncated and corrupt buffers, and concurrent loads. `test_load_memory_embed` needs `VOICEDETECT_TEST_GGUF` and `VOICEDETECT_TEST_AUDIO` and checks that path, memory and prefixed loads give bitwise-identical embeddings. + Tests labelled `model` return exit code 77 (ctest SKIP) when their required env vars or checkpoints are absent, so they never break a CI environment that has no model. `scripts/gpu_verify.sh` runs the parity gates plus the benchmark on a CUDA host (GPU only, not CI). --- diff --git a/include/voicedetect_capi.h b/include/voicedetect_capi.h index fb1dda8..f1e28e7 100644 --- a/include/voicedetect_capi.h +++ b/include/voicedetect_capi.h @@ -1,6 +1,8 @@ #ifndef VOICEDETECT_CAPI_H #define VOICEDETECT_CAPI_H +#include + #ifdef __cplusplus extern "C" { #endif @@ -23,8 +25,8 @@ extern "C" { typedef struct voicedetect_ctx voicedetect_ctx; // ABI version of this header/implementation. Bump on any breaking change to the -// function signatures or semantics below. Additive changes (new functions) are -// fine without a bump; LocalAI checks this integer for compatibility. +// function signatures or semantics below. Additive changes (new functions, such +// as the load_from_memory family) are fine without a bump; LocalAI checks this integer for compatibility. // // v1: initial flat C-API surface - load/free/last_error/free_string, // embed_path / embed_pcm (+ free_vec), verify_paths, analyze_path_json. @@ -35,6 +37,37 @@ int voicedetect_capi_abi_version(void); // voicedetect_capi_free. voicedetect_ctx* voicedetect_capi_load(const char* gguf_path); +// Load a model from a GGUF held in memory (no file, no temporary file, no file +// descriptor; works the same on every platform). `data` points to the complete +// GGUF file of `size` bytes. +// +// OWNERSHIP: the loader copies the tensor data into its own memory during the +// call. `data` is only read inside the call: the caller may free or overwrite it +// as soon as the function returns, on success and on failure. While loading, the +// buffer and the model are in memory together (peak = buffer + model). +// +// A NULL or empty buffer, a truncated buffer or a corrupt buffer gives NULL and +// a message in voicedetect_capi_last_load_error(); no byte outside +// [data, data+size) is read. Returns an owning context, released with +// voicedetect_capi_free. Calls on different threads are independent. +voicedetect_ctx* voicedetect_capi_load_from_memory(const void* data, size_t size); + +// As voicedetect_capi_load_from_memory, for a model that is one component of a +// larger GGUF (a bundle): every metadata key and every tensor name of the model +// is stored as `prefix` + name (for example "voice."). Pass the whole bundle and +// the prefix: no standalone copy of the component is needed, and only the +// tensors under the prefix are copied. Names stored as string values inside the +// model's metadata (tensor manifests) are not prefixed. Same ownership and error +// rules as above. A prefix that matches no tensor is an error. A NULL or empty +// `prefix` behaves like voicedetect_capi_load_from_memory. +voicedetect_ctx* voicedetect_capi_load_from_memory_prefixed(const void* data, size_t size, + const char* prefix); + +// Reason for the last failed voicedetect_capi_load* call made on the CALLING +// THREAD, or "" if the last such call succeeded. The pointer stays valid until +// the next load call on the same thread. Never NULL. +const char* voicedetect_capi_last_load_error(void); + // Free a context obtained from voicedetect_capi_load. Safe on NULL. void voicedetect_capi_free(voicedetect_ctx* ctx); diff --git a/src/gguf_check.cpp b/src/gguf_check.cpp new file mode 100644 index 0000000..dd3fdb9 --- /dev/null +++ b/src/gguf_check.cpp @@ -0,0 +1,116 @@ +#include "gguf_check.hpp" + +#include "ggml.h" + +#include +#include + +namespace vd { + +namespace { + +struct Cursor { + const uint8_t* p; + size_t size; + size_t pos = 0; + + bool have(uint64_t n) const { return n <= size - pos; } + bool skip(uint64_t n) { + if (!have(n)) return false; + pos += (size_t)n; + return true; + } + template + bool read(T* out) { + if (!have(sizeof(T))) return false; + std::memcpy(out, p + pos, sizeof(T)); + pos += sizeof(T); + return true; + } +}; + +// Size in bytes of a fixed-size GGUF value type, 0 for string/array/unknown. +size_t scalar_size(uint32_t t) { + switch (t) { + case 0: case 1: case 7: return 1; // u8, i8, bool + case 2: case 3: return 2; // u16, i16 + case 4: case 5: case 6: return 4; // u32, i32, f32 + case 10: case 11: case 12: return 8; // u64, i64, f64 + default: return 0; + } +} + +constexpr uint32_t kString = 8, kArray = 9; +constexpr uint64_t kMaxString = 1ull << 30; // same cap as ggml's reader + +bool skip_string(Cursor& c) { + uint64_t n; + return c.read(&n) && n <= kMaxString && c.skip(n); +} + +} // namespace + +bool gguf_precheck(const void* data, size_t size, std::string* err) { + auto fail = [&](const char* m) { + if (err) *err = m; + return false; + }; + Cursor c{static_cast(data), size}; + uint32_t version; + int64_t n_tensors, n_kv; + if (!c.have(4) || std::memcmp(c.p, "GGUF", 4) != 0) return fail("not a GGUF file (bad magic)"); + c.pos = 4; + if (!c.read(&version) || (version != 2 && version != 3)) + return fail("unsupported GGUF version"); + if (!c.read(&n_tensors) || !c.read(&n_kv) || n_tensors < 0 || n_kv < 0) + return fail("GGUF header is truncated or has a negative count"); + // Every key/value pair takes at least 13 bytes, so a count that cannot fit is + // corrupt (and must not drive a long loop). + if ((uint64_t)n_kv > (size - c.pos) / 13) return fail("GGUF metadata count exceeds the buffer size"); + + for (int64_t i = 0; i < n_kv; ++i) { + uint64_t klen; + if (!c.read(&klen) || klen > kMaxString || !c.have(klen)) + return fail("GGUF metadata key is truncated"); + if (klen == 0) return fail("GGUF metadata key has an empty name"); + c.skip(klen); + uint32_t type; + if (!c.read(&type)) return fail("GGUF metadata is truncated"); + if (type == kString) { + if (!skip_string(c)) return fail("GGUF string value is truncated"); + } else if (type == kArray) { + uint32_t et; + uint64_t n; + if (!c.read(&et) || !c.read(&n)) return fail("GGUF array header is truncated"); + if (et == kString) { + for (uint64_t j = 0; j < n; ++j) + if (!skip_string(c)) return fail("GGUF string array is truncated"); + } else { + const size_t es = scalar_size(et); + if (es == 0) return fail("GGUF array has an unsupported element type"); + if (n > size / es || !c.skip(n * es)) return fail("GGUF array is truncated"); + } + } else { + const size_t s = scalar_size(type); + if (s == 0) return fail("GGUF metadata has an unknown value type"); + if (!c.skip(s)) return fail("GGUF metadata is truncated"); + } + } + + // Tensor table: a type id outside the ggml enum is read by ggml into an enum + // before it is validated (undefined behavior), so reject it here. + if ((uint64_t)n_tensors > (size - c.pos) / 24) return fail("GGUF tensor count exceeds the buffer size"); + for (int64_t i = 0; i < n_tensors; ++i) { + uint64_t nlen; + uint32_t n_dims, type; + if (!c.read(&nlen) || nlen == 0 || nlen > kMaxString || !c.skip(nlen)) + return fail("GGUF tensor name is truncated or empty"); + if (!c.read(&n_dims) || n_dims == 0 || n_dims > GGML_MAX_DIMS) + return fail("GGUF tensor has an invalid number of dimensions"); + if (!c.skip(8ull * n_dims) || !c.read(&type) || type >= (uint32_t)GGML_TYPE_COUNT || !c.skip(8)) + return fail("GGUF tensor table is truncated or has an unknown tensor type"); + } + return true; +} + +} // namespace vd diff --git a/src/gguf_check.hpp b/src/gguf_check.hpp new file mode 100644 index 0000000..61982b1 --- /dev/null +++ b/src/gguf_check.hpp @@ -0,0 +1,21 @@ +#pragma once +#include +#include + +namespace vd { + +// Cheap structural check of a GGUF held in memory, run BEFORE the buffer is +// handed to ggml's reader. ggml validates sizes and bounds, but it still aborts +// the whole process (GGML_ASSERT) on a few well-formed-looking inputs, such as a +// metadata key with an empty name. A model buffer comes from the caller, so such +// input must become an error instead. +// +// Walks the header, the metadata key/value section and the tensor table with +// bounds checks on every read. Returns false and sets `err` on: a bad magic or +// version, an empty or oversized key, an unknown value type, a nested array, a +// tensor with an empty name, a bad dimension count or an unknown type id, or +// anything that runs past the end of the buffer. Tensor offsets and sizes are +// not checked here: the loaders check them against the buffer size. +bool gguf_precheck(const void* data, size_t size, std::string* err); + +} // namespace vd diff --git a/src/model.cpp b/src/model.cpp index be1ecea..1d78bd8 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -11,14 +11,34 @@ namespace vd { -std::unique_ptr Model::load(const std::string& gguf_path) { +std::unique_ptr Model::load(const std::string& gguf_path, std::string* err) { auto m = std::unique_ptr(new Model()); if (!m->loader_.load(gguf_path)) { + if (err) *err = m->loader_.error(); return nullptr; // bad/missing GGUF, or required KV absent } return m; } +std::unique_ptr Model::load_from_memory(const void* data, size_t size, std::string* err) { + auto m = std::unique_ptr(new Model()); + if (!m->loader_.load_from_memory(data, size)) { + if (err) *err = m->loader_.error(); + return nullptr; + } + return m; +} + +std::unique_ptr Model::load_from_memory(const void* data, size_t size, + const std::string& prefix, std::string* err) { + auto m = std::unique_ptr(new Model()); + if (!m->loader_.load_from_memory(data, size, prefix)) { + if (err) *err = m->loader_.error(); + return nullptr; + } + return m; +} + std::vector Model::embed_16k(const std::vector& pcm16k) const { // Stage 1: Kaldi-compatible FBank front end -> feat-major [n_mels, T]. FBank fbank(loader_.config()); diff --git a/src/model.hpp b/src/model.hpp index 3263c50..d795de0 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -18,7 +18,23 @@ namespace vd { class Model { public: // Loads the GGUF at `gguf_path`. Returns nullptr on failure (no throw). - static std::unique_ptr load(const std::string& gguf_path); + // On failure `*err` (if given) receives the reason. + static std::unique_ptr load(const std::string& gguf_path, std::string* err = nullptr); + + // Loads a GGUF held in memory. The tensor data is copied during the call: + // `data` may be freed or overwritten as soon as this returns. Returns nullptr + // on a NULL/empty, truncated or corrupt buffer (reason in `*err`, no throw, + // no read outside [data, data+size)). Safe to call from several threads on + // different buffers (or the same read-only buffer). + static std::unique_ptr load_from_memory(const void* data, size_t size, + std::string* err = nullptr); + + // As load_from_memory for a model stored inside a larger GGUF (a bundle): + // every key and tensor name of the model carries `prefix` (for example + // "voice."). Only the tensors under the prefix are copied. + static std::unique_ptr load_from_memory(const void* data, size_t size, + const std::string& prefix, + std::string* err = nullptr); // Compute the L2-normalized speaker embedding from raw mono float PCM. If // `sample_rate != 16000` the audio is linearly resampled to 16 kHz first. diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 17522d2..7221219 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -1,43 +1,73 @@ #include "model_loader.hpp" #include "common.hpp" #include "backend.hpp" +#include "gguf_check.hpp" #include "ggml.h" #include "ggml-backend.h" #include "ggml-alloc.h" #include "gguf.h" #include +#include #include #include namespace vd { -static uint32_t kv_u32(gguf_context* g, const char* k, uint32_t d=0){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : (uint32_t)gguf_get_val_u32(g,id); +// A GGUF plus an optional key prefix. Every key is looked up as prefix+key, so +// a model stored inside a larger GGUF (a bundle) reads like a standalone file. +struct Kv { + gguf_context* g; + std::string prefix; + mutable std::string err; // first type mismatch; ggml would abort on one + // Index of key `k` if present with scalar type `want`; -1 if absent. A key of + // another type records an error instead of reaching ggml's asserting getters. + int64_t find(const char* k, gguf_type want) const { + int64_t id = gguf_find_key(g, (prefix + k).c_str()); + if(id < 0) return -1; + if(gguf_get_kv_type(g, id) != want){ + if(err.empty()) err = std::string("GGUF key '") + prefix + k + "' has an unexpected type"; + return -1; + } + return id; + } + // Array key whose elements have type `elem`. + int64_t find_arr(const char* k, gguf_type elem) const { + int64_t id = find(k, GGUF_TYPE_ARRAY); + if(id < 0) return -1; + if(gguf_get_arr_type(g, id) != elem){ + if(err.empty()) err = std::string("GGUF key '") + prefix + k + "' has an unexpected element type"; + return -1; + } + return id; + } +}; +static uint32_t kv_u32(const Kv& kv, const char* k, uint32_t d=0){ + int64_t id = kv.find(k, GGUF_TYPE_UINT32); return id<0 ? d : gguf_get_val_u32(kv.g,id); } -static float kv_f32(gguf_context* g, const char* k, float d=0){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : gguf_get_val_f32(g,id); +static float kv_f32(const Kv& kv, const char* k, float d=0){ + int64_t id = kv.find(k, GGUF_TYPE_FLOAT32); return id<0 ? d : gguf_get_val_f32(kv.g,id); } -static bool kv_bool(gguf_context* g, const char* k, bool d=false){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : gguf_get_val_bool(g,id); +static bool kv_bool(const Kv& kv, const char* k, bool d=false){ + int64_t id = kv.find(k, GGUF_TYPE_BOOL); return id<0 ? d : gguf_get_val_bool(kv.g,id); } -static std::string kv_str(gguf_context* g, const char* k, const char* d=""){ - int64_t id = gguf_find_key(g,k); return id<0 ? std::string(d) : std::string(gguf_get_val_str(g,id)); +static std::string kv_str(const Kv& kv, const char* k, const char* d=""){ + int64_t id = kv.find(k, GGUF_TYPE_STRING); return id<0 ? std::string(d) : std::string(gguf_get_val_str(kv.g,id)); } -static std::vector kv_str_arr(gguf_context* g, const char* k){ +static std::vector kv_str_arr(const Kv& kv, const char* k){ std::vector out; - int64_t id = gguf_find_key(g,k); - if(id>=0 && gguf_get_arr_type(g,id)==GGUF_TYPE_STRING){ - size_t n = gguf_get_arr_n(g,id); + int64_t id = kv.find_arr(k, GGUF_TYPE_STRING); + if(id>=0){ + size_t n = gguf_get_arr_n(kv.g,id); out.resize(n); - for(size_t i=0;i kv_i32_arr(gguf_context* g, const char* k){ +static std::vector kv_i32_arr(const Kv& kv, const char* k){ std::vector out; - int64_t id = gguf_find_key(g,k); - if(id>=0 && gguf_get_arr_type(g,id)==GGUF_TYPE_INT32){ - size_t n = gguf_get_arr_n(g,id); - const int32_t* d = (const int32_t*)gguf_get_arr_data(g,id); + int64_t id = kv.find_arr(k, GGUF_TYPE_INT32); + if(id>=0){ + size_t n = gguf_get_arr_n(kv.g,id); + const int32_t* d = (const int32_t*)gguf_get_arr_data(kv.g,id); out.assign(d, d+n); } return out; @@ -111,107 +141,205 @@ bool ModelLoader::realize_weights(ggml_backend_t backend){ return true; } +void ModelLoader::reset_(){ + if(gguf_){ gguf_free(gguf_); gguf_ = nullptr; } + if(ctx_){ ggml_free(ctx_); ctx_ = nullptr; } + tensors_.clear(); + cfg_ = VoiceDetectConfig(); +} + +bool ModelLoader::fail_(const std::string& msg){ + error_ = msg; + VD_LOG("%s", msg.c_str()); + reset_(); + return false; +} + bool ModelLoader::load(const std::string& path){ + if(gguf_ || ctx_) return fail_("model already loaded"); + error_.clear(); struct gguf_init_params p{ /*no_alloc*/false, /*ctx*/&ctx_ }; gguf_ = gguf_init_from_file(path.c_str(), p); - if(!gguf_){ VD_LOG("gguf open failed: %s", path.c_str()); return false; } - cfg_.arch = kv_str(gguf_, "voicedetect.arch"); - cfg_.embedding_dim = kv_u32(gguf_, "voicedetect.embedding_dim"); - cfg_.l2_normalize = kv_bool(gguf_, "voicedetect.l2_normalize", true); + if(!gguf_) return fail_("gguf open failed: " + path); + return finish_(Kv{gguf_, ""}); +} + +bool ModelLoader::load_from_memory(const void* data, size_t size){ + return load_from_memory(data, size, std::string()); +} + +bool ModelLoader::load_from_memory(const void* data, size_t size, const std::string& prefix){ + if(gguf_ || ctx_) return fail_("model already loaded"); + error_.clear(); + if(!data || size == 0) return fail_("empty GGUF buffer"); + + // Parse the header and tensor table only (no_alloc), validate every tensor + // against the buffer size, then copy the selected tensors (all of them for + // an empty prefix) into a context of our own. ggml's own no_alloc=false + // reader is not used: it sizes its allocation from the tensor table before + // checking it against the buffer, so a hostile table can make it abort. + { + std::string perr; + if(!gguf_precheck(data, size, &perr)) return fail_("invalid GGUF buffer: " + perr); + } + ggml_context* meta = nullptr; + struct gguf_init_params mp{ /*no_alloc*/true, /*ctx*/&meta }; + gguf_ = gguf_init_from_buffer(data, size, mp); + if(!gguf_){ if(meta) ggml_free(meta); return fail_("invalid or truncated GGUF buffer"); } + + struct MetaGuard { ggml_context* c; ~MetaGuard(){ if(c) ggml_free(c); } } guard{meta}; + const size_t data_off = gguf_get_data_offset(gguf_); + if(data_off > size) return fail_("GGUF buffer is truncated (tensor table exceeds buffer)"); + const size_t avail = size - data_off; + const int64_t nt = gguf_get_n_tensors(gguf_); + + std::vector ids; + size_t total = 0; + for(int64_t i=0;i avail || nb > avail - off) + return fail_(std::string("GGUF buffer is truncated or corrupt (tensor '") + nm + "' is out of range)"); + const size_t padded = GGML_PAD(nb, GGML_MEM_ALIGN); + if(padded < nb || total > SIZE_MAX - padded) return fail_("GGUF tensor sizes overflow"); + total += padded; + ids.push_back(i); + } + if(ids.empty()) return fail_("no tensors with prefix '" + prefix + "' in GGUF buffer"); + // Distinct tensors never share bytes in a valid file; a larger sum means a corrupt table. + if(total > avail + ids.size()*GGML_MEM_ALIGN) return fail_("GGUF tensor table is corrupt (tensors overlap)"); + + struct ggml_init_params ip{ + /*mem_size*/ total + (ids.size()+1) * ggml_tensor_overhead() + GGML_MEM_ALIGN, + /*mem_buffer*/ nullptr, /*no_alloc*/ false }; + ctx_ = ggml_init(ip); + if(!ctx_) return fail_("out of memory"); + const uint8_t* base = static_cast(data) + data_off; + for(int64_t i : ids){ + const char* nm = gguf_get_tensor_name(gguf_, i); + ggml_tensor* src = ggml_get_tensor(meta, nm); + if(!src) return fail_(std::string("GGUF tensor '") + nm + "' has no descriptor"); + ggml_tensor* t = ggml_new_tensor(ctx_, src->type, GGML_MAX_DIMS, src->ne); + if(!t || !t->data) return fail_("out of memory"); + const size_t nb = ggml_nbytes(t); + if(nb != gguf_get_tensor_size(gguf_, i)) return fail_(std::string("GGUF tensor '") + nm + "' has inconsistent size"); + ggml_set_name(t, nm + prefix.size()); + std::memcpy(t->data, base + gguf_get_tensor_offset(gguf_, i), nb); + } + return finish_(Kv{gguf_, prefix}, /*ctx_names=*/true); +} + +bool ModelLoader::finish_(const Kv& kv, bool ctx_names){ + cfg_.arch = kv_str(kv, "voicedetect.arch"); + cfg_.embedding_dim = kv_u32(kv, "voicedetect.embedding_dim"); + cfg_.l2_normalize = kv_bool(kv, "voicedetect.l2_normalize", true); // FBank front end - cfg_.sample_rate = kv_u32(gguf_, "voicedetect.fbank.sample_rate", 16000); - cfg_.n_mels = kv_u32(gguf_, "voicedetect.fbank.n_mels", 80); - cfg_.n_fft = kv_u32(gguf_, "voicedetect.fbank.n_fft", 512); - cfg_.win_length = kv_u32(gguf_, "voicedetect.fbank.win_length", 400); - cfg_.hop_length = kv_u32(gguf_, "voicedetect.fbank.hop_length", 160); - cfg_.preemph = kv_f32(gguf_, "voicedetect.fbank.preemph", 0.97f); - cfg_.fbank_low_freq = kv_f32(gguf_, "voicedetect.fbank.low_freq", 20.0f); - cfg_.fbank_high_freq = kv_f32(gguf_, "voicedetect.fbank.high_freq", 0.0f); - cfg_.fbank_use_energy = kv_bool(gguf_, "voicedetect.fbank.use_energy", false); - cfg_.fbank_cmn = kv_bool(gguf_, "voicedetect.fbank.cmn", true); - cfg_.fbank_window = kv_str(gguf_, "voicedetect.fbank.window", "povey"); + cfg_.sample_rate = kv_u32(kv, "voicedetect.fbank.sample_rate", 16000); + cfg_.n_mels = kv_u32(kv, "voicedetect.fbank.n_mels", 80); + cfg_.n_fft = kv_u32(kv, "voicedetect.fbank.n_fft", 512); + cfg_.win_length = kv_u32(kv, "voicedetect.fbank.win_length", 400); + cfg_.hop_length = kv_u32(kv, "voicedetect.fbank.hop_length", 160); + cfg_.preemph = kv_f32(kv, "voicedetect.fbank.preemph", 0.97f); + cfg_.fbank_low_freq = kv_f32(kv, "voicedetect.fbank.low_freq", 20.0f); + cfg_.fbank_high_freq = kv_f32(kv, "voicedetect.fbank.high_freq", 0.0f); + cfg_.fbank_use_energy = kv_bool(kv, "voicedetect.fbank.use_energy", false); + cfg_.fbank_cmn = kv_bool(kv, "voicedetect.fbank.cmn", true); + cfg_.fbank_window = kv_str(kv, "voicedetect.fbank.window", "povey"); // WeSpeaker ResNet34 block manifest (only present for that arch). if(cfg_.arch == "wespeaker_resnet34"){ VoiceDetectConfig::ResNetConfig& r = cfg_.resnet; - r.stem_weight = kv_str(gguf_, "voicedetect.resnet.stem_weight"); - r.stem_bias = kv_str(gguf_, "voicedetect.resnet.stem_bias"); - r.conv1_weight = kv_str_arr(gguf_, "voicedetect.resnet.conv1_weight"); - r.conv1_bias = kv_str_arr(gguf_, "voicedetect.resnet.conv1_bias"); - r.conv2_weight = kv_str_arr(gguf_, "voicedetect.resnet.conv2_weight"); - r.conv2_bias = kv_str_arr(gguf_, "voicedetect.resnet.conv2_bias"); - r.down_weight = kv_str_arr(gguf_, "voicedetect.resnet.down_weight"); - r.down_bias = kv_str_arr(gguf_, "voicedetect.resnet.down_bias"); - r.stride = kv_i32_arr(gguf_, "voicedetect.resnet.stride"); - r.seg_weight = kv_str(gguf_, "voicedetect.resnet.seg_weight"); - r.seg_bias = kv_str(gguf_, "voicedetect.resnet.seg_bias"); - r.mean_vec = kv_str(gguf_, "voicedetect.resnet.mean_vec"); - r.var_eps = kv_f32(gguf_, "voicedetect.resnet.var_eps", 1e-8f); + r.stem_weight = kv_str(kv, "voicedetect.resnet.stem_weight"); + r.stem_bias = kv_str(kv, "voicedetect.resnet.stem_bias"); + r.conv1_weight = kv_str_arr(kv, "voicedetect.resnet.conv1_weight"); + r.conv1_bias = kv_str_arr(kv, "voicedetect.resnet.conv1_bias"); + r.conv2_weight = kv_str_arr(kv, "voicedetect.resnet.conv2_weight"); + r.conv2_bias = kv_str_arr(kv, "voicedetect.resnet.conv2_bias"); + r.down_weight = kv_str_arr(kv, "voicedetect.resnet.down_weight"); + r.down_bias = kv_str_arr(kv, "voicedetect.resnet.down_bias"); + r.stride = kv_i32_arr(kv, "voicedetect.resnet.stride"); + r.seg_weight = kv_str(kv, "voicedetect.resnet.seg_weight"); + r.seg_bias = kv_str(kv, "voicedetect.resnet.seg_bias"); + r.mean_vec = kv_str(kv, "voicedetect.resnet.mean_vec"); + r.var_eps = kv_f32(kv, "voicedetect.resnet.var_eps", 1e-8f); } // 3D-Speaker ERes2Net manifest (only present for that arch). if(cfg_.arch == "eres2net"){ VoiceDetectConfig::ERes2NetConfig& e = cfg_.eres2net; - e.conv_weight = kv_str_arr(gguf_, "voicedetect.eres2net.conv_weight"); - e.conv_bias = kv_str_arr(gguf_, "voicedetect.eres2net.conv_bias"); - e.conv_stride = kv_i32_arr(gguf_, "voicedetect.eres2net.conv_stride"); - e.num_blocks = kv_i32_arr(gguf_, "voicedetect.eres2net.num_blocks"); - e.m_channels = kv_u32(gguf_, "voicedetect.eres2net.m_channels"); - e.scale = kv_u32(gguf_, "voicedetect.eres2net.scale", 2); - e.seg_weight = kv_str(gguf_, "voicedetect.eres2net.seg_weight"); - e.seg_bias = kv_str(gguf_, "voicedetect.eres2net.seg_bias"); - e.relu_clamp = kv_f32(gguf_, "voicedetect.eres2net.relu_clamp", 20.0f); - e.var_eps = kv_f32(gguf_, "voicedetect.eres2net.var_eps", 1e-8f); + e.conv_weight = kv_str_arr(kv, "voicedetect.eres2net.conv_weight"); + e.conv_bias = kv_str_arr(kv, "voicedetect.eres2net.conv_bias"); + e.conv_stride = kv_i32_arr(kv, "voicedetect.eres2net.conv_stride"); + e.num_blocks = kv_i32_arr(kv, "voicedetect.eres2net.num_blocks"); + e.m_channels = kv_u32(kv, "voicedetect.eres2net.m_channels"); + e.scale = kv_u32(kv, "voicedetect.eres2net.scale", 2); + e.seg_weight = kv_str(kv, "voicedetect.eres2net.seg_weight"); + e.seg_bias = kv_str(kv, "voicedetect.eres2net.seg_bias"); + e.relu_clamp = kv_f32(kv, "voicedetect.eres2net.relu_clamp", 20.0f); + e.var_eps = kv_f32(kv, "voicedetect.eres2net.var_eps", 1e-8f); } // 3D-Speaker CAM++ manifest (only present for that arch). if(cfg_.arch == "campplus"){ VoiceDetectConfig::CamPPlusConfig& cp = cfg_.campplus; - cp.conv_weight = kv_str_arr(gguf_, "voicedetect.campplus.conv_weight"); - cp.conv_bias = kv_str_arr(gguf_, "voicedetect.campplus.conv_bias"); - cp.bn_prefix = kv_str_arr(gguf_, "voicedetect.campplus.bn_prefix"); - cp.emb_bn_mean = kv_str(gguf_, "voicedetect.campplus.emb_bn_mean"); - cp.emb_bn_var = kv_str(gguf_, "voicedetect.campplus.emb_bn_var"); - cp.bn_eps = kv_f32(gguf_, "voicedetect.campplus.bn_eps", 1e-5f); + cp.conv_weight = kv_str_arr(kv, "voicedetect.campplus.conv_weight"); + cp.conv_bias = kv_str_arr(kv, "voicedetect.campplus.conv_bias"); + cp.bn_prefix = kv_str_arr(kv, "voicedetect.campplus.bn_prefix"); + cp.emb_bn_mean = kv_str(kv, "voicedetect.campplus.emb_bn_mean"); + cp.emb_bn_var = kv_str(kv, "voicedetect.campplus.emb_bn_var"); + cp.bn_eps = kv_f32(kv, "voicedetect.campplus.bn_eps", 1e-5f); } // Analyze heads (phased; absent -> present=false, engine skips analyze). - cfg_.analyze_present = kv_bool(gguf_, "voicedetect.analyze.present", false); + cfg_.analyze_present = kv_bool(kv, "voicedetect.analyze.present", false); if(cfg_.analyze_present){ - cfg_.emotion_labels = kv_str_arr(gguf_, "voicedetect.analyze.emotion_labels"); - cfg_.gender_labels = kv_str_arr(gguf_, "voicedetect.analyze.gender_labels"); + cfg_.emotion_labels = kv_str_arr(kv, "voicedetect.analyze.emotion_labels"); + cfg_.gender_labels = kv_str_arr(kv, "voicedetect.analyze.gender_labels"); } // wav2vec2 analyze config (only present for the analyze GGUF). All dims/kernels/ // strides live in KV so the C++ analyze graph carries no magic numbers. Shared // by the emotion (base) and age/gender (large-robust) analyze archs. if(cfg_.arch == "wav2vec2_emotion" || cfg_.arch == "wav2vec2_age_gender"){ VoiceDetectConfig::W2V2Config& w = cfg_.w2v2; - w.hidden_size = kv_u32(gguf_, "voicedetect.w2v2.hidden_size"); - w.n_layers = kv_u32(gguf_, "voicedetect.w2v2.n_layers"); - w.n_heads = kv_u32(gguf_, "voicedetect.w2v2.n_heads"); - w.ff_dim = kv_u32(gguf_, "voicedetect.w2v2.ff_dim"); - w.num_conv_layers = kv_u32(gguf_, "voicedetect.w2v2.num_conv_layers"); - w.conv_dims = kv_i32_arr(gguf_, "voicedetect.w2v2.conv_dims"); - w.conv_kernels = kv_i32_arr(gguf_, "voicedetect.w2v2.conv_kernels"); - w.conv_strides = kv_i32_arr(gguf_, "voicedetect.w2v2.conv_strides"); - w.feat_extract_norm = kv_str(gguf_, "voicedetect.w2v2.feat_extract_norm"); - w.conv_bias = kv_bool(gguf_, "voicedetect.w2v2.conv_bias", false); - w.do_normalize = kv_bool(gguf_, "voicedetect.w2v2.do_normalize", false); - w.feat_extract_activation = kv_str(gguf_, "voicedetect.w2v2.feat_extract_activation"); - w.hidden_act = kv_str(gguf_, "voicedetect.w2v2.hidden_act"); - w.do_stable_layer_norm = kv_bool(gguf_, "voicedetect.w2v2.do_stable_layer_norm", false); - w.layer_norm_eps = kv_f32(gguf_, "voicedetect.w2v2.layer_norm_eps", 1e-5f); - w.num_conv_pos_embeddings = kv_u32(gguf_, "voicedetect.w2v2.num_conv_pos_embeddings"); + w.hidden_size = kv_u32(kv, "voicedetect.w2v2.hidden_size"); + w.n_layers = kv_u32(kv, "voicedetect.w2v2.n_layers"); + w.n_heads = kv_u32(kv, "voicedetect.w2v2.n_heads"); + w.ff_dim = kv_u32(kv, "voicedetect.w2v2.ff_dim"); + w.num_conv_layers = kv_u32(kv, "voicedetect.w2v2.num_conv_layers"); + w.conv_dims = kv_i32_arr(kv, "voicedetect.w2v2.conv_dims"); + w.conv_kernels = kv_i32_arr(kv, "voicedetect.w2v2.conv_kernels"); + w.conv_strides = kv_i32_arr(kv, "voicedetect.w2v2.conv_strides"); + w.feat_extract_norm = kv_str(kv, "voicedetect.w2v2.feat_extract_norm"); + w.conv_bias = kv_bool(kv, "voicedetect.w2v2.conv_bias", false); + w.do_normalize = kv_bool(kv, "voicedetect.w2v2.do_normalize", false); + w.feat_extract_activation = kv_str(kv, "voicedetect.w2v2.feat_extract_activation"); + w.hidden_act = kv_str(kv, "voicedetect.w2v2.hidden_act"); + w.do_stable_layer_norm = kv_bool(kv, "voicedetect.w2v2.do_stable_layer_norm", false); + w.layer_norm_eps = kv_f32(kv, "voicedetect.w2v2.layer_norm_eps", 1e-5f); + w.num_conv_pos_embeddings = kv_u32(kv, "voicedetect.w2v2.num_conv_pos_embeddings"); w.num_conv_pos_embedding_groups = - kv_u32(gguf_, "voicedetect.w2v2.num_conv_pos_embedding_groups"); - w.pos_conv_weight_norm_dim = kv_u32(gguf_, "voicedetect.w2v2.pos_conv_weight_norm_dim", 2); - w.use_weighted_layer_sum = kv_bool(gguf_, "voicedetect.w2v2.use_weighted_layer_sum", false); - w.classifier_proj_size = kv_u32(gguf_, "voicedetect.w2v2.classifier_proj_size"); + kv_u32(kv, "voicedetect.w2v2.num_conv_pos_embedding_groups"); + w.pos_conv_weight_norm_dim = kv_u32(kv, "voicedetect.w2v2.pos_conv_weight_norm_dim", 2); + w.use_weighted_layer_sum = kv_bool(kv, "voicedetect.w2v2.use_weighted_layer_sum", false); + w.classifier_proj_size = kv_u32(kv, "voicedetect.w2v2.classifier_proj_size"); + } + if(!kv.err.empty()) return fail_(kv.err); + // tensors (verbatim names). A prefixed load already stripped the prefix from + // the names in ctx_, so it walks ctx_ instead of the (prefixed) GGUF table. + if(ctx_names){ + for(ggml_tensor* t = ggml_get_first_tensor(ctx_); t; t = ggml_get_next_tensor(ctx_, t)) + tensors_[t->name] = t; + } else { + const int64_t nt = gguf_get_n_tensors(gguf_); + for(int64_t i=0;i0 || cfg_.analyze_present; + if(!(cfg_.embedding_dim>0 || cfg_.analyze_present)) + return fail_("GGUF is not a voicedetect model (no voicedetect.embedding_dim)"); + // The standalone GGUF context is only needed for the keys read above. + gguf_free(gguf_); gguf_ = nullptr; + return true; } ggml_tensor* ModelLoader::tensor(const std::string& n) const { diff --git a/src/model_loader.hpp b/src/model_loader.hpp index ad6f179..9117620 100644 --- a/src/model_loader.hpp +++ b/src/model_loader.hpp @@ -11,6 +11,7 @@ typedef struct ggml_backend_buffer* ggml_backend_buffer_t; struct ggml_backend; typedef struct ggml_backend* ggml_backend_t; namespace vd { +struct Kv; // Metadata-driven config for a speaker-recognition GGUF. All values live in GGUF // KV under the `voicedetect.*` prefix; tensor names are kept VERBATIM from the @@ -149,6 +150,19 @@ class ModelLoader { // Load a GGUF. Returns false if the file is absent/unreadable or required // KV is missing - tolerant of a not-yet-existing file (no throw, no abort). bool load(const std::string& path); + // Load from a GGUF held in memory (no file, no temporary file). The tensor + // data is copied into loader-owned memory during the call, so the caller may + // free or reuse `data` as soon as it returns. Peak memory is the buffer plus + // the model. Returns false (see error()) on a NULL/empty, truncated or + // corrupt buffer; never reads outside [data, data+size). + bool load_from_memory(const void* data, size_t size); + // As above for a model stored inside a larger GGUF (a bundle): every key and + // tensor name of the model is stored as `prefix` + name (for example + // "voice."). Only the tensors under the prefix are copied. Names inside + // metadata manifests are NOT prefixed and are used as they are. + bool load_from_memory(const void* data, size_t size, const std::string& prefix); + // Reason for the last failed load; "" after a successful one. + const std::string& error() const { return error_; } const VoiceDetectConfig& config() const { return cfg_; } ggml_tensor* tensor(const std::string& name) const; // nullptr if absent ggml_context* ggml_ctx() const { return ctx_; } @@ -161,6 +175,10 @@ class ModelLoader { bool realize_weights(ggml_backend_t backend); bool weights_realized() const { return weights_buf_ != nullptr; } private: + bool finish_(const Kv& kv, bool ctx_names = false); + bool fail_(const std::string& msg); + void reset_(); + std::string error_; VoiceDetectConfig cfg_; gguf_context* gguf_ = nullptr; ggml_context* ctx_ = nullptr; diff --git a/src/voicedetect_capi.cpp b/src/voicedetect_capi.cpp index 092285b..b1a2469 100644 --- a/src/voicedetect_capi.cpp +++ b/src/voicedetect_capi.cpp @@ -25,6 +25,21 @@ struct voicedetect_ctx { namespace { +// Load-time failures have no context to attach to yet. +thread_local std::string g_load_error; + +voicedetect_ctx* wrap_model(std::unique_ptr model, const std::string& err) { + if (!model) { + g_load_error = err.empty() ? "failed to load model" : err; + return nullptr; + } + auto* ctx = new (std::nothrow) voicedetect_ctx(); + if (!ctx) { g_load_error = "out of memory"; return nullptr; } + ctx->model = std::move(model); + g_load_error.clear(); + return ctx; +} + // malloc a copy of `s` (NUL-terminated) so a C consumer frees it with free() // (matching voicedetect_capi_free_string). Returns NULL on OOM. char* dup_to_c(const std::string& s) { @@ -68,20 +83,55 @@ extern "C" int voicedetect_capi_embedding_dim(voicedetect_ctx* ctx) { } extern "C" voicedetect_ctx* voicedetect_capi_load(const char* gguf_path) { - if (!gguf_path) return nullptr; + if (!gguf_path) { g_load_error = "model path is NULL"; return nullptr; } try { - std::unique_ptr model = vd::Model::load(gguf_path); - if (!model) return nullptr; // load failure (bad/missing GGUF) - auto* ctx = new (std::nothrow) voicedetect_ctx(); - if (!ctx) return nullptr; - ctx->model = std::move(model); - return ctx; + std::string err; + std::unique_ptr model = vd::Model::load(gguf_path, &err); + return wrap_model(std::move(model), err); + } catch (const std::exception& e) { + g_load_error = e.what(); + return nullptr; } catch (...) { // Never let an exception cross the boundary. + g_load_error = "unknown error"; return nullptr; } } +extern "C" voicedetect_ctx* voicedetect_capi_load_from_memory(const void* data, size_t size) { + try { + std::string err; + std::unique_ptr model = vd::Model::load_from_memory(data, size, &err); + return wrap_model(std::move(model), err); + } catch (const std::exception& e) { + g_load_error = e.what(); + return nullptr; + } catch (...) { + g_load_error = "unknown error"; + return nullptr; + } +} + +extern "C" voicedetect_ctx* voicedetect_capi_load_from_memory_prefixed(const void* data, size_t size, + const char* prefix) { + try { + std::string err; + std::unique_ptr model = + vd::Model::load_from_memory(data, size, prefix ? prefix : "", &err); + return wrap_model(std::move(model), err); + } catch (const std::exception& e) { + g_load_error = e.what(); + return nullptr; + } catch (...) { + g_load_error = "unknown error"; + return nullptr; + } +} + +extern "C" const char* voicedetect_capi_last_load_error(void) { + return g_load_error.c_str(); +} + extern "C" void voicedetect_capi_free(voicedetect_ctx* ctx) { delete ctx; // safe on nullptr; ~unique_ptr releases the model. } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6ae1792..29dc3ed 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,8 @@ vd_add_test(test_analyze) vd_add_test(test_age_gender) vd_add_test(test_blocked) vd_add_test(test_cache_safety) +vd_add_test(test_load_memory) +vd_add_test(test_load_memory_embed) # Manual benchmark target (not registered as a ctest; run directly). add_executable(bench_wespeaker bench_wespeaker.cpp) @@ -28,7 +30,7 @@ target_include_directories(bench_eres2net PRIVATE ${PROJECT_SOURCE_DIR}/src ${PR # Model/baseline-dependent tests read fixtures/baselines via paths relative to # the project root, and skip (exit 77) when the baseline env var is unset. -set_tests_properties(test_fbank test_encoder test_embed test_verify test_analyze test_age_gender test_cache_safety PROPERTIES +set_tests_properties(test_fbank test_encoder test_embed test_verify test_analyze test_age_gender test_cache_safety test_load_memory_embed PROPERTIES LABELS "model" WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) diff --git a/tests/gguf_bundle.hpp b/tests/gguf_bundle.hpp new file mode 100644 index 0000000..9a98954 --- /dev/null +++ b/tests/gguf_bundle.hpp @@ -0,0 +1,144 @@ +// Test helpers: build GGUF files and bundles entirely in memory (no files). +#pragma once +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include + +namespace vdtest { + +// One model to place in a bundle: a GGUF context plus the ggml context that +// owns its tensor data (as returned by gguf_init_from_*(no_alloc=false)). +struct Source { + std::string prefix; // "" = standalone + gguf_context* g; + ggml_context* ctx; +}; + +// Copy one key of `g` into `out` as `name`. Covers the types voice-detect uses. +inline void copy_kv(gguf_context* out, const gguf_context* g, int64_t i, const std::string& name) { + const char* k = name.c_str(); + switch (gguf_get_kv_type(g, i)) { + case GGUF_TYPE_UINT32: gguf_set_val_u32(out, k, gguf_get_val_u32(g, i)); break; + case GGUF_TYPE_INT32: gguf_set_val_i32(out, k, gguf_get_val_i32(g, i)); break; + case GGUF_TYPE_FLOAT32: gguf_set_val_f32(out, k, gguf_get_val_f32(g, i)); break; + case GGUF_TYPE_BOOL: gguf_set_val_bool(out, k, gguf_get_val_bool(g, i)); break; + case GGUF_TYPE_STRING: gguf_set_val_str(out, k, gguf_get_val_str(g, i)); break; + case GGUF_TYPE_UINT8: gguf_set_val_u8(out, k, gguf_get_val_u8(g, i)); break; + case GGUF_TYPE_UINT64: gguf_set_val_u64(out, k, gguf_get_val_u64(g, i)); break; + case GGUF_TYPE_ARRAY: { + const gguf_type at = gguf_get_arr_type(g, i); + const size_t n = gguf_get_arr_n(g, i); + if (at == GGUF_TYPE_STRING) { + std::vector v(n); + for (size_t j = 0; j < n; ++j) v[j] = gguf_get_arr_str(g, i, j); + gguf_set_arr_str(out, k, v.data(), n); + } else if (at != GGUF_TYPE_ARRAY) { + gguf_set_arr_data(out, k, at, gguf_get_arr_data(g, i), n); + } + break; + } + default: break; + } +} + +// Serialize `g` to bytes. `data_of(name)` returns the tensor bytes. +template +std::vector serialize(gguf_context* g, F data_of) { + const size_t meta = gguf_get_meta_size(g); + std::vector out(meta); + gguf_get_meta_data(g, out.data()); + const int64_t nt = gguf_get_n_tensors(g); + for (int64_t i = 0; i < nt; ++i) { + const size_t off = meta + gguf_get_tensor_offset(g, i); + const size_t sz = gguf_get_tensor_size(g, i); + if (out.size() < off + sz) out.resize(off + sz, 0); + std::memcpy(out.data() + off, data_of(gguf_get_tensor_name(g, i)), sz); + } + // The data section is a whole number of alignment units (ggml's file reader + // checks the size of the last tensor with padding). + const size_t al = gguf_get_alignment(g); + out.resize((out.size() + al - 1) / al * al, 0); + return out; +} + +// Build one GGUF holding every source, each under its prefix (a bundle), or a +// plain GGUF when there is one source with an empty prefix. +inline std::vector make_bundle(const std::vector& srcs) { + gguf_context* out = gguf_init_empty(); + ggml_init_params ip{ggml_tensor_overhead() * 4096 + 65536, nullptr, true}; + ggml_context* tctx = ggml_init(ip); + std::vector> data; + for (const Source& s : srcs) { + for (int64_t i = 0; i < gguf_get_n_kv(s.g); ++i) { + const std::string k = gguf_get_key(s.g, i); + if (k == "general.alignment") continue; + copy_kv(out, s.g, i, s.prefix + k); + } + for (int64_t i = 0; i < gguf_get_n_tensors(s.g); ++i) { + const char* nm = gguf_get_tensor_name(s.g, i); + ggml_tensor* src = ggml_get_tensor(s.ctx, nm); + ggml_tensor* t = ggml_new_tensor(tctx, src->type, GGML_MAX_DIMS, src->ne); + const std::string full = s.prefix + nm; + ggml_set_name(t, full.c_str()); + gguf_add_tensor(out, t); + data.emplace_back(full, src->data); + } + } + auto bytes = serialize(out, [&](const char* nm) -> const void* { + for (auto& d : data) if (d.first == nm) return d.second; + return nullptr; + }); + gguf_free(out); + ggml_free(tctx); + return bytes; +} + +// Tiny synthetic speaker model (arch wespeaker_resnet34 manifest, so the tensor +// names live in string-array metadata). Not runnable; enough to load. +struct Tiny { + gguf_context* g = nullptr; + ggml_context* ctx = nullptr; + ~Tiny() { if (g) gguf_free(g); if (ctx) ggml_free(ctx); } + Tiny(const Tiny&) = delete; + Tiny& operator=(const Tiny&) = delete; + // `seed` varies the weights; `name_tag` varies the tensor names. + Tiny(uint32_t seed, const char* name_tag, bool bad_type = false) { + ggml_init_params ip{1 << 20, nullptr, false}; + ctx = ggml_init(ip); + g = gguf_init_empty(); + gguf_set_val_str(g, "voicedetect.arch", "wespeaker_resnet34"); + if (bad_type) gguf_set_val_str(g, "voicedetect.embedding_dim", "256"); // wrong type + else gguf_set_val_u32(g, "voicedetect.embedding_dim", 256); + gguf_set_val_bool(g, "voicedetect.l2_normalize", true); + gguf_set_val_f32(g, "voicedetect.resnet.var_eps", 1e-8f); + const std::string w = std::string(name_tag) + ".w", b = std::string(name_tag) + ".b"; + gguf_set_val_str(g, "voicedetect.resnet.stem_weight", w.c_str()); + gguf_set_val_str(g, "voicedetect.resnet.stem_bias", b.c_str()); + const char* ws[] = {w.c_str()}; + const char* bs[] = {b.c_str()}; + gguf_set_arr_str(g, "voicedetect.resnet.conv1_weight", ws, 1); + gguf_set_arr_str(g, "voicedetect.resnet.conv1_bias", bs, 1); + int32_t st[] = {1}; + gguf_set_arr_data(g, "voicedetect.resnet.stride", GGUF_TYPE_INT32, st, 1); + ggml_tensor* tw = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 3, 3, 2, 5); + ggml_tensor* tb = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 5); + ggml_tensor* tq = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, 64, 3); + ggml_set_name(tw, w.c_str()); + ggml_set_name(tb, b.c_str()); + const std::string q = std::string(name_tag) + ".q"; + ggml_set_name(tq, q.c_str()); + uint32_t x = seed * 2654435761u + 1; + for (ggml_tensor* t : {tw, tb, tq}) { + uint8_t* p = (uint8_t*)t->data; + for (size_t i = 0; i < ggml_nbytes(t); ++i) { x = x * 1664525u + 1013904223u; p[i] = (uint8_t)(x >> 24); } + gguf_add_tensor(g, t); + } + } +}; + +} // namespace vdtest diff --git a/tests/test_load_memory.cpp b/tests/test_load_memory.cpp new file mode 100644 index 0000000..d4626ef --- /dev/null +++ b/tests/test_load_memory.cpp @@ -0,0 +1,296 @@ +// Loading a model from a memory buffer (no model files needed: the GGUFs are +// synthesized in memory, so this runs in CI). +// +// * same tensors and config through the path, memory and prefixed loaders +// * the bundle's other component is not loaded +// * the buffer is not referenced after the call (poisoned, freed) +// * NULL, empty, truncated (every length) and corrupt buffers are rejected +// with a message, never a crash (run under ASan/UBSan to prove no OOB read) +// * concurrent loads +#include "gguf_bundle.hpp" + +#include "model_loader.hpp" +#include "model.hpp" +#include "voicedetect_capi.h" + +#include +#include +#include +#include +#include +#include + +using namespace vdtest; + +static int g_fail = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); ++g_fail; } } while (0) + +static const char* kNames[] = {"stem.w", "stem.b", "stem.q"}; + +// Compare the three tensors of a loader with those of a Tiny. +static bool same_tensors(const vd::ModelLoader& L, const Tiny& t, const char* tag) { + for (const char* suffix : {".w", ".b", ".q"}) { + const std::string nm = std::string(tag) + suffix; + ggml_tensor* a = L.tensor(nm); + ggml_tensor* b = ggml_get_tensor(t.ctx, nm.c_str()); + if (!a || !b || a->type != b->type || ggml_nbytes(a) != ggml_nbytes(b)) return false; + for (int d = 0; d < GGML_MAX_DIMS; ++d) if (a->ne[d] != b->ne[d]) return false; + if (std::memcmp(a->data, b->data, ggml_nbytes(a)) != 0) return false; + } + return true; +} + +static bool config_ok(const vd::ModelLoader& L, const char* tag) { + const auto& c = L.config(); + return c.arch == "wespeaker_resnet34" && c.embedding_dim == 256 && c.resnet.conv1_weight.size() == 1 && + c.resnet.conv1_weight[0] == std::string(tag) + ".w" && c.resnet.stem_bias == std::string(tag) + ".b" && + c.resnet.stride.size() == 1 && L.tensor(c.resnet.conv1_weight[0]) != nullptr; +} + +static std::string write_tmp(const std::vector& b) { + std::string p = std::string("vd_test_load_memory_") + std::to_string((long)std::random_device()()) + ".gguf"; + FILE* f = std::fopen(p.c_str(), "wb"); + if (f) { std::fwrite(b.data(), 1, b.size(), f); std::fclose(f); } + return p; +} + +static void test_equivalence() { + Tiny voice(1, "stem"), other(2, "other"); + auto solo = make_bundle({{"", voice.g, voice.ctx}}); + auto bundle = make_bundle({{"other.", other.g, other.ctx}, {"voice.", voice.g, voice.ctx}}); + + // Path loader (reference) + std::string path = write_tmp(solo); + vd::ModelLoader P; + CHECK(P.load(path)); + std::remove(path.c_str()); + CHECK(same_tensors(P, voice, "stem") && config_ok(P, "stem")); + + // Memory loader; buffer is poisoned and freed before anything is compared. + { + auto* heap = new std::vector(solo); + vd::ModelLoader M; + CHECK(M.load_from_memory(heap->data(), heap->size())); + std::memset(heap->data(), 0xAB, heap->size()); + delete heap; + CHECK(same_tensors(M, voice, "stem") && config_ok(M, "stem")); + CHECK(M.error().empty()); + CHECK(!M.load_from_memory(solo.data(), solo.size())); // second load refused + } + // Prefixed loader on the whole bundle. + { + auto* heap = new std::vector(bundle); + vd::ModelLoader B; + CHECK(B.load_from_memory(heap->data(), heap->size(), "voice.")); + std::memset(heap->data(), 0xCD, heap->size()); + delete heap; + CHECK(same_tensors(B, voice, "stem") && config_ok(B, "stem")); + CHECK(B.tensor("other.stem.w") == nullptr && B.tensor("voice.stem.w") == nullptr); + CHECK(B.tensor("other.w") == nullptr); + // the other component loads on its own from the same bundle + vd::ModelLoader O; + CHECK(O.load_from_memory(bundle.data(), bundle.size(), "other.")); + CHECK(same_tensors(O, other, "other") && config_ok(O, "other")); + CHECK(!same_tensors(O, voice, "stem")); + } + // Empty prefix == standalone + { + vd::ModelLoader E; + CHECK(E.load_from_memory(solo.data(), solo.size(), "")); + CHECK(same_tensors(E, voice, "stem")); + } + // Weights can be realized on the CPU backend after a prefixed load (the + // tensors are laid out in one contiguous ctx buffer). + // (covered by the embedding test with real models) + (void)kNames; +} + +static void test_bad_inputs() { + Tiny voice(1, "stem"), other(2, "other"); + auto solo = make_bundle({{"", voice.g, voice.ctx}}); + auto bundle = make_bundle({{"other.", other.g, other.ctx}, {"voice.", voice.g, voice.ctx}}); + std::string msg; + { + vd::ModelLoader L; + CHECK(!L.load_from_memory(nullptr, 100)); CHECK(!L.error().empty()); + CHECK(!L.load_from_memory(solo.data(), 0)); CHECK(!L.error().empty()); + CHECK(!L.load_from_memory(solo.data(), 1)); + CHECK(!L.load_from_memory(nullptr, 0, "voice.")); + CHECK(!L.load_from_memory(solo.data(), 0, "voice.")); + } + // wrong / partial / unknown prefix + for (const char* pre : {"nope.", "voic", "voice", "v", "voice.stem.", "other.stem.", "VOICE."}) { + vd::ModelLoader L; + bool ok = L.load_from_memory(bundle.data(), bundle.size(), pre); + CHECK(!ok); + CHECK(!L.error().empty()); + } + { vd::ModelLoader L; CHECK(!L.load_from_memory(solo.data(), solo.size(), "voice.")); } // solo has no prefix + { vd::ModelLoader L; CHECK(!L.load_from_memory(bundle.data(), bundle.size())); } // bundle is not a model unprefixed + + // wrong key type must be an error, not an abort in ggml + { + Tiny bad(3, "stem", /*bad_type=*/true); + auto b = make_bundle({{"", bad.g, bad.ctx}}); + vd::ModelLoader L; + CHECK(!L.load_from_memory(b.data(), b.size())); + CHECK(L.error().find("unexpected type") != std::string::npos); + auto bb = make_bundle({{"voice.", bad.g, bad.ctx}}); + vd::ModelLoader L2; + CHECK(!L2.load_from_memory(bb.data(), bb.size(), "voice.")); + CHECK(L2.error().find("unexpected type") != std::string::npos); + } + + // Truncation at EVERY length, both loaders. `voice.` is the last component, + // so any cut removes bytes the prefixed load needs. + // Only the alignment padding after the last tensor may be missing. + auto first_ok = [&](const std::vector& full, const char* prefix) { + size_t first = full.size() + 1; + for (size_t len = 0; len <= full.size(); ++len) { + std::vector cut(full.begin(), full.begin() + len); // exact-size heap block: ASan sees any overread + vd::ModelLoader L; + bool ok = prefix ? L.load_from_memory(cut.data(), cut.size(), prefix) + : L.load_from_memory(cut.data(), cut.size()); + if (!ok) CHECK(!L.error().empty()); + if (ok && first > len) first = len; + if (!ok) CHECK(first > len); // once a length loads, every longer one does too + } + return first; + }; + const size_t fs = first_ok(solo, nullptr), fb = first_ok(bundle, "voice."); + CHECK(fs <= solo.size() && solo.size() - fs < gguf_get_alignment(voice.g)); + CHECK(fb <= bundle.size() && bundle.size() - fb < gguf_get_alignment(voice.g)); + // Extra trailing bytes are harmless. + { auto big = solo; big.resize(big.size() + 100, 0); vd::ModelLoader L; CHECK(L.load_from_memory(big.data(), big.size())); } + + // Targeted corruption of the bundle header. + auto patch64 = [&](size_t off, uint64_t v) { auto b = bundle; std::memcpy(b.data() + off, &v, 8); return b; }; + std::vector> bad; + { auto b = bundle; b[0] = 'X'; bad.push_back(b); } // magic + { auto b = bundle; b[4] = 99; bad.push_back(b); } // version + bad.push_back(patch64(8, ~0ull)); // n_tensors huge + bad.push_back(patch64(8, 1ull << 40)); + bad.push_back(patch64(16, ~0ull)); // n_kv huge + bad.push_back(patch64(16, 1ull << 40)); + // tensor-info offset out of range: find the info record of "voice.stem.w" + { + const std::string nm = "voice.stem.w"; + auto it = std::search(bundle.begin(), bundle.end(), nm.begin(), nm.end()); + CHECK(it != bundle.end()); + size_t p = (it - bundle.begin()) + nm.size(); + uint32_t nd; std::memcpy(&nd, &bundle[p], 4); + size_t off_pos = p + 4 + 8 * nd + 4; + bad.push_back(patch64(off_pos, 1ull << 40)); + bad.push_back(patch64(off_pos, ~0ull)); + bad.push_back(patch64(off_pos, bundle.size())); + bad.push_back(patch64(off_pos, bundle.size() - 8)); + // dimension grows: size no longer fits the data + bad.push_back(patch64(p + 4, 1ull << 40)); + bad.push_back(patch64(p + 4, ~0ull)); + // type id out of range + { auto b = bundle; uint32_t t = 9999; std::memcpy(&b[p + 4 + 8 * nd], &t, 4); bad.push_back(b); } + } + for (size_t i = 0; i < bad.size(); ++i) { + std::vector b = bad[i]; + vd::ModelLoader L1, L2; + bool a = L1.load_from_memory(b.data(), b.size(), "voice."); + bool c = L2.load_from_memory(b.data(), b.size()); + if (a || c) std::fprintf(stderr, "corrupt case %zu accepted (prefixed=%d plain=%d)\n", i, a, c); + CHECK(!a); + CHECK(!c); + CHECK((!a && !L1.error().empty())); + } + + // A metadata key with an EMPTY name makes ggml's reader abort the process; + // the loader must refuse it first. Build the buffer by hand. + { + auto b = solo; + // first key is "voicedetect.arch" after the 24-byte header: len(8)+name + uint64_t zero = 0; + std::memcpy(b.data() + 24, &zero, 8); // key length 0 + vd::ModelLoader L1, L2; + CHECK(!L1.load_from_memory(b.data(), b.size())); + CHECK(!L2.load_from_memory(b.data(), b.size(), "voice.")); + CHECK(L1.error().find("empty name") != std::string::npos); + } + // Random byte flips across the header/table region and the data: must never + // crash; success is allowed only when the result is a valid model. + std::mt19937 rng(1234); + const size_t hdr = std::min(bundle.size(), 1500); + int accepted = 0; + for (int it = 0; it < 6000; ++it) { + std::vector b = bundle; + int flips = 1 + rng() % 4; + for (int f = 0; f < flips; ++f) b[rng() % hdr] ^= (uint8_t)(1u << (rng() % 8)) | (uint8_t)(rng() % 255); + vd::ModelLoader L; + if (L.load_from_memory(b.data(), b.size(), "voice.")) { + ++accepted; + CHECK(L.config().embedding_dim > 0 || L.config().analyze_present); + } + vd::ModelLoader L2; + (void)L2.load_from_memory(b.data(), b.size()); + } + std::fprintf(stderr, "fuzz: %d of 6000 mutated buffers still loaded as valid models\n", accepted); +} + +static void test_capi() { + Tiny voice(1, "stem"), other(2, "other"); + auto solo = make_bundle({{"", voice.g, voice.ctx}}); + auto bundle = make_bundle({{"other.", other.g, other.ctx}, {"voice.", voice.g, voice.ctx}}); + CHECK(voicedetect_capi_load_from_memory(nullptr, 10) == nullptr); + CHECK(std::strlen(voicedetect_capi_last_load_error()) > 0); + CHECK(voicedetect_capi_load_from_memory(solo.data(), 0) == nullptr); + CHECK(voicedetect_capi_load_from_memory(solo.data(), solo.size() / 2) == nullptr); + CHECK(voicedetect_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "nope.") == nullptr); + CHECK(std::string(voicedetect_capi_last_load_error()).find("nope.") != std::string::npos); + CHECK(voicedetect_capi_load("/nonexistent/model.gguf") == nullptr); + CHECK(std::strlen(voicedetect_capi_last_load_error()) > 0); + CHECK(voicedetect_capi_load(nullptr) == nullptr); + // These synthetic models load but cannot embed (no real weights); the + // context must still be valid, report errors cleanly, and free. + voicedetect_ctx* c = voicedetect_capi_load_from_memory(solo.data(), solo.size()); + CHECK(c != nullptr); + CHECK(std::strlen(voicedetect_capi_last_load_error()) == 0); + voicedetect_capi_free(c); + c = voicedetect_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "voice."); + CHECK(c != nullptr); + voicedetect_capi_free(c); + c = voicedetect_capi_load_from_memory_prefixed(solo.data(), solo.size(), nullptr); // NULL prefix == plain + CHECK(c != nullptr); + voicedetect_capi_free(c); + voicedetect_capi_free(nullptr); +} + +static void test_threads() { + Tiny voice(1, "stem"), other(2, "other"); + auto bundle = make_bundle({{"other.", other.g, other.ctx}, {"voice.", voice.g, voice.ctx}}); + auto solo = make_bundle({{"", voice.g, voice.ctx}}); + std::atomic bad{0}; + std::vector th; + for (int t = 0; t < 8; ++t) + th.emplace_back([&, t] { + for (int i = 0; i < 30; ++i) { + vd::ModelLoader L; + // each thread also loads from its own private copy of the buffer + std::vector mine = (t & 1) ? bundle : solo; + bool ok = (t & 1) ? L.load_from_memory(mine.data(), mine.size(), "voice.") + : L.load_from_memory(mine.data(), mine.size()); + std::fill(mine.begin(), mine.end(), 0); + if (!ok || !same_tensors(L, voice, "stem") || !config_ok(L, "stem")) ++bad; + // a failing load on the same thread must not disturb the others + voicedetect_ctx* x = voicedetect_capi_load_from_memory(solo.data(), 7); + if (x) ++bad; + } + }); + for (auto& x : th) x.join(); + CHECK(bad == 0); +} + +int main() { + test_equivalence(); + test_bad_inputs(); + test_capi(); + test_threads(); + std::fprintf(stderr, g_fail ? "FAILED (%d)\n" : "PASS\n", g_fail); + return g_fail ? 1 : 0; +} diff --git a/tests/test_load_memory_embed.cpp b/tests/test_load_memory_embed.cpp new file mode 100644 index 0000000..bc068ab --- /dev/null +++ b/tests/test_load_memory_embed.cpp @@ -0,0 +1,113 @@ +// A real model loaded by path, from memory and from a bundle (with a prefix) +// must give bitwise-identical embeddings. The memory buffers are poisoned and +// freed before the first embedding runs. +// +// Needs VOICEDETECT_TEST_GGUF (speaker encoder GGUF) and VOICEDETECT_TEST_AUDIO +// (WAV). Exits 77 (skip) when unset. +#include "gguf_bundle.hpp" + +#include "backend.hpp" +#include "model.hpp" +#include "voicedetect_capi.h" + +#include +#include + +using namespace vdtest; + +static int g_fail = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); ++g_fail; } } while (0) + +static bool bitwise_equal(const std::vector& a, const std::vector& b) { + return !a.empty() && a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0; +} + +static std::vector read_file(const char* p) { + std::vector b; + FILE* f = std::fopen(p, "rb"); + if (!f) return b; + uint8_t tmp[1 << 16]; + size_t n; + while ((n = std::fread(tmp, 1, sizeof tmp, f)) > 0) b.insert(b.end(), tmp, tmp + n); + std::fclose(f); + return b; +} + +int main() { + const char* gguf = std::getenv("VOICEDETECT_TEST_GGUF"); + const char* audio = std::getenv("VOICEDETECT_TEST_AUDIO"); + if (!gguf || !audio) return 77; + + std::string err; + auto ref = vd::Model::load(gguf, &err); + if (!ref) { std::fprintf(stderr, "load failed: %s\n", err.c_str()); return 77; } + const std::vector e_path = ref->embed_path(audio); + CHECK((int)e_path.size() == ref->embedding_dim()); + + // Memory, unprefixed. + { + auto* buf = new std::vector(read_file(gguf)); + CHECK(!buf->empty()); + auto m = vd::Model::load_from_memory(buf->data(), buf->size(), &err); + std::memset(buf->data(), 0xAB, buf->size()); + delete buf; // freed before the embedding runs + CHECK(m != nullptr); + if (m) CHECK(bitwise_equal(m->embed_path(audio), e_path)); + } + + // Prefixed, inside a bundle next to another component. + { + ggml_context* ctx = nullptr; + gguf_init_params ip{false, &ctx}; + gguf_context* g = gguf_init_from_file(gguf, ip); + CHECK(g != nullptr); + Tiny other(2, "other"); + auto bundle = make_bundle({{"other.", other.g, other.ctx}, {"v.", g, ctx}}); + gguf_free(g); + ggml_free(ctx); + auto* buf = new std::vector(std::move(bundle)); + auto m = vd::Model::load_from_memory(buf->data(), buf->size(), "v.", &err); + std::memset(buf->data(), 0xCD, buf->size()); + delete buf; + CHECK(m != nullptr); + if (m) CHECK(bitwise_equal(m->embed_path(audio), e_path)); + // wrong prefix on the same data + auto b2 = vd::Model::load_from_memory(nullptr, 0, "v.", &err); + CHECK(b2 == nullptr && !err.empty()); + } + + // C API, memory + path equality (embed_pcm on the same samples) + { + std::vector buf = read_file(gguf); + voicedetect_ctx* c = voicedetect_capi_load_from_memory(buf.data(), buf.size()); + CHECK(c != nullptr); + std::fill(buf.begin(), buf.end(), 0xEE); + buf.clear(); buf.shrink_to_fit(); + voicedetect_ctx* p = voicedetect_capi_load(gguf); + CHECK(p != nullptr); + if (c && p) { + float *vc = nullptr, *vp = nullptr; int dc = 0, dp = 0; + CHECK(voicedetect_capi_embed_path(c, audio, &vc, &dc) == 0); + CHECK(voicedetect_capi_embed_path(p, audio, &vp, &dp) == 0); + CHECK(dc == dp && dc > 0 && std::memcmp(vc, vp, dc * sizeof(float)) == 0); + voicedetect_capi_free_vec(vc); + voicedetect_capi_free_vec(vp); + } + voicedetect_capi_free(c); + voicedetect_capi_free(p); + } + + // Truncated real model: clear failure at a handful of lengths. + { + std::vector full = read_file(gguf); + for (double f : {0.0, 0.001, 0.01, 0.1, 0.5, 0.9, 0.999}) { + std::vector cut(full.begin(), full.begin() + (size_t)(full.size() * f)); + std::string e; + auto m = vd::Model::load_from_memory(cut.data(), cut.size(), &e); + CHECK(m == nullptr && !e.empty()); + } + } + vd::shutdown_backend(); + std::fprintf(stderr, g_fail ? "FAILED (%d)\n" : "PASS (embedding dim %d)\n", g_fail ? g_fail : (int)e_path.size()); + return g_fail ? 1 : 0; +}