chore(vendor): track zvec v0.8.1; feat: adapt to grouped search and IVF PQ

Cargo.toml

@@ -4,7 +4,7 @@ exclude = ["fuzz"]
 resolver = "2"
 
 [workspace.package]
-version = "0.8.0"
+version = "0.8.1"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/zvec-ai/zvec-rust"

Makefile

@@ -3,6 +3,7 @@
        bench coverage coverage-html coverage-lcov coverage-text \
        fuzz fuzz-types fuzz-schema fuzz-doc fuzz-query fuzz-collection \
        example example-basic example-schema example-search example-crud example-config \
+       example-group-by \
        lint setup help \
        submodule-init submodule-update zvec-build zvec-clean zvec-rebuild
 
@@ -158,6 +159,9 @@ example-crud:  ## Run crud_operations example
 example-config:  ## Run config_logging example
 	$(CARGO) run --example config_logging
 
+example-group-by:  ## Run group_by_search example
+	$(CARGO) run --example group_by_search
+
 # ---------------------------------------------------------------------------
 # Documentation
 # ---------------------------------------------------------------------------

README.md

@@ -17,6 +17,7 @@ Safe, idiomatic Rust bindings for the [zvec](https://github.com/alibaba/zvec) ve
 - **Zero-Copy Where Possible** — Minimizes data copying across the FFI boundary
 - **Prebuilt Libraries** — Automatically downloads prebuilt `libzvec_c_api` from GitHub Releases; advanced users can override with `ZVEC_LIB_DIR`
 - **FTS with jieba Out of the Box** — `libzvec_c_api` embeds cppjieba's default dictionary and HMM model (LZ4-compressed), so the `jieba` FTS tokenizer needs no dictionary files on disk and no registration step
+- **Grouped Vector Search** — `collection.group_by_query(&query)` returns the best matches per distinct value of a scalar field, so one dense category cannot sweep the whole result page
 
 ## Supported Platforms
 
@@ -41,10 +42,11 @@ Safe, idiomatic Rust bindings for the [zvec](https://github.com/alibaba/zvec) ve
 
 > The dynamic library name varies by platform: `libzvec_c_api.dylib` (macOS), `libzvec_c_api.so` (Linux), `zvec_c_api.dll` (Windows).
 
-> The Windows ARM64 prebuilt ships without RaBitQ and DiskANN: upstream zvec
-> gates RaBitQ on x86_64 (its kernels are AVX2/AVX-512) and its Windows
-> DiskANN backend on x86_64 too, so those index types are unavailable on that
-> target.
+> The Windows ARM64 prebuilt ships without RaBitQ: since zvec v0.8.1 upstream
+> builds RaBitQ for ARM64 (NEON) on Linux, macOS, Android and iOS and for
+> x86-64 (runtime AVX2/AVX-512 dispatch) on Linux, macOS and Windows, but not
+> for Windows ARM64, so that index type is unavailable on that target. DiskANN
+> is built for every prebuilt target listed above.
 
 ## Architecture
 
@@ -68,7 +70,7 @@ Add `zvec-rust` to your `Cargo.toml`. The default `bundled` feature automaticall
 
 ```toml
 [dependencies]
-zvec-rust = "0.8.0"
+zvec-rust = "0.8.1"
 ```
 
 `cargo run` / `cargo test` work out of the box because Cargo passes the
@@ -81,10 +83,10 @@ the executable:
 
 ```toml
 [dependencies]
-zvec-rust = "0.8.0"
+zvec-rust = "0.8.1"
 
 [build-dependencies]
-zvec-rust-build = "0.8.0"
+zvec-rust-build = "0.8.1"
 ```
 
 ```rust
@@ -229,6 +231,7 @@ Run any example with `cargo run --example <name>`:
 | `basic` | End-to-end workflow: schema → insert → query → fetch → delete |
 | `schema_builder` | Various schema configurations: field types, index types, quantization |
 | `vector_search` | Vector query patterns: simple, builder, filter, output fields, HNSW params |
+| `group_by_search` | Grouped vector search: best matches per category, filters, and a flat top-k comparison |
 | `crud_operations` | Full CRUD: insert, fetch, update, upsert, delete, stats, flush |
 | `config_logging` | Library configuration: memory limits, thread counts, logging |
 
@@ -313,6 +316,7 @@ let schema = CollectionSchema::builder("name")
 | `collection.delete(&pks)` | Delete by primary keys |
 | `collection.delete_by_filter(filter)` | Delete documents matching a filter expression |
 | `collection.query(&query)` | Scalar filtering, full-text, or vector similarity search |
+| `collection.group_by_query(&query)` | Vector search grouped by a scalar field (best matches per group) |
 | `collection.multi_query(&query)` | Multi-route search with RRF / weighted rerank |
 | `collection.fetch(&pks)` | Fetch by primary keys |
 | `collection.fetch_with_options(&pks, fields, include_vector)` | Fetch with output-field control |
@@ -385,6 +389,31 @@ vector query (adding an FTS clause makes it a hybrid search), an FTS clause
 without a vector builds a keyword-only query, and neither builds a scalar query.
 `SearchQuery::fts(field, fts, topk)` is the direct keyword-only constructor.
 
+### Grouped Search
+
+`GroupBySearchQuery` runs a vector search grouped by a scalar field: instead of one flat top-k
+list it returns the best matches for every distinct value of that field.
+
+```rust
+// The 2 best matches in each of up to 3 categories
+let mut query = GroupBySearchQuery::new("embedding", "category", &query_vec, 3, 2)?;
+query.set_output_fields(&["title", "price"])?;
+query.set_filter("price <= 100.0")?;   // applied before grouping
+query.set_include_vector(true)?;       // return the vectors as well
+
+for group in collection.group_by_query(&query)? {
+    println!("{}: {} hit(s)", group.group_by_value, group.docs.len());
+    for doc in &group.docs {
+        println!("  {} scored {:.4}", doc.get_pk().unwrap_or(""), doc.get_score());
+    }
+}
+```
+
+Groups come back ordered by their best match, each holding at most `group_topk` documents in
+similarity order. A group the filter empties out is dropped rather than returned empty, and a
+query that matches nothing returns an empty `Vec` instead of an error. The documents are owned
+by Rust, so the groups stay usable after the query — and the collection — are dropped.
+
 ### Multi-Query (Hybrid Search)
 
 `MultiQuery` combines multiple sub-queries (dense vector, sparse vector, or FTS) with RRF or weighted reranking:
@@ -434,15 +463,35 @@ Available distance metrics: `L2`, `Ip`, `Cosine`, `MipsL2`.
 | HNSW | `IndexParams::hnsw(metric, m, ef)` | Graph index (recommended) |
 | HNSW+Q | `IndexParams::hnsw_with_quantize(...)` | HNSW with quantization |
 | IVF | `IndexParams::ivf(metric, nlist, niters, soar)` | Inverted file index |
+| IVF PQ | `IndexParams::ivf(...)` + `set_quantize_type(Pq)` + `set_quantizer_pq(&pq)` | IVF compressed with PQ4/PQ8 product quantization, optionally learned OPQ |
 | IVF RaBitQ | `IndexParams::ivf_rabitq(metric, nlist, total_bits, sample_count)` | IVF with RaBitQ quantization |
 | Flat | `IndexParams::flat(metric)` | Brute-force index |
-| DiskANN | `IndexParams::diskann(metric, max_degree, list_size, pq_chunk_num)` | Disk-based graph index for large datasets (Linux x86_64/ARM64, macOS ARM64, Windows x86_64, 64-bit Android/iOS) |
+| DiskANN | `IndexParams::diskann(metric, max_degree, list_size, pq_chunk_num)` | Disk-based graph index for large datasets (Linux x86_64/ARM64, macOS x86_64/ARM64, Windows x86_64/ARM64, 64-bit Android/iOS) |
 | Invert | `IndexParams::invert(range, wildcard)` | Scalar field index |
 | FTS | `IndexParams::fts(tokenizer, filters, extra)` | Full-text search index |
 
 With `Int8` / `Int4` quantization, `params.set_quantizer_enable_rotate(true)` randomly rotates
-vectors before quantization to reduce the quantization error. The rotation matrix is stored with
-the index and applied to query vectors at search time.
+vectors before quantization to reduce the quantization error; with `Pq` the same flag selects a
+learned OPQ rotation instead. The rotation matrix is stored with the index and applied to query
+vectors at search time.
+
+An IVF index can also be compressed with product quantization. Set the quantize type to
+`QuantizeType::Pq`, then describe the layout with `PqQuantizerParams`:
+
+```rust
+let mut params = IndexParams::ivf(MetricType::L2, 100, 10, false)?;
+params.set_quantize_type(QuantizeType::Pq)?;
+params.set_quantizer_pq(&PqQuantizerParams {
+    num_chunk: 16,        // the vector dimension must be divisible by it
+    num_bits: 8,          // 4 or 8 bits per sub-quantizer code
+    fast_scan: false,     // SIMD layout, requires num_bits == 4
+    enable_rotate: true,  // learned OPQ instead of plain PQ
+    ..Default::default()  // opq_iter: 5, opq_pq_iter: 4
+})?;
+```
+
+PQ supports FP32 IVF fields with the `L2`, `Ip` and `Cosine` metrics; the dimension is checked
+against `num_chunk` when the collection is created.
 
 ## Testing
 

README_CN.md

@@ -17,6 +17,7 @@
 - **尽可能零拷贝** — 最小化 FFI 边界的数据拷贝
 - **预编译库支持** — 自动从 GitHub Releases 下载预编译的 `libzvec_c_api`；高阶用户可通过 `ZVEC_LIB_DIR` 覆盖
 - **FTS jieba 开箱即用** — `libzvec_c_api` 内嵌 cppjieba 默认词表与 HMM 模型（LZ4 压缩），`jieba` 分词器不再依赖磁盘上的词表文件，也无需注册步骤
+- **分组向量检索** — `collection.group_by_query(&query)` 按标量字段的每个取值分别返回最佳匹配，避免单一类别占满整个结果页
 
 ## 支持的平台
 
@@ -40,9 +41,10 @@
 
 > 动态库文件名因平台而异：`libzvec_c_api.dylib`（macOS）、`libzvec_c_api.so`（Linux）、`zvec_c_api.dll`（Windows）。
 
-> Windows ARM64 预编译产物不含 RaBitQ 与 DiskANN：上游 zvec 将 RaBitQ 限定在
-> x86_64（其内核为 AVX2/AVX-512），Windows 的 DiskANN 后端同样只支持 x86_64，
-> 因此该目标上这两类索引不可用。
+> Windows ARM64 预编译产物不含 RaBitQ：自 zvec v0.8.1 起，上游在 Linux、macOS、
+> Android、iOS 的 ARM64（NEON）以及 Linux、macOS、Windows 的 x86-64（运行时
+> AVX2/AVX-512 分发）上构建 RaBitQ，但不包含 Windows ARM64，因此该目标上这类
+> 索引不可用。DiskANN 在上表所有预编译目标上均可用。
 
 ## 架构
 
@@ -66,7 +68,7 @@ Rust SDK 依赖 zvec C 库（`libzvec_c_api`）。可通过以下任一方式提
 
 ```toml
 [dependencies]
-zvec-rust = "0.8.0"
+zvec-rust = "0.8.1"
 ```
 
 `cargo run` / `cargo test` 可开箱即用，因为 Cargo 会把解析出的库目录传给链接器。
@@ -77,10 +79,10 @@ zvec-rust = "0.8.0"
 
 ```toml
 [dependencies]
-zvec-rust = "0.8.0"
+zvec-rust = "0.8.1"
 
 [build-dependencies]
-zvec-rust-build = "0.8.0"
+zvec-rust-build = "0.8.1"
 ```
 
 ```rust
@@ -221,6 +223,7 @@ fn main() -> zvec_rust::Result<()> {
 | `basic` | 端到端流程：schema → 插入 → 查询 → 获取 → 删除 |
 | `schema_builder` | 各种 Schema 配置：字段类型、索引类型、量化 |
 | `vector_search` | 向量查询模式：简单查询、Builder、过滤器、输出字段、HNSW 参数 |
+| `group_by_search` | 分组向量检索：按类别取最佳匹配、过滤器，以及与扁平 top-k 的对比 |
 | `crud_operations` | 完整 CRUD：插入、获取、更新、upsert、删除、统计、刷新 |
 | `config_logging` | 库配置：内存限制、线程数、日志 |
 
@@ -303,6 +306,7 @@ let schema = CollectionSchema::builder("name")
 | `collection.delete(&pks)` | 按主键删除 |
 | `collection.delete_by_filter(filter)` | 按过滤表达式批量删除 |
 | `collection.query(&query)` | 标量过滤、全文搜索或向量相似度搜索 |
+| `collection.group_by_query(&query)` | 按标量字段分组的向量检索（每组取最佳匹配） |
 | `collection.multi_query(&query)` | 多路检索 + RRF / 加权重排 |
 | `collection.fetch(&pks)` | 按主键获取 |
 | `collection.fetch_with_options(&pks, fields, include_vector)` | 按主键获取并控制输出字段 |
@@ -373,6 +377,30 @@ Builder 会根据已设置的字段推断查询类型：设置 `vector(..)` 构
 两者都不设置则构建标量查询。纯关键词查询也可以直接用
 `SearchQuery::fts(field, fts, topk)` 构造。
 
+### 分组检索
+
+`GroupBySearchQuery` 按某个标量字段分组执行向量检索：返回的不是一个扁平的 top-k 列表，
+而是该字段每个取值下的最佳匹配。
+
+```rust
+// 最多 3 个类别，每个类别取最佳的 2 条
+let mut query = GroupBySearchQuery::new("embedding", "category", &query_vec, 3, 2)?;
+query.set_output_fields(&["title", "price"])?;
+query.set_filter("price <= 100.0")?;   // 分组前生效
+query.set_include_vector(true)?;       // 同时返回向量
+
+for group in collection.group_by_query(&query)? {
+    println!("{}: {} 条命中", group.group_by_value, group.docs.len());
+    for doc in &group.docs {
+        println!("  {} 得分 {:.4}", doc.get_pk().unwrap_or(""), doc.get_score());
+    }
+}
+```
+
+分组按组内最佳匹配排序返回，每组最多 `group_topk` 条文档并按相似度排序。被过滤器清空的组
+会被直接丢弃，而不是返回一个空组；没有任何命中时返回空 `Vec` 而不是错误。文档所有权归
+Rust，因此查询对象乃至集合被 drop 之后，分组结果依然可用。
+
 ### 多路检索（混合搜索）
 
 `MultiQuery` 可以组合多个子查询（稠密向量、稀疏向量或全文检索），并使用 RRF 或加权策略重排：
@@ -422,14 +450,34 @@ mq.set_rerank_weighted(&[0.7, 0.3])?;  // 每个子查询的权重
 | HNSW | `IndexParams::hnsw(metric, m, ef)` | 图索引（推荐） |
 | HNSW+量化 | `IndexParams::hnsw_with_quantize(...)` | 带量化的 HNSW |
 | IVF | `IndexParams::ivf(metric, nlist, niters, soar)` | 倒排文件索引 |
+| IVF PQ | `IndexParams::ivf(...)` + `set_quantize_type(Pq)` + `set_quantizer_pq(&pq)` | 使用 PQ4/PQ8 乘积量化压缩的 IVF 索引，可选学习式 OPQ |
 | IVF RaBitQ | `IndexParams::ivf_rabitq(metric, nlist, total_bits, sample_count)` | 带 RaBitQ 量化的 IVF 索引 |
 | Flat | `IndexParams::flat(metric)` | 暴力搜索索引 |
-| DiskANN | `IndexParams::diskann(metric, max_degree, list_size, pq_chunk_num)` | 基于磁盘的图索引，适合大规模数据集（Linux x86_64/ARM64、macOS ARM64、Windows x86_64、64 位 Android/iOS） |
+| DiskANN | `IndexParams::diskann(metric, max_degree, list_size, pq_chunk_num)` | 基于磁盘的图索引，适合大规模数据集（Linux x86_64/ARM64、macOS x86_64/ARM64、Windows x86_64/ARM64、64 位 Android/iOS） |
 | Invert | `IndexParams::invert(range, wildcard)` | 标量字段索引 |
 | FTS | `IndexParams::fts(tokenizer, filters, extra)` | 全文检索索引 |
 
 使用 `Int8` / `Int4` 量化时，`params.set_quantizer_enable_rotate(true)` 会在量化前对向量做随机旋转，
-以降低量化误差。旋转矩阵随索引一起保存，并在检索时自动作用于查询向量。
+以降低量化误差；使用 `Pq` 时，同一开关改为启用学习式 OPQ 旋转。旋转矩阵随索引一起保存，
+并在检索时自动作用于查询向量。
+
+IVF 索引还可以用乘积量化（PQ）压缩：先把量化类型设为 `QuantizeType::Pq`，再用
+`PqQuantizerParams` 描述编码布局：
+
+```rust
+let mut params = IndexParams::ivf(MetricType::L2, 100, 10, false)?;
+params.set_quantize_type(QuantizeType::Pq)?;
+params.set_quantizer_pq(&PqQuantizerParams {
+    num_chunk: 16,        // 向量维度必须能被它整除
+    num_bits: 8,          // 每个子量化器 4 或 8 bit
+    fast_scan: false,     // SIMD 布局，要求 num_bits == 4
+    enable_rotate: true,  // 使用学习式 OPQ 而非普通 PQ
+    ..Default::default()  // opq_iter: 5, opq_pq_iter: 4
+})?;
+```
+
+PQ 支持使用 `L2`、`Ip`、`Cosine` 度量的 FP32 IVF 字段；创建集合时会校验向量维度与
+`num_chunk` 是否匹配。
 
 ## 测试
 

zvec-sys/src/lib.rs

@@ -349,6 +349,8 @@ pub const ZVEC_QUANTIZE_TYPE_INT8: zvec_quantize_type_t = 2;
 pub const ZVEC_QUANTIZE_TYPE_INT4: zvec_quantize_type_t = 3;
 /// RaBitQ quantization.
 pub const ZVEC_QUANTIZE_TYPE_RABITQ: zvec_quantize_type_t = 4;
+/// Product quantization (PQ), optionally with learned OPQ rotation.
+pub const ZVEC_QUANTIZE_TYPE_PQ: zvec_quantize_type_t = 8;
 
 // =============================================================================
 // Log Level Constants
@@ -462,6 +464,31 @@ pub struct zvec_write_result_t {
     pub message: *const c_char,
 }
 
+/// Product quantization (PQ) configuration of an IVF index.
+///
+/// The C default initializer `ZVEC_PQ_QUANTIZER_PARAMS_DEFAULT` expands to
+/// `{8, 8, false, false, 5, 4}`.
+#[repr(C)]
+pub struct zvec_pq_quantizer_params_t {
+    pub num_chunk: c_int,
+    pub num_bits: c_int,
+    pub enable_rotate: bool,
+    pub fast_scan: bool,
+    pub opq_iter: u32,
+    pub opq_pq_iter: u32,
+}
+
+/// One group of grouped vector search results.
+///
+/// Every member is owned by the result array it came from: release the whole
+/// array with `zvec_group_results_free` and never free a member on its own.
+#[repr(C)]
+pub struct zvec_group_result_t {
+    pub group_by_value: *mut c_char,
+    pub docs: *mut *mut zvec_doc_t,
+    pub doc_count: usize,
+}
+
 /// Detailed error information including source location.
 #[repr(C)]
 pub struct zvec_error_details_t {
@@ -664,6 +691,14 @@ extern "C" {
     pub fn zvec_index_params_get_quantizer_enable_rotate(
         params: *const zvec_index_params_t,
     ) -> bool;
+    pub fn zvec_index_params_set_quantizer_pq(
+        params: *mut zvec_index_params_t,
+        pq: *const zvec_pq_quantizer_params_t,
+    ) -> zvec_error_code_t;
+    pub fn zvec_index_params_get_quantizer_pq(
+        params: *const zvec_index_params_t,
+        pq: *mut zvec_pq_quantizer_params_t,
+    ) -> zvec_error_code_t;
     pub fn zvec_index_params_set_hnsw_params(
         params: *mut zvec_index_params_t,
         m: c_int,
@@ -1087,6 +1122,13 @@ extern "C" {
         results: *mut *mut *mut zvec_doc_t,
         result_count: *mut usize,
     ) -> zvec_error_code_t;
+    pub fn zvec_collection_group_by_query(
+        collection: *const zvec_collection_t,
+        query: *const zvec_group_by_vector_query_t,
+        results: *mut *mut zvec_group_result_t,
+        result_count: *mut usize,
+    ) -> zvec_error_code_t;
+    pub fn zvec_group_results_free(results: *mut zvec_group_result_t, result_count: usize);
     pub fn zvec_collection_multi_query(
         collection: *const zvec_collection_t,
         query: *const zvec_multi_query_t,