From ea9d3ec5bc0bd436764a1c799fd926b214df7301 Mon Sep 17 00:00:00 2001 From: Mohamed Arbi Date: Tue, 24 Mar 2026 08:14:26 +0100 Subject: [PATCH 01/38] fix: pass timeout parameter to query_points in Qdrant clients (#725) --- vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py | 1 + vectordb_bench/backend/clients/qdrant_local/qdrant_local.py | 1 + 2 files changed, 2 insertions(+) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index a2c8f9020..e7d7054d9 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -205,6 +205,7 @@ def search_embedding( query_filter=self.query_filter, search_params=self.db_case_config.search_param(), with_payload=self.db_case_config.with_payload, + timeout=timeout, ) res = points_res.points diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py index d1cf736d9..15c790c61 100644 --- a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -227,6 +227,7 @@ def search_embedding( limit=k, query_filter=f, search_params=SearchParams(**self.search_parameter), + timeout=timeout, ).points return [result.id for result in res] From 99c311510648dff36f57960c2beeaae3ecb9ad61 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Tue, 31 Mar 2026 18:02:43 +0800 Subject: [PATCH 02/38] enhance: Migrate PyMilvus orm to MilvusClient (#738) * enhance: Migrate PyMilvus orm to MilvusClient * chore: add .worktrees/ to .gitignore Signed-off-by: yangxuan --- .gitignore | 3 + README.md | 13 +- pyproject.toml | 33 +-- tests/pytest.ini | 3 +- tests/test_bench_runner.py | 3 +- tests/test_milvus.py | 39 ++++ .../backend/clients/milvus/milvus.py | 204 +++++++++--------- 7 files changed, 147 insertions(+), 151 deletions(-) create mode 100644 tests/test_milvus.py diff --git a/.gitignore b/.gitignore index 3ddb942c4..8985eeb4d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ venv/ results/ logs/ +# Worktrees +.worktrees/ + # AI rules CLAUDE.md AGENTS.md diff --git a/README.md b/README.md index ef498c263..fe215e7a7 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,6 @@ python >= 3.11 pip install vectordb-bench ``` -**Install all database clients** - -``` shell -pip install 'vectordb-bench[all]' -``` **Install the specific database client** ```shell @@ -42,7 +37,6 @@ All the database client supported | Optional database client | install command | |--------------------------|---------------------------------------------| | pymilvus, zilliz_cloud (*default*) | `pip install vectordb-bench` | -| all (*clients requirements might be conflict with each other*) | `pip install vectordb-bench[all]` | | qdrant | `pip install vectordb-bench[qdrant]` | | pinecone | `pip install vectordb-bench[pinecone]` | | weaviate | `pip install vectordb-bench[weaviate]` | @@ -225,7 +219,6 @@ Options: --ondisk Ondisk mode with binary quantization(32x compression) --oversample-factor Controls the degree of oversampling applied to minority classes in imbalanced datasets to improve model performance by balancing class distributions.(default 1.0) - # Quantization Type --quantization-type TEXT which type of quantization to use valid values [fp32, fp16, bq] @@ -294,13 +287,13 @@ Options: # Connection --cloud-id TEXT Elastic Cloud ID [required] --password TEXT Elastic Cloud password [required] - + # HNSW Index Parameters --m INTEGER HNSW M parameter [default: 16] --ef-construction INTEGER HNSW efConstruction parameter [default: 100] --num-candidates INTEGER Number of candidates for search [default: 100] --element-type [float|byte] Element type for vectors (float: 4 bytes, byte: 1 byte) [default: float] - + # Index Configuration --number-of-shards INTEGER Number of shards [default: 1] --number-of-replicas INTEGER Number of replicas [default: 0] @@ -311,7 +304,7 @@ Options: --use-routing BOOLEAN Whether to use routing [default: False] --use-rescore BOOLEAN Whether to use rescore [default: False] --oversample-ratio FLOAT Oversample ratio for rescore [default: 2.0] - + # Common Options --case-type [CapacityDim128|CapacityDim960|Performance768D100M|...] Case type diff --git a/pyproject.toml b/pyproject.toml index c5e30aa6b..905996f5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ "pydantic=0.10.1", ] dynamic = ["version"] @@ -51,37 +50,7 @@ test = [ "ruff", "pytest", ] -restful = [ "flask" ] - -all = [ - "grpcio==1.53.0", # for qdrant-client and pymilvus - "grpcio-tools==1.53.0", # for qdrant-client and pymilvus - "qdrant-client", - "pinecone", - "weaviate-client", - "elasticsearch", - "sqlalchemy", - "redis", - "chromadb", - "pgvector", - "psycopg", - "psycopg-binary", - "pgvecto_rs[psycopg3]>=0.2.2", - "opensearch-dsl", - "opensearch-py", - "memorydb", - "alibabacloud_ha3engine_vector", - "mariadb", - "PyMySQL", - "clickhouse-connect", - "pyvespa", - "lancedb", - "mysql-connector-python", - "turbopuffer[fast]", - 'zvec', - "endee==0.1.10", # compatible with pydantic<2 -] - +restful = [ "flask" ] qdrant = [ "qdrant-client" ] pinecone = [ "pinecone" ] weaviate = [ "weaviate-client" ] diff --git a/tests/pytest.ini b/tests/pytest.ini index ad082137c..e5915e89e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,4 +1,5 @@ [pytest] -filterwarnings = +filterwarnings = ignore::UserWarning + ignore::DeprecationWarning diff --git a/tests/test_bench_runner.py b/tests/test_bench_runner.py index 5fab91067..7aff0c27e 100644 --- a/tests/test_bench_runner.py +++ b/tests/test_bench_runner.py @@ -1,5 +1,7 @@ import time import logging + +import ujson from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import ( DB, IndexType, CaseType, TaskConfig, CaseConfig, @@ -55,6 +57,5 @@ def test_performance_case_no_error(self): d = t.json(exclude={'db_config': {'password', 'api_key'}}) log.info(f"{d}") - import ujson loads = ujson.loads(d) log.info(f"{loads}") diff --git a/tests/test_milvus.py b/tests/test_milvus.py new file mode 100644 index 000000000..8cc391acc --- /dev/null +++ b/tests/test_milvus.py @@ -0,0 +1,39 @@ +"""E2E test for Milvus client using MilvusClient API. + +Requires a running Milvus instance at localhost:19530. +""" + +import logging + +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.clients.milvus.config import MilvusConfig +from vectordb_bench.backend.cases import CaseType +from vectordb_bench.interface import BenchMarkRunner +from vectordb_bench.models import CaseConfig, TaskConfig + + +log = logging.getLogger(__name__) + + +class TestMilvus: + """E2E test for Milvus using Performance1536D50K (OpenAI 50K dataset).""" + + def test_performance_1536d_50k(self): + """Full benchmark: download dataset, insert, optimize (force merge), search.""" + runner = BenchMarkRunner() + + task_config = TaskConfig( + db=DB.Milvus, + db_config=MilvusConfig(uri=SecretStr("http://localhost:19530")), + db_case_config=DB.Milvus.case_config_cls(index_type=IndexType.Flat)(), + case_config=CaseConfig(case_id=CaseType.Performance1536D50K), + ) + + runner.run([task_config]) + runner._sync_running_task() + result = runner.get_results() + log.info(f"test result: {result}") + assert len(result) > 0 diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index b177af332..ead2979ff 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from contextlib import contextmanager -from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, MilvusException, utility +from pymilvus import DataType, MilvusClient, MilvusException from vectordb_bench.backend.filter import Filter, FilterOp @@ -51,76 +51,72 @@ def __init__( self._scalar_id_index_name = "id_sort_idx" self._scalar_labels_index_name = "labels_idx" - from pymilvus import connections - - connections.connect( + client = MilvusClient( uri=self.db_config.get("uri"), user=self.db_config.get("user"), password=self.db_config.get("password"), timeout=30, ) - if drop_old and utility.has_collection(self.collection_name): + + if drop_old and client.has_collection(self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") - utility.drop_collection(self.collection_name) + client.drop_collection(self.collection_name) + + if not client.has_collection(self.collection_name): + schema = MilvusClient.create_schema() + schema.add_field(self._primary_field, DataType.INT64, is_primary=True) + schema.add_field(self._scalar_id_field, DataType.INT64) + schema.add_field(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) - if not utility.has_collection(self.collection_name): - fields = [ - FieldSchema(self._primary_field, DataType.INT64, is_primary=True), - FieldSchema(self._scalar_id_field, DataType.INT64), - FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim), - ] if self.with_scalar_labels: is_partition_key = db_case_config.use_partition_key log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") - fields.append( - FieldSchema( - self._scalar_label_field, - DataType.VARCHAR, - max_length=256, - is_partition_key=is_partition_key, - ) + schema.add_field( + self._scalar_label_field, + DataType.VARCHAR, + max_length=256, + is_partition_key=is_partition_key, ) log.info(f"{self.name} create collection: {self.collection_name}") - # Create the collection - col = Collection( - name=self.collection_name, - schema=CollectionSchema(fields), - consistency_level="Session", + index_params = self._build_index_params() + client.create_collection( + collection_name=self.collection_name, + schema=schema, num_shards=self.db_config.get("num_shards", 1), + consistency_level="Session", + ) + client.create_index(self.collection_name, index_params) + client.load_collection( + self.collection_name, + replica_number=self.db_config.get("replica_number", 1), ) - self.create_index() - col.load(replica_number=self.db_config.get("replica_number", 1)) - - connections.disconnect("default") + client.close() - def create_index(self): - col = Collection(self.collection_name) - # vector index - col.create_index( - self._vector_field, - self.case_config.index_param(), + def _build_index_params(self): + index_params = MilvusClient.prepare_index_params() + vec_idx = self.case_config.index_param() + index_params.add_index( + field_name=self._vector_field, index_name=self._vector_index_name, + index_type=vec_idx.get("index_type", ""), + metric_type=vec_idx.get("metric_type", ""), + params=vec_idx.get("params", {}), ) - # scalar index for range-expr (int-filter) - col.create_index( - self._scalar_id_field, - index_params={ - "index_type": "STL_SORT", - }, + index_params.add_index( + field_name=self._scalar_id_field, index_name=self._scalar_id_index_name, + index_type="STL_SORT", ) - # scalar index for varchar (label-filter) if self.with_scalar_labels: - col.create_index( - self._scalar_label_field, - index_params={ - "index_type": "BITMAP", - }, + index_params.add_index( + field_name=self._scalar_label_field, index_name=self._scalar_labels_index_name, + index_type="BITMAP", ) + return index_params @contextmanager def init(self): @@ -130,65 +126,58 @@ def init(self): >>> self.insert_embeddings() >>> self.search_embedding() """ - from pymilvus import connections - - self.col: Collection | None = None - - connections.connect(**self.db_config, timeout=60) - # Grab the existing colection with connections - self.col = Collection(self.collection_name) - + self.client: MilvusClient | None = None + self.client = MilvusClient( + uri=self.db_config.get("uri"), + user=self.db_config.get("user"), + password=self.db_config.get("password"), + timeout=60, + ) yield - connections.disconnect("default") + self.client.close() + self.client = None + + def _wait_for_index(self): + while True: + info = self.client.describe_index(self.collection_name, self._vector_index_name) + if info.get("pending_index_rows", -1) == 0: + break + time.sleep(5) + + def _wait_for_compaction(self, compaction_id: int): + while True: + state = self.client.get_compaction_state(compaction_id) + if state == "Completed": + break + time.sleep(0.5) def _optimize(self): log.info(f"{self.name} optimizing before search") - self._post_insert() try: - self.col.load(refresh=True) - except Exception as e: - log.warning(f"{self.name} optimize error: {e}") - raise e from None - - def _post_insert(self): - try: - self.col.flush() - # wait for index done and load refresh - self.create_index() - - utility.wait_for_index_building_complete(self.collection_name, index_name=self._vector_index_name) - - def wait_index(): - while True: - progress = utility.index_building_progress(self.collection_name, index_name=self._vector_index_name) - if progress.get("pending_index_rows", -1) == 0: - break - time.sleep(5) - - wait_index() - - # Skip compaction if use GPU indexType + self.client.flush(self.collection_name) + self._wait_for_index() if self.case_config.is_gpu_index: - log.debug("skip compaction for gpu index type.") + log.debug("skip force merge compaction for gpu index type.") else: try: - self.col.compact() - self.col.wait_for_compaction_completed() - log.info("compactation completed. waiting for the rest of index buliding.") + compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) + if compaction_id > 0: + self._wait_for_compaction(compaction_id) + log.info(f"{self.name} force merge compaction completed.") + self._wait_for_index() except Exception as e: log.warning(f"{self.name} compact error: {e}") - if hasattr(e, "code"): - if e.code().name == "PERMISSION_DENIED": - log.warning("Skip compact due to permission denied.") + if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": + log.warning("Skip compact due to permission denied.") else: - raise e from e - wait_index() + raise e from None + self.client.refresh_load(self.collection_name) except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None def optimize(self, data_size: int | None = None): - assert self.col, "Please call self.init() before" + assert self.client, "Please call self.init() before" self._optimize() def need_normalize_cosine(self) -> bool: @@ -207,22 +196,24 @@ def insert_embeddings( **kwargs, ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" - # use the first insert_embeddings to init collection - assert self.col is not None + assert self.client is not None assert len(embeddings) == len(metadata) insert_count = 0 try: for batch_start_offset in range(0, len(embeddings), self.batch_size): batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) - insert_data = [ - metadata[batch_start_offset:batch_end_offset], - metadata[batch_start_offset:batch_end_offset], - embeddings[batch_start_offset:batch_end_offset], - ] - if self.with_scalar_labels: - insert_data.append(labels_data[batch_start_offset:batch_end_offset]) - res = self.col.insert(insert_data) - insert_count += len(res.primary_keys) + batch_data = [] + for i in range(batch_start_offset, batch_end_offset): + row = { + self._primary_field: metadata[i], + self._scalar_id_field: metadata[i], + self._vector_field: embeddings[i], + } + if self.with_scalar_labels: + row[self._scalar_label_field] = labels_data[i] + batch_data.append(row) + res = self.client.insert(self.collection_name, batch_data) + insert_count += res["insert_count"] except MilvusException as e: log.info(f"Failed to insert data: {e}") return insert_count, e @@ -246,16 +237,15 @@ def search_embedding( timeout: int | None = None, ) -> list[int]: """Perform a search on a query embedding and return results.""" - assert self.col is not None + assert self.client is not None - # Perform the search. - res = self.col.search( + res = self.client.search( + collection_name=self.collection_name, data=[query], anns_field=self._vector_field, - param=self.case_config.search_param(), + search_params=self.case_config.search_param(), limit=k, - expr=self.expr, + filter=self.expr, ) - # Organize results. - return [result.id for result in res[0]] + return [result[self._primary_field] for result in res[0]] From 6f7a1534a3cdf753285296dc6c8a9d87587e7a83 Mon Sep 17 00:00:00 2001 From: nanlongyu Date: Wed, 1 Apr 2026 11:49:16 +0800 Subject: [PATCH 03/38] feat: add support for PolarDB (#737) - add PolarDB vector search client with FAISS_HNSW_FLAT, FAISS_HNSW_PQ, and FAISS_HNSW_SQ index types - add CLI integration with hnswflat, hnswpq, and hnswsq benchmark commands - add frontend (Streamlit) UI support with index type selection, HNSW/PQ/SQ parameter configuration --- README.md | 42 +++ install/requirements_py3.11.txt | 3 +- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 16 + .../backend/clients/polardb/__init__.py | 0 vectordb_bench/backend/clients/polardb/cli.py | 248 +++++++++++++++ .../backend/clients/polardb/config.py | 148 +++++++++ .../backend/clients/polardb/polardb.py | 286 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 8 + .../frontend/config/dbCaseConfigs.py | 127 ++++++++ vectordb_bench/frontend/config/styles.py | 1 + vectordb_bench/models.py | 5 + 12 files changed, 884 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/polardb/__init__.py create mode 100644 vectordb_bench/backend/clients/polardb/cli.py create mode 100644 vectordb_bench/backend/clients/polardb/config.py create mode 100644 vectordb_bench/backend/clients/polardb/polardb.py diff --git a/README.md b/README.md index fe215e7a7..1b0f46309 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ All the database client supported | hologres | `pip install vectordb-bench[hologres]` | | tencent_es | `pip install vectordb-bench[tencent_es]` | | alisql | `pip install 'vectordb-bench[alisql]'` | +| polardb | `pip install vectordb-bench[polardb]` | | doris | `pip install vectordb-bench[doris]` | | zvec | `pip install vectordb-bench[zvec]` | | endee | `pip install vectordb-bench[endee]` | @@ -520,6 +521,47 @@ To list the options for Lindorm, execute `vectordbbench lindormhnsw --help`, The --ef-search INTEGER hnsw ef-search [required] ``` +### Run PolarDB from command line + +PolarDB supports index types: faiss_hnsw_flat, faiss_hnsw_pq, and faiss_hnsw_sq. + +**Example: Run faiss_hnsw_flat benchmark** + +```shell +vectordbbench polardbhnswflat \ + --case-type Performance768D1M \ + --username \ + --password '' \ + --host \ + --port 3306 \ + --m 16 \ + --ef-construction 256 \ + --ef-search 256 \ + --insert-workers 64 \ + --num-concurrency '10,20,40,60,80' \ + --concurrency-duration 60 \ + --task-label \ + --db-label \ + --skip-search-serial \ + --post-load-index +``` + +To list the options for PolarDB, execute `vectordbbench polardbhnswflat --help`. The following are some PolarDB-specific command-line options. + +```text + --username TEXT Username [required] + --password TEXT Password + --host TEXT Db host [default: 127.0.0.1] + --port INTEGER Db Port [default: 3306] + --database TEXT Database name [default: vectordbbench] + --m INTEGER M parameter (max_degree) in HNSW + --ef-construction INTEGER ef_construction parameter in HNSW + --ef-search INTEGER polar_vector_index_hnsw_ef_search session variable + --insert-workers INTEGER Number of concurrent threads for data insertion + --post-load-index / --inline-index + Create index after load or inline at table creation +``` + #### Using a configuration file. The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file. diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 5d5702492..4214267a3 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -26,5 +26,6 @@ pymilvus clickhouse_connect pyvespa mysql-connector-python +PyMySQL packaging -hdrhistogram>=0.10.1 \ No newline at end of file +hdrhistogram>=0.10.1 diff --git a/pyproject.toml b/pyproject.toml index 905996f5e..d7bf42633 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ vespa = [ "pyvespa" ] lancedb = [ "lancedb" ] oceanbase = [ "mysql-connector-python" ] alisql = [ "mysql-connector-python" ] +polardb = [ "PyMySQL" ] doris = [ "doris-vector-search" ] turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index b5f3a4d6c..214d85e96 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -59,6 +59,7 @@ class DB(Enum): Zvec = "Zvec" Endee = "Endee" Lindorm = "Lindorm" + PolarDB = "PolarDB" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -246,6 +247,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormVector + if self == DB.PolarDB: + from .polardb.polardb import PolarDB + + return PolarDB + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -435,6 +441,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormConfig + if self == DB.PolarDB: + from .polardb.config import PolarDBConfig + + return PolarDBConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -581,6 +592,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return AliSQLIndexConfig + if self == DB.PolarDB: + from .polardb.config import _polardb_case_config + + return _polardb_case_config.get(index_type) + if self == DB.Doris: from .doris.config import DorisCaseConfig diff --git a/vectordb_bench/backend/clients/polardb/__init__.py b/vectordb_bench/backend/clients/polardb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/polardb/cli.py b/vectordb_bench/backend/clients/polardb/cli.py new file mode 100644 index 000000000..6f7a8f79c --- /dev/null +++ b/vectordb_bench/backend/clients/polardb/cli.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, Unpack + +if TYPE_CHECKING: + from .config import PolarDBConfig + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class PolarDBTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option( + "--username", + type=str, + help="Username", + required=True, + ), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Password", + default="", + ), + ] + + host: Annotated[ + str, + click.option( + "--host", + type=str, + help="Db host", + default="127.0.0.1", + ), + ] + + port: Annotated[ + int, + click.option( + "--port", + type=int, + default=3306, + help="Db Port", + ), + ] + + database: Annotated[ + str, + click.option( + "--database", + type=str, + help="Database name", + default="vectordbbench", + ), + ] + + unix_socket: Annotated[ + str, + click.option( + "--unix-socket", + type=str, + help="Unix socket path (overrides host/port if set)", + default="", + ), + ] + + +class PolarDBHNSWTypedDict(PolarDBTypedDict): + m: Annotated[ + int, + click.option( + "--m", + type=int, + help="M parameter (max_degree) in HNSW", + default=16, + ), + ] + + ef_construction: Annotated[ + int, + click.option( + "--ef-construction", + type=int, + help="ef_construction parameter in HNSW", + default=200, + ), + ] + + ef_search: Annotated[ + int, + click.option( + "--ef-search", + type=int, + help="polar_vector_index_hnsw_ef_search session variable", + default=64, + ), + ] + + insert_workers: Annotated[ + int, + click.option( + "--insert-workers", + type=int, + help="Number of concurrent threads for data insertion", + default=10, + ), + ] + + post_load_index: Annotated[ + bool, + click.option( + "--post-load-index/--inline-index", + type=bool, + help="Create vector index via ALTER TABLE after data load; " + "otherwise create index inline during table creation", + default=False, + ), + ] + + +class PolarDBHNSWPQTypedDict(PolarDBHNSWTypedDict): + pq_m: Annotated[ + int, + click.option( + "--pq-m", + type=int, + help="PQ subquantizer count (must divide dimension)", + default=1, + ), + ] + + pq_nbits: Annotated[ + int, + click.option( + "--pq-nbits", + type=int, + help="PQ bits per subquantizer (max 24)", + default=8, + ), + ] + + +class PolarDBHNSWSQTypedDict(PolarDBHNSWTypedDict): + sq_type: Annotated[ + str, + click.option( + "--sq-type", + type=str, + help="SQ quantizer type (8bit, 4bit, fp16, bf16, 6bit, etc.)", + default="8bit", + ), + ] + + +def _build_db_config(parameters: dict) -> PolarDBConfig: + from .config import PolarDBConfig + + pwd = parameters["password"] + sock = parameters["unix_socket"] + return PolarDBConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(pwd) if pwd else None, + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + unix_socket=sock if sock else None, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PolarDBHNSWTypedDict) +def PolarDBHNSWFlat( + **parameters: Unpack[PolarDBHNSWTypedDict], +): + from .config import PolarDBHNSWFlatConfig + + run( + db=DB.PolarDB, + db_config=_build_db_config(parameters), + db_case_config=PolarDBHNSWFlatConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + insert_workers=parameters["insert_workers"], + post_load_index=parameters["post_load_index"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PolarDBHNSWPQTypedDict) +def PolarDBHNSWPQ( + **parameters: Unpack[PolarDBHNSWPQTypedDict], +): + from .config import PolarDBHNSWPQConfig + + run( + db=DB.PolarDB, + db_config=_build_db_config(parameters), + db_case_config=PolarDBHNSWPQConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + insert_workers=parameters["insert_workers"], + post_load_index=parameters["post_load_index"], + pq_m=parameters["pq_m"], + pq_nbits=parameters["pq_nbits"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PolarDBHNSWSQTypedDict) +def PolarDBHNSWSQ( + **parameters: Unpack[PolarDBHNSWSQTypedDict], +): + from .config import PolarDBHNSWSQConfig + + run( + db=DB.PolarDB, + db_config=_build_db_config(parameters), + db_case_config=PolarDBHNSWSQConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + insert_workers=parameters["insert_workers"], + post_load_index=parameters["post_load_index"], + sq_type=parameters["sq_type"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/polardb/config.py b/vectordb_bench/backend/clients/polardb/config.py new file mode 100644 index 000000000..c75448c49 --- /dev/null +++ b/vectordb_bench/backend/clients/polardb/config.py @@ -0,0 +1,148 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class PolarDBConfigDict(TypedDict): + user: str + password: str + host: str + port: int + database: str + unix_socket: str | None + + +class PolarDBConfig(DBConfig): + user_name: str = "root" + password: SecretStr | None = None + host: str = "127.0.0.1" + port: int = 3306 + database: str = "vectordbbench" + unix_socket: str | None = None + + @staticmethod + def common_long_configs() -> list[str]: + return ["note", "unix_socket"] + + def to_dict(self) -> PolarDBConfigDict: + pwd_str = self.password.get_secret_value() if self.password else "" + return { + "host": self.host, + "port": self.port, + "user": self.user_name, + "password": pwd_str, + "database": self.database, + "unix_socket": self.unix_socket or None, + } + + +class PolarDBIndexConfig(BaseModel): + """Base config for PolarDB vector index""" + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "EUCLIDEAN" + if self.metric_type == MetricType.COSINE: + return "COSINE" + if self.metric_type == MetricType.IP: + return "INNER_PRODUCT" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + def parse_metric_for_distance(self) -> str: + """Return the metric name used in DISTANCE() function""" + if self.metric_type == MetricType.L2: + return "EUCLIDEAN" + if self.metric_type == MetricType.COSINE: + return "COSINE" + if self.metric_type == MetricType.IP: + return "DOT" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + +class PolarDBHNSWBaseConfig(PolarDBIndexConfig, DBCaseConfig): + """Shared HNSW config fields for all PolarDB HNSW variants.""" + + M: int = 16 + ef_construction: int = 200 + ef_search: int = 64 + insert_workers: int = 10 + post_load_index: bool = False # If True, create index after data load via ALTER TABLE + index: IndexType = IndexType.HNSW + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric_for_distance(), + "ef_search": self.ef_search, + } + + +class PolarDBHNSWFlatConfig(PolarDBHNSWBaseConfig): + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "metric_type_distance": self.parse_metric_for_distance(), + "index_type": "FAISS_HNSW_FLAT", + "M": self.M, + "ef_construction": self.ef_construction, + "vector_index_comment": ( + f"imci_vector_index=FAISS_HNSW_FLAT(" + f"metric={self.parse_metric()}," + f"max_degree={self.M}," + f"ef_construction={self.ef_construction})" + ), + } + + +class PolarDBHNSWPQConfig(PolarDBHNSWBaseConfig): + pq_m: int = 1 + pq_nbits: int = 8 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "metric_type_distance": self.parse_metric_for_distance(), + "index_type": "FAISS_HNSW_PQ", + "M": self.M, + "ef_construction": self.ef_construction, + "vector_index_comment": ( + f"imci_vector_index=FAISS_HNSW_PQ(" + f"metric={self.parse_metric()}," + f"max_degree={self.M}," + f"ef_construction={self.ef_construction}," + f"pq_m={self.pq_m}," + f"pq_nbits={self.pq_nbits})" + ), + } + + +class PolarDBHNSWSQConfig(PolarDBHNSWBaseConfig): + sq_type: str = "8bit" + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "metric_type_distance": self.parse_metric_for_distance(), + "index_type": "FAISS_HNSW_SQ", + "M": self.M, + "ef_construction": self.ef_construction, + "vector_index_comment": ( + f"imci_vector_index=FAISS_HNSW_SQ(" + f"metric={self.parse_metric()}," + f"max_degree={self.M}," + f"ef_construction={self.ef_construction}," + f"sq_type={self.sq_type})" + ), + } + + +_polardb_case_config = { + IndexType.HNSW: PolarDBHNSWFlatConfig, + IndexType.HNSW_PQ: PolarDBHNSWPQConfig, + IndexType.HNSW_SQ: PolarDBHNSWSQConfig, +} diff --git a/vectordb_bench/backend/clients/polardb/polardb.py b/vectordb_bench/backend/clients/polardb/polardb.py new file mode 100644 index 000000000..f42b6fca5 --- /dev/null +++ b/vectordb_bench/backend/clients/polardb/polardb.py @@ -0,0 +1,286 @@ +import concurrent.futures +import logging +import time +from contextlib import contextmanager + +import numpy as np +import pymysql + +from ..api import VectorDB +from .config import PolarDBConfigDict, PolarDBIndexConfig + +log = logging.getLogger(__name__) + + +class PolarDB(VectorDB): + def __init__( + self, + dim: int, + db_config: PolarDBConfigDict, + db_case_config: PolarDBIndexConfig, + collection_name: str = "vec_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "PolarDB" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + conn, cursor = self._create_connection() + + if drop_old: + log.info(f"PolarDB dropping old table: {self.table_name}") + self._create_db_table(cursor, dim) + + cursor.close() + conn.close() + + def _create_connection(self): + connect_kwargs = { + "user": self.db_config["user"], + "password": self.db_config["password"], + "autocommit": True, + } + if self.db_config.get("unix_socket"): + connect_kwargs["unix_socket"] = self.db_config["unix_socket"] + else: + connect_kwargs["host"] = self.db_config["host"] + connect_kwargs["port"] = self.db_config["port"] + + conn = pymysql.connect(**connect_kwargs) + cursor = conn.cursor() + # Disable query cache to ensure accurate benchmarking + cursor.execute("SET query_cache_type = OFF") + return conn, cursor + + def _create_db_table(self, cursor: pymysql.cursors.Cursor, dim: int) -> None: + index_param = self.case_config.index_param() + vector_index_comment = index_param["vector_index_comment"] + post_load_index = getattr(self.case_config, "post_load_index", False) + + try: + log.info(f"PolarDB creating database: {self.db_config['database']}") + cursor.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_config['database']}") + cursor.execute(f"USE {self.db_config['database']}") + cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + if post_load_index: + # Post-load mode: create table without vector index, will add via ALTER TABLE later + create_sql = ( + f"CREATE TABLE {self.table_name} (" + f"id INT PRIMARY KEY, " + f"v VECTOR({dim}) NOT NULL" + f") ENGINE=InnoDB COMMENT 'COLUMNAR=1'" + ) + log.info(f"PolarDB creating table (post-load index mode): {create_sql}") + else: + # Inline mode: create table with vector index comment + create_sql = ( + f"CREATE TABLE {self.table_name} (" + f"id INT PRIMARY KEY, " + f'v VECTOR({dim}) NOT NULL COMMENT "{vector_index_comment}"' + f") ENGINE=InnoDB COMMENT 'COLUMNAR=1'" + ) + log.info(f"PolarDB creating table: {create_sql}") + cursor.execute(create_sql) + except Exception as e: + log.warning(f"Failed to create table: {self.table_name} error: {e}") + raise + + @contextmanager + def init(self): + self.conn, self.cursor = self._create_connection() + + search_param = self.case_config.search_param() + + # Force PolarDB vector search to use the IMCI engine. + self.cursor.execute("SET use_imci_engine = FORCED") + self.cursor.execute("SET imci_enable_vector_search = ON") + self.cursor.execute("SET imci_max_dop = 1") + self.cursor.execute("SET cost_threshold_for_imci = 0") + + # Set ef_search + if search_param.get("ef_search") is not None: + self.cursor.execute(f"SET polar_vector_index_hnsw_ef_search = {search_param['ef_search']}") + + metric_type = search_param["metric_type"] + db_name = self.db_config["database"] + hint = "/*+ SET_VAR(imci_enable_fast_vector_search=on) */" + + self.insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + self.select_sql = ( + f"SELECT {hint} id FROM {db_name}.{self.table_name} " # noqa: S608 + f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " + f"LIMIT %s" + ) + self.select_sql_with_filter = ( + f"SELECT id FROM {db_name}.{self.table_name} " # noqa: S608 + f"WHERE id >= %s " + f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " + f"LIMIT %s" + ) + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def ready_to_load(self) -> bool: + pass + + def optimize(self, data_size: int | None = None) -> None: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + db_name = self.db_config["database"] + index_param = self.case_config.index_param() + post_load_index = getattr(self.case_config, "post_load_index", False) + + # Ensure vector index builds even for small datasets + try: + self.cursor.execute("SET GLOBAL imci_vector_index_dump_rows_threshold = 1") + except Exception as e: + log.warning(f"Cannot SET GLOBAL imci_vector_index_dump_rows_threshold (need SUPER): {e}") + + start_time = time.time() + + if post_load_index: + # Post-load mode: issue ALTER TABLE to add vector index after data load + vector_index_comment = index_param["vector_index_comment"] + alter_sql = ( + f"ALTER TABLE {db_name}.{self.table_name} " + f'MODIFY COLUMN v VECTOR({self.dim}) NOT NULL COMMENT "{vector_index_comment}"' + ) + log.info(f"PolarDB creating vector index via ALTER TABLE: {alter_sql}") + self.cursor.execute(alter_sql) + log.info("ALTER TABLE completed.") + + analyze_sql = f"/* FORCE_IMCI_NODES */ ANALYZE TABLE {db_name}.{self.table_name}" + log.info(f"PolarDB analyzing table: {analyze_sql}") + analyze_start = time.time() + self.cursor.execute(analyze_sql) + analyze_elapsed = time.time() - analyze_start + + log.info(f"ANALYZE TABLE completed in {analyze_elapsed:.1f}s, waiting for vector index to be built...") + + last_vectors = 0 + while True: + self.cursor.execute( + "SELECT VECTORS FROM information_schema.imci_vector_index_stats " + "WHERE SCHEMA_NAME=%s AND TABLE_NAME=%s", + (db_name, self.table_name), + ) + result = self.cursor.fetchone() + + if result is None: + log.info("Vector index stats not yet available, waiting...") + time.sleep(2) + continue + + vectors = int(result[0]) + + if vectors != last_vectors: + elapsed = max(0.0, time.time() - start_time - analyze_elapsed) + log.info( + f"Vector index building: {vectors} vectors indexed " + f"(target: {data_size}), elapsed: {elapsed:.1f}s" + ) + last_vectors = vectors + + if data_size is not None and vectors >= data_size: + break + + # Also check imci_index_stats for vector_rows as secondary indicator + self.cursor.execute( + "SELECT VECTOR_ROWS FROM information_schema.imci_index_stats WHERE SCHEMA_NAME=%s AND TABLE_NAME=%s", + (db_name, self.table_name), + ) + idx_result = self.cursor.fetchone() + if idx_result and data_size is not None and int(idx_result[0]) >= data_size: + break + + time.sleep(2) + + total_time = max(0.0, time.time() - start_time - analyze_elapsed) + log.info(f"PolarDB vector index build completed in {total_time:.1f}s") + + @staticmethod + def vector_to_hex(v: list[float]) -> bytes: + return np.array(v, "float32").tobytes() + + def _insert_batch(self, embeddings: list[list[float]], metadata: list[int], offset: int, size: int) -> None: + """Insert a batch of embeddings using a dedicated connection.""" + conn, cursor = self._create_connection() + try: + db_name = self.db_config["database"] + insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + batch_data = [] + for i in range(offset, offset + size): + batch_data.append((int(metadata[i]), self.vector_to_hex(embeddings[i]))) + + cursor.executemany(insert_sql, batch_data) + finally: + cursor.close() + conn.close() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + workers = self.case_config.insert_workers + total = len(embeddings) + batch_size = max(1, total // workers) + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [] + for i in range(0, total, batch_size): + offset = i + size = min(batch_size, total - i) + future = executor.submit(self._insert_batch, embeddings, metadata, offset, size) + futures.append(future) + + done, pending = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_EXCEPTION) + for future in done: + future.result() + for future in pending: + future.cancel() + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + if filters: + self.cursor.execute( + self.select_sql_with_filter, + (filters.get("id"), self.vector_to_hex(query), k), + ) + else: + self.cursor.execute(self.select_sql, (self.vector_to_hex(query), k)) + return [row[0] for row in self.cursor.fetchall()] + except Exception: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index dd704ae91..b48d5900c 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -25,6 +25,11 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.pinecone.cli import Pinecone +from ..backend.clients.polardb.cli import ( + PolarDBHNSWFlat, + PolarDBHNSWPQ, + PolarDBHNSWSQ, +) from ..backend.clients.qdrant_cloud.cli import QdrantCloud from ..backend.clients.qdrant_local.cli import QdrantLocal from ..backend.clients.redis.cli import Redis @@ -82,6 +87,9 @@ cli.add_command(LindormHNSW) cli.add_command(LindormIVFBQ) cli.add_command(Pinecone) +cli.add_command(PolarDBHNSWFlat) +cli.add_command(PolarDBHNSWPQ) +cli.add_command(PolarDBHNSWSQ) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index e8c81c1d1..387f7fb4a 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -2847,6 +2847,129 @@ class FilterType(Enum): CaseConfigParamInput_NumberOfRegions_Lindorm, ] +# PolarDB configs +CaseConfigParamInput_IndexType_PolarDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_SQ.value, + ], + }, +) + +CaseConfigParamInput_M_PolarDB = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 2, + "max": 1024, + "value": 16, + }, + inputHelp="HNSW M parameter", +) + +CaseConfigParamInput_EFConstruction_PolarDB = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 8192, + "value": 200, + }, + inputHelp="ef_construction", +) + +CaseConfigParamInput_EFSearch_PolarDB = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 8192, + "value": 200, + }, + inputHelp="ef_search", +) + +CaseConfigParamInput_InsertWorkers_PolarDB = CaseConfigInput( + label=CaseConfigParamType.insert_workers, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1024, + "value": 10, + }, + inputHelp="Number of insert workers", +) + +CaseConfigParamInput_PostLoadIndex_PolarDB = CaseConfigInput( + label=CaseConfigParamType.post_load_index, + inputType=InputType.Bool, + inputConfig={ + "value": True, + }, + inputHelp="Create index after data load via ALTER TABLE", +) + +CaseConfigParamInput_PQM_PolarDB = CaseConfigInput( + label=CaseConfigParamType.pq_m, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 16383, + "value": 1, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW_PQ.value, + inputHelp="PQ M parameter", +) + +CaseConfigParamInput_PQNbits_PolarDB = CaseConfigInput( + label=CaseConfigParamType.pq_nbits, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 24, + "value": 8, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW_PQ.value, + inputHelp="PQ nbits parameter", +) + +CaseConfigParamInput_SQType_PolarDB = CaseConfigInput( + label=CaseConfigParamType.sq_type, + inputType=InputType.Option, + inputConfig={ + "options": [ + "8bit", + "4bit", + "8bit_uniform", + "4bit_uniform", + "8bit_direct", + "8bit_direct_signed", + "fp16", + "bf16", + "6bit", + ], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW_SQ.value, + inputHelp="Scalar quantizer type", +) + +PolarDBConfig = [ + CaseConfigParamInput_IndexType_PolarDB, + CaseConfigParamInput_M_PolarDB, + CaseConfigParamInput_EFConstruction_PolarDB, + CaseConfigParamInput_EFSearch_PolarDB, + CaseConfigParamInput_InsertWorkers_PolarDB, + CaseConfigParamInput_PostLoadIndex_PolarDB, + CaseConfigParamInput_PQM_PolarDB, + CaseConfigParamInput_PQNbits_PolarDB, + CaseConfigParamInput_SQType_PolarDB, +] + # Map DB to config CASE_CONFIG_MAP = { DB.Milvus: { @@ -2937,6 +3060,10 @@ class FilterType(Enum): CaseLabel.Load: LindormLoadConfig, CaseLabel.Performance: LindormPerformanceConfig, }, + DB.PolarDB: { + CaseLabel.Load: PolarDBConfig, + CaseLabel.Performance: PolarDBConfig, + }, } diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index fabe9bdcf..268a2cd7d 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -74,6 +74,7 @@ def getPatternShape(i): DB.Zvec: "https://zvec.org/img/zvec-logo-light.svg", DB.Endee: "data:image/svg+xml,%3c?xml%20version=%271.0%27%20encoding=%27UTF-8%27?%3e%3csvg%20id=%27Layer_1%27%20xmlns=%27http://www.w3.org/2000/svg%27%20version=%271.1%27%20viewBox=%270%200%20600%20600%27%3e%3c!--%20Generator:%20Adobe%20Illustrator%2030.0.0,%20SVG%20Export%20Plug-In%20.%20SVG%20Version:%202.1.1%20Build%20123)%20--%3e%3cdefs%3e%3cstyle%3e%20.st0%20{%20fill:%20%233266a4;%20}%20%3c/style%3e%3c/defs%3e%3cpath%20class=%27st0%27%20d=%27M106.22,490.02H10.36l-.04-184.85c11.19-163.31,232.1-211.74,306.42-61.94,23.96,48.3,15.59,99.83,17,152.01h61.62c15.25,0,42.7-13.75,54.75-23.2,66.11-51.86,57.28-158.2-16.28-198.49-9.65-5.28-33.16-14.19-43.74-14.19h-154.31v-91.09c0-.87,2.55-1.86,3.63-1.63,104.05,4.23,201.15-21.64,284.48,55.84,119.18,110.8,69.12,325.47-91.33,362.6-28.65,6.63-85.47,7.76-115.02,4.8-19.71-1.97-43.29-16.57-55.97-31.45-37.98-44.56-20.77-98.07-24.7-151.16-5.18-69.99-100-85.31-125.9-20.47-1.16,2.92-4.76,13.88-4.76,16.3v186.91Z%27/%3e%3c/svg%3e", DB.Lindorm: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJgAAACKCAYAAABW3IOxAAAMT2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSIQQIREBK6E0QqQGkhNACSC+CqIQkQCgxJgQVO7K4gmsXESwrugqi2FZAFhvqqiuLgr0uFlSUdXFd7MqbEECXfeV7831z57//nPnnnHPn3rkDAL2LL5XmopoA5EnyZbEhAazJySksUg8gAmNABjZAiy+QSznR0REAluH27+X1NYAo28sOSq1/9v/XoiUUyQUAINEQpwvlgjyIfwQAbxFIZfkAEKWQN5+VL1XidRDryKCDENcocaYKtyhxugpfGrSJj+VC/AgAsjqfL8sEQKMP8qwCQSbUocNogZNEKJZA7A+xb17eDCHEiyC2gTZwTrpSn53+lU7m3zTTRzT5/MwRrIplsJADxXJpLn/O/5mO/13ychXDc1jDqp4lC41Vxgzz9ihnRrgSq0P8VpIeGQWxNgAoLhYO2isxM0sRmqCyR20Eci7MGWBCPFGeG8cb4mOF/MBwiA0hzpDkRkYM2RRliIOVNjB/aIU4nxcPsR7ENSJ5UNyQzQnZjNjhea9lyLicIf4pXzbog1L/syIngaPSx7SzRLwhfcyxMCs+CWIqxIEF4sRIiDUgjpTnxIUP2aQWZnEjh21kilhlLBYQy0SSkACVPlaeIQuOHbLfnScfjh07kSXmRQ7hzvys+FBVrrBHAv6g/zAWrE8k4SQM64jkkyOGYxGKAoNUseNkkSQhTsXjetL8gFjVWNxOmhs9ZI8HiHJDlLwZxPHygrjhsQX5cHGq9PESaX50vMpPvDKbHxat8gffDyIAFwQCFlDAmg5mgGwgbu9t7IV3qp5gwAcykAlEwGGIGR6RNNgjgdc4UAh+h0gE5CPjAgZ7RaAA8p9GsUpOPMKprg4gY6hPqZIDHkOcB8JBLrxXDCpJRjxIBI8gI/6HR3xYBTCGXFiV/f+eH2a/MBzIRAwxiuEZWfRhS2IQMZAYSgwm2uIGuC/ujUfAqz+szjgb9xyO44s94TGhg/CAcJXQRbg5XVwkG+XlJNAF9YOH8pP+dX5wK6jphgfgPlAdKuNM3AA44K5wHg7uB2d2gyx3yG9lVlijtP8WwVdPaMiO4kRBKWMo/hSb0SM17DTcRlSUuf46Pypf00fyzR3pGT0/96vsC2EbPtoS+xY7hJ3FTmLnsRasEbCw41gT1oYdVeKRFfdocMUNzxY76E8O1Bm9Zr48WWUm5U51Tj1OH1V9+aLZ+cqXkTtDOkcmzszKZ3HgjiFi8SQCx3EsZydnNwCU+4/q8/YqZnBfQZhtX7glvwHgc3xgYOCnL1zYcQAOeMBPwpEvnA0bbi1qAJw7IlDIClQcrrwQ4JeDDt8+fbi/mcP9zQE4A3fgDfxBEAgDUSAeJINp0PssuM5lYBaYBxaDElAGVoH1oBJsBdtBDdgLDoJG0AJOgp/BBXAJXAW34erpBs9BH3gNPiAIQkJoCAPRR0wQS8QecUbYiC8ShEQgsUgykoZkIhJEgcxDliBlyBqkEtmG1CIHkCPISeQ80oHcRO4jPcifyHsUQ9VRHdQItULHo2yUg4aj8ehUNBOdiRaixegKtAKtRvegDehJ9AJ6Fe1Cn6P9GMDUMCZmijlgbIyLRWEpWAYmwxZgpVg5Vo3VY83wOV/GurBe7B1OxBk4C3eAKzgUT8AF+Ex8Ab4cr8Rr8Ab8NH4Zv4/34Z8JNIIhwZ7gReARJhMyCbMIJYRywk7CYcIZ+C51E14TiUQm0ZroAd/FZGI2cS5xOXEzcR/xBLGD+JDYTyKR9En2JB9SFIlPyieVkDaS9pCOkzpJ3aS3ZDWyCdmZHExOIUvIReRy8m7yMXIn+Qn5A0WTYknxokRRhJQ5lJWUHZRmykVKN+UDVYtqTfWhxlOzqYupFdR66hnqHeorNTU1MzVPtRg1sdoitQq1/Wrn1O6rvVPXVrdT56qnqivUV6jvUj+hflP9FY1Gs6L501Jo+bQVtFraKdo92lsNhoajBk9DqLFQo0qjQaNT4wWdQrekc+jT6IX0cvoh+kV6ryZF00qTq8nXXKBZpXlE87pmvxZDa4JWlFae1nKt3VrntZ5qk7SttIO0hdrF2tu1T2k/ZGAMcwaXIWAsYexgnGF06xB1rHV4Otk6ZTp7ddp1+nS1dV11E3Vn61bpHtXtYmJMKyaPmctcyTzIvMZ8P8ZoDGeMaMyyMfVjOse80Rur568n0ivV26d3Ve+9Pks/SD9Hf7V+o/5dA9zAziDGYJbBFoMzBr1jdcZ6jxWMLR17cOwtQ9TQzjDWcK7hdsM2w34jY6MQI6nRRqNTRr3GTGN/42zjdcbHjHtMGCa+JmKTdSbHTZ6xdFkcVi6rgnWa1WdqaBpqqjDdZtpu+sHM2izBrMhsn9ldc6o52zzDfJ15q3mfhYnFJIt5FnUWtywplmzLLMsNlmct31hZWyVZLbVqtHpqrWfNsy60rrO+Y0Oz8bOZaVNtc8WWaMu2zbHdbHvJDrVzs8uyq7K7aI/au9uL7Tfbd4wjjPMcJxlXPe66g7oDx6HAoc7hviPTMcKxyLHR8cV4i/Ep41ePPzv+s5ObU67TDqfbE7QnhE0omtA84U9nO2eBc5XzFReaS7DLQpcml5eu9q4i1y2uN9wYbpPclrq1un1y93CXude793hYeKR5bPK4ztZhR7OXs895EjwDPBd6tni+83L3yvc66PWHt4N3jvdu76cTrSeKJu6Y+NDHzIfvs82ny5flm+b7vW+Xn6kf36/a74G/ub/Qf6f/E44tJ5uzh/MiwClAFnA44A3XizufeyIQCwwJLA1sD9IOSgiqDLoXbBacGVwX3BfiFjI35EQoITQ8dHXodZ4RT8Cr5fWFeYTNDzsdrh4eF14Z/iDCLkIW0TwJnRQ2ae2kO5GWkZLIxigQxYtaG3U32jp6ZvRPMcSY6JiqmMexE2LnxZ6NY8RNj9sd9zo+IH5l/O0EmwRFQmsiPTE1sTbxTVJg0pqkrsnjJ8+ffCHZIFmc3JRCSklM2ZnSPyVoyvop3aluqSWp16ZaT5099fw0g2m5045Op0/nTz+URkhLStud9pEfxa/m96fz0jel9wm4gg2C50J/4Tphj8hHtEb0JMMnY03G00yfzLWZPVl+WeVZvWKuuFL8Mjs0e2v2m5yonF05A7lJufvyyHlpeUck2pIcyekZxjNmz+iQ2ktLpF0zvWaun9knC5ftlCPyqfKmfB34o9+msFF8o7hf4FtQVfB2VuKsQ7O1Zktmt82xm7NszpPC4MIf5uJzBXNb55nOWzzv/nzO/G0LkAXpC1oXmi8sXti9KGRRzWLq4pzFvxY5Fa0p+mtJ0pLmYqPiRcUPvwn5pq5Eo0RWcn2p99Kt3+Lfir9tX+aybOOyz6XC0l/KnMrKyz4uFyz/5bsJ31V8N7AiY0X7SveVW1YRV0lWXVvtt7pmjdaawjUP105a27COta503V/rp68/X+5avnUDdYNiQ1dFREXTRouNqzZ+rMyqvFoVULVvk+GmZZvebBZu7tziv6V+q9HWsq3vvxd/f2NbyLaGaqvq8u3E7QXbH+9I3HH2B/YPtTsNdpbt/LRLsqurJrbmdK1Hbe1uw90r69A6RV3PntQ9l/YG7m2qd6jfto+5r2w/2K/Y/+xA2oFrB8MPth5iH6r/0fLHTYcZh0sbkIY5DX2NWY1dTclNHUfCjrQ2ezcf/snxp10tpi1VR3WPrjxGPVZ8bOB44fH+E9ITvSczTz5snd56+9TkU1dOx5xuPxN+5tzPwT+fOss5e/ycz7mW817nj/zC/qXxgvuFhja3tsO/uv16uN29veGix8WmS56Xmjsmdhzr9Os8eTnw8s9XeFcuXI282nEt4dqN66nXu24Ibzy9mXvz5a2CWx9uL7pDuFN6V/Nu+T3De9W/2f62r8u96+j9wPttD+Ie3H4oePj8kfzRx+7ix7TH5U9MntQ+dX7a0hPcc+nZlGfdz6XPP/SW/K71+6YXNi9+/MP/j7a+yX3dL2UvB/5c/kr/1a6/XP9q7Y/uv/c67/WHN6Vv9d/WvGO/O/s+6f2TD7M+kj5WfLL91Pw5/POdgbyBASlfxh/8FcCA8miTAcCfuwCgJQPAgOdG6hTV+XCwIKoz7SAC/wmrzpCDxR2AevhPH9ML/26uA7B/BwBWUJ+eCkA0DYB4T4C6uIzU4bPc4LlTWYjwbPD9tE/peeng3xTVmfQrv0e3QKnqCka3/wLmpoMnuLWGFQAAAIplWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACQAAAAAQAAAJAAAAABAAOShgAHAAAAEgAAAHigAgAEAAAAAQAAAJigAwAEAAAAAQAAAIoAAAAAQVNDSUkAAABTY3JlZW5zaG90rCu3yAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAdZpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+MTM4PC9leGlmOlBpeGVsWURpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6UGl4ZWxYRGltZW5zaW9uPjE1MjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpf6GgjAAAAHGlET1QAAAACAAAAAAAAAEUAAAAoAAAARQAAAEUAACFD9bmq1QAAIQ9JREFUeAGcncu2JFdxhk+LBQJzsw1mgO0ZSIwl8RDm5ocw6pY9smH5LUD2DHXDU3DxQ4AaewYC7JFnXJbBmJuE2vFF5rfPX3Eyz2nYUtXeEfHHH7FzR+3MyqpTfe/tt99+clXt3r17V0+ePLno0duw0cCVT/fasseeTT90jJ955pk03znWP3kzfuqNISm29Gc88WKzT5/UOyb+XfM4iiWvPGe5pK8+ic35n3HJkf6Oz3zUZ6/PWXz1E4eM7V4lu1VOsaLAwMMDKJB+NvH22pFp00e9uLPemNjnYibnEV8e/OQ5wh7Fx4cH8zdW+h7pkgc7eHoeHseJUU5udNPvrvkbTz5leeS3N4b41KuTA/mu+OnjGM4VvwZdObkwAukzgR26dMriJT7zm3r95El/bfZpMyf9znj1fRrc2fzlmPHh5GEu5pCxjnTixck7uYxrL05O+vTJsT639TM+2Iwxfc1bPf48jl5AYujvFagLbO/axlhCe+30Sao+SZv4KXcx/TMOOuXJexb/NnxyPA3uaeObe/LfNTb+bb5PGz9jySd/2o7G4rXph/4oftqPxvLMfp0icZLYfoLV2087sjsBfI7RTxnd5EH21OQkwM02/bCjs+lL/NteDMbTXznznpxgxGnLXt85X2Xzkcdc6dGJS7382ulnEy8G+5z/lBPLmAf5OQc45GVsA2fTfqRrnnpaaME6Z58JpH6OM9C03SbjR/y7cjiz628M85h49eK0e1CR5Uqs+umnbJ8+6rI33hEubcZLHT5ZoPBOnLHkT3/xYrI/m7/+YOFUPuLXJpb+3u9///vCXieOkgYYvUQ5MW3g0u5YPTgSp+mfSbRhPMlhjMRjU04cFMg8jCPW+PoZbsrypV1O+ne84x1Xr//gN1eP3/jd1YPPfmDNO3mSgzE2F8685NeesmN6fM8w2sTpl/GnTYz95FZvj51H5i3/UfyJd95dYDjYACKrqwLsIMrgkswFNBGTkEc5OWcsZfqV2J7TEY84YyJnHGViwocNLHLi5BZvn7ky/+/+6M2rr37zf6vAfntVR+bq/mfeT8CrB5/7IC4XTX65nU/migN2HuqnHzmIO8o7fRu4P8mDKEfaHYM74kBnvNnLeeSLTju9814X+Sgld0x/V3MS6Tt9ZnDt0yfl6XOXvCa0L0zGkNcDhi3zVoZjLvhXv/XLq4ff/EUfGznFP/jMB7ZiK8WMb75g/5j4+NGSZ9McPxtDa85P2x8yf3nsZx7JD8YY4rXfWmBHThlIEklv6/VzktnrJyblXPCMN3ObcnI4tgebvOjhzgLjdHj/Sz++urffGG7+2r3qdd80nQvD2mgotJc//b5VtADAn7Xb4jtH+skxdcqzJy462uRQd9f8xSWPcbAd6Y/s/S5SQzpBQlO3SduzSU8/rL6SwWA/wugv520YbROLjM14jGnKM34b60lc5mAMek6Dng7bZy8i/amvLjIKiJh7jy+F9vlPvXfFyFiO4TG3HKdOPT1+t9mcr/zK+qiHKxt2mnb8LLrEMRbL2HwcGw/ZJob+4jYFgCRLh7SZlHZtEiPnGE6TP0pIPL3NPPSl59FJs7B7E4eYMXOcfo7TD9/v/uh3V4++sV1nIa/WC1HxCEkO1XEdxsgebGdUed3/9PuvXnz+XVcvPfdu1J0zseb8yU87vXIr68n80lddYtXJoez8sweDXZ0+6lNmTLPw8NP3KP6RDf8+RWqchA2IxUSmgbfPYOhMXk7ldgj79MMub/qc4RIjN736s/hgtIl/+I1fXD2qi/iukp7bXlBiMWz1UM6lXOMS5vHZ7fc/9b6r+/Vu03yIZSO+BafO3gVVtj86NnMeR9ipQ57xk2fG10ZPy/VQd6TXr99FAnDC6ZSOOqATc9vBEwPehs44qXNM7wT0T/lsnP7G0P/MBpfXWR13v9ZKfI/ruG671dXVS8+/++rF557ti35tq9h03At02+u267N+11n2zC3nguuZLK07PzjnNsdgZ4Hoj89RfPA01yW5py/xzDNtjNUbf/G89dZbTzAaQMeckDp69YwlZSwhY1rakLWjZzztYGzYxaSf9uTgNgKy+U+8XNrh+M4bv67rrF9utx3K96LtO9ATDnzZXICv/fNHrl78+LMN5Z3la9/4edXW9c4247rTPan72G8/efvq7z/3p/2OM3PPuOmfY+ObP7bkYP407diygdcHvb5iJn/aGWPHX365jDP95bW/2MF01jhl9OiyGUgsAdGpF6vf1GunnxjkqdNfvb0HYMbPA/D4h7+9esh11hu/WWHlawVzY4H2KVb0q5c51dV9L/nFE5fT6qNv1am1GnIXHM6xyMTHhj9vAl58/tm6PtsKtR33JzDNMY5d5j/xyPjQyIuHHK2sp4wvjl6/5NfXOYKb/uhoYtIfvRyMaes2hQYDb+ZLIieBbeLE2x9NVpuxlOmnTn4nknZt+KU9ZcbpQzH0rsPisyjd19Bqgqv+U/5knQ7vf7Yu2GvXgkeuo3hyE9PWxdahLLwtFv4P6o3Ay1W0cuEjPz0tbcgsJDoeYtHb9FMWhyyXGHpfMOrEKeuDHh0PdKnHZtNPWdx6F4kBpcBJqF5HZQnF26tPvGNt9MmT9jygZ3j0+ohXJ6/vDr/z/V8D3vCs4X4m6VPh5tQF9uJz71qFBbc8QNgBv/uDN3tHU298TrvYHtapc8UpH3E1uNbXkELOG7VbCpfx5Ma2eBCqpQ3Z+YObNuypTzt+NHQ8Ms6UxbXDeJrxjXFxHwxyK1t/A85gEpzhEp9j8fo7cXrHYNKunHZ50GlP3eMfctvhF+vjHRY0W/LXoa2AdUFeOwvv/Gjy6rOuu2oRLIzMR/zC7ZyNqfGMLy/9K5/9YN+oTb7MDwxy2tHRjLtJlzht+oqhl18dWB5z/bUf9fJjSz70xryxg02iJJFIgsQm7sgudgWuA0ZLP23oTTjtqWdM0y6ed4bcdnj8g99tgPFMMbHY4vGnYPgAOxt6MP1O88s/WSaLcfqZB0DGX/u3/zv+iOmk2Lw+e/Hj7+pYmd8KXgP16jJu6iZOG3p8pv1Ip89RP+PKhz65btwHk0wCE0KPbla4OP0MpHzWZxJzjI882HgQl96WdnXzohu9uN7A9tNi44uL2w5cC7GoC7eTcTpcN17JAX31+dERKvwoDj4qUqY3V3Lis8xZ2HCVcwEbXU81rv+5UettDSw0uIjjcVDXxv3J/I2rnBh50HlKA8f4bF2Ni89R/BnH+OBp6yIfIRMQKMGUwdNMVF/xm/X6+cz/GrHFV5Zn+hlHHPG5znLXyh0GjDw1uc2lDij3sh7sF/Dy0MPtNRsfGWW7yKOoehekIqiQ4swbq/iJZ0yREb8LjfttpLK7XvZ7IZfZ0zD+2ZyPx8HjD2ba0g8czULS3x7bHKNLzilj08ceDGPbKrALZTnO+0s6SopscMnEpE3eI+wRXq55QNSnTxfW17frrLQzvhFvLwqud+YOIWfuNr3wbYCsHrv/2omw1XGyOU8Lw/jqkeXXJ/vEEYt7Z+j+4W//bH2QjizvXB/95VS2oNCnP8cXOe36GkM5/dTN+2/q9XX91ikSAEYM2esggf2RnkRMBjtjW+LFYNPHiWpbCe6LqL+cFNd9ro32he84xiufxiPv/p56zEs+/LjO4nTIdduTWlh8ln3nvJ5JRwp7yVVnnVfE98J9Q2/H1jFvPh6yq1mf5Qd/776MKv4qgBo//OKH12eb5uXxyePmsQGjnS9K0tLmmB6svX70PGjyGKeVQz/55GneIug7+YIkmL0BTeYIL2b66pP69Mcv5cQdjcHfuPjmgGyrdFF03M96+TPvW3fhzZF4XGf1Hf03fluulyUkbsb3Dr96cT1HKoZ1KSr4uKHa13nxVZ7G7fNlR+sbtTsezuRTdldEZsHFaKe3JT86sOrE3NaDpWWMxBt/2o2RffPUVtcFJomOEFG1ZwHV2+tHbxA57RPreGLlmfHl0L4KzAPCtY2tdFxnccsh75rnweGabbv4Lqc9Z9y3XYQRB3ovGMS9mbcy9UTblmVfmIq/eW/+Fgi+PPK48tUg4nMTmGZ8dtJ792pORaE/9hkfHc3jYgxkdRvi2jf14FNO7Jl+xtPHXk781ykS4Sj5qTeoJJDmOIMw1l8/dBnnTA/OBt4tWt8+RX5pu32Quw98uSBy2Hdh8TmiBbnXUdsrTiUs9LovvQWksvNWWS7mpd2egsHbvHi3mfMBh9zfnN1v0uprn/fn1M1+xs/jeoQ9ssOhXj5kx/Ck3TF6MalrvDuYBoFtjIOtHhxjHvpMLLJ4x2ATr789OFr6GSv9wLATUWAPXv3pwudCPn7tL4GtBqenw+/U55Bg+X+rmpOiwpsC2nGWWMfp3QUOjHsrrBhVq/dY7ccu32jkrvrg1Z/0jeFrvxpViPmC8RjN46Ie/7SpT50xsD2NPjkY65e+6uTuPGqCpb8Okg4AsNmmTb198ug3faYemUfuUOkjnj713Km//+UfG3rrybUWMQuMQlz3sy7RXUBdFHDvO1rHsQALb/wupb1AoFlFunM2Lnh29Wm3fTFx++DbGBZYcmOjwPxKdh4DbMqMeeRxPA1eBv3AJI8+6GZLn2lDPuK5uJMPaJIYaOrB3tZu8zMRMfDIr03uxKijZ0d68OXawbZtqE29u5ScBQbvxe0BjpsbD4vSniP+UYEVD818Ot/g6rzDb6fdusB1wKLi5i5vPj75/Hsag/9tBeatlduOkzYIzZMxemXH2YOZTbz65Fb3NP26BgMsKWSTMG1s675SDKKdHl8wNHDKcopF5qFP6vVPDHzi120KlLWAq1Qq/uOHf9Xx4TM+MD8nrID7Ou/zjIIDV0Eu8rco2kYs7lG1sOVDYVu0vsvsuezx4SMmvYX1wse2Tw/Iz3nff/XH9YF5fcQ14h+dIjkONHwdT1lej5k9+lw/ORLf5Af86F0bxnDKi0xDpjWv12CtiSdBAsO0CFJncupIAg4noh2dttRlPDjAYHcBtNuzg/V9sD2gXIjsYB4E/PXBxm722tf/p7hLqOPwDAdjPyDYbenf4B1DHAqKG6E0ue3Nwzn2/PdYj+p+Fl//0SZWX3YwTv20jH/bRf7kaOc7nuB2XczFHKYr/DzEMwZrftPvhlwOvhgXtyQophmCqQMncdrUyZO8acOeTY7EpI7xLLD09xSZ/tjlYNz3oKrY3PkoGsb2YHKMTENXTEy4ZTmPYunvDgRWnH369ymy7sl1CNgrDDnpjwq/LJD0x24zVtrViaFPu/rE/TF2eejXR0UqIeRhxTqZGQiZydrr7ysCOXkSl2P97LHpR08zjmN67oN5DeZCoi/nPkVuwydX//6fb/af+3sNo56eltdni6fnVqeuXmHmuY8Lb2G0814AXXRbvXX8AvUc+G4Z39Lwq9b6Mi9O8fwMQX5Ifp8djM9Ae97srDWs3eaV+rq1uI4becAlLzZlesfoE4NsA0PTrt9cf/GzzxjYUu5xPXUECyMDJRn6HdrJMOZhIomdY/3QT355wTA+sssnD5jeweoPY7fDsyEsEHcwtF535a0B9JPr9e/V13z2rz9jt5lP11o9tbwXVpGsd5+NJ5kqCm7u+unB8peweosa3MMvfLhzIZ9X/uWnl7cpdh92MH8PQz7wPDz+jLHRZxOv7sx+tv7pb4zkSrv67NcOZuCjJNMB3JxU2udYXvUmNPVpT9vEG59vO3BKSawcWWAupjyebvRTjyy2SHklNF3aUezatrHDaW9FPXmdNfmxL/4dTIG99k8f2qW63+V9sIiP0ZwFGpMYOdaevXZ15qWsXT2yYzDTnjo5busvblNATPHQT2IDZw+xyST+aCw2bcYzQeN6WtaHfvpx6uMi3/hyUAC8i6TB03fI/T4WF+Z74bBonHZ8sYA3fn8YHX/Mgc3TYMfj+NRxohnfHTJ3AvnYbfuPTepr2/rhyyn00Rf+gmE33kVy2ixWNsxuzNsCMxa9eavb4cuHQR5HZPNxnDJxlKcf+Gy3xZQDfI+LrOeiwYWUMMkYOzHti2hfuNTfNpbXeMaffIlzDGbdpqjsXQ6LgB1M7Nw1MicW7qVPPNvXSBlfDL7rM0LmV/PPhQfXn3nW/az8S270zIvC8iav+ThfMJ4iGWPvU6QX+fu8wPtiyByT54j7SEccm3ZlejkzTtpTrz8+qU98c5bRY9Y2RR11eJrgM5A+cMgrn6+SxCQOPA8LeuL4Iwt2sAt94dkhLDBsFthRfAry3jPbAvqVaXHG/4//eqt3FT7D3M+PVPTVCx9/Z/9xiDdKM3fGD7/+876mMz/75OebFo/qGoyGnVMkH2XROD51AK6eqa/bUGDzGqzthfP44AN3xmFsPPXgsk17ysmXPjmeeG3kR8x1DXbDUEaahaCdPknVq1Omz0mZrDh6Hn5fST8Ts0efB1Hct7//q6tXXv3Z9aLvBmKeFRgF5R7UebAg3Cejcup/FtIfL8Fu/vQWKsXFYv/d3/xJ5yXGvPxu2be/9ysOwMq9cfuuBBZ+TpFf/eJHVpwusP2vnzo+SVXAfBdpPL7wx3FRNr69+dPTjnBHNo+7eHnktVc/OZTJD451DYagkyTZ65iBsacsx9TLK/aMVxz2u+J5H2zhtqDtR4EZy3ebpWCtupCAXox3mTv0L33iPX3qytsajb/jicLyr8WbLhe2xlvorZCxH31t+/oarADUVjXmR+Gbj/ParNfHCflpjv86Xvt6y4NvbiaJA3Mm648dDvNb+CLtEs/kcFJOB8fYJWScbRGzoNWUxRzxYjvS4+vuNeNthVMX+fWfOxCriJwFZty1A6Eo3l7BfREbg28VmBfhXvfknBt38HR9/2qjBeK89U/57M58F1j+NVTlSaYWmFzw2+ZxyThglMWr47imDW53L7HYz44/Nnxmn76MewdLIEqdGJ81MSbl5JXxU8cYPC11yPLYo6Mlftqwcw3G13W6RvZibr869/MuUp85Ny/czaNxxbK+4AdJFJ6Li9pmboff4WKe5LP68mpx+4br/BKknBy3V/71Z3W9d/ntWl487F5eI4p3fh7vi/nEwou/q4fnrJiOfI1vL0Z55VWDXvlMEHDKR+MkdAyOADR9GKfOBNDT0rZptmf19EcT907+KoZ9QcG7gx1xMHHegfZfYZ/8NGbm0YVSQVhg76bzeWH/SJ3v+HDY4y9fDsNeqP3X4lUkvNMkpzw24tGv+2Aq6UvPNRhFduTrHIEe8W4U1zGPOMTQ2+RKfI7F2R/ZWldPtE6Onia5zvRHNv2m/QwrX/oRSzn9HOMjxjE9BZY/cYmuW03h9dc+ujgpBm4HyCcXfZ82fXdYftvpFpaqjL04OrcSsLGbwcePp+BfpCS3xfWZQ4hqt7kDHsXnNO9vX8DH99s27uuPpqD1Hhtjmlyd2x6/89nMCzN1+k6/lHMnC7obQ3zSL2Olbf26jgD7ybi2vHlAdyCkNHp3nN209NroiUPPg2Zc+9SLw6b9xg7WLBWfHerRXy9eTmN8vvcC378aN1ZxgZvd6LXazWjyt8BTpUdxFbCLiVMWcuNIfdRX40r5Uv2KDh8D2ZwDx4bC4sYrrl/5xz9vCHxHO5inSHYwc5vHBoK0OW7ieMIPW/ZyuWbKk0Mf6ZQn3gJd+lJwmFYzARQ5XoBdPwtuEdYEaMg8SJw+E9amHXza9c/42uXyIt8FRm/zFInsxT12dhNOdeRuk5f+ckfbCkqc/D27fY5ddzxFY7fkV3m88YpJX8YP6vPGx9zrKo6jj4r6Jw8Iwnz2OOTt6RkOc2Yst7opg6GlnvkrW1gbauPGNtdXOz2xpl0+ccrrNgUGlDibrDqd6LW5SCaor72T0K6fXPqrp9dXDD06mrgW6qkLLH4zon2r2rhY5yJffBZY8xSG3cAFE2f87cJ9280sXvzO8lC/FdYHrl742Ds7trzazYMdSd5ZYP2Fwx++WcGIWG17rfYLY+a7AW7mNRf+CEdO5mWe4NAheyyU5aAX7/q5vmKNb39RYBInEeOZjGRpY0wzAcbTTx16E0NHS85Nc/k87X0NFnfyMxY7mG3tSihqwRrHgaydNXeGzBuoBdGLjR+7Hgcfkn3hwdG8ztqky2dPh7w7XH4scMHyTj559e42frIAoG8wZo5Ect5GnRjsqUPm4fFPf3DKRz7oxKQ980g9XBfXYDpnsjPgmdxklQBt8ky+mUTaHSefuuxvFNgqgOsPu8FfF0qt1H4As0j4TQna/Fc7jK//nDc+ng7zQl0/CsvPIcHWynX8bVxi/Yd/XqddFFjgbytg8zIu/Lcd38SBdSfCZ67blMHbMgY6dyzG6bfug2HI4CaOnpZOKRtIvHI71ZOc2tVPHHqxjsXSG1+/s6/rUDyPH97cwbY945pRno5ZPp421V8jt9OxtzXQUxjzV3mcH/58bklhErP5qraMLz94eB59cXybYr/RmnxPU2DmKz9ycjgWZy8eOw/laVeWB1ziU5/YdYqcxDrrqJPEKTMWp10+9YmXW91RL+bIH7zXYEf217/y0XWgJo4CXAtfPOmPjULjQXMujikaGnb9cp7udmXEubFpR0EM/sefwskbqOsTgcJhF/uHFpi5dQLjKeeEKfM7sw2KNffpr5zx+8NutjfOyReG/QDh1JPdKxb5qJmotuRCN+1uqeiNr+/sj+Kvr+tMcMl5DYYvC++OIryXnzmyu/AR0f5n+uadi4puHp/FUxw3TodllGfhqlzcxbYdcPuKT84/v02x/Iqfj5bYMc1D2x/Tmxc9sTO+usmr3h4742zw0NQrr1MkhiQQkCQSYBNroWBLHwNNfzH62yf39ElZfO9M/dMB+27BhJlkdXmKNA967nfxtWh3hi6uerqWLYEtIkVGY8c6ypubro/q/lnfWoj4RbgO9MZUz9irPaifj2LXch722F588N+kdNGISx63vYsEkzwXBLuA3ZZ4/eh5ePEv9qzX72j9teF7eorEkZYB0zHH4JCzGfjM/4h/+iO7c7jA6IhFgfFZ5GzmMe9+6wfPOpWFs34dp6bin6U9UzsbBeFpUx6LVQr9lTNfdPOLieLwm/msggdUuRA7i7LV5UeMGVfeGV/97Ke/fkfrA9aYfh0n1ze55e0CQxAoiQEMqPOU1eO3SGvi+vt9r7Q5pp8FhI4Y+DsmRuaHvE6R1PW2O/cu0dzl33kWFzvGvGaSdy0suTcNRDWK/PvvJkuLjp3khefqi4Zxe2R33OZePN0iPrciuPHKH9rSyMv4vEge8gN6+41XCovTNbE8ztwe6dN1Fbn+PYinnnPJ9vhmnID20PgIjJXTx/UzjyMOdK4f44y//EvZR0UjQNptxCaSOPzTRz6x2iaug9VT4tFNWRyJw9WnyLjRir0XqFacnv9p8HANw30vC82csHtjdSux63nP+MpFCAjXi9bzKxM85pF/AOL8cepTa73T9Gc6xSeh+M4/3gw4/7Trhw68NvWpy7F2evS06dvKeDrzD0gPF18lfPEDdAa4i0iCSewk0cslBh8e7kbojSOfPhzIxCWH496B9t/Vulh0DnIVFLvLKpxa9gd1qvn8/nmk8eAi5vXvhW3Fkxf+YBKPvJoLQ7xq4Py+1zwWzIlT66P6fNS8yuGiYPvFsTHVM78nu/1uvx89mUceJ+DKjMGcHbtpQ555ypGc6I7azAdM6tZXplHyOEtsks/gSTqx2tQfxUGX+vQ505vDOtWNyXU8Xpix4cBFAeSNVWPRz58f7xjktidvTLCrGHZ+3x1y41UcbmDN8QZfscA+9f5LI5xa8Z/r0vH3nZTxbBnfHMSc8aU+ORnLZ5+6xGo31sW7SJQTIFAS7HObTsz0xy99Zwxs+ExM6icn8eXRxgLyrQn/aEJ9A12APU4XRhUF1zZ+B79x9YQfp9/Xv1934ouzi7PWzxLbimqfU2HBe+OVfs6DTxzYHbePijY81dpFRXGNay5+HCU/LPc4mB89uqknD2ODyflP/LSlrzb5s5d38qVef3S03sEk2VTbM7rZZiLpN4MaaPKol/vML+2MMxayucgnT19T7V+9EUdvHomnWLgu48I9/zpIvLsOchEQtIc8daGV+PALH1o/DZA58iYkPyqa8ZvIwi3u/GzUHMHol9zoEyMOjDudfuKO5LQdjeWlp4nZpMs1gT/ji1mnSBX2JqQseerVgUGPrF3bmV4f+e31U5ZPOfvEzjgWR2NcyHK2MJq3crZo5m2NjCtXxhafOGIhg/dfYWufg/ibnn1s++jpwee237DAf84r4zpODLozP3HmmTI6C1Leu3huw8kthn7tYAiZhKfBmQA4bGKxQ6xsguqU7adeecaXD3tiGBufsfmJhwe9fP6Vdmt2fSW7FRarW21bZgZ1fVanTf+l2rbFglM4vPN7+dPvXbsWsYzXhZi/s1q+nRdx95iN3cfbvbHtXxoxFn1yItOcHzbGxtys13Zt4pTt1ae/NnrHaZ/xPf7o5/qjw1dMf5tikgEiEA0CgyI75kYbzQVuIZ7kFC8fesf2YsO9E0Se8dGBZwI0459xob84bfa0eKpFZ93hKb7ZcoeCI+NkfMZcs+WfrCUXvuyaFjFj/k3v+1XIXLMxj6P5p865wZv6GSdlcR0/jjl6deCNn3j0yowzvjaPf+K00WPvOPW0v46vi6cNBwcdR5pJMTaANMrYZjviTT/tqZscyOIcH2HMI7nYzfxqND5z4SkCdfizmx19RGN8CiuvszKfjl9H1sLqHOtQe+P3KL/G1JP8N/h2QB5/fY56YuT8HYs1B2Xj2quf/ZFdbjnFrGswFfQ85ivWINhskiEf6dElRr/spx8yD3cusLdxGEMeZXzUTQ7veWUejXGn6byf6e/Vf75Oh/MNQPptp8XrnxVwR+ycmcsOZrfyT9bMEZM5OkdkHh5/islx4hk7R3vtyeXYXgw9Tb05zfjqN/T5MzgbnPKsa7C7iHSCJJOSNPWpY5zBtcmh7KtS/fSZ8Z1AHny56PXXT97EWByUQZ8k68DQtmLY/rXbxE9OZHjhodFfl9Tm2R8VxY+jbNqbz85/Wo7yFmN8ZXpzVKe/emXt+sg17eknJn08/mnTTr8+7EawTbB6emySKqf9LMHEMD7DoSfGWcv4d+WZHDOeNncz7Z4WlcWZ04yvnb4LNm6R+FERtsmHLpv86HJe0y9t6e84edDpr175NnzG0E/85JRP3A25Xjm9mhgktk/SHEuWOol9JSIzzmLUZpKTBxl8xhdjfviKc0yfLXOZ8ZXhSB4Kjest7JmnvHIim5O27LFxD8w/stXm8ZCHHqwyYx63zR+uxCkbQxnOyZ82x8bTH1k/deanj30eR7H2+nAc/x8AAP//4/JLOgAAIrJJREFUnZwJ1GVVdedPzSNUUQwWQ0xWEzEYjVFBMYDGqEA3Dq0SVBBMbBEihIA4QZYDmABL44AE0RZjWu2lcaVNFCFZ6IqmUUHRdEgMajRGoxQWWEzFUHPl/M59v/vtt+ve7/voE9/bZ+/9///3Pvedd+999yuyYNeuXbvLZCxYsMDpoN29e3cRw5yBPxRXIOPH4mqIF2cdfDHMI07MbLGMyX49Dk0TDeuIsZ5xfHPiI8Z5xDGPI/ZKPNaPuKE65CN/CBPzUc955BDDz5wYYx79iCXOGIzVZL/BAOlieS1cuJBwI5tDaOfOnS1mHox55kN54g6biRxyxjnguT55YmDIM8Rjoxb16c3+zIlv5PpmXD/mYy7qj2HAm8sbxrh1tLEGsYiLOeL6ERN1yMf1Rpxc8TFHzHyME4u+uBwzHvF+/gtqcGqDeWAEY2laGOLMEVi0aNFgAxRUJzYjl/zY3JwbCN+DxpxhfeLqa+1zaIOJ6VSmD6o8chFHH0PrFxd56mLz+qOmOLnknJOL2LnqD2kRQy/q6FvHmmKMwzU2pGNsCEfM0a+/TtoGywUFxsLELJ7jMScXC46Xm0Se9cTqz6YPdiyPrjk1xY/lIo55xNmnmCFtc1gPqLGIVyvGxEXrZiKWOfPxx/Qz15rEI2eovlhtxBNTmzk5fXELZttggiEzJDG3OS2xoaGGXHxe8awALzanTtZWg7y5ISsfGznG5ehrY5x59Id05GHFYhkZH+NiGzC85Q8YXPxiRs0xjSA3NR2qnzWG6seaca541oh1wPSXyCguOdpMBG9BrPkmOvHNEzMvNubIz1ZfLjj5cR5jOY4PP9bLehkzlAfjMI9mPA7m7WeoprGoAQ8/5pybi9rMrStPvHEwxsQQY1i78zrcbMc/crKm2mpGH2y/wQQoYPFIYD72jXJhmZ+bG8ON1Xfh5O1lqMZsurFvca4va9mHeawYc/rkjGGJxxx5hhhy4mKsQ3XvOZ7Xn4+/eNjqO8eS5yUvrx9MHLl/uOpGnchhnnXl9BsMUG6GmIMcwwYydigfsepgwbpgfbHxgJIzzjyOXD/mmLtg+GDVmQ9vrDdrqIfNw3rErZkxkZcx/7/rV1O9uP5YHxwvcVoxMWcsWzHYPNTr69fJnqjKEqhAT6gfGMMi5mOxyI1x5nx4kYsuww814omrFTnEs0+MYXxIJ+aiLrwxX82MyXjyDGq4li4y8259I7HHMb0Ydw5/Li21hzhyxaAXcfh5yCGe5xmLltpTN/kC+2QAxhwCczUkHqsec4vPxfc5Sv6w8kZH85GOoQMU+yFvHeu7BrlaeeajT0x+7FFujuHDJ58fs4i1Lyza1jOvRQMMj5JyPeLEGPaXdTz+xrXWV9O4etFvmEpolUhkEA0YYx6HQjHGvIlWLUfmj/EyFx4vD0DWG9PJ9eWNWfvL68/1xanjgbY/62rJM8zLG7PWow97IZb5xBi5vrqxfuTH40VcHePW1CdPDevHOLXkm8ePGq6/vweLAJvFSnIec8wtZAMxP5az1hBfjjpD9TNGLDb2Ic6YPpaY8cgfm8sxr5b+fLTkYOMHMx8NN5R11Brz0QRj3hpj8YzN+rm+euL0c73+EgmApIRcMBMVFB/5UUec+egzd6fnAz5WP9aLWhE/VH+ufF5fxMc6zof6sG7WguMHxDzmXT+xGM/1rScm+1k/57MefTDEMVebuXrGxOmDYRiP+hEzdQYD5Afd0WcEIClGTpEYM24xc5krDhuxzB9pfTQYsVYX6d6jfozbf4yJzTH8ITzxsbrk4hBnTD1rYp2LiVZ8jMW5+uDU0UbcbPOIj3pyjOHbT4wZjzpTG0ySgti5BHJezpAWuXgAxGIZQ5ysn/kdc6bPnJ8vX51s5Q/1BnYoTyzjIy7mjaNl78wZ0c96HWLmPepE/bl4MwrTs6hBRn17Utd4ZBPzRNE2GKdDA7MRyCmsoHjj8/XVyg0P6aqJtc+M07cPfbnWMY4Vay9acvKcy5dDnKEf8V2mezdvTJy1yHv8jYnFEjOOnWv9cq0Lh5H9GHsk9dVpouEtx/u6VXy3SYOB10/F9IHJhObMadGJ88whFzHmY8xeso6+HOvLzXl1tOQzBi3yvPwAxVsnWvjko7UPY+KtFfXEZBvry488YuqZt679RE0wmU8sasjD8nL94BhjfHnWB5t1W6wC2xZXKIIAMBTrvJmiLsZ4tOrlhiOGuTjjQ/XN8UDl4a27y6rlMx+wufnoxHVwYBhz9QdG3oNbdpcHt3TPkJYs2lXWrl68B98+PPBRXx0085Dn+mfrL2PRMqau/lh99a2nhedcLW3M+ZxMfeuJVaNdIk0SjCKAbSQLKQBGvpacPB70MV79J3eV7TvatLz0WSvLf3vaqubAseZQfTVv+NaWcun/vqc88PDu8rRfWVauPHe/sqj7t5BTOjjoZC11jIMjxrqsbwyMw3WAO/nijeUHG7a31NG/urRccc5+UxsMHbWcu358hvVjjaGcdcVpI7YJTt6sF2sQG6sPjrz9+PlSl1iuNynTx+1PXq6vbn+TrwBARi4QfedysLFZxYkz51nuEb93O24bZz1/7/Ka5+2t26x1p4LV6bRKeeb5t7fN1T76qnnpq9eV449Y0cOt3wfCJObiPEBGp7Gvky/ZWP7t9rrBav1jn7C8vO/sfaeOk+vGOuDnmuLARKwcbKyLL84NQCwO87mW9eNGEAtffK4XtZnLsb6+vOg7b7xK6O/BomguHEkZF33mcuWwZ4846/Ya7y5LZ71gTb/BxOZGWcjMQSnl6b9/e9m2feb0fcFvrymnPHv14Aeh5lBf5NSNvQ5h6d++yLcNtqE7DbvBMg8/1pCPFmtiUN84vseJucP1ixOjL07fNeHzij5Y1xJzxKmjtjjjsU8x1tOPa4KX60+dwUhKBMzIBAuQExt5MS+GM9iRr92A2/TOPHF1OfMFa5tvDK3MFUD8musfKB/6/OaGWb1iQbn+sgPL6hXdNTLWjxzm9hjj4P0AiMvHOiLPeNxgz/i1Fe0MJn+IZyxi8jr1tXJyfX170Y/azI1HHHPj4rM/FFdPLX1sHFmfnJz2JD8Wi0SBOe+ujXHnCkcdNthT2WCTTRQvkX6DhvjEYvM/u3tn+cldO8qTfnlJWVivQnKoFef2bdyesh75sfpDemywH27YWXvaVY594opyxdn7zRzI1CvaDuq7DnX151tfLWzkjK3NOHhrWZuYQy39aCNePWJj88jtMXXCmPUDgjiEiYJijE03Vy+R4R7szOft1V8i5UW8MbWwP7pjR7n1h9taH0uXLCgnHrWqX+h3frytfO8n25u/csWidm9224+2lxu/vaXc+oNtTebQgxaXo+t909MftzzKtjlrc8C75V+3ln/6t23l/od2lafWHxTPeuLycujBi8tL33Fn+eEdOxv0GO7BXruuP25otFdZWG7854fLv/xoW/nuj7eXLdt2N+7hj15ajn788rLfmu5Hj8cTu/GeHeXm27o+ET/+yJX1Xm9b+cQXHygbNu0qh+y/qJz6nJXlcb+4rPBj5+H6S7bUlp902LJ2Fv/8zQ+Wb353a9lZL3dPPHRZeXKNH/nYmXXe/J2t5evf2VK+XY/fsnrsDv+lpeVFx6wqB+073YvHYMjmjTj0ebEWrwzMGVOXyCjcA+qOfSRjiEetfoPV+ZnP7zYY2NyotdTRv/pzm8tH/qa7RPLr8RsfOLhxwb3rL+4rn/rSA6X+9inLl5Xy8t9aXT76tw9IbR9G/fibf2y9tL3rzHVl6eLps+POepZ996fvK3/x5Qcbvkq1YR8XnrK2fPrLD8xssMcva78iLQLuxxt3lvOu2tTOsvLI0xf1+WJc8sp15bgjV3SbcbL+r9Yvwh9cdbdS5U0vW1su/+Q9zff4nP3CvcornrO63ove0ePY5N+oG4t7U+pZB8BLnrG6vPGla8r5tZ+bbtva6hMHw/84hhe+fE150bGrCc854noA25fEVj/sFfFTjyn6YAVmAYWw4DJWHwtXTKezoBzJGWwSz/dgURM8L74xaPCNwP/AZ+9vG4x9smjh7nLzVQe1lsizwT75d5ub3/bFhF8Fum9U9Zlb/7Uv2Ku86r/u1XS7/ko5+3131Q9iS4uBmxrsTUNtn+6uZ6Nl5b31DGZ///D9re1RjHWaLjUZVW83N/gT3dOfs6qc99v7dLn6zgb7/St/3npsvIpj/Yx2RqgyZ//3yQY7hw1WA7HH6voFsj7HbsXSUrZur42DDetnvqAeN2Q++ZYDymGHLGm14DA8JsyJ4cecvZFv/TGZDPG4bV4fmA3+ihwqogjWIhEX87Gh2vLUGew1J67qb/JtaBo/rU/ug9c+0G2wWmRRPV43X3VgK7fHBpsc+KcfvrScUr/x1P7c1x4qX/jWw1179RguXLCr8g+pB647eF//7rby2rrBGKyHb/cFJ68tj//FJWXrjt3lw9fdX88UM5cwPpijH7+0f0yxY+fucuJFG8vP7+sun+gcf8TK8vzfWFmWL11Q/l+9TF/11/cRrt3wYe0qn3rr+vbBsrav/cvWboORn/TP8eWMd8A+i8vGTTvK+fVX80uOXdWdwdgHkw2/ZtXC8saXrSn77b2wfPEfHq5n2XoGrgM+WhyfY+ql+bTjVrczHWfoG299qObqIqvGr9f72Y+8/oDGGXqjP172BYZ5ftA6xG3YSu627QgipxHPBaEO4YxTof2KZFL53IOdcWJ3BpE3l65nMD4gzmBfr5dIh5dI/aN/dXl7EGt97IUfvrvc8M2HWn38a/94fTl4v8Wt7xe/bWP5j3p544CzeT9zyaPqfc9iYC1P23/wp3eXr9UzHJuLDfKMJ67sf0V+9G83lyv/arKB6vpOf+7qcu6L9576UG79wdbyu++8sztzVN3DDl7Szh6sv10iqz5nIdbH/06rZ7lzX7y2bpDWRnvbXjf7UWdvaPU5jnwRrr9sfbuvcwO85c/uKdd/46HWN1qH1y/JJy7qNpCf2yl/dGf53k/r87y6lqX1C3DTld3VYKbSnrP8OYmwrn62g/dgNgKYbwLfglggi+Cbl+uG6XJusI7JPRgbLA+bVYO89TkDcA8GhgP79XqJtMaffPr+/hJZGyl/efH68l8OXNL3hA6XsNe8exPT9kF+4Lz9y1GHL6t/etpVjj53Q7cZ6gF/4TEry1tesbbzK9b6P62/Xl/4lo29ZnxMcfrld5Vv//vWps1N9P9930Flcb1/Zh1eQphf9JF7yg231DNp3UCMr77/oHrT3V0iz/3TTTXcba4jH7u0fPD8/ftaHhc22NPOnnlg/cKjV5W3njbTK5pf/seHywUfrJu11qtv5ZJX7Vued9RKUv245vrN5erP3d/7N15xUFm5bNJUjcbj77zp1Zy9GO9FJpMYb/P6xpgSBRuF9MHFOH48gOSIiW+T+kboKWf9VLedwc58/preZ+IpfSoYHG7yr7m+OyiL6zOKb1x9cF/LMxgf0MJ6dvMHgHR64hHHiRf9rO/fvwR8vz6Zf1n9deg90mX1LwTHhb8QqMHanvP6DeXuzfXSU//v2F9b3u7BiD/z/A3trwycEZ77lOXlsjPWSWv1PG6cqdhIjk9cuH89wyxtZ7BzuQerWoz3/N6+5Zn1lysjHs9t23eVo86pX4b6f5ztzj9p73Lac7u/iFjjth9tLa+4rLvcw/+fr9u/POWwpf26iXHLcPHHuh8R9Px371lf9qrPFlkLL4Z1m1PfYnxoLm4PXv1gux0holpAceOEVJsOFdhDeNIohPlssMhnbg1re4lEbHE9hbHBOu2ZX5H4K+o38Sv1GylfXe6Pjn/Tnhvs72/dUl539ab+gF7z+v3Lrx+6pOe3IpO3V07OVNy/HFM32BX1T0U8huAM6Di93uuc+6LpP4ORo59/r49aTqp/z3Rc/Dv7tLNL3nh/dfEB5dGPmr7xhtNdIm+vWt118031V+DJv9n9CnSd3/2P7eXUS++0RGE9T/rl6Q32+ZsfKm/7826DwfvSew4se6/sNOlTrV6kTjyentFjLs4zt/8n0wprJSmsj0VkKJ5z4iq8/amoMvnC9Gcw8/AY+tou2r33G6y6i+rNeb4Ha48Xam75kt3lK+/vHmHEs+Km+3eV497QPexF8bIz9i3H1bMNN+BnvPvn3UGtjV513gH10jn9gYBnvKw+aP0+fyqqOB538LfI+hup3l/OXLZe/qzV5Q31MQMjfxicXU67vJ6p6mCN73zNPuXZT17ZncE4s9UYm+faPz6gHLhvdw/YwPWN4902WD2DgeM4vvnla/fYYDwT5AzGWa4CyzVvOKBtMHQ8Hmywt/+ve5tOFW4bjDMYPXk7NPb5ojP0+RCLOfgNV4t2mZrNolko+01xHm/U9jkYGtzk50vkXNptg3GJrI17D0Zpen7np+7tn1/xHIwzWB6ewYyzwfhj+V337iwnvPln/SXyD0/dpz5D6v6lh1gs/R1z7u1lCz/764h/izzhzXeUO++p/++s6v8d8dhl5UOvm37CDx7+Z+ul6R0frx/sxP/r+mPiFw5YPHWJROPzlz6qbTA4DtbJJdLnYOTyBgPDBuMM5mfpGSxq9RtsIs4ZbHW9IsORZ91o0RjLm7OOuP4mPyfY7bz85x4QMiYWZ24R5v6M7XgzG4xc/BWJz7ChWCPqscG4BwPnYwryfOO4B+sekNZnP8sXtg1mfS/1d927Y+oS+UevWltOqE/M0XtqPQPt7B47lcc9ekn5WL03QpuX6+dh5dlX1DNDxXP64B6MMxjjnCt+Xm6qT8sroX0Brrt0fdl/cotpfXgvefvGepmsv97qwL+lXub5k1e8RFITPmcw5q1eY5SZDVbL7Kq/ZNlgL33WzI8l8N/7yY7uEjnpZWyDvfWjd7cvxILaQLxEomF/uf6kjSkjnmDEe7bcY4MBZFECIHnaNKdoXDy5OPiAGR2322B8Ozm5/4/6kPN3T+j+1MMHGHWsa2xpvVIw739FVg3+idlNV65vcXL9Bqv1vAcb2mAnvKne/3QnoMIG4xJJ/Us+fk/57FfrI4w6uNk/76S15dRnr2y9E9t4z65y+uV3tudcrIH/HfOE+iS//i2Sccv3tpaz3ttd+vB/oT7iuOaCdWWfvbr/B3h8Zpd/6r7yf/7+wbZ+MDxpv+iUNW0NX/lnnuRPfuHW+tfVP+R7ifRDw3KJfDoPWmv9XfX4Xnjquv4SiSajvwejaD02cYNxrNDhDMYG4wuxsK4/brB8/K3fqc+8G8cOjV6nThrC4tg4xgQihrkF5cvDp0T3HKwCla9V2Wzi6xSVGcAE960Pdjfz/QarevkS2X5F8iS/5txgqDHog9fdm3eX499YL4VdoXLZq+s92BHLW/3u/mzyhHzS4GPq3x6fcOjScm/91cjfNPlzDMN+j6lP8uO/B7vg6rvKl/5xa59fUr8Y/MPINasW1Q24pdx5b/f3Q+rT4xfe1T0aoDfOYDxnc/3X1Wdbj9qnu+luRSd1+3uwSZAz2EnP6B5BoENvbjD7/PAF+/U/Woy1m/yP1pv8yTF2g7EpPOOi90iH+pE3eJMvIBdBIMb0o83cLjd9iawiFdadzci30dZT3/QJVty3PnRIS7PB/uxv6t8XK9wNJtfHFOydFfWfU8d7MPtlE/ErsvlV97Iz9us3GAW+8M2Hyx9+ZFOpf9do9cTFfjx7AWCD8S9aHfc+sLM+jb+73FbvgRjWtUdxy+ufb95/zr710cHMY4i4wTgMXCLXr5v+T/7R2WOD1R8TJz2zu8xTD0y+B2OD8SvSAebamx4sb//zei84xwZTM3JdFzG0MkasduoSCWFoZBF8Y1q5+lr0+jMYTj2CnkVwG4+jSmKyYvNspFuu7jbYh6/bXP9cVG/yKyw+6+Jb9/7P3F8+/sXuTyT+irS+B8ENZn2fg1FZzA/rYwQuVXds4p/ktKZqR/WY1P+98vi9yuYHd5XP3Nhd5rgH4xLZXwrawS7lYzfcX66+dnPZzhmP4zlZFnr8q4y3/c66/pkTtYnfUv8MddZ7uTHvzlo3vOvAsm/9008c4OIlkkv5m0/Zp53B6L8dx0r4wU+3lpPfUZ+D0X+NX1P/DPTkx8xsMDTbBuNXZB3o8qub4+bZyzjWY6M+MQY8Yq6/i87EzfcbTEC2Q6fNWBQhR26COHleNp8bypyol3XNRY4xsPaV57PVzzXw+Y87vvPjreWO+nD20AMXl8ccsrQsmfzrC7XljdXnksg/IeI/EjmsXm5/af2Ser8384G4hsjP2vjkXVdcR4zbizh1IsZ6Hn99MHHk+FDNiJdvLXP20jaYScEmAVtQIlacPGIRZ96YPjhG5umLkxexMedcXhOdvMElbk4tOeblmLcWdjaMvIiXYy7WMqa1N79wxO2VefxC4882Ii/WjHP4Eaevrmv1OIhVA5yYPBdrHMuQC29B/bXVtrAFtBEkaSiXcfkAmVeDA0sMLax5tbXgHeLIySMX5xFrDTFYNfIHK09r/dhXnIvT5lxevzgsWHu2DniGfWW9lqxvcvWzlYdVC4zxXNec+cgxh5UHjrl+zDnH5vVP3eQDYFg0i5FDwEL4ecglzpzGYwxubiJryAXrUAOrRuxDrDj9zJfrAR3Cx5hzdbBR27wxfXDMXb954kPrlyfOxyz6WnDM/RzQGxriyanNXK4xj0PEyyEmjphDDfzME4NtvdZG+zOYYpGUY9mPgsxdOJbhAppT32w61jDXGgqbyjjWusb01dFaX38IH+tEnTjPPP3ZdMFYP68/8mJ9dYnx8njpy9OKRz/iiYOxvngwcRBXW46WuDw5Q3xyc9Xv+6iTvoMsHotYPGNswHj04xwtNXLcOtFmTN/wHBvQPmK9qJvjuY6+HPWIO1cDa0yevpghfwgLnjGUyzF98FEff7YhD+tGFk9sNq18/NWSny1aTTPeg8UCFsxCEYNozJOzkVxQPfmRB9a4PHWMi9cXpxWPDybjrS9eC4/hAZcnfqyeOPNz1TcvXr59DMWJiRvKw41x8cbUxlqfecyrPxQfwhmLPLj45LTEGP1/tkYiJhXqYN17zBPBz2OoSMSqKzf6Ud941JdDzHyMGVfHHFjn6kW+eXnqYOWJkaeOdgynpnnxQzoZIxY7hI/5obl69h4xWQ8srxyXMxa3xhCu6dWd3e+S2EgWVMi4vsLRkvOMYFx85kefecYN8cFEnhhtzBvTWmOIbw7sUD7G1dPaNz5z1x/j5KyhPrE4Il4s+TF85DqPGsa0amY9OOYyFj/jxeRae2hUAKMXYB6HBK05Cw7hxWSb60QN53LUta7WfMYb10a+sWhjL1w+8oZQHxwv8mqiY17NmDOWcWDkecnCt37Mq0feOTb3GWvNNrdu1CKmfrRZJ3PyujI++lNP8hWaAtQm4gBjs8Q9UMZjLvLAMTxAMedcDXx7yXoRI27o4GR+9merSc66mTdW3+NgL+qrk9evTrTMPT7M59OHddXJ9e0jWjH2Rg5+rB/x5o1l3pBej62ibSUT0+IQfA7jgiVowStMLBYVg1XXvL5WfX04YpkzyM0Vi3w4+QPNfDCMzBvC5fpyItZYpzqzBuNi9XN/maePhSPfOJ8Pw+NnXNyQ/pCO/chTJ/LHeHDky9Ma7/9URAICwtFaWKtAtDbQi07OevpqY4lFPAdI3FDduNBYM8/VJa4OMf/BYOzf+mDlkecVc+RzffKMqIevjjm18BnixZmPNs471p71Yxwt1mdPWDR40bd5OGJyDeK83KjMI585w3xz0pvaWms0rfo2ddOV3KkDg24jTYri24DxaM1HTfHkhoZYcfpi1c/56DOPPDlqiMU3F/HisGJj3hj5HFePnCPG4tx8tOrFGuSNMycXfWKMzOmi01xx8ufi5LwnILW14tB13mpVwuB/2Y1Q3LU2pOBs1iKxEHibi3GxWc/4WF010GTYK3i5YOQzz1h49sScEXXVJB518B1qyiMONnLFEs85/MiN2BgHxzCmrzWOHdO0V3qTFzVbgckb2LE1RJw6YvF56feXSBuTbMP4EKIvRhvzzBkZH+MRD9ZcjqtjXl9OrCHGWPSzbvbVI+6BIeYYw1sLHJg4xnLEI1YcMV65vnFx2lgrztWOOGL6s+llnLrEh/pSExyYOMy1f02BY0CgfiQ5j5g4J599Y8RjHbXyGcQ4NvYQdZkP6RmDK975UO2YEx9rkh8aYK0V8WrAMS7WWIwzV0eueesO5cewcHIOX41cL/rW08rRGsdmTXPEHa6jP4PlhGCBeSMYlxeLGos242MO7mx5sNaPuMgbqy+ePCP6cd6SKW8MG2vhw1UTn5ExxKzBfGxEXpyDjzVizSHdjI38Ibz5WHM2XK4feWph4+ifg9mcBcZ8yeL0LSbPODZjzcnRn82qq1bm6otTK+KJRR+sp355Ma/GmAUrT+3oGxvjg7WemBxTT1z25WlzPuuJw4qNMevEWJxHzhA25uFNncFMQuSMweADwI855tGPheKZJmKYi4vzVmTyFvHWh5Prk3NjRD7zXN9aWjDMebk2Ygzj1uyi3Ts5BrnZ6oMDw4h6xl2LmAZMWGuQG1rnbPVdv7Vdo3E0yTGoYz/4OU7MEXNo4cPN/VlH3f4eTCEtQAYCiuFDZJjX18ZGGnDgLerFeYS6iFwfjLUi3nmur36MexDMRb2Iy5r6EW9szFoj6hqD4zzmiXt88/rFgxka6mjdAPr0rrZaQ+uJMbnWixrqmxMrv79EArBgBpnDKp6FyTGGuF1mJodvA+aw1s9zfaz1h/jk4wAT+3EORr4xfXIxFufkGHxAY+vvEDNriXznYob6yzGxcsn7BTGHjf3HeOQxF+c85uWZ08dGXMwbj1jm1uk3mCQsLw6gZMDOFVJAX+vi5Qzh0MrxGHNuTbWsoTWuJS43z2O9jAFrnhwv1+8cDAOfIb454Y28/TjPeDUCrU0jz/pyhzjiYy+xZuREDJoxZw3j5PwCqSd+SMc+5OMb22ODAZrvsCh4CxvTn48WHF6RE+dqRIycsQMhRx3wQyPmmYuLcXiz5cxj5TNnRJ5+S6Q3v5gp3LtRxzm1xtYPxgEu+sTt03jGmFcj48jzsr64PXg1sMeRJ6SgRG3OSReffXljVnzME5ut8VgrzqOG8/nmxVM71s/9xRwc83PVUV+cvlYdfOaz4Tw2ERf5aMg3rk+OMRaPuQacvMmPtwexvlh19afOYCRpfogIwXgUsbAxMcRjM/Dzt1QsOfD6ahnHxjGUH4rNVj/WQxs+L9dvPdcnRp75bNEAwyuuN/vquVn0xcX4WA05WHj6Q/Oc02+k+gbH3rURM5aXr43ctv4aaGcwD4bNSdAC4+XCJ7Q9FmZcnno5bj5ascbgxFj01TMfc/CH8upi5RkbWn/UZM5raP2xlnN157L2AW9oHvkRM594xIzNY7/Wj9iYZ+76I2ZoLm/wv4uUIAif4vrMh/49Umwg520+alhnzIKVJ8ZY1icf64uTpyXOUFffvJYNx/Cf+xifTTfnrCF3yOb6kZP1om9/8QOfSyvWVyvWI68Glpf64uRFLXkxx+cDt/+PPiREEDEWApA4g0LMe4Hqx2Ej5Jnra7OOcTVy/eyDm0/9IZ5cLHXB5Lk8rfkGnLxFnnFj+ti8tphjLif2Qjzy7CNiwYwdf3Fz6bhB8xeIOPW16LjJmDOskfvsst27/KkzGMExMcmKWsQ41lyMgeNFLuejBjlxmR/92eZRX2114cX8kM5QfXFRj9iQH/XNiyU3Vy+PpH7uS649zKe+nKg1xDePHctnLXGzbrDYZBRnrqCW2NAY0pATczRkXJ2YJybGvL6LMY6Vay76uY68HI9+nKuvtvxowY8NcvmLDHa2GjmntnFtjDvPFqxXpbgG4nFkzYzFFxMtGmL/E7AKXaYE1pgUAAAAAElFTkSuQmCC", + DB.PolarDB: "data:image/svg+xml;base64,PHN2ZyB0PSIxNzczNTU4MDAyMzQ5IiBjbGFzcz0iaWNvbiIgdmlld0JveD0iMCAwIDMwNTEgMTAyNCIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHAtaWQ9IjE3MzgiIHdpZHRoPSIyMDAiIGhlaWdodD0iMjAwIj48cGF0aCBkPSJNMCAwaDQ3Ny40NzA3MmMyNjMuMjgwNjQgMCA0NzYuNzIzMiAyMTMuNDMyMzIgNDc2LjcyMzIgNDc2LjcyMzJ2NjguMDg1NzZjMCAyNjMuMjgwNjQtMjEzLjQ0MjU2IDQ3Ni43MjMyLTQ3Ni43MjMyIDQ3Ni43MjMySDBWMHoiIGZpbGw9IiNGRTY5MDIiIHAtaWQ9IjE3MzkiPjwvcGF0aD48cGF0aCBkPSJNMzY0LjkwMjQgMTAyNGMyNC43ODA4LTEzMC4yMzIzMiA2Ni44MzY0OC0yMzIuNzk2MTYgMTI2LjE3NzI4LTMwNy42NzEwNEM1OTkuODQ4OTYgNTc5LjA1MTUyIDcwNi41NiA1MTAuMDQ0MTYgNzE1LjAwOCA0OTkuMTU5MDRjMTguNjQ3MDQtMjQuMDQzNTIgOS4wNDE5Mi01MC4wODM4NCAxNC4yMDI4OC02MC45NDg0OHMzNS4zMjgtMzEuMTkxMDQgNDEuMDAwOTYtNDkuMDQ5NmM1LjY3Mjk2LTE3LjgzODA4IDUuNjcyOTYtOTEuMjA3NjgtMjIuODk2NjQtMTA1LjQwMDMyLTE5LjA0NjQtOS40NzItNjcuNzY4MzItMi4zNTUyLTE0Ni4xODYyNCAyMS4zNTA0bC0xMjkuMDU0NzItMzEuNDA2MDhjLTk1Ljg3NzEyLTEwLjQ4NTc2LTE1Ni44MTUzNi0xMC40ODU3Ni0xODIuNzg0IDAtMjUuOTg5MTIgMTAuNDc1NTItMTIyLjQxOTIgNzMuOTYzNTItMjg5LjI5MDI0IDE5MC40NjRWMTAyNGgzNjQuOTAyNHoiIGZpbGw9IiNGRUZGRkEiIHAtaWQ9IjE3NDAiPjwvcGF0aD48cGF0aCBkPSJNMzM5LjY3MTA0IDM2Mi4xODg4YzM1LjE3NDQtMjkuNzQ3MiA1Ni4wMTI4LTQ0LjYxNTY4IDYyLjUyNTQ0LTQ0LjYxNTY4IDExLjQ3OTA0IDAgMTkuMjIwNDggNS43MjQxNiAyNS40NjY4OCAxMy41ODg0OGE0MjkuMDU2IDQyOS4wNTYgMCAwIDEgMTQuMjMzNiAxOS41ODkxMiA1LjE4MTQ0IDUuMTgxNDQgMCAwIDEtNC4wNDQ4IDguMTUxMDRjLTI2LjM1Nzc2IDAuOTYyNTYtNDUuOTY3MzYgMi4zMjQ0OC01OC44MTg1NiA0LjA3NTUyLTkuMjg3NjggMS4yNjk3Ni0yMC44MDc2OCAzLjk3MzEyLTM0LjU3MDI0IDguMTEwMDh2MC4wMTAyNGE1LjE4MTQ0IDUuMTgxNDQgMCAwIDEtNC44MTI4LTguOTI5Mjh6TTYyMS41MTY4IDMzMS4yMjMwNGMzLjUwMjA4LTE4LjczOTIgOTUuOTQ4OC01NC4zNDM2OCAxMTUuNTg5MTItNDAuNDI3NTIgMTkuNjQwMzIgMTMuOTI2NCAxMC40MTQwOCA0MC40Mjc1Mi0xMi45NjM4NCA2OS40Mzc0NC0zMC41NjY0IDMzLjI4LTEwNi4xMTcxMi0xMC4yODA5Ni0xMDIuNjI1MjgtMjkuMDA5OTJ6IiBmaWxsPSIjRkU2OTAyIiBwLWlkPSIxNzQxIj48L3BhdGg+PHBhdGggZD0iTTExOS42MzM5MiA0NjMuOTg0NjRsLTQ4LjQ4NjQtNDYuNTkyYy0yNi43ODc4NC0yNS43MjI4OC0yNS45MTc0NC02OS44NDcwNCAxLjkzNTM2LTk4LjUzOTUyIDI3Ljg1MjgtMjguNzAyNzIgNzIuMTQwOC0zMS4wOTg4OCA5OC45MTg0LTUuMzc2bDQ4LjUwNjg4IDQ2LjU5MiIgZmlsbD0iI0ZFRkZGQSIgcC1pZD0iMTc0MiI+PC9wYXRoPjxwYXRoIGQ9Ik02OS4zOTY0OCAzMTUuMjM4NGMyOS43ODgxNi0zMC43MDk3NiA3Ny4zMjIyNC0zMy4zMTA3MiAxMDYuMjA5MjgtNS41Mjk2bDQ4LjQ1NTY4IDQ2LjU5Mi03LjE4ODQ4IDcuNDc1Mi00OC40NTU2OC00Ni41OTJjLTI0LjY0NzY4LTIzLjY5NTM2LTY1LjY1ODg4LTIxLjQ1MjgtOTEuNTc2MzIgNS4yNzM2LTI1LjkwNzIgMjYuNzE2MTYtMjYuNzI2NCA2Ny41NjM1Mi0yLjA5OTIgOTEuMjM4NGw0OC40NTU2OCA0Ni41OTItNy4xODg0OCA3LjQ3NTItNDguNDU1NjgtNDYuNTkyYy0yOC44OTcyOC0yNy43ODExMi0yNy45NTUyLTc1LjIxMjggMS44NDMyLTEwNS45MzI4eiIgZmlsbD0iI0ZCNkQwMSIgcC1pZD0iMTc0MyI+PC9wYXRoPjxwYXRoIGQ9Ik00NDQuNzMzNDQgMjk0LjA3MjMyYTYyLjIyODQ4IDU2Ljc1MDA4IDAgMSAwIDEyNC40NTY5NiAwIDYyLjIyODQ4IDU2Ljc1MDA4IDAgMSAwLTEyNC40NTY5NiAwWiIgZmlsbD0iI0ZFRkZGQSIgcC1pZD0iMTc0NCI+PC9wYXRoPjxwYXRoIGQ9Ik0xOTEuMzY1MTIgNjEwLjg1Njk2YzAgOS44OTE4NCA2MC44MjU2IDU1LjI5NiAxMDAuNzIwNjQgNjIuOTU1NTIgMzkuODg0OCA3LjY2OTc2IDg2LjI3Mi0yNi4yNzU4NCA4Ni4yNzItMzIuODI5NDQgMC02LjU1MzYtNDcuODIwOCAxMC4zNDI0LTg2LjI3MiA0LjE0NzItMzguNDUxMi02LjE5NTItMTAwLjcyMDY0LTQ0LjE3NTM2LTEwMC43MjA2NC0zNC4yNzMyOHpNNzI3LjM3NzkyIDQ1NC4yODczNmMtMC4wNjE0NCAxMi42MjU5Mi0wLjE5NDU2IDI5LjE3Mzc2LTEyLjM2OTkyIDQ0Ljg3MTY4LTguNDM3NzYgMTAuODg1MTItMTE1LjE1OTA0IDc5Ljg5MjQ4LTIyMy45MjgzMiAyMTcuMTY5OTItNTkuMzQwOCA3NC44NzQ4OC0xMDEuMzk2NDggMTc3LjQzODcyLTEyNi4xNzcyOCAzMDcuNjcxMDRINzIuMzA0NjRjMTExLjYxNi0xNzQuMjc0NTYgMjIxLjMzNzYtMzA3LjA5NzYgMzI5LjE0NDMyLTM5OC40OTk4NEM1MjUuMjA5NiA1MjAuNjAxNiA2MzUuMjU4ODggNDYzLjM3MDI0IDczMS42Mjc1MiA0NTMuODQ3MDR6IiBmaWxsPSIjRkVEQkJCIiBwLWlkPSIxNzQ1Ij48L3BhdGg+PHBhdGggZD0iTTExNTMuODYzNjggMzMwLjM0MjR2MzYwLjk5MDcyaDU1LjM5ODRWNTUwLjc3ODg4aDk0LjAxMzQ0Yzg2LjkwNjg4IDAgMTMwLjYxMTItMzYuOTA0OTYgMTMwLjYxMTItMTEwLjcyNTEyIDAtNzMuMzE4NC00My4xOTIzMi0xMDkuNzIxNi0xMjkuNTg3Mi0xMDkuNzIxNmgtMTUwLjQyNTZ6IG01NS4zOTg0IDQ3LjAxMTg0aDkwLjQ2MDE2YzI2LjkzMTIgMCA0Ni43NTU4NCA1LjA1ODU2IDU5LjQ2MzY4IDE1LjE3NTY4IDEyLjY5NzYgOS4wOTMxMiAxOS4zMDI0IDI1LjI3MjMyIDE5LjMwMjQgNDcuNTEzNiAwIDIyLjI1MTUyLTYuNjA0OCAzOC40MzA3Mi0xOC44MDA2NCA0OC41Mzc2LTEyLjY5NzYgMTAuMTE3MTItMzIuNTIyMjQgMTUuMTc1NjgtNTkuOTY1NDQgMTUuMTc1NjhoLTkwLjQ2MDE2VjM3Ny4zNTQyNHpNMTYwMS4wODU0NCA0MjIuODYwOGMtMzkuNjI4OCAwLTcxLjY0OTI4IDEzLjE0ODE2LTk1LjUzOTIgMzkuNDM0MjQtMjMuODg5OTIgMjUuNzg0MzItMzUuNTczNzYgNTguNjU0NzItMzUuNTczNzYgOTguNjAwOTYgMCAzOS40MjQgMTEuNjgzODQgNzIuMjk0NCAzNS4wNjE3NiA5Ny41NzY5NiAyNC40MDE5MiAyNi4yOTYzMiA1Ni40MjI0IDM5LjkzNiA5Ni4wNTEyIDM5LjkzNiAzOS42MzkwNCAwIDcxLjY1OTUyLTEzLjYzOTY4IDk2LjA1MTItMzkuOTM2IDIzLjM3NzkyLTI1LjI4MjU2IDM1LjA3Mi01OC4xNDI3MiAzNS4wNzItOTcuNTg3MiAwLTM5LjkzNi0xMi4xOTU4NC03Mi44MDY0LTM1LjU3Mzc2LTk4LjU5MDcyLTIzLjg4OTkyLTI2LjI4NjA4LTU1LjkxMDQtMzkuNDM0MjQtOTUuNTM5Mi0zOS40MzQyNHogbTAgNDMuOTkxMDRjMjQuOTAzNjggMCA0NC4yMTYzMiA5LjYwNTEyIDU4LjQ0OTkyIDI5LjMxNzEyIDEyLjE4NTYgMTYuNjkxMiAxOC4yODg2NCAzOC40MzA3MiAxOC4yODg2NCA2NC43MTY4IDAgMjUuNzk0NTYtNi4wOTI4IDQ3LjAyMjA4LTE4LjI4ODY0IDY0LjIxNTA0LTE0LjIzMzYgMTkuMjIwNDgtMzMuNTQ2MjQgMjkuMzI3MzYtNTguNDQ5OTIgMjkuMzI3MzYtMjQuOTAzNjggMC00NC4yMDYwOC0xMC4xMDY4OC01Ny45Mjc2OC0yOS4zMjczNi0xMi4yMDYwOC0xNi42OTEyLTE3Ljc4Njg4LTM3LjkxODcyLTE3Ljc4Njg4LTY0LjIwNDggMC0yNi4yOTYzMiA1LjU4MDgtNDguMDM1ODQgMTcuNzg2ODgtNjQuNzE2OCAxMy43MjE2LTE5LjcyMjI0IDMzLjAyNC0yOS4zMjczNiA1Ny45Mjc2OC0yOS4zMjczNnpNMTc4OS42MzQ1NiAzMjMuMjU2MzJ2MzY4LjA3NjhoNTMuODYyNFYzMjMuMjU2MzJ6TTIwMjkuMDA0OCA0MjIuODYwOGMtMzIuNTMyNDggMC01OC45NTE2OCA1LjU2MDMyLTc4LjI2NDMyIDE3LjY5NDcyLTIyLjM2NDE2IDEzLjE0ODE2LTM2LjU5Nzc2IDM0LjM4NTkyLTQyLjE4ODggNjIuNjk5NTJsNTMuMzcwODggNC41NDY1NmMzLjA0MTI4LTE0LjY2MzY4IDEwLjY3MDA4LTI1LjI4MjU2IDIyLjg2NTkyLTMyLjM1ODQgMTAuMTY4MzItNi4wNjIwOCAyMy44ODk5Mi05LjEwMzM2IDQwLjY1MjgtOS4xMDMzNiAzOS42MzkwNCAwIDU5LjQ2MzY4IDE4LjIwNjcyIDU5LjQ2MzY4IDU0LjYwOTkydjEwLjYxODg4bC01OC45NTE2OCAxLjUxNTUyYy0zOC42MjUyOCAxLjAxMzc2LTY5LjEyIDguNjAxNi05MC40NjAxNiAyMy43NTY4LTIzLjM3NzkyIDE1LjY3NzQ0LTM1LjA3MiAzOC40MzA3Mi0zNS4wNzIgNjcuNzU4MDggMCAyMS43Mzk1MiA4LjEzMDU2IDM5LjQzNDI0IDI0LjkwMzY4IDUzLjA4NDE2IDE1LjI1NzYgMTMuNjQ5OTIgMzYuNTk3NzYgMjAuNzM2IDY0LjA0MDk2IDIwLjczNiAyMy4zNzc5MiAwIDQzLjcwNDMyLTQuNTU2OCA2MC45NzkyLTEyLjY0NjRhMTA5LjMxMiAxMDkuMzEyIDAgMCAwIDM4LjExMzI4LTMxLjM0NDY0djM2LjkwNDk2aDQ5LjgwNzM2VjUyNC40OTI4YzAtMzEuODU2NjQtOC4xMzA1Ni01Ni4xMjU0NC0yMy44Nzk2OC03Mi44MDY0LTE4LjI5ODg4LTE5LjIyMDQ4LTQ2Ljc1NTg0LTI4LjgyNTYtODUuMzgxMTItMjguODI1NnogbTU1LjkwMDE2IDE0Ny42NDAzMnYxNS4xNjU0NGMwIDIwLjIyNC04LjY0MjU2IDM3LjQxNjk2LTI0LjkwMzY4IDUxLjA2Njg4LTE2LjI2MTEyIDEzLjY0OTkyLTM1LjU3Mzc2IDIwLjcyNTc2LTU4LjQzOTY4IDIwLjcyNTc2LTEzLjcyMTYgMC0yNC45MDM2OC0zLjUzMjgtMzMuMDM0MjQtMTAuMTA2ODgtOC42NDI1Ni02LjU3NDA4LTEyLjcwNzg0LTE0LjY2MzY4LTEyLjcwNzg0LTI0Ljc4MDggMC0zMi4zNTg0IDI0LjM5MTY4LTQ5LjU0MTEyIDczLjY4NzA0LTUwLjU1NDg4bDU1LjM5ODQtMS41MTU1MnpNMjMyMS43MzU2OCA0MjIuODYwOGMtMTYuMjcxMzYgMC0zMC40OTQ3MiA0LjU0NjU2LTQyLjcwMDggMTQuNjYzNjgtMTAuMTU4MDggNy4wNzU4NC0xOC44MDA2NCAxNy42OTQ3Mi0yNS4zOTUyIDMxLjg0NjR2LTM5LjQyNGgtNTMuODgyODh2MjYxLjM4NjI0aDUzLjg3MjY0VjU1Mi44MDY0YzAtMjIuNzUzMjggNi42MDQ4LTQxLjQ3MiAyMC4zMjY0LTU1LjYyMzY4IDEyLjcwNzg0LTEzLjE0ODE2IDI3LjQ0MzItMTkuNzEyIDQzLjcwNDMyLTE5LjcxMiAxMi4yMDYwOCAwIDI0LjkwMzY4IDEuNTE1NTIgMzguMTIzNTIgNS41NjAzMnYtNTMuNTk2MTZjLTkuMTU0NTYtNC41NDY1Ni0yMC44Mzg0LTYuNTc0MDgtMzQuMDQ4LTYuNTc0MDh6TTIzOTMuODk2OTYgMzMwLjM0MjR2MzYwLjk5MDcyaDEzMS4xMTI5NmM1OC40NDk5MiAwIDEwMi42NjYyNC0xNi4xNzkyIDEzMy4xNTA3Mi00OC41Mzc2IDI4Ljk3OTItMzEuMzM0NCA0My43MTQ1Ni03NS4zMzU2OCA0My43MTQ1Ni0xMzEuOTYyODggMC01Ny4xMjg5Ni0xNC4yMzM2LTEwMS4xMi00Mi43MDA4LTEzMS40NTA4OC0zMC40ODQ0OC0zMi44NzA0LTc0LjcwMDgtNDkuMDQ5Ni0xMzMuMTQwNDgtNDkuMDQ5NmgtMTMyLjEzNjk2eiBtNTUuMzk4NCA0Ny4wMTE4NGg2Ni41NzAyNGM0NS43NDIwOCAwIDc5LjI3ODA4IDEwLjYxODg4IDEwMC42Mjg0OCAzMi4zNTg0IDIwLjMyNjQgMjEuMjM3NzYgMzAuOTk2NDggNTQuNjA5OTIgMzAuOTk2NDggMTAxLjEyIDAgNDUuNTA2NTYtMTAuNjcwMDggNzguODc4NzItMzEuNTA4NDggMTAwLjYxODI0LTIxLjM0MDE2IDIxLjczOTUyLTU1LjM5ODQgMzIuODcwNC0xMDEuMTMwMjQgMzIuODcwNGgtNjUuNTU2NDh2LTI2Ni45NTY4ek0yNzU4LjI4NzM2IDMzMC4zNDI0djM2MC45OTA3MmgxNjUuNjcyOTZjMzguNjI1MjggMCA2OC42MDgtNy4wNjU2IDg5Ljk0ODE2LTIxLjIyNzUyIDI0LjkwMzY4LTE3LjIwMzIgMzcuNjExNTItNDMuNDg5MjggMzcuNjExNTItNzkuODkyNDggMC0yNC4yNjg4LTYuMTAzMDQtNDMuOTgwOC0xOC4yOTg4OC01OC42NDQ0OC0xMi4xODU2LTE0LjY2MzY4LTMwLjQ4NDQ4LTI0LjI2ODgtNTQuMzc0NC0yOC44MjU2IDE4LjI5ODg4LTYuNTc0MDggMzIuMDIwNDgtMTYuNjgwOTYgNDIuMTg4OC0yOS44MjkxMiAxMC4xNTgwOC0xNC4xNTE2OCAxNS4yMzcxMi0zMS4zNDQ2NCAxNS4yMzcxMi01MS41Njg2NCAwLTI3LjgxMTg0LTkuNjU2MzItNDkuNTUxMzYtMjguOTY4OTYtNjUuNzMwNTYtMjAuMzI2NC0xNy4xOTI5Ni00Ny43Njk2LTI1LjI4MjU2LTgzLjM1MzYtMjUuMjgyNTZoLTE2NS42NjI3MnogbTU1LjM5ODQgNDUuNDk2MzJoOTYuNTUyOTZjMjQuMzkxNjggMCA0Mi42OTA1NiA0LjA0NDggNTMuODYyNCAxMi42NDY0IDExLjE5MjMyIDguMDg5NiAxNi43NzMxMiAyMS4yMjc1MiAxNi43NzMxMiAzOS40MjQgMCAxOS4yMjA0OC01LjU4MDggMzMuMzgyNC0xNi43NjI4OCA0Mi40NzU1Mi0xMS4xODIwOCA4LjYwMTYtMjkuNDgwOTYgMTMuMTQ4MTYtNTQuODg2NCAxMy4xNDgxNmgtOTUuNTM5MlYzNzUuODM4NzJ6IG0wIDE1Mi42OTg4OGgxMDQuMTcxNTJjMjYuNDI5NDQgMCA0Ni4yNTQwOCA0LjU0NjU2IDU4Ljk1MTY4IDE0LjE1MTY4IDEyLjcwNzg0IDkuNjA1MTIgMTkuMzEyNjQgMjUuMjgyNTYgMTkuMzEyNjQgNDYuNTIwMzIgMCAyMC43MjU3Ni04LjYzMjMyIDM1Ljg5MTItMjQuOTAzNjggNDUuNDk2MzItMTMuMjA5NiA3LjA4NjA4LTMxLjUwODQ4IDExLjEzMDg4LTU0Ljg4NjQgMTEuMTMwODhoLTEwMi42NTZWNTI4LjUzNzZ6IiBmaWxsPSIjMTExMTExIiBwLWlkPSIxNzQ2Ij48L3BhdGg+PC9zdmc+", } # RedisCloud color: #0D6EFD diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 27be85b6e..0369b2e33 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -155,6 +155,11 @@ class CaseConfigParamType(Enum): optimize_after_write = "optimize_after_write" read_dur_after_write = "read_dur_after_write" + # PolarDB parameters + insert_workers = "insert_workers" + post_load_index = "post_load_index" + pq_nbits = "pq_nbits" + # Lindorm parameters efSearch = "efSearch" pq_m = "pq_m" From 7e251b6a96654116ed13d98c1a8f8cf16d1061ec Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Wed, 1 Apr 2026 13:39:04 +0800 Subject: [PATCH 04/38] Add concurrent insert in performence case (#741) 1. Fix concurrent insert memory and process cleanup 2. Add configurable load concurrency for performance cases 3. Make CLI Ctrl+C work by polling has_running() instead of blocking on concurrent.futures.wait(), which swallows SIGINT. 4. Remove perf-case insert from SerialInsertRunner 5. Ignore S608 lint rule and fix formatting Signed-off-by: yangxuan --- pyproject.toml | 2 +- tests/test_concurrent_runner.py | 159 ++++++++++ vectordb_bench/__init__.py | 1 + .../backend/clients/alisql/alisql.py | 8 +- vectordb_bench/backend/clients/api.py | 5 + vectordb_bench/backend/clients/doris/doris.py | 2 + .../backend/clients/mariadb/mariadb.py | 6 +- .../backend/clients/milvus/milvus.py | 11 + .../backend/clients/oceanbase/oceanbase.py | 4 +- .../backend/clients/pgvector/pgvector.py | 1 + vectordb_bench/backend/clients/tidb/tidb.py | 10 +- vectordb_bench/backend/clients/vespa/vespa.py | 2 +- vectordb_bench/backend/runner/__init__.py | 2 + .../backend/runner/concurrent_runner.py | 278 ++++++++++++++++++ vectordb_bench/backend/runner/executor.py | 170 +++++++++++ .../backend/runner/serial_runner.py | 93 +----- vectordb_bench/backend/task_runner.py | 18 +- vectordb_bench/backend/utils.py | 41 +++ vectordb_bench/cli/cli.py | 26 +- .../components/run_test/submitTask.py | 11 +- vectordb_bench/interface.py | 31 +- vectordb_bench/models.py | 1 + 22 files changed, 730 insertions(+), 152 deletions(-) create mode 100644 tests/test_concurrent_runner.py create mode 100644 vectordb_bench/backend/runner/concurrent_runner.py create mode 100644 vectordb_bench/backend/runner/executor.py diff --git a/pyproject.toml b/pyproject.toml index d7bf42633..6706a3d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ lint.ignore = [ "INP001", # TODO "TID252", # TODO "N801", "N802", "N815", - "S101", "S108", "S603", "S311", + "S101", "S108", "S603", "S311", "S608", "PLR2004", "RUF017", "C416", diff --git a/tests/test_concurrent_runner.py b/tests/test_concurrent_runner.py new file mode 100644 index 000000000..c9e5d9267 --- /dev/null +++ b/tests/test_concurrent_runner.py @@ -0,0 +1,159 @@ +"""Tests for ConcurrentInsertRunner against a running Milvus instance. + +Includes: + - Correctness tests (threading & async backends) + - Parameterized benchmark: serial vs concurrent across (batch_size, workers) matrix + +NUM_PER_BATCH is set via os.environ before each run. Since runners execute +task() in a spawn subprocess that re-imports config, the env var takes effect. + +Requires: + - Milvus running at localhost:19530 + - Network access to download OpenAI 50K dataset + +Usage: + pytest tests/test_concurrent_runner.py -v -s # correctness tests only + python tests/test_concurrent_runner.py # full benchmark matrix +""" + +# ruff: noqa: T201 + +from __future__ import annotations + +import logging +import os +import time + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.milvus.config import FLATConfig +from vectordb_bench.backend.dataset import Dataset, DatasetSource +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner, ExecutorBackend +from vectordb_bench.backend.runner.serial_runner import SerialInsertRunner + +log = logging.getLogger("vectordb_bench") +log.setLevel(logging.INFO) + +DATASET_SIZE = 50_000 + + +# ── Shared helpers ────────────────────────────────────────────────────── + + +def get_milvus_db(collection_name: str): + return DB.Milvus.init_cls( + dim=1536, + db_config={"uri": "http://localhost:19530", "user": "", "password": ""}, + db_case_config=FLATConfig(metric_type="COSINE"), + collection_name=collection_name, + drop_old=True, + ) + + +def prepare_dataset(): + dataset = Dataset.OPENAI.manager(DATASET_SIZE) + dataset.prepare(DatasetSource.AliyunOSS) + return dataset + + +def set_batch_size(batch_size: int) -> None: + os.environ["NUM_PER_BATCH"] = str(batch_size) + + +def timed_run(runner: SerialInsertRunner | ConcurrentInsertRunner) -> tuple[int, float]: + start = time.perf_counter() + count = runner.run() + return count, time.perf_counter() - start + + +# ── Correctness tests (pytest) ────────────────────────────────────────── + + +def test_concurrent_insert_threading(): + """Test concurrent insert with threading backend.""" + db = get_milvus_db("test_conc_threading") + runner = ConcurrentInsertRunner( + db=db, + dataset=prepare_dataset(), + normalize=False, + max_workers=4, + backend=ExecutorBackend.THREADING, + ) + count = runner.run() + assert count == DATASET_SIZE, f"Expected {DATASET_SIZE}, got {count}" + + +def test_concurrent_insert_async(): + """Test concurrent insert with async backend.""" + db = get_milvus_db("test_conc_async") + runner = ConcurrentInsertRunner( + db=db, + dataset=prepare_dataset(), + normalize=False, + max_workers=4, + backend=ExecutorBackend.ASYNC, + ) + count = runner.run() + assert count == DATASET_SIZE, f"Expected {DATASET_SIZE}, got {count}" + + +# ── Parameterized benchmark ──────────────────────────────────────────── + + +def run_serial(batch_size: int) -> tuple[int, float]: + set_batch_size(batch_size) + runner = SerialInsertRunner( + db=get_milvus_db(f"bench_serial_b{batch_size}"), + dataset=prepare_dataset(), + normalize=False, + ) + return timed_run(runner) + + +def run_concurrent(batch_size: int, workers: int) -> tuple[int, float]: + set_batch_size(batch_size) + runner = ConcurrentInsertRunner( + db=get_milvus_db(f"bench_conc_b{batch_size}_w{workers}"), + dataset=prepare_dataset(), + normalize=False, + max_workers=workers, + backend=ExecutorBackend.THREADING, + ) + return timed_run(runner) + + +def bench_matrix(): + batch_sizes = [100, 500, 1000, 5000] + worker_counts = [1, 2, 4, 8] + + conc_headers = [f"conc({w}w)" for w in worker_counts] + speedup_headers = [f"speedup({w}w)" for w in worker_counts] + print(f"\n{'Batch':>6} {'#Bat':>5} {'serial':>8}", end="") + for h in conc_headers: + print(f" {h:>10}", end="") + for h in speedup_headers: + print(f" {h:>12}", end="") + print() + print("-" * (22 + 10 * len(worker_counts) + 12 * len(worker_counts))) + + for bs in batch_sizes: + n_batches = DATASET_SIZE // bs + _, dur_s = run_serial(bs) + + conc_durs = [] + for w in worker_counts: + _, dur_c = run_concurrent(bs, w) + conc_durs.append(dur_c) + + print(f"{bs:>6} {n_batches:>5} {dur_s:>7.2f}s", end="") + for dur_c in conc_durs: + print(f" {dur_c:>9.2f}s", end="") + for dur_c in conc_durs: + print(f" {dur_s / dur_c:>11.2f}x", end="") + print() + + # restore default + set_batch_size(100) + + +if __name__ == "__main__": + bench_matrix() diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 07f77bb02..fc1813b38 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -20,6 +20,7 @@ class config: DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3" or "AliyunOSS" DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) + LOAD_CONCURRENCY = env.int("LOAD_CONCURRENCY", 0) # 0 = cpu_count TIME_PER_BATCH = 1 # 1s. for streaming insertion. MAX_INSERT_RETRY = 5 MAX_SEARCH_RETRY = 5 diff --git a/vectordb_bench/backend/clients/alisql/alisql.py b/vectordb_bench/backend/clients/alisql/alisql.py index f88cf9d88..6d1fbaefe 100644 --- a/vectordb_bench/backend/clients/alisql/alisql.py +++ b/vectordb_bench/backend/clients/alisql/alisql.py @@ -107,15 +107,13 @@ def init(self): self.cursor.execute(f"SET SESSION vidx_hnsw_ef_search = {search_param['ef_search']}") self.cursor.execute("COMMIT") - self.insert_sql = ( - f'INSERT INTO {self.db_config["database"]}.{self.table_name} (id, v) VALUES (%s, %s)' # noqa: S608 - ) + self.insert_sql = f'INSERT INTO {self.db_config["database"]}.{self.table_name} (id, v) VALUES (%s, %s)' self.select_sql = ( - f'SELECT id FROM {self.db_config["database"]}.{self.table_name} ' # noqa: S608 + f'SELECT id FROM {self.db_config["database"]}.{self.table_name} ' f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" ) self.select_sql_with_filter = ( - f'SELECT id FROM {self.db_config["database"]}.{self.table_name} WHERE id >= %s ' # noqa: S608 + f'SELECT id FROM {self.db_config["database"]}.{self.table_name} WHERE id >= %s ' f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" ) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 82eda1824..80709e8e3 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -140,6 +140,11 @@ class VectorDB(ABC): supported_filter_types: list[FilterOp] = [FilterOp.NonFilter] name: str = "" + # Whether the client can share a single connection across threads. + # If False, concurrent runners will deep-copy the instance and call + # init() per thread instead of sharing the parent connection. + thread_safe: bool = True + @classmethod def filter_supported(cls, filters: Filter) -> bool: """Ensure that the filters are supported before testing filtering cases.""" diff --git a/vectordb_bench/backend/clients/doris/doris.py b/vectordb_bench/backend/clients/doris/doris.py index 82b3a12da..01984d665 100644 --- a/vectordb_bench/backend/clients/doris/doris.py +++ b/vectordb_bench/backend/clients/doris/doris.py @@ -13,6 +13,8 @@ class Doris(VectorDB): + thread_safe: bool = False + def __init__( self, dim: int, diff --git a/vectordb_bench/backend/clients/mariadb/mariadb.py b/vectordb_bench/backend/clients/mariadb/mariadb.py index db3863c85..e6053a0d8 100644 --- a/vectordb_bench/backend/clients/mariadb/mariadb.py +++ b/vectordb_bench/backend/clients/mariadb/mariadb.py @@ -108,13 +108,13 @@ def init(self): self.cursor.execute(f"SET mhnsw_ef_search = {search_param['ef_search']}") self.cursor.execute("COMMIT") - self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" # noqa: S608 + self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" self.select_sql = ( - f"SELECT id FROM {self.db_name}.{self.table_name}" # noqa: S608 + f"SELECT id FROM {self.db_name}.{self.table_name}" f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %d" ) self.select_sql_with_filter = ( - f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d " # noqa: S608 + f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d " f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %d" ) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index ead2979ff..9e9dfb7f9 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -137,6 +137,16 @@ def init(self): self.client.close() self.client = None + def _wait_for_segments_sorted(self): + while True: + segments = self.client.list_persistent_segments(self.collection_name) + unsorted = [s for s in segments if not s.is_sorted] + if not unsorted: + log.info(f"{self.name} all persistent segments are sorted.") + break + log.debug(f"{self.name} waiting for {len(unsorted)} segments to be sorted...") + time.sleep(5) + def _wait_for_index(self): while True: info = self.client.describe_index(self.collection_name, self._vector_index_name) @@ -155,6 +165,7 @@ def _optimize(self): log.info(f"{self.name} optimizing before search") try: self.client.flush(self.collection_name) + self._wait_for_segments_sorted() self._wait_for_index() if self.case_config.is_gpu_index: log.debug("skip force merge compaction for gpu index type.") diff --git a/vectordb_bench/backend/clients/oceanbase/oceanbase.py b/vectordb_bench/backend/clients/oceanbase/oceanbase.py index 93c42aac1..bf615e4d0 100644 --- a/vectordb_bench/backend/clients/oceanbase/oceanbase.py +++ b/vectordb_bench/backend/clients/oceanbase/oceanbase.py @@ -186,7 +186,7 @@ def insert_embeddings( batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) self._cursor.execute( - f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" # noqa: S608 + f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" ) insert_count += len(batch) except mysql.Error: @@ -217,7 +217,7 @@ def search_embedding( packed = struct.pack(f"<{len(query)}f", *query) hex_vec = packed.hex() query_str = ( - f"SELECT id FROM {self.table_name} " # noqa: S608 + f"SELECT id FROM {self.table_name} " f"{self.expr} ORDER BY " f"{self.db_case_config.parse_metric_func_str()}(embedding, X'{hex_vec}') " f"APPROXIMATE LIMIT {k}" diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 42fa7533d..30c797c38 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -21,6 +21,7 @@ class PgVector(VectorDB): """Use psycopg instructions""" + thread_safe: bool = False supported_filter_types: list[FilterOp] = [ FilterOp.NonFilter, FilterOp.NumGE, diff --git a/vectordb_bench/backend/clients/tidb/tidb.py b/vectordb_bench/backend/clients/tidb/tidb.py index a5c99bbe4..fba60d41c 100644 --- a/vectordb_bench/backend/clients/tidb/tidb.py +++ b/vectordb_bench/backend/clients/tidb/tidb.py @@ -119,7 +119,7 @@ def _optimize_check_tiflash_replica_progress(self): cursor.execute(f""" SELECT PROGRESS FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = "{database}" AND TABLE_NAME = "{self.table_name}" - """) # noqa: S608 + """) result = cursor.fetchone() return result[0] except Exception as e: @@ -131,7 +131,7 @@ def _optimize_wait_tiflash_catch_up(self): with self._get_connection() as (conn, cursor): cursor.execute('SET @@TIDB_ISOLATION_READ_ENGINES="tidb,tiflash"') conn.commit() - cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") # noqa: S608 + cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") result = cursor.fetchone() return result[0] except Exception as e: @@ -155,7 +155,7 @@ def _optimize_get_tiflash_index_pending_rows(self): SELECT SUM(ROWS_STABLE_NOT_INDEXED) FROM information_schema.tiflash_indexes WHERE TIDB_DATABASE = "{database}" AND TIDB_TABLE = "{self.table_name}" - """) # noqa: S608 + """) result = cursor.fetchone() return result[0] except Exception as e: @@ -172,7 +172,7 @@ def _insert_embeddings_serial( try: with self._get_connection() as (conn, cursor): buf = io.StringIO() - buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") # noqa: S608 + buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") for i in range(offset, offset + size): if i > offset: buf.write(",") @@ -220,6 +220,6 @@ def search_embedding( self.cursor.execute(f""" SELECT id FROM {self.table_name} ORDER BY {self.search_fn}(embedding, "{query!s}") LIMIT {k}; - """) # noqa: S608 + """) result = self.cursor.fetchall() return [int(i[0]) for i in result] diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py index 5288bc04c..1f2e1b883 100644 --- a/vectordb_bench/backend/clients/vespa/vespa.py +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -107,7 +107,7 @@ def search_embedding( embedding_field = "embedding" if self.case_config.quantization_type == "none" else "embedding_binary" yql = ( - f"select id from {self.schema_name} where " # noqa: S608 + f"select id from {self.schema_name} where " f"{{targetHits: {k}, hnsw.exploreAdditionalHits: {extra_ef}}}" f"nearestNeighbor({embedding_field}, query_embedding)" ) diff --git a/vectordb_bench/backend/runner/__init__.py b/vectordb_bench/backend/runner/__init__.py index 4af583773..d56fe0ff8 100644 --- a/vectordb_bench/backend/runner/__init__.py +++ b/vectordb_bench/backend/runner/__init__.py @@ -1,8 +1,10 @@ +from .concurrent_runner import ConcurrentInsertRunner from .mp_runner import MultiProcessingSearchRunner from .read_write_runner import ReadWriteRunner from .serial_runner import SerialInsertRunner, SerialSearchRunner __all__ = [ + "ConcurrentInsertRunner", "MultiProcessingSearchRunner", "ReadWriteRunner", "SerialInsertRunner", diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py new file mode 100644 index 000000000..6ed8e39fb --- /dev/null +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -0,0 +1,278 @@ +"""Concurrent insert runner with configurable executor backend. + +Replaces SerialInsertRunner for faster data loading in performance cases. + +Auto-detects thread-unsafe DBs via VectorDB.thread_safe and +falls back to single-worker mode. +""" + +from __future__ import annotations + +import concurrent.futures +import logging +import multiprocessing as mp +import threading +import time +from copy import deepcopy +from enum import StrEnum +from typing import TYPE_CHECKING + +import numpy as np + +from vectordb_bench.backend.filter import Filter, FilterOp, non_filter +from vectordb_bench.backend.utils import kill_proc_tree, time_it + +from ... import config +from ...models import PerformanceTimeoutError +from .executor import AsyncExecutor, ThreadExecutor + +if TYPE_CHECKING: + from vectordb_bench.backend.clients import api + from vectordb_bench.backend.dataset import DatasetManager + + from .executor import TaskExecutor + +log = logging.getLogger(__name__) + + +class ExecutorBackend(StrEnum): + THREADING = "threading" + ASYNC = "async" + + +class ConcurrentInsertRunner: + """Concurrent insert runner with pluggable executor backend. + + Thread-safety: If db.thread_safe is False, max_workers is clamped to 1 + and each worker thread gets a deep-copied DB instance with its own connection. + + Args: + db: VectorDB instance. + dataset: DatasetManager for batch iteration. + normalize: Whether to L2-normalize embeddings. + filters: Filter configuration. + timeout: Timeout in seconds for the overall operation. + max_workers: Number of concurrent workers (default: cpu_count). + backend: Executor backend to use ('threading' or 'async'). + """ + + def __init__( + self, + db: api.VectorDB, + dataset: DatasetManager, + normalize: bool, + filters: Filter = non_filter, + timeout: float | None = None, + max_workers: int | None = None, + backend: ExecutorBackend = ExecutorBackend.THREADING, + ): + self.timeout = timeout if isinstance(timeout, int | float) else None + self.dataset: DatasetManager = dataset + self.db = db + self.normalize = normalize + self.filters = filters + self.backend = backend + + effective_workers = max_workers or mp.cpu_count() + if not db.thread_safe: + log.info(f"DB {db.name} is not thread-safe, falling back to max_workers=1") + effective_workers = 1 + self.max_workers = effective_workers + + def __getstate__(self): + """Exclude unpicklable thread-local state for ProcessPoolExecutor(spawn).""" + state = self.__dict__.copy() + state.pop("_local", None) + state.pop("_ctx_lock", None) + state.pop("_thread_contexts", None) + state.pop("_iter_lock", None) + state.pop("_dataset_iter", None) + return state + + def __setstate__(self, state: dict): + self.__dict__.update(state) + self._local = threading.local() + self._ctx_lock = threading.Lock() + self._thread_contexts = [] + + def _create_executor(self) -> TaskExecutor: + if self.backend == ExecutorBackend.ASYNC: + return AsyncExecutor(max_workers=self.max_workers) + return ThreadExecutor(max_workers=self.max_workers) + + def _get_thread_db(self) -> api.VectorDB: + """Get or create a per-thread DB instance. + + Thread-safe DBs reuse self.db (connection opened in task()). + Non-thread-safe DBs get a deep-copied instance with its own connection, + cached in thread-local storage so it is created once per thread. + """ + if not hasattr(self._local, "db"): + if self.db.thread_safe: + self._local.db = self.db + else: + db = deepcopy(self.db) + # Manual __enter__/__exit__ because enter and exit happen in + # different scopes (here vs _cleanup_thread_contexts). + ctx = db.init() + ctx.__enter__() + self._local.db = db + with self._ctx_lock: + self._thread_contexts.append(ctx) + return self._local.db + + def _cleanup_thread_contexts(self) -> None: + """Close per-thread DB connections opened for non-thread-safe clients.""" + for ctx in self._thread_contexts: + try: + ctx.__exit__(None, None, None) + except Exception: + log.warning("Failed to close per-thread DB connection", exc_info=True) + self._thread_contexts.clear() + + def _insert_batch_with_retry( + self, + db: api.VectorDB, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + retry_idx: int = 0, + ) -> int: + """Insert a single batch with retry logic. Returns inserted count.""" + insert_count, error = db.insert_embeddings( + embeddings=embeddings, + metadata=metadata, + labels_data=labels_data, + ) + if error is not None: + log.warning(f"Insert failed, try_idx={retry_idx}, Exception: {error}") + retry_idx += 1 + if retry_idx <= config.MAX_INSERT_RETRY: + time.sleep(retry_idx) + return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, retry_idx) + msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" + raise RuntimeError(msg) + return insert_count + + def _worker_insert( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + ) -> int: + """Worker function: insert a batch with retry. + + Thread-safe DBs: reuse self.db whose connection is already open + via task()'s `with self.db.init()` — all threads share it safely. + + Non-thread-safe DBs: use a per-thread deep-copied instance with + its own connection, cached via threading.local. + """ + db = self._get_thread_db() + return self._insert_batch_with_retry(db, embeddings, metadata, labels_data) + + def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None] | None: + """Pull the next batch from the shared dataset iterator. + + Thread-safe: only one thread reads from the iterator at a time. + Returns None when the iterator is exhausted. + """ + with self._iter_lock: + try: + data_df = next(self._dataset_iter) + except StopIteration: + return None + + all_metadata = data_df[self.dataset.data.train_id_field].tolist() + emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + if self.normalize: + all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() + else: + all_embeddings = emb_np.tolist() + del emb_np + + labels_data = None + if self.filters.type == FilterOp.StrEqual: + if self.dataset.data.scalar_labels_file_separated: + labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() + else: + labels_data = data_df[self.filters.label_field].tolist() + + return all_embeddings, all_metadata, labels_data + + def _worker_loop(self) -> int: + """Worker loop: pull batches from the shared iterator and insert them.""" + total = 0 + while True: + batch = self._next_batch() + if batch is None: + break + embeddings, metadata, labels_data = batch + total += self._worker_insert(embeddings, metadata, labels_data) + return total + + def task(self) -> int: + """Insert entire dataset using concurrent executor. Runs in subprocess.""" + count = 0 + self._local = threading.local() + self._ctx_lock = threading.Lock() + self._thread_contexts = [] + self._iter_lock = threading.Lock() + self._dataset_iter = iter(self.dataset) + + with self.db.init(): + log.info( + f"({mp.current_process().name:16}) Start concurrent insert, " + f"batch_size={config.NUM_PER_BATCH}, max_workers={self.max_workers}" + ) + start = time.perf_counter() + + try: + with self._create_executor() as executor: + for _ in range(self.max_workers): + executor.submit(self._worker_loop) + + batch_results = executor.wait_all() + + # Log all errors, then raise the first one + errors = [r.error for r in batch_results if r.error is not None] + if errors: + for err in errors: + log.warning(f"Batch insert error: {err}") + raise errors[0] + + count = sum(r.value for r in batch_results) + finally: + self._cleanup_thread_contexts() + + log.info( + f"({mp.current_process().name:16}) Finish concurrent insert, " + f"count={count}, dur={time.perf_counter() - start:.2f}s" + ) + return count + + @time_it + def _insert_all_batches(self) -> int: + """Performance case only: run task() in subprocess with timeout.""" + with concurrent.futures.ProcessPoolExecutor( + mp_context=mp.get_context("spawn"), + max_workers=1, + ) as executor: + future = executor.submit(self.task) + try: + count = future.result(timeout=self.timeout) + except TimeoutError as e: + msg = f"VectorDB load dataset timeout in {self.timeout}" + log.warning(msg) + kill_proc_tree(pids=list(executor._processes.keys())) + raise PerformanceTimeoutError(msg) from e + except Exception as e: + log.warning(f"VectorDB load dataset error: {e}") + raise e from e + else: + return count + + def run(self) -> int: + """Insert full dataset concurrently. Returns total inserted count.""" + count, _ = self._insert_all_batches() + return count diff --git a/vectordb_bench/backend/runner/executor.py b/vectordb_bench/backend/runner/executor.py new file mode 100644 index 000000000..0bff2dc2a --- /dev/null +++ b/vectordb_bench/backend/runner/executor.py @@ -0,0 +1,170 @@ +"""Task executor abstraction with threading and async backends. + +Provides a unified interface for submitting callables with controlled +concurrency. Two implementations: + - ThreadExecutor: backed by ThreadPoolExecutor + - AsyncExecutor: backed by asyncio with semaphore-based concurrency control +""" + +from __future__ import annotations + +import asyncio +import logging +from abc import ABC, abstractmethod +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + +log = logging.getLogger(__name__) + + +@dataclass +class TaskResult: + """Result of a single submitted task.""" + + value: Any = None + error: Exception | None = None + + @property + def success(self) -> bool: + return self.error is None + + +class TaskExecutor(ABC): + """Abstract executor that accepts callables and controls concurrency.""" + + @abstractmethod + def start(self) -> None: + """Initialize executor resources.""" + raise NotImplementedError + + @abstractmethod + def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None: + """Submit a task for execution.""" + raise NotImplementedError + + @abstractmethod + def wait_all(self) -> list[TaskResult]: + """Block until all submitted tasks complete. Return results in submission order.""" + raise NotImplementedError + + @abstractmethod + def shutdown(self) -> None: + """Release executor resources. Safe to call multiple times.""" + raise NotImplementedError + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: object) -> bool: + self.shutdown() + return False + + +class ThreadExecutor(TaskExecutor): + """ThreadPoolExecutor-backed implementation.""" + + def __init__(self, max_workers: int): + self._max_workers = max(1, max_workers) + self._executor: ThreadPoolExecutor | None = None + self._futures: list[Future] = [] + + def start(self) -> None: + self._executor = ThreadPoolExecutor(max_workers=self._max_workers) + self._futures = [] + + def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None: + if self._executor is None: + raise RuntimeError("Executor not started. Call start() or use as context manager.") + future = self._executor.submit(fn, *args, **kwargs) + self._futures.append(future) + + def wait_all(self) -> list[TaskResult]: + results = [] + for future in self._futures: + try: + value = future.result() + results.append(TaskResult(value=value)) + except Exception as e: + results.append(TaskResult(error=e)) + self._futures = [] + return results + + def shutdown(self) -> None: + if self._executor is not None: + self._executor.shutdown(wait=True) + self._executor = None + + +class AsyncExecutor(TaskExecutor): + """asyncio-backed implementation for async DB clients. + + Accepts coroutine functions (async def), runs them on a single event + loop thread with semaphore-based concurrency control. No thread pool. + """ + + def __init__(self, max_workers: int): + self._max_workers = max(1, max_workers) + self._loop: asyncio.AbstractEventLoop | None = None + self._semaphore: asyncio.Semaphore | None = None + self._coros: list = [] + self._owns_loop = False + + def start(self) -> None: + try: + self._loop = asyncio.get_running_loop() + except RuntimeError: + self._loop = asyncio.new_event_loop() + self._owns_loop = True + self._semaphore = asyncio.Semaphore(self._max_workers) + self._coros = [] + + def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None: + """Submit a callable for execution. + + Accepts both coroutine functions (async def) and regular functions. + Sync functions are offloaded to a thread via run_in_executor. + """ + if self._loop is None or self._semaphore is None: + raise RuntimeError("Executor not started. Call start() or use as context manager.") + + async def _run(): + async with self._semaphore: + if asyncio.iscoroutinefunction(fn): + return await fn(*args, **kwargs) + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, lambda: fn(*args, **kwargs)) + + self._coros.append(_run()) + + def wait_all(self) -> list[TaskResult]: + if not self._coros: + return [] + + async def _gather(): + gathered = await asyncio.gather(*self._coros, return_exceptions=True) + results = [] + for item in gathered: + if isinstance(item, Exception): + results.append(TaskResult(error=item)) + else: + results.append(TaskResult(value=item)) + return results + + if self._owns_loop: + results = self._loop.run_until_complete(_gather()) + else: + results = asyncio.run_coroutine_threadsafe(_gather(), self._loop).result() + + self._coros = [] + return results + + def shutdown(self) -> None: + if self._owns_loop and self._loop is not None: + self._loop.close() + self._loop = None + self._semaphore = None diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 300553a4e..be0c6322d 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -1,4 +1,4 @@ -import concurrent +import concurrent.futures import logging import math import multiprocessing as mp @@ -6,14 +6,13 @@ import traceback import numpy as np -import psutil from vectordb_bench.backend.dataset import DatasetManager -from vectordb_bench.backend.filter import Filter, FilterOp, non_filter +from vectordb_bench.backend.filter import Filter, non_filter from ... import config from ...metric import calc_ndcg, calc_recall, get_ideal_dcg -from ...models import LoadTimeoutError, PerformanceTimeoutError +from ...models import LoadTimeoutError from .. import utils from ..clients import api @@ -38,66 +37,6 @@ def __init__( self.normalize = normalize self.filters = filters - def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): - _, error = db.insert_embeddings(**kwargs) - if error is not None: - log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") - retry_idx += 1 - if retry_idx <= config.MAX_INSERT_RETRY: - time.sleep(retry_idx) - self.retry_insert(db, retry_idx=retry_idx, **kwargs) - else: - msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" - raise RuntimeError(msg) from None - - def task(self) -> int: - count = 0 - with self.db.init(): - log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") - start = time.perf_counter() - for data_df in self.dataset: - all_metadata = data_df[self.dataset.data.train_id_field].tolist() - - emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) - if self.normalize: - log.debug("normalize the 100k train data") - all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() - else: - all_embeddings = emb_np.tolist() - del emb_np - log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") - - labels_data = None - if self.filters.type == FilterOp.StrEqual: - if self.dataset.data.scalar_labels_file_separated: - labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() - else: - labels_data = data_df[self.filters.label_field].tolist() - - insert_count, error = self.db.insert_embeddings( - embeddings=all_embeddings, - metadata=all_metadata, - labels_data=labels_data, - ) - if error is not None: - self.retry_insert( - self.db, - embeddings=all_embeddings, - metadata=all_metadata, - labels_data=labels_data, - ) - - assert insert_count == len(all_metadata) - count += insert_count - if count % 100_000 == 0: - log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") - - log.info( - f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " - f"dur={time.perf_counter() - start}" - ) - return count - def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: with self.db.init(): # unique id for endlessness insertion @@ -147,28 +86,6 @@ def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: ) return count - @utils.time_it - def _insert_all_batches(self) -> int: - """Performance case only""" - with concurrent.futures.ProcessPoolExecutor( - mp_context=mp.get_context("spawn"), - max_workers=1, - ) as executor: - future = executor.submit(self.task) - try: - count = future.result(timeout=self.timeout) - except TimeoutError as e: - msg = f"VectorDB load dataset timeout in {self.timeout}" - log.warning(msg) - for pid, _ in executor._processes.items(): - psutil.Process(pid).kill() - raise PerformanceTimeoutError(msg) from e - except Exception as e: - log.warning(f"VectorDB load dataset error: {e}") - raise e from e - else: - return count - def run_endlessness(self) -> int: """run forever util DB raises exception or crash""" # datasets for load tests are quite small, can fit into memory @@ -204,10 +121,6 @@ def run_endlessness(self) -> int: else: raise LoadTimeoutError(self.timeout) - def run(self) -> int: - count, _ = self._insert_all_batches() - return count - class SerialSearchRunner: def __init__( diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 8224a0415..6b51d1277 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -6,7 +6,6 @@ from enum import Enum, auto import numpy as np -import psutil from ..base import BaseModel from ..metric import Metric @@ -15,7 +14,14 @@ from .cases import Case, CaseLabel, StreamingPerformanceCase from .clients import DB, MetricType, api from .data_source import DatasetSource -from .runner import MultiProcessingSearchRunner, ReadWriteRunner, SerialInsertRunner, SerialSearchRunner +from .runner import ( + ConcurrentInsertRunner, + MultiProcessingSearchRunner, + ReadWriteRunner, + SerialInsertRunner, + SerialSearchRunner, +) +from .utils import kill_proc_tree log = logging.getLogger(__name__) @@ -241,14 +247,15 @@ def _run_streaming_case(self) -> Metric: @utils.time_it def _load_train_data(self): - """Insert train data and get the insert_duration""" + """Insert train data concurrently and get the insert_duration""" try: - runner = SerialInsertRunner( + runner = ConcurrentInsertRunner( self.db, self.ca.dataset, self.normalize, self.ca.filters, self.ca.load_timeout, + max_workers=self.config.load_concurrency or None, ) runner.run() except Exception as e: @@ -299,8 +306,7 @@ def _optimize(self) -> float: return future.result(timeout=self.ca.optimize_timeout)[1] except TimeoutError as e: log.warning(f"VectorDB optimize timeout in {self.ca.optimize_timeout}") - for pid, _ in executor._processes.items(): - psutil.Process(pid).kill() + kill_proc_tree(pids=list(executor._processes.keys())) raise PerformanceTimeoutError from e except Exception as e: log.warning(f"VectorDB optimize error: {e}") diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index 86c4faf5e..432f0d1d1 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -1,6 +1,47 @@ +import contextlib +import logging +import signal import time from functools import wraps +import psutil + +log = logging.getLogger(__name__) + + +def kill_proc_tree(pids: list[int] | None = None, grace: float = 2, timeout: float = 3): + """Kill child processes with SIGTERM, then SIGKILL for survivors. + + Args: + pids: Specific PIDs to kill. If None, kills all children of the + current process (recursive). + grace: Seconds to wait after SIGTERM before sending SIGKILL. + timeout: Seconds to wait for processes to fully exit after SIGKILL. + """ + if pids is not None: + targets = [] + for pid in pids: + with contextlib.suppress(psutil.NoSuchProcess): + targets.append(psutil.Process(pid)) + else: + targets = psutil.Process().children(recursive=True) + + for p in targets: + try: + log.warning(f"sending SIGTERM to child process: {p}") + p.send_signal(signal.SIGTERM) + except psutil.NoSuchProcess: + pass + + _, alive = psutil.wait_procs(targets, timeout=grace) + for p in alive: + try: + log.warning(f"force killing child process: {p}") + p.kill() + except psutil.NoSuchProcess: + pass + psutil.wait_procs(alive, timeout=timeout) + def numerize(n: int) -> str: """display positive number n for readability diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 12bb4be9b..94b13762a 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -1,7 +1,6 @@ import logging import time from collections.abc import Callable -from concurrent.futures import wait from datetime import datetime from pathlib import Path from pprint import pformat @@ -20,7 +19,7 @@ from .. import config from ..backend.clients import DB from ..backend.clients.api import MetricType -from ..interface import benchmark_runner, global_result_future +from ..interface import benchmark_runner from ..models import ( CaseConfig, CaseType, @@ -231,6 +230,16 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + load_concurrency: Annotated[ + int, + click.option( + "--load-concurrency", + type=int, + default=config.LOAD_CONCURRENCY, + show_default=True, + help="Number of concurrent workers for data loading in performance cases (0 = cpu_count)", + ), + ] search_serial: Annotated[ bool, click.option( @@ -643,15 +652,16 @@ def run( parameters["search_serial"], parameters["search_concurrent"], ), + load_concurrency=parameters["load_concurrency"], ) task_label = parameters["task_label"] log.info(f"Task:\n{pformat(task)}\n") if not parameters["dry_run"]: benchmark_runner.run([task], task_label) - time.sleep(5) - if global_result_future: - wait([global_result_future]) - - while benchmark_runner.has_running(): - time.sleep(1) + try: + while benchmark_runner.has_running(): + time.sleep(1) + except KeyboardInterrupt: + log.warning("Ctrl+C received, stopping benchmark...") + benchmark_runner.stop_running() diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 01d0c5876..e5c2a1e42 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -61,11 +61,17 @@ def advancedSettings(st): "Concurrency Duration", value=config.CONCURRENCY_DURATION, label_visibility="collapsed" ) container[1].caption("concurrency duration for each concurrency search test") - return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration + + container = st.columns([1, 2]) + load_concurrency = container[0].number_input( + "Load Concurrency", min_value=0, value=config.LOAD_CONCURRENCY, label_visibility="collapsed" + ) + container[1].caption("number of concurrent workers for data loading in performance cases (0 = cpu_count)") + return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid): - index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration = advancedSettings(st) + index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency = advancedSettings(st) def runHandler(): benchmark_runner.set_drop_old(not index_already_exists) @@ -80,6 +86,7 @@ def runHandler(): task.case_config.k = k task.case_config.concurrency_search_config.num_concurrency = concurrentInput_list task.case_config.concurrency_search_config.concurrency_duration = concurrency_duration + task.load_concurrency = load_concurrency benchmark_runner.set_download_address(use_aliyun) benchmark_runner.run(tasks, taskLabel) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 42dc876b0..0d4119e93 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -2,20 +2,17 @@ import logging import multiprocessing as mp import pathlib -import signal import traceback import uuid -from collections.abc import Callable from enum import Enum from multiprocessing.connection import Connection -import psutil - from . import config from .backend.assembler import Assembler, FilterNotSupportedError from .backend.data_source import DatasetSource from .backend.result_collector import ResultCollector from .backend.task_runner import TaskRunner +from .backend.utils import kill_proc_tree from .metric import Metric from .models import ( CaseResult, @@ -240,7 +237,7 @@ def _clear_running_task(self): for r in self.running_task.case_runners: r.stop() - self.kill_proc_tree(timeout=5) + kill_proc_tree() self.running_task = None if self.receive_conn: @@ -261,29 +258,5 @@ def _run_async(self, conn: Connection) -> bool: return True - def kill_proc_tree( - self, - sig: int = signal.SIGTERM, - timeout: float | None = None, - on_terminate: Callable | None = None, - ): - """Kill a process tree (including grandchildren) with signal - "sig" and return a (gone, still_alive) tuple. - "on_terminate", if specified, is a callback function which is - called as soon as a child terminates. - """ - children = psutil.Process().children(recursive=True) - for p in children: - try: - log.warning(f"sending SIGTERM to child process: {p}") - p.send_signal(sig) - except psutil.NoSuchProcess: - pass - _, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) - - for p in alive: - log.warning(f"force killing child process: {p}") - p.kill() - benchmark_runner = BenchMarkRunner() diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 0369b2e33..cdc64b9d7 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -243,6 +243,7 @@ class TaskConfig(BaseModel): db_case_config: DBCaseConfig case_config: CaseConfig stages: list[TaskStage] = ALL_TASK_STAGES + load_concurrency: int = config.LOAD_CONCURRENCY @property def db_name(self): From 243eb2e94c3f0c7298e3c41a2b9f3236071f10e6 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 2 Apr 2026 18:17:19 +0800 Subject: [PATCH 05/38] fix: Add back ujson in the requirements (#744) * fix: Add back ujson in the requirements * fix the coding style Signed-off-by: yangxuan --- pyproject.toml | 3 ++- vectordb_bench/backend/clients/polardb/polardb.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6706a3d4e..2baeb16e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,9 @@ dependencies = [ "environs", "pydantic=0.10.1", + "ujson", ] dynamic = ["version"] diff --git a/vectordb_bench/backend/clients/polardb/polardb.py b/vectordb_bench/backend/clients/polardb/polardb.py index f42b6fca5..d53638b0e 100644 --- a/vectordb_bench/backend/clients/polardb/polardb.py +++ b/vectordb_bench/backend/clients/polardb/polardb.py @@ -109,14 +109,14 @@ def init(self): db_name = self.db_config["database"] hint = "/*+ SET_VAR(imci_enable_fast_vector_search=on) */" - self.insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + self.insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" self.select_sql = ( - f"SELECT {hint} id FROM {db_name}.{self.table_name} " # noqa: S608 + f"SELECT {hint} id FROM {db_name}.{self.table_name} " f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " f"LIMIT %s" ) self.select_sql_with_filter = ( - f"SELECT id FROM {db_name}.{self.table_name} " # noqa: S608 + f"SELECT id FROM {db_name}.{self.table_name} " f"WHERE id >= %s " f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " f"LIMIT %s" @@ -218,7 +218,7 @@ def _insert_batch(self, embeddings: list[list[float]], metadata: list[int], offs conn, cursor = self._create_connection() try: db_name = self.db_config["database"] - insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" batch_data = [] for i in range(offset, offset + size): batch_data.append((int(metadata[i]), self.vector_to_hex(embeddings[i]))) From 10ffbcb7060645e5dc5133b41e580108938fcf04 Mon Sep 17 00:00:00 2001 From: jamesgao-jpg Date: Wed, 1 Apr 2026 12:51:14 +0000 Subject: [PATCH 06/38] feat: add region parameter and fix SDK compatibility for turbopuffer client - Add --region CLI parameter (required) for region-based API routing - Change --api-base-url to optional override for private networking - Rename write(columns=...) to write(upsert_columns=...) per current SDK - Fix docstring referencing wrong database name Signed-off-by: jamesgao-jpg --- .../backend/clients/turbopuffer/cli.py | 18 ++++++++++++++---- .../backend/clients/turbopuffer/config.py | 4 +++- .../backend/clients/turbopuffer/turbopuffer.py | 16 +++++++++------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/vectordb_bench/backend/clients/turbopuffer/cli.py b/vectordb_bench/backend/clients/turbopuffer/cli.py index 6fd91f2a8..d510889a0 100644 --- a/vectordb_bench/backend/clients/turbopuffer/cli.py +++ b/vectordb_bench/backend/clients/turbopuffer/cli.py @@ -17,15 +17,24 @@ class TurboPufferTypedDict(TypedDict): str, click.option("--api-key", type=str, help="TurboPuffer API key", required=True), ] + region: Annotated[ + str, + click.option( + "--region", + type=str, + help="TurboPuffer region (e.g. aws-us-east-1, gcp-us-central1)", + required=True, + ), + ] api_base_url: Annotated[ str, click.option( "--api-base-url", type=str, - help="TurboPuffer API base URL", + help="Override the region-based API URL", required=False, - default="https://api.turbopuffer.com", - show_default=True, + default="", + show_default=False, ), ] namespace: Annotated[ @@ -54,7 +63,8 @@ def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): db_config=TurboPufferConfig( db_label=parameters["db_label"], api_key=SecretStr(parameters["api_key"]), - api_base_url=parameters["api_base_url"], + region=parameters["region"], + api_base_url=parameters["api_base_url"] or None, namespace=parameters["namespace"], ), db_case_config=TurboPufferIndexConfig(), diff --git a/vectordb_bench/backend/clients/turbopuffer/config.py b/vectordb_bench/backend/clients/turbopuffer/config.py index c552299e3..88e797351 100644 --- a/vectordb_bench/backend/clients/turbopuffer/config.py +++ b/vectordb_bench/backend/clients/turbopuffer/config.py @@ -5,12 +5,14 @@ class TurboPufferConfig(DBConfig): api_key: SecretStr - api_base_url: str + region: str + api_base_url: str | None = None namespace: str = "vdbbench_test" def to_dict(self) -> dict: return { "api_key": self.api_key.get_secret_value(), + "region": self.region, "api_base_url": self.api_base_url, "namespace": self.namespace, } diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 241551792..3e1f32af4 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -1,4 +1,4 @@ -"""Wrapper around the Pinecone vector database over VectorDB""" +"""Wrapper around the TurboPuffer vector database over VectorDB""" import logging import time @@ -30,9 +30,9 @@ def __init__( with_scalar_labels: bool = False, **kwargs, ): - """Initialize wrapper around the milvus vector database.""" self.api_key = db_config.get("api_key", "") - self.api_base_url = db_config.get("api_base_url", "") + self.region = db_config.get("region", "") + self.api_base_url = db_config.get("api_base_url") self.namespace = db_config.get("namespace", "") self.db_case_config = db_case_config self.metric = db_case_config.parse_metric() @@ -43,8 +43,10 @@ def __init__( self.with_scalar_labels = with_scalar_labels - # Initialize client with new SDK pattern - self.client = tpuf.Turbopuffer(api_key=self.api_key, base_url=self.api_base_url) + client_kwargs = {"api_key": self.api_key, "region": self.region} + if self.api_base_url: + client_kwargs["base_url"] = self.api_base_url + self.client = tpuf.Turbopuffer(**client_kwargs) if drop_old: log.info(f"Drop old. delete the namespace: {self.namespace}") @@ -78,7 +80,7 @@ def insert_embeddings( try: if self.with_scalar_labels: self.ns.write( - columns={ + upsert_columns={ self._scalar_id_field: metadata, self._vector_field: embeddings, self._scalar_label_field: labels_data, @@ -87,7 +89,7 @@ def insert_embeddings( ) else: self.ns.write( - columns={ + upsert_columns={ self._scalar_id_field: metadata, self._vector_field: embeddings, }, From 337d156f3b766609d234ff584b5ed47158a757e4 Mon Sep 17 00:00:00 2001 From: jamesgao-jpg Date: Wed, 1 Apr 2026 12:51:14 +0000 Subject: [PATCH 07/38] fix: turbopuffer client pickle/ID compatibility and add benchmark results - Defer tpuf.Turbopuffer client creation to init() to avoid pickle errors with ProcessPoolExecutor(spawn) - Cast search result IDs to int for ground truth recall comparison - Update leaderboard_v2.json with 20 TurboPuffer filter performance entries Signed-off-by: jamesgao-jpg --- .../clients/turbopuffer/turbopuffer.py | 18 +- vectordb_bench/results/leaderboard_v2.json | 200 ++++++++++++++++++ 2 files changed, 211 insertions(+), 7 deletions(-) diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 3e1f32af4..6de0df21d 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -43,21 +43,25 @@ def __init__( self.with_scalar_labels = with_scalar_labels - client_kwargs = {"api_key": self.api_key, "region": self.region} - if self.api_base_url: - client_kwargs["base_url"] = self.api_base_url - self.client = tpuf.Turbopuffer(**client_kwargs) - if drop_old: log.info(f"Drop old. delete the namespace: {self.namespace}") - ns = self.client.namespace(self.namespace) + tmp_client = self._create_client() + ns = tmp_client.namespace(self.namespace) try: ns.delete_all() except Exception as e: log.warning(f"Failed to delete all. Error: {e}") + tmp_client = None + + def _create_client(self) -> tpuf.Turbopuffer: + client_kwargs = {"api_key": self.api_key, "region": self.region} + if self.api_base_url: + client_kwargs["base_url"] = self.api_base_url + return tpuf.Turbopuffer(**client_kwargs) @contextmanager def init(self): + self.client = self._create_client() self.ns = self.client.namespace(self.namespace) yield @@ -110,7 +114,7 @@ def search_embedding( top_k=k, filters=self.expr, ) - return [row.id for row in res.rows] if res.rows is not None else [] + return [int(row.id) for row in res.rows] if res.rows is not None else [] def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index e4f8dece5..6dfb6a47d 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -2858,5 +2858,205 @@ "latency": 463.9, "recall": 0.8478, "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:01:48.528365", + "db_name": "TurboPuffer-2026-03-31T10:01:48.528365", + "qps": 346.5847, + "latency": 42.7, + "recall": 0.9631, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T11:53:58.907951", + "db_name": "TurboPuffer-2026-03-31T11:53:58.907951", + "qps": 369.4921, + "latency": 41.6, + "recall": 0.779, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:35:50.149485", + "db_name": "TurboPuffer-2026-03-31T09:35:50.149485", + "qps": 310.957, + "latency": 49.4, + "recall": 0.9698, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:18:27.391390", + "db_name": "TurboPuffer-2026-03-31T09:18:27.391390", + "qps": 798.328, + "latency": 56.7, + "recall": 0.8993, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T10:33:52.971649", + "db_name": "TurboPuffer-2026-03-31T10:33:52.971649", + "qps": 649.8781, + "latency": 55.2, + "recall": 0.8352, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:00:07.738985", + "db_name": "TurboPuffer-2026-03-31T12:00:07.738985", + "qps": 370.7241, + "latency": 49.6, + "recall": 0.7177, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:33:40.019128", + "db_name": "TurboPuffer-2026-03-31T12:33:40.019128", + "qps": 100.0554, + "latency": 69.3, + "recall": 0.9638, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:08:01.370057", + "db_name": "TurboPuffer-2026-03-31T10:08:01.370057", + "qps": 284.6367, + "latency": 47.6, + "recall": 0.9788, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:12:22.420710", + "db_name": "TurboPuffer-2026-03-31T12:12:22.420710", + "qps": 81.8678, + "latency": 105.6, + "recall": 0.8751, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:42:19.988467", + "db_name": "TurboPuffer-2026-03-31T09:42:19.988467", + "qps": 260.4031, + "latency": 48.3, + "recall": 0.9828, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T11:47:52.080875", + "db_name": "TurboPuffer-2026-03-31T11:47:52.080875", + "qps": 365.2505, + "latency": 34.9, + "recall": 0.8251, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:20:29.187645", + "db_name": "TurboPuffer-2026-03-31T10:20:29.187645", + "qps": 471.553, + "latency": 44.1, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:19:46.258746", + "db_name": "TurboPuffer-2026-03-31T12:19:46.258746", + "qps": 91.8612, + "latency": 85.7, + "recall": 0.8799, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:48:48.830799", + "db_name": "TurboPuffer-2026-03-31T09:48:48.830799", + "qps": 206.0934, + "latency": 56.7, + "recall": 0.9795, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T11:41:42.263276", + "db_name": "TurboPuffer-2026-03-31T11:41:42.263276", + "qps": 351.7114, + "latency": 46.7, + "recall": 0.8735, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:26:48.189944", + "db_name": "TurboPuffer-2026-03-31T12:26:48.189944", + "qps": 96.592, + "latency": 76.9, + "recall": 0.9178, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:29:40.287089", + "db_name": "TurboPuffer-2026-03-31T09:29:40.287089", + "qps": 802.6923, + "latency": 48.1, + "recall": 0.935, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:55:20.689312", + "db_name": "TurboPuffer-2026-03-31T09:55:20.689312", + "qps": 184.5363, + "latency": 53.4, + "recall": 0.9681, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:14:19.197970", + "db_name": "TurboPuffer-2026-03-31T10:14:19.197970", + "qps": 323.0238, + "latency": 50.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:06:15.579552", + "db_name": "TurboPuffer-2026-03-31T12:06:15.579552", + "qps": 382.5332, + "latency": 54.7, + "recall": 0.6135, + "filter_ratio": 0.98 } ] \ No newline at end of file From b39689bf3c030c1da97bbeda175325d286730d00 Mon Sep 17 00:00:00 2001 From: jamesgao-jpg Date: Wed, 1 Apr 2026 12:51:14 +0000 Subject: [PATCH 08/38] feat: add consolidated turbopuffer results and update streaming leaderboard Merge 22 individual TurboPuffer result files into single consolidated result file. Add streaming benchmark entries (500/1000 rows/s) to leaderboard_v2_streaming.json. Normalize TurboPuffer db_name and label in both leaderboard files. Signed-off-by: jamesgao-jpg --- ...lt_20260331_standard_2025_turbopuffer.json | 3072 +++++++++++++++++ vectordb_bench/results/leaderboard_v2.json | 80 +- .../results/leaderboard_v2_streaming.json | 18 + 3 files changed, 3130 insertions(+), 40 deletions(-) create mode 100644 vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json diff --git a/vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json b/vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json new file mode 100644 index 000000000..3f6294f67 --- /dev/null +++ b/vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json @@ -0,0 +1,3072 @@ +{ + "run_id": "45faeeb9909d4c20982712629b7109f8", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3636.6248, + "optimize_duration": 60.1176, + "load_duration": 3696.7424, + "qps": 649.8781, + "serial_latency_p99": 0.0552, + "serial_latency_p95": 0.0323, + "recall": 0.8352, + "ndcg": 0.8489, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 5.0181, + 136.3428, + 401.9281, + 631.8033, + 649.8781, + 644.0239, + 638.2568, + 619.0102 + ], + "conc_latency_p99_list": [ + 0.7842317419981555, + 0.1055335910506619, + 0.04784800433084457, + 0.05269464588025578, + 0.1962103870000271, + 1.0594186752803216, + 1.0895923817619768, + 1.105607972859143 + ], + "conc_latency_p95_list": [ + 0.4166556664986274, + 0.06923620555135131, + 0.03216669879984694, + 0.04018809000044712, + 0.07153936980175785, + 0.08024858280041376, + 0.13536313344884537, + 1.0559816184473676 + ], + "conc_latency_avg_list": [ + 0.19879915163556802, + 0.03653489135292955, + 0.024727361365498506, + 0.031303927102332416, + 0.04542742658123844, + 0.06072973767027509, + 0.0906762097364659, + 0.12303716134739207 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 351.7114, + "serial_latency_p99": 0.0467, + "serial_latency_p95": 0.0306, + "recall": 0.8735, + "ndcg": 0.8847, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 39.0694, + 207.8572, + 315.6016, + 351.7114, + 350.2248, + 341.017, + 342.4504, + 333.8541 + ], + "conc_latency_p99_list": [ + 0.04308568556152748, + 0.03936693226052748, + 0.04930899777031294, + 0.08584493122052665, + 1.0702309198401785, + 1.1136302413583326, + 1.2335058578685432, + 2.3380776378013257 + ], + "conc_latency_p95_list": [ + 0.02841167169972323, + 0.02715096870197158, + 0.03799563039938221, + 0.07426544004883909, + 0.12973272005019681, + 0.21250388589869676, + 1.0903613543001485, + 1.1055554908001795 + ], + "conc_latency_avg_list": [ + 0.025532494878335706, + 0.02396293916721007, + 0.03149805626143546, + 0.056246979911343725, + 0.08423275960825198, + 0.11371272981539232, + 0.16889921984869344, + 0.22411732816279095 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 365.2505, + "serial_latency_p99": 0.0349, + "serial_latency_p95": 0.0247, + "recall": 0.8251, + "ndcg": 0.8424, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 41.7453, + 217.4248, + 327.3086, + 365.2505, + 360.5899, + 361.9818, + 353.6243, + 351.7834 + ], + "conc_latency_p99_list": [ + 0.03651399044829308, + 0.03795003006755009, + 0.04851697580088516, + 0.0817924941409364, + 1.069550116062965, + 1.1113606851896825, + 1.2188607804002098, + 2.3407695170007345 + ], + "conc_latency_p95_list": [ + 0.025673050000477815, + 0.025719283201397047, + 0.0362045420006325, + 0.07120951019969653, + 0.12899555909789345, + 0.17483308564860484, + 1.085536041801606, + 1.1010552038016612 + ], + "conc_latency_avg_list": [ + 0.023894811787483095, + 0.02290646905055157, + 0.03037167207873524, + 0.05417950100621118, + 0.08193541962097284, + 0.1079860384930871, + 0.1626031282242101, + 0.21517478888871755 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.8 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 369.4921, + "serial_latency_p99": 0.0416, + "serial_latency_p95": 0.026, + "recall": 0.779, + "ndcg": 0.8011, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 44.1762, + 217.1941, + 333.4918, + 368.3695, + 369.4921, + 361.0564, + 358.8268, + 351.6052 + ], + "conc_latency_p99_list": [ + 0.037339031600567986, + 0.04319219880198944, + 0.04170634405851157, + 0.08140748869991507, + 1.075197636150333, + 1.113729548148076, + 1.185450099499576, + 2.3347086974994453 + ], + "conc_latency_p95_list": [ + 0.025345681600447277, + 0.026001171001553303, + 0.035399802850224656, + 0.0708943822002766, + 0.11817662325156564, + 0.16750261040106082, + 1.0850413800017122, + 1.1004046580001159 + ], + "conc_latency_avg_list": [ + 0.022580311149034315, + 0.022936271871885103, + 0.02982460058318166, + 0.05370093295877385, + 0.07984126562141454, + 0.10845894286435549, + 0.16060839830850934, + 0.21328552188517172 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 370.7241, + "serial_latency_p99": 0.0496, + "serial_latency_p95": 0.0256, + "recall": 0.7177, + "ndcg": 0.7469, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 42.3641, + 213.6356, + 337.7677, + 367.4005, + 370.7241, + 352.4176, + 344.8198, + 354.0423 + ], + "conc_latency_p99_list": [ + 0.04112669600999651, + 0.04059317171937442, + 0.049354484229988935, + 0.08914589621916108, + 1.0625034953784052, + 1.1084485840195337, + 1.1761026821609268, + 2.329700628177888 + ], + "conc_latency_p95_list": [ + 0.026073782299499724, + 0.026734366401069565, + 0.03505219174985541, + 0.07247991124786494, + 0.11844929140133904, + 0.19264427510115556, + 1.0887599481509824, + 1.1004490541015912 + ], + "conc_latency_avg_list": [ + 0.023546151951321655, + 0.02331067781469216, + 0.02942534475558775, + 0.05384849155748122, + 0.07935110206611184, + 0.10908346295012428, + 0.16543306380794237, + 0.21503563146286667 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 382.5332, + "serial_latency_p99": 0.0547, + "serial_latency_p95": 0.0261, + "recall": 0.6135, + "ndcg": 0.6532, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 42.6797, + 211.9221, + 340.0303, + 381.3494, + 382.5332, + 377.5534, + 372.3789, + 370.273 + ], + "conc_latency_p99_list": [ + 0.045958382528879155, + 0.04791345044010085, + 0.050254700950063146, + 0.08182212736079236, + 1.0455512913525304, + 1.1066815286008933, + 1.1499066802006566, + 2.330022009398708 + ], + "conc_latency_p95_list": [ + 0.027506814953085268, + 0.030191246797767208, + 0.035157976998561935, + 0.06934378335081418, + 0.1208363270012342, + 0.17324353699950734, + 1.0816264840002987, + 1.0978374554006223 + ], + "conc_latency_avg_list": [ + 0.023371494825615652, + 0.023497298190272724, + 0.02923274826642055, + 0.051893872435525415, + 0.07698122536272771, + 0.1032508273402585, + 0.15527307758066816, + 0.20625486973445892 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.98 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 81.8678, + "serial_latency_p99": 0.1056, + "serial_latency_p95": 0.1021, + "recall": 0.8751, + "ndcg": 0.8923, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 10.0132, + 45.2108, + 69.7393, + 81.8678, + 81.5054, + 79.5748, + 80.1286, + 76.405 + ], + "conc_latency_p99_list": [ + 0.11135667605140044, + 0.12466917503959847, + 0.17545606489948118, + 0.4186298333996091, + 1.4131757669402578, + 2.561454358698395, + 4.161437169238486, + 6.53146800307746 + ], + "conc_latency_p95_list": [ + 0.10567549315092037, + 0.11936616160019184, + 0.15836084999864397, + 0.3964126334994944, + 1.2486386469990973, + 1.4115760019994923, + 2.527067465399887, + 2.6629810965998333 + ], + "conc_latency_avg_list": [ + 0.09962701216235118, + 0.11013113512517288, + 0.14239347650973677, + 0.24165083264509568, + 0.36060090070201456, + 0.4901049624103434, + 0.720209461182002, + 0.9583824911724366 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 91.8612, + "serial_latency_p99": 0.0857, + "serial_latency_p95": 0.0809, + "recall": 0.8799, + "ndcg": 0.8975, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 12.999, + 57.8487, + 82.2415, + 91.8612, + 90.3749, + 90.0362, + 89.1634, + 87.8603 + ], + "conc_latency_p99_list": [ + 0.08638456059743475, + 0.09575280532066244, + 0.13928119900861935, + 0.4007615396678755, + 1.3269691034200513, + 2.511851395880221, + 4.069974964441136, + 4.318752965519816 + ], + "conc_latency_p95_list": [ + 0.08070138149923878, + 0.09151377140078694, + 0.13317929814766102, + 0.33960816225135204, + 1.222629544099982, + 1.2648818186004065, + 2.463837313051044, + 2.5308212226009346 + ], + "conc_latency_avg_list": [ + 0.07674188703336754, + 0.08605192841317442, + 0.12095865782566981, + 0.2152256534213578, + 0.3258099637445097, + 0.4338084209719558, + 0.6348769175872554, + 0.8261389732993693 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.995 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 96.592, + "serial_latency_p99": 0.0769, + "serial_latency_p95": 0.0671, + "recall": 0.9178, + "ndcg": 0.9309, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 15.4173, + 68.7506, + 91.1578, + 95.9584, + 96.592, + 96.5377, + 92.9689, + 92.3363 + ], + "conc_latency_p99_list": [ + 0.07026230202154693, + 0.09216422941106428, + 0.1530633716819285, + 0.37221308532047254, + 1.379136629800488, + 2.4759424806782047, + 4.150151355230372, + 4.311918809250528 + ], + "conc_latency_p95_list": [ + 0.06769757184974878, + 0.07770936130109475, + 0.1223220942003536, + 0.3146231973501927, + 1.2210356577998025, + 1.2695832258003064, + 2.432397950649647, + 2.5293083527494673 + ], + "conc_latency_avg_list": [ + 0.06470390199121445, + 0.072410190861118, + 0.10899080468085105, + 0.20620261645924234, + 0.3044200164964746, + 0.40215828176873775, + 0.5961679787750713, + 0.7680528748023352 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.998 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 100.0554, + "serial_latency_p99": 0.0693, + "serial_latency_p95": 0.0649, + "recall": 0.9638, + "ndcg": 0.9702, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 16.4825, + 73.9278, + 94.48, + 100.0554, + 99.5101, + 96.945, + 94.0424, + 86.2962 + ], + "conc_latency_p99_list": [ + 0.06726060309974854, + 0.08376507117056463, + 0.12645121190198552, + 0.39609605101897616, + 1.3409943886408304, + 2.4752014657589463, + 4.116221232489005, + 4.324473266399982 + ], + "conc_latency_p95_list": [ + 0.06371058149852615, + 0.07187465289953253, + 0.11859881825057528, + 0.2999235360995954, + 1.2061096817997168, + 1.2548316742995667, + 2.412594861300204, + 2.5453477221013596 + ], + "conc_latency_avg_list": [ + 0.06052078211502204, + 0.06738729749599892, + 0.10525026012149993, + 0.19776199416695284, + 0.2947393509041799, + 0.39903049677303315, + 0.5802193109362205, + 0.8205917814459879 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.999 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 237.1288, + "optimize_duration": 60.1231, + "load_duration": 297.2519, + "qps": 798.328, + "serial_latency_p99": 0.0567, + "serial_latency_p95": 0.0373, + "recall": 0.8993, + "ndcg": 0.909, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 7.2334, + 233.3539, + 418.0091, + 779.6834, + 798.328, + 721.0519, + 665.1265, + 584.2167 + ], + "conc_latency_p99_list": [ + 0.16710825715950703, + 0.06532752437056843, + 0.06613579651964772, + 0.06110905407032987, + 0.14510449524073932, + 1.0521202659806477, + 1.0834973058097968, + 1.1199298221199219 + ], + "conc_latency_p95_list": [ + 0.1503542814499269, + 0.04019359780040751, + 0.03864274919997114, + 0.03302840365022349, + 0.06032294020042168, + 0.07371430989933284, + 0.12393024720067815, + 1.0562808335987939 + ], + "conc_latency_avg_list": [ + 0.13791148913299642, + 0.021343171084931285, + 0.023774816974899124, + 0.025365002319208883, + 0.03695695700041601, + 0.05426009768172259, + 0.08686200089993441, + 0.13066586199842864 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 802.6923, + "serial_latency_p99": 0.0481, + "serial_latency_p95": 0.0323, + "recall": 0.935, + "ndcg": 0.9417, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 35.1162, + 166.394, + 441.2273, + 754.7672, + 802.6923, + 758.6779, + 709.8788, + 667.6315 + ], + "conc_latency_p99_list": [ + 0.05474694928023395, + 0.05710166565986581, + 0.08122192963965079, + 0.053906371219891225, + 0.1059555222807467, + 1.0329836665192853, + 1.0822717395997459, + 1.1019392383902227 + ], + "conc_latency_p95_list": [ + 0.033523448400228514, + 0.03988007639854914, + 0.03580541750015981, + 0.03533476559914561, + 0.05827280570065341, + 0.07033409700015901, + 0.10542240839895384, + 1.0461306155005332 + ], + "conc_latency_avg_list": [ + 0.028406082829701498, + 0.0299218017148686, + 0.02252275923560204, + 0.02620027550993044, + 0.03675704353476552, + 0.051493407375862076, + 0.0814369894123999, + 0.114157400844387 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 310.957, + "serial_latency_p99": 0.0494, + "serial_latency_p95": 0.0478, + "recall": 0.9698, + "ndcg": 0.9735, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 22.9937, + 114.2135, + 201.3758, + 310.957, + 284.36, + 278.919, + 275.2043, + 268.6154 + ], + "conc_latency_p99_list": [ + 0.05187608798047224, + 0.0676384811005301, + 0.061836029370570034, + 0.10488168669999139, + 1.0839724454297168, + 1.1247000296189251, + 1.1835066912906216, + 2.350891308639548 + ], + "conc_latency_p95_list": [ + 0.04523930709929118, + 0.046784667498923224, + 0.05690364935007892, + 0.09261150339998493, + 0.13370714204984316, + 0.30792148500040484, + 1.109438732399667, + 1.1224438496010407 + ], + "conc_latency_avg_list": [ + 0.04338349602020482, + 0.04359157635044708, + 0.04934177400048448, + 0.06363149687529962, + 0.10389666149204274, + 0.13943306853059848, + 0.2093151750419116, + 0.28126611132382795 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.8 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 260.4031, + "serial_latency_p99": 0.0483, + "serial_latency_p95": 0.0461, + "recall": 0.9828, + "ndcg": 0.9852, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.2538, + 105.0145, + 165.942, + 260.4031, + 255.1092, + 249.9039, + 247.1553, + 222.8946 + ], + "conc_latency_p99_list": [ + 0.04928288878063535, + 0.05744525966034426, + 0.08332391123971328, + 0.12499408589974335, + 1.1001869340804298, + 1.1539904669602765, + 2.271939874058515, + 2.3963180435212417 + ], + "conc_latency_p95_list": [ + 0.04398724759921606, + 0.05287559080006758, + 0.06577553119968797, + 0.11041119500077912, + 0.1910506329497366, + 1.079135947499708, + 1.123120172900053, + 1.156959662200461 + ], + "conc_latency_avg_list": [ + 0.042898485931390104, + 0.0473868923309176, + 0.059896585720115844, + 0.07594923477056366, + 0.11576974236313346, + 0.15613794048676635, + 0.23384194864694827, + 0.34093762910663383 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 206.0934, + "serial_latency_p99": 0.0567, + "serial_latency_p95": 0.0493, + "recall": 0.9795, + "ndcg": 0.9827, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.9853, + 111.9502, + 167.2983, + 206.0934, + 202.6785, + 200.7624, + 198.7852, + 176.9053 + ], + "conc_latency_p99_list": [ + 0.04505532840961678, + 0.0549183023203659, + 0.07183914795945383, + 0.16050115711950638, + 1.1392679925593985, + 1.1783225936005692, + 2.391149005459738, + 2.4627247357385564 + ], + "conc_latency_p95_list": [ + 0.04352937004987325, + 0.049765212599595536, + 0.0665941942007521, + 0.14062045170048804, + 0.25411106819992707, + 1.1207731039994542, + 1.1548708326008637, + 1.2187981356008997 + ], + "conc_latency_avg_list": [ + 0.04158898759147668, + 0.044488593964723565, + 0.05943009694892961, + 0.09602951410439428, + 0.145682298229123, + 0.19324255140426388, + 0.2893373419111252, + 0.4256243749569817 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 184.5363, + "serial_latency_p99": 0.0534, + "serial_latency_p95": 0.0447, + "recall": 0.9681, + "ndcg": 0.9731, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.3311, + 105.7534, + 154.8527, + 184.5363, + 179.2525, + 179.9984, + 173.6043, + 172.2228 + ], + "conc_latency_p99_list": [ + 0.06133968915926745, + 0.05785074116065515, + 0.07778257604990356, + 0.1836007817494645, + 1.1589217726791685, + 1.2315237454791341, + 2.4227708060203077, + 2.5051683164208227 + ], + "conc_latency_p95_list": [ + 0.044938461699348405, + 0.053342464800516604, + 0.07181120149953131, + 0.1550450290005756, + 0.37187840719961957, + 1.1339543323992984, + 1.174229825800103, + 1.2181544718499933 + ], + "conc_latency_avg_list": [ + 0.04275554558972205, + 0.047119999140656205, + 0.06418707822424495, + 0.1072447022017609, + 0.1639824330546471, + 0.21581640828899204, + 0.3248775175758776, + 0.4320007027329427 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.98 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 346.5847, + "serial_latency_p99": 0.0427, + "serial_latency_p95": 0.0306, + "recall": 0.9631, + "ndcg": 0.9691, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 24.832, + 105.4399, + 268.1794, + 342.1315, + 346.5847, + 339.9961, + 331.4736, + 318.4889 + ], + "conc_latency_p99_list": [ + 0.0524532141194868, + 0.124523086280969, + 0.05130710799858207, + 0.09639830237927527, + 0.6372997376802239, + 1.1167713518602072, + 1.1833148257201527, + 2.346942362739901 + ], + "conc_latency_p95_list": [ + 0.04208997349924175, + 0.054497509400971464, + 0.04311577799853694, + 0.0820058475001133, + 0.15190685399975346, + 0.24409308345066127, + 1.094288076799785, + 1.113402110500283 + ], + "conc_latency_avg_list": [ + 0.040171743147231896, + 0.04722931638843159, + 0.03705184992629306, + 0.05782180853289151, + 0.0850777291892508, + 0.11466982697445866, + 0.17464619983809074, + 0.23867489188873692 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 284.6367, + "serial_latency_p99": 0.0476, + "serial_latency_p95": 0.0343, + "recall": 0.9788, + "ndcg": 0.9824, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 33.6033, + 154.3359, + 234.9368, + 284.6367, + 281.0354, + 282.1947, + 275.5716, + 254.759 + ], + "conc_latency_p99_list": [ + 0.04858522409849683, + 0.048488989419420224, + 0.05480865936007828, + 0.11453194788005944, + 1.0936976923593464, + 1.1293674966106844, + 2.3225755058392124, + 2.393918312400474 + ], + "conc_latency_p95_list": [ + 0.03065781050008809, + 0.037750776849679826, + 0.04890473074956389, + 0.09784386339888443, + 0.1834858106001775, + 0.5145842184007118, + 1.1164754623508997, + 1.13096353059982 + ], + "conc_latency_avg_list": [ + 0.02968536912167731, + 0.03225200436006818, + 0.04233185695698942, + 0.06951284309111169, + 0.10508535649574902, + 0.1382588987117255, + 0.2097049643681289, + 0.2930996819200318 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.995 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 323.0238, + "serial_latency_p99": 0.0504, + "serial_latency_p95": 0.0276, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 33.45, + 152.7353, + 227.8618, + 275.5285, + 291.2986, + 323.0238, + 318.2655, + 312.4427 + ], + "conc_latency_p99_list": [ + 0.04559309834949112, + 0.048093286539624296, + 0.05670377755039221, + 0.11922176679041513, + 1.0838092030408004, + 1.1185716670006514, + 1.246932903139767, + 2.348052036049994 + ], + "conc_latency_p95_list": [ + 0.031013834000077622, + 0.03764021150072949, + 0.05072674379944146, + 0.10094266020023496, + 0.18840500040023464, + 0.29478117500002554, + 1.0974889842496849, + 1.115618919398821 + ], + "conc_latency_avg_list": [ + 0.029821064443336008, + 0.03260137427613773, + 0.0436201719452207, + 0.07183805259952356, + 0.10142021420372482, + 0.12089794632971683, + 0.18129370678602072, + 0.24532422969222006 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.998 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 471.553, + "serial_latency_p99": 0.0441, + "serial_latency_p95": 0.0247, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 49.1528, + 228.9819, + 363.0592, + 468.8616, + 471.553, + 461.5481, + 437.5487, + 394.213 + ], + "conc_latency_p99_list": [ + 0.041978935619772575, + 0.04080429583897057, + 0.0448176010107636, + 0.08228810615008104, + 0.3177955880392612, + 1.090591485319801, + 1.134414788300455, + 2.3114677489898723 + ], + "conc_latency_p95_list": [ + 0.021382112400533514, + 0.02570402879937319, + 0.033227993899890854, + 0.06001253969943718, + 0.10880423700054961, + 0.13627656039943753, + 1.060789735999606, + 1.0911676228000942 + ], + "conc_latency_avg_list": [ + 0.020293962421219093, + 0.0217530582419813, + 0.02738142802709893, + 0.04220026652024836, + 0.06239387785636437, + 0.08461281376066486, + 0.13169568263657166, + 0.18839915684310343 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.999 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 10000.231680193, + "optimize_duration": 60.23235460700016, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_p95_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1001.3061, + 2032.9136, + 3065.0184, + 4086.7507, + 5107.3667, + 6128.9501, + 7149.9124, + 8171.4762, + 9192.0596, + 10223.3342, + 10444.8913 + ], + "st_max_qps_list_list": [ + 362.5803, + 334.5913, + 235.4548, + 319.031, + 459.0171, + 305.2895, + 336.8119, + 317.4398, + 442.5824, + 830.8648, + 837.1841 + ], + "st_recall_list": [ + 0.1028, + 0.1942, + 0.2856, + 0.3713, + 0.4495, + 0.5356, + 0.6176, + 0.7044, + 0.7795, + 0.8358, + 0.8358 + ], + "st_ndcg_list": [ + 0.1033, + 0.1954, + 0.2882, + 0.3754, + 0.4551, + 0.5423, + 0.626, + 0.7145, + 0.7914, + 0.849, + 0.849 + ], + "st_serial_latency_p99_list": [ + 0.162, + 0.2745, + 0.1655, + 0.154, + 0.1724, + 0.2706, + 0.2654, + 0.1929, + 0.0724, + 0.033, + 0.0413 + ], + "st_serial_latency_p95_list": [ + 0.138, + 0.1042, + 0.1335, + 0.1357, + 0.0747, + 0.1695, + 0.1519, + 0.1432, + 0.0571, + 0.0211, + 0.0213 + ], + "st_conc_failed_rate_list": [ + 1.621796951021732e-05, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 7.329873632978568e-06, + 0.0, + 0.0 + ], + "st_conc_num_list_list": [ + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ] + ], + "st_conc_qps_list_list": [ + [ + 362.5803, + 305.9887, + 211.6508 + ], + [ + 313.8881, + 334.5913, + 295.2759 + ], + [ + 235.4548, + 187.7903, + 223.1515 + ], + [ + 195.555, + 255.0244, + 319.031 + ], + [ + 459.0171, + 448.8463, + 326.9188 + ], + [ + 264.3766, + 302.7591, + 305.2895 + ], + [ + 212.8641, + 336.8119, + 265.5836 + ], + [ + 164.1314, + 174.9407, + 317.4398 + ], + [ + 280.9929, + 234.8017, + 442.5824 + ], + [ + 793.081, + 830.8648, + 830.5245 + ], + [ + 790.8788, + 837.1841, + 831.3462 + ] + ], + "st_conc_latency_p99_list_list": [ + [ + 0.125567, + 1.124351, + 2.306047 + ], + [ + 0.140031, + 1.105919, + 1.235967 + ], + [ + 0.201983, + 1.233919, + 2.289663 + ], + [ + 0.270847, + 1.172479, + 1.171455 + ], + [ + 0.098623, + 1.064959, + 1.157119 + ], + [ + 0.156671, + 1.128447, + 1.165311 + ], + [ + 0.239615, + 1.102847, + 1.522687 + ], + [ + 0.300031, + 1.263615, + 1.216511 + ], + [ + 0.163583, + 1.207295, + 1.197055 + ], + [ + 0.042719, + 0.201983, + 1.076223 + ], + [ + 0.046463, + 0.231167, + 1.074175 + ] + ], + "st_conc_latency_p95_list_list": [ + [ + 0.094463, + 0.180351, + 1.168383 + ], + [ + 0.106879, + 0.162175, + 1.087487 + ], + [ + 0.143487, + 1.043455, + 1.123327 + ], + [ + 0.193151, + 0.275455, + 1.065983 + ], + [ + 0.068607, + 0.115583, + 1.075199 + ], + [ + 0.118655, + 0.174463, + 1.084415 + ], + [ + 0.162047, + 0.147711, + 1.101823 + ], + [ + 0.225023, + 1.132543, + 1.075199 + ], + [ + 0.119295, + 0.299263, + 0.238591 + ], + [ + 0.032479, + 0.062495, + 0.081855 + ], + [ + 0.034303, + 0.059647, + 0.075775 + ] + ], + "st_conc_latency_avg_list_list": [ + [ + 0.05500603695334308, + 0.09764594385089817, + 0.18784171186688073 + ], + [ + 0.06353041564585622, + 0.08927120173312625, + 0.13474179712740827 + ], + [ + 0.08473482217393206, + 0.15917699015615014, + 0.17835078322515305 + ], + [ + 0.10202393731369927, + 0.11714498760979741, + 0.12473291585827614 + ], + [ + 0.043456775709144496, + 0.06659470207128572, + 0.12171512893789603 + ], + [ + 0.07546223394734103, + 0.09867369148091878, + 0.1303126478911198 + ], + [ + 0.0937178086011191, + 0.0887245961053375, + 0.14989714770693796 + ], + [ + 0.12154873281339651, + 0.17085041091090097, + 0.12538872906231163 + ], + [ + 0.07100562943422423, + 0.1272563421099916, + 0.08991327977599742 + ], + [ + 0.024614513704250777, + 0.03488576620155038, + 0.04598832608029421 + ], + [ + 0.024699256453538636, + 0.03461592000922972, + 0.04602557062363449 + ] + ] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_streaming_test3" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "bulk_insert_ratio": 0.0 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 6900.174543077999, + "optimize_duration": 60.1439380869997, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_p95_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 90, + 100, + 110 + ], + "st_search_time_list": [ + 4901.1288, + 6922.6153, + 7155.3455 + ], + "st_max_qps_list_list": [ + 536.0198, + 1027.3522, + 1020.8349 + ], + "st_recall_list": [ + 0.7646, + 0.8365, + 0.836 + ], + "st_ndcg_list": [ + 0.7768, + 0.8503, + 0.8497 + ], + "st_serial_latency_p99_list": [ + 0.3132, + 0.0532, + 0.0406 + ], + "st_serial_latency_p95_list": [ + 0.2025, + 0.0385, + 0.0219 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0 + ], + "st_conc_num_list_list": [ + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ] + ], + "st_conc_qps_list_list": [ + [ + 404.5543, + 429.4512, + 536.0198 + ], + [ + 944.4992, + 1025.9923, + 1027.3522 + ], + [ + 930.6361, + 1020.5733, + 1020.8349 + ] + ], + "st_conc_latency_p99_list_list": [ + [ + 0.139007, + 1.069055, + 1.088511 + ], + [ + 0.038335, + 0.077695, + 0.569855 + ], + [ + 0.042943, + 0.078015, + 0.783871 + ] + ], + "st_conc_latency_p95_list_list": [ + [ + 0.103295, + 0.115455, + 0.121151 + ], + [ + 0.027967, + 0.044607, + 0.063839 + ], + [ + 0.028175, + 0.045951, + 0.065151 + ] + ], + "st_conc_latency_avg_list_list": [ + [ + 0.04936760449523388, + 0.06972336395202725, + 0.07444538541092975 + ], + [ + 0.02068985204345577, + 0.02825433607535321, + 0.037233642103957326 + ], + [ + 0.020995807770388515, + 0.028403830945016105, + 0.03740811619860986 + ] + ] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_streaming_500" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "bulk_insert_ratio": 0.8 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1775001600.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index 6dfb6a47d..c463463fc 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -2862,8 +2862,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:01:48.528365", - "db_name": "TurboPuffer-2026-03-31T10:01:48.528365", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 346.5847, "latency": 42.7, "recall": 0.9631, @@ -2872,8 +2872,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T11:53:58.907951", - "db_name": "TurboPuffer-2026-03-31T11:53:58.907951", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 369.4921, "latency": 41.6, "recall": 0.779, @@ -2882,8 +2882,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:35:50.149485", - "db_name": "TurboPuffer-2026-03-31T09:35:50.149485", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 310.957, "latency": 49.4, "recall": 0.9698, @@ -2892,8 +2892,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:18:27.391390", - "db_name": "TurboPuffer-2026-03-31T09:18:27.391390", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 798.328, "latency": 56.7, "recall": 0.8993, @@ -2902,8 +2902,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T10:33:52.971649", - "db_name": "TurboPuffer-2026-03-31T10:33:52.971649", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 649.8781, "latency": 55.2, "recall": 0.8352, @@ -2912,8 +2912,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:00:07.738985", - "db_name": "TurboPuffer-2026-03-31T12:00:07.738985", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 370.7241, "latency": 49.6, "recall": 0.7177, @@ -2922,8 +2922,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:33:40.019128", - "db_name": "TurboPuffer-2026-03-31T12:33:40.019128", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 100.0554, "latency": 69.3, "recall": 0.9638, @@ -2932,8 +2932,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:08:01.370057", - "db_name": "TurboPuffer-2026-03-31T10:08:01.370057", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 284.6367, "latency": 47.6, "recall": 0.9788, @@ -2942,8 +2942,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:12:22.420710", - "db_name": "TurboPuffer-2026-03-31T12:12:22.420710", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 81.8678, "latency": 105.6, "recall": 0.8751, @@ -2952,8 +2952,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:42:19.988467", - "db_name": "TurboPuffer-2026-03-31T09:42:19.988467", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 260.4031, "latency": 48.3, "recall": 0.9828, @@ -2962,8 +2962,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T11:47:52.080875", - "db_name": "TurboPuffer-2026-03-31T11:47:52.080875", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 365.2505, "latency": 34.9, "recall": 0.8251, @@ -2972,8 +2972,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:20:29.187645", - "db_name": "TurboPuffer-2026-03-31T10:20:29.187645", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 471.553, "latency": 44.1, "recall": 1.0, @@ -2982,8 +2982,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:19:46.258746", - "db_name": "TurboPuffer-2026-03-31T12:19:46.258746", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 91.8612, "latency": 85.7, "recall": 0.8799, @@ -2992,8 +2992,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:48:48.830799", - "db_name": "TurboPuffer-2026-03-31T09:48:48.830799", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 206.0934, "latency": 56.7, "recall": 0.9795, @@ -3002,8 +3002,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T11:41:42.263276", - "db_name": "TurboPuffer-2026-03-31T11:41:42.263276", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 351.7114, "latency": 46.7, "recall": 0.8735, @@ -3012,8 +3012,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:26:48.189944", - "db_name": "TurboPuffer-2026-03-31T12:26:48.189944", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 96.592, "latency": 76.9, "recall": 0.9178, @@ -3022,8 +3022,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:29:40.287089", - "db_name": "TurboPuffer-2026-03-31T09:29:40.287089", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 802.6923, "latency": 48.1, "recall": 0.935, @@ -3032,8 +3032,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:55:20.689312", - "db_name": "TurboPuffer-2026-03-31T09:55:20.689312", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 184.5363, "latency": 53.4, "recall": 0.9681, @@ -3042,8 +3042,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:14:19.197970", - "db_name": "TurboPuffer-2026-03-31T10:14:19.197970", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 323.0238, "latency": 50.4, "recall": 1.0, @@ -3052,8 +3052,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:06:15.579552", - "db_name": "TurboPuffer-2026-03-31T12:06:15.579552", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 382.5332, "latency": 54.7, "recall": 0.6135, diff --git a/vectordb_bench/results/leaderboard_v2_streaming.json b/vectordb_bench/results/leaderboard_v2_streaming.json index 224c78d12..222c68bf7 100644 --- a/vectordb_bench/results/leaderboard_v2_streaming.json +++ b/vectordb_bench/results/leaderboard_v2_streaming.json @@ -124,5 +124,23 @@ "insert_rate": 1000, "streaming_qps": 167.2689, "streaming_latency": 0.5048 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 500, + "streaming_qps": 536.0198, + "streaming_latency": 0.3132 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 1000, + "streaming_qps": 442.5824, + "streaming_latency": 0.0724 } ] \ No newline at end of file From 0fef7dd9ef8a5ee2a04dc9b4f17e90dccd7dc602 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 2 Apr 2026 07:35:42 +0000 Subject: [PATCH 09/38] feat: add SQ4U scalar quantization type for Milvus HNSW index Co-Authored-By: Claude Opus 4.6 (1M context) --- vectordb_bench/backend/clients/api.py | 1 + vectordb_bench/backend/clients/milvus/cli.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 80709e8e3..5511f18db 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -49,6 +49,7 @@ class IndexType(StrEnum): class SQType(StrEnum): + SQ4U = "SQ4U" SQ6 = "SQ6" SQ8 = "SQ8" BF16 = "BF16" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index af31fe50d..2f2a286be 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -242,8 +242,8 @@ class MilvusHNSWSQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusHNSWTypedDic str | None, click.option( "--sq-type", - type=click.Choice(["SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), - help="Scalar quantizer type. Supported values: SQ6,SQ8,BF16,FP16,FP32", + type=click.Choice(["SQ4U", "SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), + help="Scalar quantizer type. Supported values: SQ4U,SQ6,SQ8,BF16,FP16,FP32", required=True, ), ] From 46cc146ce6c541acfb01c9c5b9d7a752e99f7616 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 09:06:08 +0000 Subject: [PATCH 10/38] Update benchmark results: Milvus 2.6.14, ElasticCloud 8.17, ZillizCloud Milvus results (16c64g, force_merge, v2.6.14): - 1M Cohere: SQ4U+FP16 (sweep refine_k) + SQ8 (sweep ef), 8 points each - 10M Cohere: SQ4U+FP16 + SQ8 (sweep ef), 8 points each - Total 32 benchmark configurations ElasticCloud and ZillizCloud results from standard benchmark runs. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...result_20260209_standard_elasticcloud.json | 1556 +++++ .../result_20260403_standard_milvus.json | 4074 +++++++++++++ .../result_20260209_standard_zillizcloud.json | 2814 +++++++++ vectordb_bench/results/leaderboard_v2.json | 5200 +++++++---------- 4 files changed, 10584 insertions(+), 3060 deletions(-) create mode 100644 vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json create mode 100644 vectordb_bench/results/Milvus/result_20260403_standard_milvus.json create mode 100644 vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json diff --git a/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json new file mode 100644 index 000000000..213c2d374 --- /dev/null +++ b/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json @@ -0,0 +1,1556 @@ +{ + "run_id": "80696b60e39749b295273db3cdba1b69", + "task_label": "standard_20260209", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 2030.4249, + "serial_latency_p99": 0.0106, + "serial_latency_p95": 0.0073, + "recall": 0.925, + "ndcg": 0.9306, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 139.9315, + 742.8751, + 1246.354, + 1773.6358, + 1944.4702, + 1997.4895, + 2030.4249, + 2030.061 + ], + "conc_latency_p99_list": [ + 0.011573358500827451, + 0.01055288627227128, + 0.013919824328950206, + 0.021638741448914516, + 0.035741090469564335, + 0.045256564997544046, + 0.06317665028094777, + 0.08284782179980536 + ], + "conc_latency_p95_list": [ + 0.00789401560141414, + 0.007500608301234024, + 0.010187717052576773, + 0.017150824350028415, + 0.026996734849308254, + 0.03459080500033451, + 0.04788784860138549, + 0.0629504940006882 + ], + "conc_latency_avg_list": [ + 0.0071435065296698895, + 0.006726900525604191, + 0.008019201478369918, + 0.011268319014980386, + 0.01541493476050156, + 0.02000300411234457, + 0.029459945652882687, + 0.039275078762117686 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 1804.8996, + "serial_latency_p99": 0.0123, + "serial_latency_p95": 0.0079, + "recall": 0.9365, + "ndcg": 0.9405, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 128.0906, + 646.8272, + 1197.2061, + 1629.7064, + 1728.435, + 1774.0467, + 1804.8996, + 1802.14 + ], + "conc_latency_p99_list": [ + 0.0134131232793152, + 0.012170731302467179, + 0.01435177448063773, + 0.023948891401232696, + 0.038592138251187846, + 0.04927967675830586, + 0.06325175963895165, + 0.08297362611905544 + ], + "conc_latency_p95_list": [ + 0.008920073449553456, + 0.008726374499019585, + 0.011022335800225845, + 0.019354423999175197, + 0.029856205349278752, + 0.03790496999936294, + 0.05230563919976702, + 0.06659373179936665 + ], + "conc_latency_avg_list": [ + 0.007804807226078364, + 0.007726593180097387, + 0.008348314213725788, + 0.01226467845434359, + 0.017345920270812453, + 0.022531855009387067, + 0.033063582489267676, + 0.04425664062925568 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 250 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 2353.8935, + "serial_latency_p99": 0.0171, + "serial_latency_p95": 0.0071, + "recall": 0.9056, + "ndcg": 0.9143, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 158.2504, + 791.7118, + 1438.1982, + 2003.6548, + 2210.8624, + 2291.9799, + 2345.97, + 2353.8935 + ], + "conc_latency_p99_list": [ + 0.010709055121405905, + 0.01030786765921221, + 0.01125256240178715, + 0.0189343996412208, + 0.02869696417925298, + 0.04078966366032545, + 0.05675600830181786, + 0.07136866949964316 + ], + "conc_latency_p95_list": [ + 0.0071951633981370815, + 0.00706048959873442, + 0.00878364119926118, + 0.014748687801329647, + 0.022636354199676134, + 0.031156333000035372, + 0.04301601800034405, + 0.05551979320152896 + ], + "conc_latency_avg_list": [ + 0.006316858815334563, + 0.006312276763547482, + 0.006949238477192474, + 0.009956655466073081, + 0.01355885582338353, + 0.017437786081889343, + 0.025496532582614126, + 0.033849936560945634 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 1623.8421, + "serial_latency_p99": 0.0118, + "serial_latency_p95": 0.0094, + "recall": 0.945, + "ndcg": 0.9479, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 129.0095, + 638.0261, + 1077.0653, + 1477.0665, + 1561.7802, + 1595.9667, + 1623.8421, + 1621.924 + ], + "conc_latency_p99_list": [ + 0.011765603999083404, + 0.012711298419817475, + 0.01651381758929347, + 0.026589661599427918, + 0.039944376738385454, + 0.049404421698636715, + 0.06832758593183824, + 0.08929936919903413 + ], + "conc_latency_p95_list": [ + 0.008827744000882376, + 0.008996219901018777, + 0.012427216298965503, + 0.021186420002777595, + 0.03210561599898938, + 0.04080166250059847, + 0.055962507547701525, + 0.07124235199989926 + ], + "conc_latency_avg_list": [ + 0.00774891642235306, + 0.007833377676777789, + 0.009280148463738688, + 0.01353283532887281, + 0.019196317583287527, + 0.025042119929708374, + 0.0368172028168529, + 0.04911914661560091 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 2808.2421, + "serial_latency_p99": 0.0095, + "serial_latency_p95": 0.0068, + "recall": 0.8674, + "ndcg": 0.8815, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 155.3245, + 826.7189, + 1600.1053, + 2225.9447, + 2513.1208, + 2695.4952, + 2765.1543, + 2808.2421 + ], + "conc_latency_p99_list": [ + 0.010928568999224795, + 0.010829480089960267, + 0.01071714987985615, + 0.01612071243016544, + 0.02644522499940649, + 0.032847231551932046, + 0.04966854284037254, + 0.06645170240146402 + ], + "conc_latency_p95_list": [ + 0.007179388998338254, + 0.006837483900562801, + 0.007670438798959367, + 0.012552767749366466, + 0.020286900500650518, + 0.02540200199928222, + 0.038052106797840664, + 0.05054814299946883 + ], + "conc_latency_avg_list": [ + 0.0064359595901721245, + 0.006044790967625746, + 0.0062457260062389625, + 0.008978903692601561, + 0.011928161713624801, + 0.01482735261744678, + 0.021619448376489415, + 0.028383759514242164 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 1482.3772, + "serial_latency_p99": 0.0121, + "serial_latency_p95": 0.0094, + "recall": 0.9523, + "ndcg": 0.9546, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 123.3087, + 580.5824, + 1004.7055, + 1371.9591, + 1426.106, + 1442.8227, + 1482.3772, + 1482.2809 + ], + "conc_latency_p99_list": [ + 0.011828215117784533, + 0.01352971964137398, + 0.0166967718025262, + 0.02956553739913943, + 0.041614612400007904, + 0.05316882036993774, + 0.07466962000107744, + 0.09735826710020776 + ], + "conc_latency_p95_list": [ + 0.009158867201040264, + 0.009766673699778041, + 0.013278135998916696, + 0.023815533299421075, + 0.03488031349843368, + 0.04589065709897113, + 0.06176861299900338, + 0.07706757499909145 + ], + "conc_latency_avg_list": [ + 0.008107228449720227, + 0.008606959040297272, + 0.009948278934310932, + 0.014567306633654263, + 0.02102471552485447, + 0.027699906799067583, + 0.0403594785833045, + 0.05377204408997558 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 350 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1721.5416, + "serial_latency_p99": 0.0096, + "serial_latency_p95": 0.0084, + "recall": 0.876, + "ndcg": 0.8855, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 18.9602, + 421.7232, + 1110.2953, + 1558.5395, + 1655.8557, + 1687.2055, + 1707.2641, + 1721.5416 + ], + "conc_latency_p99_list": [ + 0.0890053563169205, + 0.06768262584373579, + 0.01639450403279624, + 0.025886146546399667, + 0.03685992147773504, + 0.04851100179657815, + 0.06766616894965408, + 0.08664867073734057 + ], + "conc_latency_p95_list": [ + 0.07462253859848716, + 0.04929285799589706, + 0.011980525049875722, + 0.02050993259763345, + 0.030178915952274107, + 0.039007512998068705, + 0.054956941800628524, + 0.06864605330047198 + ], + "conc_latency_avg_list": [ + 0.0527364081070003, + 0.011852286945072127, + 0.008997451744878971, + 0.012824499661199576, + 0.018105110192998403, + 0.023662544243058387, + 0.03503091284714358, + 0.04628931630721377 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1032.9696, + "serial_latency_p99": 0.0148, + "serial_latency_p95": 0.0127, + "recall": 0.9299, + "ndcg": 0.933, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 94.6407, + 456.8911, + 819.2779, + 986.623, + 1016.3699, + 1020.5033, + 1032.9696, + 1017.9882 + ], + "conc_latency_p99_list": [ + 0.01443464591226075, + 0.01668544119311263, + 0.020597075797559213, + 0.04234821415491753, + 0.055556238333374516, + 0.07370169115238243, + 0.10577300176868448, + 0.14174547339440327 + ], + "conc_latency_p95_list": [ + 0.012447767957200992, + 0.01300238400872331, + 0.016574700995988678, + 0.03295493289260776, + 0.04768081354850436, + 0.064796744252817, + 0.09333402040065265, + 0.12088888059952296 + ], + "conc_latency_avg_list": [ + 0.010563127625240488, + 0.010939355078447704, + 0.012200001814344278, + 0.020260497643984427, + 0.02949441625551025, + 0.03917002076625185, + 0.05792509511531056, + 0.07840660381120702 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 350 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1150.4393, + "serial_latency_p99": 0.0134, + "serial_latency_p95": 0.012, + "recall": 0.9225, + "ndcg": 0.9265, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 104.4067, + 517.3208, + 889.9421, + 1094.8411, + 1095.6488, + 1137.8826, + 1150.4393, + 1141.58 + ], + "conc_latency_p99_list": [ + 0.012909343038918442, + 0.01400524774362566, + 0.019253388400829866, + 0.038147892540728215, + 0.05472438236000016, + 0.06572198009467672, + 0.0952600136399269, + 0.1264052395534234 + ], + "conc_latency_p95_list": [ + 0.011359572803485206, + 0.011475628245534608, + 0.01540375875265454, + 0.03010115729921381, + 0.044985946398810484, + 0.05843431264365791, + 0.08308516180550213, + 0.10663665049651172 + ], + "conc_latency_avg_list": [ + 0.009575407694901247, + 0.009661187708110561, + 0.011232427720710226, + 0.018257383714569215, + 0.027368236626444314, + 0.035086554527816706, + 0.05202559829357753, + 0.06982070456346111 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1452.1536, + "serial_latency_p99": 0.0108, + "serial_latency_p95": 0.0092, + "recall": 0.8973, + "ndcg": 0.9042, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 49.0384, + 580.9947, + 986.4739, + 1351.5216, + 1402.1557, + 1419.7304, + 1450.6627, + 1452.1536 + ], + "conc_latency_p99_list": [ + 0.039148551175603646, + 0.013189356601214931, + 0.017704268741654233, + 0.030597097602731108, + 0.04256812395251478, + 0.05555199954222194, + 0.07699622272266429, + 0.0999706184824754 + ], + "conc_latency_p95_list": [ + 0.03537960539688356, + 0.010066490995814093, + 0.013690835254237753, + 0.024310099499416538, + 0.035451141749945236, + 0.04658339719389914, + 0.06417367161193396, + 0.08166611165797803 + ], + "conc_latency_avg_list": [ + 0.020388936871343147, + 0.008602396555117218, + 0.010126462637783804, + 0.014789400451352687, + 0.02138135761146721, + 0.028131951981904997, + 0.04123096160957269, + 0.0548839897278163 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 2181.3939, + "serial_latency_p99": 0.0094, + "serial_latency_p95": 0.0077, + "recall": 0.8353, + "ndcg": 0.8501, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 3.646, + 29.6904, + 333.5573, + 1900.0888, + 2079.4449, + 2131.3329, + 2181.3939, + 2019.3786 + ], + "conc_latency_p99_list": [ + 0.4185616793336521, + 0.3197556457712197, + 0.22267254341524675, + 0.020585235283360776, + 0.0324391679285327, + 0.04552019087132067, + 0.06043267271961665, + 0.08661333356372784 + ], + "conc_latency_p95_list": [ + 0.38934891536191574, + 0.2844508775517169, + 0.1684353517535783, + 0.01616000900394283, + 0.024561249790713186, + 0.0332476719981059, + 0.04531921409798087, + 0.06508466459927148 + ], + "conc_latency_avg_list": [ + 0.2742520264012538, + 0.1683271812554273, + 0.029971614569954105, + 0.010518879796956216, + 0.014417846781857802, + 0.018753533944310726, + 0.02740638227517153, + 0.039480719052758934 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1295.5543, + "serial_latency_p99": 0.0112, + "serial_latency_p95": 0.0102, + "recall": 0.9126, + "ndcg": 0.9176, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 108.6988, + 554.0053, + 962.0715, + 1222.1509, + 1262.7418, + 1287.8623, + 1295.5543, + 1295.4142 + ], + "conc_latency_p99_list": [ + 0.012900208660430504, + 0.013385620156768804, + 0.017747372422018085, + 0.033394850781187424, + 0.04573169803712508, + 0.05876307546350308, + 0.08581843032181498, + 0.1129824651160743 + ], + "conc_latency_p95_list": [ + 0.010918107806355692, + 0.01064183129929006, + 0.014414189194212666, + 0.02686949290437041, + 0.03948758620244917, + 0.051792045756883454, + 0.0734679256027448, + 0.09282746819080781 + ], + "conc_latency_avg_list": [ + 0.009197126244932163, + 0.00902096815373196, + 0.010389166613132592, + 0.016355007809079263, + 0.02374101070396333, + 0.03104020946657135, + 0.046198905251943534, + 0.06155043230356708 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 250 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1770595200.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json new file mode 100644 index 000000000..e10404530 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json @@ -0,0 +1,4074 @@ +{ + "run_id": "c11e83b51ff14060a08f06d58f801214", + "task_label": "standard_20260403", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, + "qps": 3917.2035, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9203, + "ndcg": 0.9238, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 467.4942, + 1828.2004, + 2454.4495, + 2976.5539, + 3178.6052, + 3494.4787, + 3755.8025, + 3917.2035 + ], + "conc_latency_p99_list": [ + 0.0025411477516172455, + 0.0033511776782688685, + 0.005052069290541106, + 0.011894863871420993, + 0.016992485875962308, + 0.020195355401374387, + 0.026863074758439298, + 0.03365033164591297 + ], + "conc_latency_p95_list": [ + 0.0023739541989925782, + 0.00314145510783419, + 0.004699915152013999, + 0.009829005991923623, + 0.014131764802732504, + 0.017079706999356854, + 0.022949057801451987, + 0.02835416550078662 + ], + "conc_latency_avg_list": [ + 0.002136165356290855, + 0.002730801989459182, + 0.004067402028788586, + 0.006701902388970856, + 0.009401196782828802, + 0.011384491649831373, + 0.015778383442210466, + 0.020076245655094027 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3628.8527, + "serial_latency_p99": 0.0026, + "serial_latency_p95": 0.0024, + "recall": 0.9318, + "ndcg": 0.9346, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 456.622, + 1760.1234, + 2305.1866, + 2811.6811, + 3062.7639, + 3254.2934, + 3483.1948, + 3628.8527 + ], + "conc_latency_p99_list": [ + 0.0025775732805777807, + 0.003496330338239204, + 0.005392098429438191, + 0.012580982464569393, + 0.01744679359835574, + 0.02114546248485566, + 0.02862863955655484, + 0.03660006429607165 + ], + "conc_latency_p95_list": [ + 0.002428409402637044, + 0.003272143194044474, + 0.005011953243229073, + 0.010387669454212298, + 0.01447147800354287, + 0.017890810401149794, + 0.023953950349823568, + 0.030202264491526864 + ], + "conc_latency_avg_list": [ + 0.002187054468852311, + 0.002836289985030498, + 0.004330527318313066, + 0.007095944638723138, + 0.009756847450075634, + 0.012226813030036315, + 0.017037973309470753, + 0.021662613936531003 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3250.1112, + "serial_latency_p99": 0.0027, + "serial_latency_p95": 0.0025, + "recall": 0.9443, + "ndcg": 0.9463, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 434.6843, + 1686.1263, + 2163.5936, + 2555.03, + 2793.5899, + 2921.7137, + 3130.5251, + 3250.1112 + ], + "conc_latency_p99_list": [ + 0.0027417352329939604, + 0.003703351480362472, + 0.005958503000438213, + 0.013709455646312565, + 0.01829364382574568, + 0.022580389242502868, + 0.03056172190743382, + 0.037927262283337766 + ], + "conc_latency_p95_list": [ + 0.0025719269993714987, + 0.0034471726998162922, + 0.005409308793605305, + 0.011277809000603156, + 0.015221869490051177, + 0.019072343403240672, + 0.02558471505108173, + 0.03225500610860763 + ], + "conc_latency_avg_list": [ + 0.002297469730699608, + 0.0029608701952901183, + 0.004614075755959777, + 0.007808470701124476, + 0.010702304970719298, + 0.013606579627839316, + 0.018957654525715156, + 0.02420684028195088 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2762.4144, + "serial_latency_p99": 0.0031, + "serial_latency_p95": 0.0029, + "recall": 0.9556, + "ndcg": 0.9567, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 395.9308, + 1515.3208, + 1959.8545, + 2234.9654, + 2427.3893, + 2537.495, + 2657.5964, + 2762.4144 + ], + "conc_latency_p99_list": [ + 0.003096595037495717, + 0.00413693475740729, + 0.007250679123098961, + 0.01500825069088023, + 0.01980871459498304, + 0.024630421155015937, + 0.033659081743244314, + 0.041983861521002835 + ], + "conc_latency_p95_list": [ + 0.0028621868113987148, + 0.003837919446232263, + 0.006172719193273224, + 0.012415598499501357, + 0.0168672572079231, + 0.021068831244338074, + 0.028828703101316914, + 0.03613693019142374 + ], + "conc_latency_avg_list": [ + 0.002522697462250955, + 0.0032950399942241306, + 0.0050943009057731, + 0.008926197356642889, + 0.012320074532162776, + 0.01567786154805744, + 0.022343385743465827, + 0.028501836863104958 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2384.6245, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.003, + "recall": 0.9627, + "ndcg": 0.9632, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 368.492, + 1356.984, + 1780.9947, + 1978.8754, + 2086.4087, + 2180.2788, + 2347.7647, + 2384.6245 + ], + "conc_latency_p99_list": [ + 0.0033670899452408775, + 0.004818720840266903, + 0.008331618664960839, + 0.01588158588972873, + 0.02218642996114793, + 0.02756158859701827, + 0.036660355595813585, + 0.047074296680802936 + ], + "conc_latency_p95_list": [ + 0.0031033190454763816, + 0.004360820600413717, + 0.00704125490374281, + 0.013423533007153307, + 0.019178588101203785, + 0.023883526999270543, + 0.03231190550286556, + 0.041070348300854674 + ], + "conc_latency_avg_list": [ + 0.0027103553220750067, + 0.0036796698451320694, + 0.005606053561698588, + 0.010084455073144883, + 0.01433518456445194, + 0.0182535198619555, + 0.025308713260092305, + 0.03299686872047841 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2134.1717, + "serial_latency_p99": 0.0038, + "serial_latency_p95": 0.0036, + "recall": 0.9671, + "ndcg": 0.9672, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 342.9302, + 1297.1692, + 1605.3373, + 1723.4931, + 1874.7201, + 1950.9837, + 2026.1279, + 2134.1717 + ], + "conc_latency_p99_list": [ + 0.003612513002881314, + 0.005075874237372772, + 0.009592544495098993, + 0.01732624325857614, + 0.02383941526291892, + 0.02994993883999997, + 0.04171322006412084, + 0.052276343395351435 + ], + "conc_latency_p95_list": [ + 0.00337228730058996, + 0.00455044719419675, + 0.008192990009411006, + 0.015064282899402313, + 0.02108702180557884, + 0.02638680679956451, + 0.036932101499405685, + 0.04595883999718353 + ], + "conc_latency_avg_list": [ + 0.0029124934026024873, + 0.0038495151919032532, + 0.0062189602467817824, + 0.011577066999218889, + 0.01595138406671783, + 0.02037925821852399, + 0.029283526202598515, + 0.03685219368742522 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1641.3478, + "serial_latency_p99": 0.0041, + "serial_latency_p95": 0.0039, + "recall": 0.9729, + "ndcg": 0.9726, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 300.7136, + 1134.4852, + 1338.9285, + 1408.0122, + 1523.2338, + 1540.9389, + 1631.3282, + 1641.3478 + ], + "conc_latency_p99_list": [ + 0.004302493912400677, + 0.006291621240816314, + 0.011818032589217181, + 0.020626583240227772, + 0.028327662804658774, + 0.03604436940222513, + 0.0508815360846347, + 0.06704054794870894 + ], + "conc_latency_p95_list": [ + 0.003943610056012403, + 0.0053649904970370695, + 0.010212573444005101, + 0.018129582199617286, + 0.02555973254638957, + 0.03308224900683854, + 0.04562628499115817, + 0.05833644095400814 + ], + "conc_latency_avg_list": [ + 0.0033214706270654664, + 0.004401974502845957, + 0.00745933768945177, + 0.014171396440932295, + 0.0196399944581788, + 0.025851743210982894, + 0.03640642504236049, + 0.048033687367785266 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1488.5841, + "serial_latency_p99": 0.0047, + "serial_latency_p95": 0.0043, + "recall": 0.9764, + "ndcg": 0.976, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 259.0154, + 1019.3401, + 1241.5382, + 1301.0531, + 1358.779, + 1399.0002, + 1425.3615, + 1488.5841 + ], + "conc_latency_p99_list": [ + 0.004889184155326802, + 0.007301273914636109, + 0.01256226149998838, + 0.02185779360006564, + 0.03058185266534565, + 0.03868378391998703, + 0.0577071424780297, + 0.07162654809217199 + ], + "conc_latency_p95_list": [ + 0.004537451700889505, + 0.006121107403305359, + 0.010819275506946724, + 0.019440887503151316, + 0.02824652445488027, + 0.035727200949622784, + 0.05152663829067023, + 0.0646723014942836 + ], + "conc_latency_avg_list": [ + 0.0038569616304989004, + 0.0048995726665189525, + 0.008043409835523815, + 0.015340665489496967, + 0.022014652133720863, + 0.02845767912486287, + 0.04164023567491625, + 0.05289991318053246 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, + "qps": 2747.3167, + "serial_latency_p99": 0.0033, + "serial_latency_p95": 0.003, + "recall": 0.9204, + "ndcg": 0.9262, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 412.8127, + 1516.5887, + 1934.7028, + 2345.2563, + 2531.5491, + 2627.1159, + 2747.3167, + 2733.8527 + ], + "conc_latency_p99_list": [ + 0.0030919767648447303, + 0.004120720853097738, + 0.006685402100338251, + 0.014304348317091375, + 0.019310851126065252, + 0.024482081828900833, + 0.03435190891788809, + 0.04423686145673856 + ], + "conc_latency_p95_list": [ + 0.0028119500013417563, + 0.003827417498541763, + 0.006025877799402224, + 0.011986060402705334, + 0.016484533800394274, + 0.020710799152220714, + 0.028733705398917665, + 0.037459268098609756 + ], + "conc_latency_avg_list": [ + 0.002419260628861797, + 0.003292419872587543, + 0.005160383001607833, + 0.00850752764269055, + 0.011804369542313493, + 0.01513907190518365, + 0.021623125977333776, + 0.028768996633767287 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2514.4481, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.003, + "recall": 0.9303, + "ndcg": 0.9357, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 376.3101, + 1413.4044, + 1804.5963, + 2063.4126, + 2224.2985, + 2285.9557, + 2428.4015, + 2514.4481 + ], + "conc_latency_p99_list": [ + 0.003350270938608448, + 0.004419036731342202, + 0.007495493090173118, + 0.01604453914129406, + 0.021470454052177963, + 0.027058087129116764, + 0.03722573504091981, + 0.048193111100408685 + ], + "conc_latency_p95_list": [ + 0.003067891455066274, + 0.004094811100731022, + 0.006531247201928636, + 0.013379140298275157, + 0.018289522749910248, + 0.0233742457050539, + 0.03192761289792543, + 0.040280850498675136 + ], + "conc_latency_avg_list": [ + 0.002653342368273733, + 0.0035319332925499757, + 0.005532113228559582, + 0.009670992982743579, + 0.013421869019516723, + 0.017384391084778364, + 0.024445988357111505, + 0.03128177971695877 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2177.2345, + "serial_latency_p99": 0.0034, + "serial_latency_p95": 0.0031, + "recall": 0.9408, + "ndcg": 0.9456, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 354.703, + 1270.9673, + 1660.8118, + 1799.9874, + 1911.9995, + 2018.5829, + 2119.3303, + 2177.2345 + ], + "conc_latency_p99_list": [ + 0.0035591462109005084, + 0.0051245430819108154, + 0.008888348671316635, + 0.017485948742978506, + 0.023954762732464586, + 0.02942745841770374, + 0.04045989539969013, + 0.052306493496216695 + ], + "conc_latency_p95_list": [ + 0.003282636954463669, + 0.004633509999257512, + 0.0075649314512702395, + 0.014845420597703196, + 0.02076956499913649, + 0.025784255946928167, + 0.035505612393899356, + 0.045118153299335974 + ], + "conc_latency_avg_list": [ + 0.002815711151857267, + 0.003928179410303787, + 0.006011238757150647, + 0.011084966830100315, + 0.015638666422418603, + 0.019708495546663755, + 0.02799275543572887, + 0.03611913807969698 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1833.2575, + "serial_latency_p99": 0.0039, + "serial_latency_p95": 0.0035, + "recall": 0.951, + "ndcg": 0.9555, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 329.7676, + 1176.835, + 1426.5695, + 1546.3292, + 1656.2697, + 1716.0803, + 1790.6121, + 1833.2575 + ], + "conc_latency_p99_list": [ + 0.003868887719290797, + 0.005920497829865769, + 0.01097595489642117, + 0.01919553377847477, + 0.02603815700131236, + 0.03326150514345494, + 0.04663379819947293, + 0.05838948019809325 + ], + "conc_latency_p95_list": [ + 0.003534871701413067, + 0.005110333902484853, + 0.009471108402794925, + 0.01673110910487594, + 0.023566940006276127, + 0.02969893909685197, + 0.04118938999890815, + 0.05255015200236812 + ], + "conc_latency_avg_list": [ + 0.0030290844068619504, + 0.004243610521352684, + 0.007000278697486464, + 0.012908119052890047, + 0.018050427070200076, + 0.023198493359582885, + 0.03316593436696198, + 0.04298056582898082 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1552.4803, + "serial_latency_p99": 0.004, + "serial_latency_p95": 0.0037, + "recall": 0.9565, + "ndcg": 0.9605, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 303.2446, + 1078.6424, + 1299.206, + 1360.6235, + 1425.268, + 1465.3767, + 1520.0788, + 1552.4803 + ], + "conc_latency_p99_list": [ + 0.0040712328813970085, + 0.006725932629851741, + 0.012099390296789348, + 0.021067992354510352, + 0.02953478858660674, + 0.037649333699373524, + 0.05402578063920373, + 0.07030989054946984 + ], + "conc_latency_p95_list": [ + 0.0038339799008099357, + 0.005704106903795035, + 0.01050544159807032, + 0.01850984884877107, + 0.02700225284861517, + 0.034401998404064216, + 0.04840153990226099, + 0.06195298564853147 + ], + "conc_latency_avg_list": [ + 0.003293997160806082, + 0.004629127288807816, + 0.00768599232973781, + 0.014665701743116042, + 0.02097794137202591, + 0.027194933866063295, + 0.03908773235524631, + 0.050673599669096146 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1355.3121, + "serial_latency_p99": 0.0044, + "serial_latency_p95": 0.0042, + "recall": 0.9602, + "ndcg": 0.964, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 279.2851, + 975.7699, + 1150.3744, + 1187.2494, + 1247.0213, + 1283.8337, + 1302.6721, + 1355.3121 + ], + "conc_latency_p99_list": [ + 0.004451236832683208, + 0.0076792708405992016, + 0.01329627803701441, + 0.023810231103198035, + 0.033509768832373055, + 0.04278365236037641, + 0.06259672742118709, + 0.08097375019555315 + ], + "conc_latency_p95_list": [ + 0.004183817799275858, + 0.006473346400889567, + 0.011516073600796517, + 0.0212969502517808, + 0.030741239200142444, + 0.03883912599849282, + 0.056272352899395625, + 0.07115194579964736 + ], + "conc_latency_avg_list": [ + 0.003576748584184128, + 0.005118136384013237, + 0.008682482330775591, + 0.0168131587141993, + 0.023978672854972592, + 0.031007006045867456, + 0.04562714668667405, + 0.058088526012853484 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1079.2123, + "serial_latency_p99": 0.0053, + "serial_latency_p95": 0.0049, + "recall": 0.9648, + "ndcg": 0.9686, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 241.6992, + 799.4681, + 939.7848, + 975.2814, + 1001.75, + 1029.2059, + 1065.2548, + 1079.2123 + ], + "conc_latency_p99_list": [ + 0.005288620950886979, + 0.009748364380211563, + 0.015409245888222355, + 0.028667413447983563, + 0.04076468282910357, + 0.05202346699952616, + 0.07527647967857774, + 0.09870467029977587 + ], + "conc_latency_p95_list": [ + 0.004910965752060292, + 0.008243431097071152, + 0.013543863647282705, + 0.025803527399330048, + 0.03817865899909521, + 0.048153773248486686, + 0.06777279375382932, + 0.08822071449685609 + ], + "conc_latency_avg_list": [ + 0.004132684600647634, + 0.006247207634111248, + 0.010625959006772904, + 0.02046250279903178, + 0.02985517599151899, + 0.0386749397755516, + 0.055773659676497576, + 0.07301672729002262 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 876.5772, + "serial_latency_p99": 0.0063, + "serial_latency_p95": 0.0059, + "recall": 0.9676, + "ndcg": 0.9713, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 213.9083, + 704.2348, + 797.531, + 821.1706, + 825.533, + 797.2598, + 858.8645, + 876.5772 + ], + "conc_latency_p99_list": [ + 0.0060647999984212225, + 0.01099963705062691, + 0.017907913918024848, + 0.03398729390319204, + 0.049453794501459925, + 0.06846078443864825, + 0.09411621719773387, + 0.12295867912092945 + ], + "conc_latency_p95_list": [ + 0.005586325001786463, + 0.009474276846958667, + 0.015768809196015348, + 0.03062209299969254, + 0.046334437996847555, + 0.06321956860047066, + 0.08318591200077208, + 0.10695994940178935 + ], + "conc_latency_avg_list": [ + 0.004670567833335547, + 0.007091383770451675, + 0.01252405204148396, + 0.02430557731179879, + 0.036228697626746305, + 0.04987607783688658, + 0.06916092553517646, + 0.09005697652328008 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 10663.1231, + "serial_latency_p99": 0.002, + "serial_latency_p95": 0.0018, + "recall": 0.8405, + "ndcg": 0.8674, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 592.3235, + 2657.0693, + 4857.6653, + 7731.1954, + 9035.8688, + 9746.5046, + 10547.8051, + 10663.1231 + ], + "conc_latency_p99_list": [ + 0.0019481061986880378, + 0.002309793634340169, + 0.0025996991485590097, + 0.003907884397631278, + 0.005842388768505771, + 0.007729610755923204, + 0.011498715590278154, + 0.01563908824173267 + ], + "conc_latency_p95_list": [ + 0.0018476900004316121, + 0.0021391708025475962, + 0.0023516209941590203, + 0.0031451117574761156, + 0.004541207106376533, + 0.006081955801346338, + 0.008940340152912542, + 0.012104344801628029 + ], + "conc_latency_avg_list": [ + 0.0016856015045043139, + 0.0018777718208453423, + 0.0020532305285396453, + 0.0025771131947367317, + 0.0033031913740212363, + 0.0040746336526161255, + 0.005611474910509973, + 0.007361856948830913 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 10333.9072, + "serial_latency_p99": 0.002, + "serial_latency_p95": 0.0019, + "recall": 0.889, + "ndcg": 0.9058, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 568.5874, + 2594.6923, + 4693.0426, + 7429.976, + 8564.4439, + 9136.6911, + 9985.356, + 10333.9072 + ], + "conc_latency_p99_list": [ + 0.0020693473634310062, + 0.0023700819991063315, + 0.0026944835495669377, + 0.004012327503005508, + 0.006051957674062573, + 0.008206313409027643, + 0.012019852105004216, + 0.015895357320114278 + ], + "conc_latency_p95_list": [ + 0.0019474557979265227, + 0.0021944244945188984, + 0.002450034000503365, + 0.003276585503044771, + 0.004752537995955206, + 0.006455344991991296, + 0.009426155498658773, + 0.012384038396703547 + ], + "conc_latency_avg_list": [ + 0.0017559580574677634, + 0.0019229909223785742, + 0.0021257891514691474, + 0.0026812402175934932, + 0.0034838524012537763, + 0.00433970032247638, + 0.005933638012347279, + 0.007582591068540877 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.2 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 9575.6863, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.0021, + "recall": 0.9189, + "ndcg": 0.93, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 535.4941, + 2437.6236, + 4429.593, + 6980.413, + 8035.6356, + 8594.0728, + 9041.3709, + 9575.6863 + ], + "conc_latency_p99_list": [ + 0.002218716748757288, + 0.002563155780226227, + 0.0028770195422111954, + 0.004238974919717297, + 0.006498919794103134, + 0.008666419192886682, + 0.013239830499514937, + 0.016983359853475096 + ], + "conc_latency_p95_list": [ + 0.002100886751577491, + 0.0023667280438530724, + 0.0026158143016800747, + 0.0034943125079735177, + 0.005161623004823923, + 0.006949284041184, + 0.010531335494306404, + 0.013410061004833551 + ], + "conc_latency_avg_list": [ + 0.0018646425533687709, + 0.0020474016047948205, + 0.00225211486396273, + 0.00285420751423595, + 0.003711133378722069, + 0.004623532072278411, + 0.006552865012667485, + 0.008189225702132959 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.5 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 8596.7694, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0022, + "recall": 0.9416, + "ndcg": 0.9493, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 497.6866, + 2259.6612, + 4114.0036, + 6455.3584, + 7309.2383, + 7686.907, + 8294.3018, + 8596.7694 + ], + "conc_latency_p99_list": [ + 0.002440318200387992, + 0.0028159428038634342, + 0.0031061971996678038, + 0.004590758396079762, + 0.00719438069412717, + 0.009845319669257151, + 0.014379469840059753, + 0.018956097210466392 + ], + "conc_latency_p95_list": [ + 0.0022961719951126724, + 0.002580139000201598, + 0.0028258040038053878, + 0.003789236750890268, + 0.005791235491051338, + 0.00791677404558868, + 0.011584387852053624, + 0.015152870106976477 + ], + "conc_latency_avg_list": [ + 0.0020063849993894605, + 0.002208682703794502, + 0.002425080457737401, + 0.003086665132816786, + 0.00408523815973775, + 0.005170043147020008, + 0.007138938063080818, + 0.009125464665924427 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 2.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7704.3625, + "serial_latency_p99": 0.0027, + "serial_latency_p95": 0.0025, + "recall": 0.9541, + "ndcg": 0.96, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 464.498, + 2087.0727, + 3811.3426, + 5897.8764, + 6613.5685, + 6995.184, + 7496.7496, + 7704.3625 + ], + "conc_latency_p99_list": [ + 0.002632544774387497, + 0.0030649999552406367, + 0.003373088193475267, + 0.004990858241653769, + 0.00786220946611138, + 0.010790031323267592, + 0.015690509527339604, + 0.020764024818781757 + ], + "conc_latency_p95_list": [ + 0.002470412495313212, + 0.002798333394457586, + 0.0030606225445808377, + 0.004139978906459873, + 0.006436757100163957, + 0.008721163390146102, + 0.012695113198424209, + 0.016752441306016403 + ], + "conc_latency_avg_list": [ + 0.002149911288992391, + 0.0023915170520859654, + 0.002617775435786274, + 0.0033781344953593517, + 0.0045130488491076, + 0.005680457334630225, + 0.007906434767685636, + 0.010188877722018175 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 2.5 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7023.6735, + "serial_latency_p99": 0.003, + "serial_latency_p95": 0.0028, + "recall": 0.962, + "ndcg": 0.9667, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 437.0253, + 1938.4624, + 3454.7847, + 5460.2663, + 6058.6283, + 6398.7693, + 6828.1471, + 7023.6735 + ], + "conc_latency_p99_list": [ + 0.0028729987128463105, + 0.0033484641878749246, + 0.0037352688424289217, + 0.0053953273908700725, + 0.008702208310278365, + 0.011712039967096652, + 0.0172460880043218, + 0.02189830483126571 + ], + "conc_latency_p95_list": [ + 0.00266674381273333, + 0.003031579001981299, + 0.0033792859961977225, + 0.004472345393151045, + 0.007129233997693518, + 0.009540854604711059, + 0.014027368000824936, + 0.018085699503717478 + ], + "conc_latency_avg_list": [ + 0.0022852531678980575, + 0.0025748727575029204, + 0.0028880249144625355, + 0.003650684004208795, + 0.004931203325481902, + 0.006211804309036586, + 0.008678531268037587, + 0.011200709193523 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 3.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6031.3725, + "serial_latency_p99": 0.0033, + "serial_latency_p95": 0.003, + "recall": 0.971, + "ndcg": 0.9743, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 375.0077, + 1749.9239, + 3171.188, + 4757.8121, + 5337.8052, + 5585.8874, + 5830.76, + 6031.3725 + ], + "conc_latency_p99_list": [ + 0.0034118053846759725, + 0.00371807467265171, + 0.004087487124343162, + 0.006445972005603836, + 0.0099661920015933, + 0.013419415393291265, + 0.01954109726633761, + 0.02450512952229475 + ], + "conc_latency_p95_list": [ + 0.0031440883423783815, + 0.0033821857992734294, + 0.003687061999517027, + 0.0053249524033162745, + 0.008198342995456187, + 0.011055400453187756, + 0.01616062365719699, + 0.020329035600298084 + ], + "conc_latency_avg_list": [ + 0.002663293590200931, + 0.0028528372994040497, + 0.0031468442372841717, + 0.004191480480374251, + 0.005595629037850519, + 0.007119491187741896, + 0.010171118781282724, + 0.013044592076065837 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 4.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5258.1868, + "serial_latency_p99": 0.0036, + "serial_latency_p95": 0.0033, + "recall": 0.9768, + "ndcg": 0.9793, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 351.2889, + 1555.9976, + 2891.7532, + 4328.3041, + 4710.0472, + 4944.6567, + 5159.0143, + 5258.1868 + ], + "conc_latency_p99_list": [ + 0.0036150889145210364, + 0.004169191400287673, + 0.004474782356992364, + 0.00714936985692475, + 0.011309100479702464, + 0.014964991490269298, + 0.02106605595399745, + 0.02670282432547537 + ], + "conc_latency_p95_list": [ + 0.0032958149939076972, + 0.0038005771988537163, + 0.004026532001444138, + 0.0059051988064311445, + 0.0093500265997136, + 0.012421341851586474, + 0.017624663742026314, + 0.022111607256374555 + ], + "conc_latency_avg_list": [ + 0.0028433795541688497, + 0.003208622344277306, + 0.0034516157786571304, + 0.004607007276075149, + 0.006345752606166889, + 0.008043607402151223, + 0.01149538342406618, + 0.014941377751891149 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 5.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5973.0024, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9192, + "ndcg": 0.9299, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 192.2873, + 780.772, + 1243.1343, + 2209.7863, + 5135.3255, + 5268.6684, + 5595.5025, + 5973.0024 + ], + "conc_latency_p99_list": [ + 0.006922555523342447, + 0.011980964867980226, + 0.01822498522611567, + 0.021657125554483937, + 0.011112995611620146, + 0.015273953418945892, + 0.021470034882659094, + 0.025694710013340227 + ], + "conc_latency_p95_list": [ + 0.006153562400140799, + 0.009661811696423684, + 0.013727128539176193, + 0.01456564510299358, + 0.00905536999925971, + 0.012242838197562377, + 0.017436827097844797, + 0.020891863998258486 + ], + "conc_latency_avg_list": [ + 0.005195547580597013, + 0.00639603304659674, + 0.0080334040769494, + 0.009027100221592798, + 0.0058122537237125, + 0.007549590091448817, + 0.010597991937403054, + 0.01314884199753361 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5416.5758, + "serial_latency_p99": 0.0026, + "serial_latency_p95": 0.0024, + "recall": 0.9334, + "ndcg": 0.9421, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 468.5468, + 2023.0586, + 3273.7331, + 4338.5873, + 4722.177, + 4899.851, + 5215.5063, + 5416.5758 + ], + "conc_latency_p99_list": [ + 0.002587841608328745, + 0.0031619464718096405, + 0.003964070154324872, + 0.007774805837543681, + 0.012349921874993024, + 0.01622475358992233, + 0.022378725241142112, + 0.02744747619624828 + ], + "conc_latency_p95_list": [ + 0.0024390827456954867, + 0.0029180134042690042, + 0.0035825525046675466, + 0.006318293401272964, + 0.009966622248612111, + 0.013235174745204858, + 0.018063901497225743, + 0.02230343740739044 + ], + "conc_latency_avg_list": [ + 0.0021313683553780983, + 0.0024673246161570224, + 0.0030484556683506147, + 0.0045937434301476935, + 0.006330211445594135, + 0.008107756352776998, + 0.01136996968107513, + 0.014491805104902885 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4771.4324, + "serial_latency_p99": 0.0028, + "serial_latency_p95": 0.0025, + "recall": 0.9479, + "ndcg": 0.9545, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 446.4305, + 1857.0762, + 3066.2065, + 3959.9145, + 4119.6984, + 4404.2396, + 4654.3656, + 4771.4324 + ], + "conc_latency_p99_list": [ + 0.0027465902952826583, + 0.003445194798405282, + 0.004290146040148099, + 0.008809843383787666, + 0.014246009238704564, + 0.017628938370035024, + 0.024053513460094107, + 0.029474502794328145 + ], + "conc_latency_p95_list": [ + 0.0025784781464608377, + 0.0031829175946768372, + 0.003835800604429096, + 0.0071477785022580065, + 0.011466846610710487, + 0.014443101210054009, + 0.019454453799698967, + 0.023967151201213708 + ], + "conc_latency_avg_list": [ + 0.002237092280444995, + 0.0026882400384546624, + 0.0032547429679731454, + 0.005034559716886033, + 0.007252702527297999, + 0.009024968635487724, + 0.012743531884434339, + 0.016486166204689928 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4006.3994, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.003, + "recall": 0.9609, + "ndcg": 0.966, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 396.0794, + 1682.937, + 2770.1408, + 3415.7329, + 3586.5656, + 3732.9793, + 3908.8395, + 4006.3994 + ], + "conc_latency_p99_list": [ + 0.0031631914011086342, + 0.003759940078307404, + 0.005007598159427288, + 0.010577416090527546, + 0.01645218172343448, + 0.0200205380024272, + 0.02614469664360513, + 0.03247302199597469 + ], + "conc_latency_p95_list": [ + 0.0029416235047392547, + 0.0034802541093085894, + 0.004256061747582862, + 0.008588075407897121, + 0.01299767559976317, + 0.016106622002553195, + 0.021291244098392777, + 0.026077511996845715 + ], + "conc_latency_avg_list": [ + 0.002521469348982409, + 0.0029665274119998614, + 0.0036030252400985627, + 0.005839950739688844, + 0.008331452198117406, + 0.010640699049339198, + 0.015161825352942679, + 0.019627049620518894 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3441.7597, + "serial_latency_p99": 0.0035, + "serial_latency_p95": 0.0032, + "recall": 0.9682, + "ndcg": 0.9725, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 363.4573, + 1530.4211, + 2519.3499, + 2941.0933, + 3130.5066, + 3239.8074, + 3357.9662, + 3441.7597 + ], + "conc_latency_p99_list": [ + 0.003447128901898394, + 0.004104390731372401, + 0.005795499211671998, + 0.012608604685810857, + 0.018390358952165124, + 0.02161771130544366, + 0.029535210086614822, + 0.03676844512228854 + ], + "conc_latency_p95_list": [ + 0.0032040981095633465, + 0.0037800333026098087, + 0.004730919548455857, + 0.010123601651866907, + 0.014520535804331302, + 0.017753759350307517, + 0.023507353749300814, + 0.029112758993869645 + ], + "conc_latency_avg_list": [ + 0.0027478315701259456, + 0.0032620836103758183, + 0.003962182206516774, + 0.006784177567489496, + 0.009538412010322416, + 0.012277581280440339, + 0.017678232976666337, + 0.022830987265851796 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3040.6216, + "serial_latency_p99": 0.0037, + "serial_latency_p95": 0.0035, + "recall": 0.9734, + "ndcg": 0.9771, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 323.1632, + 1408.5716, + 2291.0458, + 2658.8647, + 2809.4563, + 2889.303, + 2958.6865, + 3040.6216 + ], + "conc_latency_p99_list": [ + 0.0038972479960648343, + 0.004427719511440956, + 0.006464274920726896, + 0.013835774816980114, + 0.01854686296428553, + 0.023528500349348183, + 0.030925163915235258, + 0.037629884978523494 + ], + "conc_latency_p95_list": [ + 0.0036286949907662347, + 0.00409807980468031, + 0.005293481396074639, + 0.01114347539114533, + 0.015239884803304439, + 0.01894600450323196, + 0.02535856414833688, + 0.03149616299779154 + ], + "conc_latency_avg_list": [ + 0.0030909596088423805, + 0.0035447166881325863, + 0.004357504468202672, + 0.007503025706852371, + 0.010637931999990373, + 0.013780835663880903, + 0.02002760247152969, + 0.02587108250217688 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2446.7373, + "serial_latency_p99": 0.0043, + "serial_latency_p95": 0.004, + "recall": 0.9791, + "ndcg": 0.9822, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 290.4004, + 1199.0654, + 1943.7472, + 2207.5563, + 2272.2082, + 2361.8647, + 2425.272, + 2446.7373 + ], + "conc_latency_p99_list": [ + 0.004225126459205057, + 0.005225188090844314, + 0.007753408416756429, + 0.015723756006627808, + 0.021543593837413895, + 0.025226465429732312, + 0.0348595633450895, + 0.04409603113395864 + ], + "conc_latency_p95_list": [ + 0.003936908097239211, + 0.004869158701330889, + 0.006411225302144882, + 0.012806271501176525, + 0.017710478595108724, + 0.02138321524907951, + 0.02917539790214505, + 0.037459137551195454 + ], + "conc_latency_avg_list": [ + 0.003439496089142086, + 0.004164736080481502, + 0.005136606676356694, + 0.0090353538125237, + 0.013157439387184814, + 0.01685513374865052, + 0.024467911853133933, + 0.03215223014404708 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2084.6245, + "serial_latency_p99": 0.005, + "serial_latency_p95": 0.0046, + "recall": 0.9819, + "ndcg": 0.9847, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 257.7354, + 1074.5131, + 1699.1152, + 1906.3127, + 1964.552, + 1992.5973, + 2059.3939, + 2084.6245 + ], + "conc_latency_p99_list": [ + 0.004679669999168255, + 0.0057560302416095515, + 0.008956946645048449, + 0.016920548141788453, + 0.022776664053235435, + 0.02843890285366797, + 0.039300132958451285, + 0.0494756092831085 + ], + "conc_latency_p95_list": [ + 0.004424713403568603, + 0.005422145003103651, + 0.007496941405406687, + 0.014108828103053384, + 0.019311829509388187, + 0.024556438496802002, + 0.033556375399348325, + 0.04314066854931297 + ], + "conc_latency_avg_list": [ + 0.00387603491228106, + 0.004647482651951955, + 0.005875930555797928, + 0.010467915104743828, + 0.015220352811585824, + 0.019988946757462604, + 0.028783135202314097, + 0.037787432564769345 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ] +} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json new file mode 100644 index 000000000..619b5fbf4 --- /dev/null +++ b/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json @@ -0,0 +1,2814 @@ +{ + "run_id": "80696b60e39749b295273db3cdba1b69", + "task_label": "standard_20260209", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 9441.1235, + "serial_latency_p99": 0.0052, + "serial_latency_p95": 0.0039, + "recall": 0.9589, + "ndcg": 0.9658, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 306.3773, + 1519.6034, + 3129.1309, + 5457.1507, + 6585.7082, + 7420.0391, + 8468.6183, + 9441.1235 + ], + "conc_latency_p99_list": [ + 0.004086580532602967, + 0.00566743115196005, + 0.005243223664583638, + 0.0099503791576717, + 0.009588507658627355, + 0.011322382240905426, + 0.01665949933376398, + 0.018269318575912616 + ], + "conc_latency_p95_list": [ + 0.0036428007049835284, + 0.003802539707976394, + 0.003850769612472504, + 0.004741756187286226, + 0.006738374719861894, + 0.00850677301059477, + 0.01233988689491525, + 0.01412305135163478 + ], + "conc_latency_avg_list": [ + 0.0032598009878015348, + 0.003285434242748037, + 0.0031902432390411135, + 0.0036557288568108254, + 0.004539560740384104, + 0.005359882091644495, + 0.007008278288237143, + 0.008346175450269581 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 2, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 6125.6146, + "serial_latency_p99": 0.0049, + "serial_latency_p95": 0.0047, + "recall": 0.9919, + "ndcg": 0.9936, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 238.8712, + 1196.2193, + 2440.8391, + 4133.8486, + 4858.2331, + 5446.0047, + 6000.1652, + 6125.6146 + ], + "conc_latency_p99_list": [ + 0.004919703922932965, + 0.010525701696751709, + 0.006511175064661075, + 0.011745981796411804, + 0.01397663167037535, + 0.014958455199375772, + 0.02016973898542343, + 0.026595940839324612 + ], + "conc_latency_p95_list": [ + 0.0046619078202638775, + 0.004602618556236848, + 0.004652375826844946, + 0.006574279977940023, + 0.009700259550299961, + 0.011811009392840788, + 0.016332678495382422, + 0.02156840759853367 + ], + "conc_latency_avg_list": [ + 0.004181373075740294, + 0.004174027799032302, + 0.004089987135078621, + 0.004828215031413884, + 0.006158122789762511, + 0.007310655337680553, + 0.009892293927032914, + 0.012884722355037423 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 7, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 5502.1797, + "serial_latency_p99": 0.0038, + "serial_latency_p95": 0.0035, + "recall": 0.9452, + "ndcg": 0.9509, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 309.5267, + 1357.6945, + 2213.2564, + 3349.9157, + 3910.9875, + 4385.5899, + 5039.0299, + 5502.1797 + ], + "conc_latency_p99_list": [ + 0.004089714956353409, + 0.0056684831657912264, + 0.012667869632714424, + 0.01000201591959918, + 0.015246839204337462, + 0.01789945445081684, + 0.02318606456159614, + 0.02852195684099569 + ], + "conc_latency_p95_list": [ + 0.003446204590727575, + 0.00420360880671069, + 0.0055909310030983735, + 0.008043184356938578, + 0.011835442011943087, + 0.014750372216803953, + 0.01907540229440201, + 0.023490797984413794 + ], + "conc_latency_avg_list": [ + 0.0032266616589272513, + 0.0036776357130507064, + 0.004511172191326527, + 0.0059575964382477965, + 0.007649578970207299, + 0.009076414605964197, + 0.011790618147195132, + 0.01435249249351469 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 2, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 1827.5849, + "serial_latency_p99": 0.0054, + "serial_latency_p95": 0.0052, + "recall": 0.9903, + "ndcg": 0.9918, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 218.8679, + 861.6609, + 1195.6337, + 1339.6438, + 1475.742, + 1546.5953, + 1747.9077, + 1827.5849 + ], + "conc_latency_p99_list": [ + 0.006139998821017798, + 0.008449624522763766, + 0.014799785086070211, + 0.02650444130937102, + 0.03739181953598747, + 0.04722500271425814, + 0.05959104605717584, + 0.0715404962032335 + ], + "conc_latency_p95_list": [ + 0.005324425446451642, + 0.006872303040290717, + 0.011261535996163731, + 0.021800653115496966, + 0.03018094879225827, + 0.038646315991354645, + 0.04888677580165675, + 0.06127824939903802 + ], + "conc_latency_avg_list": [ + 0.004563713319136163, + 0.0057948937011398916, + 0.00835191496328548, + 0.014905808578982384, + 0.020272801297484298, + 0.025762202275047597, + 0.034040973878552726, + 0.043269632825280596 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 8, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 1938.1932, + "serial_latency_p99": 0.0056, + "serial_latency_p95": 0.0053, + "recall": 0.989, + "ndcg": 0.9906, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 252.1234, + 954.3561, + 1355.554, + 1490.2007, + 1619.1811, + 1720.5715, + 1860.9305, + 1938.1932 + ], + "conc_latency_p99_list": [ + 0.0050690684028086245, + 0.008166075175395238, + 0.011429371475242079, + 0.024431421170011097, + 0.03405256026599093, + 0.043166847461543534, + 0.05706310785026292, + 0.07251818811346307 + ], + "conc_latency_p95_list": [ + 0.004786731592321303, + 0.006139315699692815, + 0.009358109103050082, + 0.019912595101050083, + 0.028057811519829556, + 0.035778356752416585, + 0.04874245898681693, + 0.061324251000769436 + ], + "conc_latency_avg_list": [ + 0.003961802805693023, + 0.005232545384579334, + 0.007366828470110833, + 0.01339630708221878, + 0.018484241519041742, + 0.023149088452180534, + 0.03196686416892556, + 0.04073973314917765 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 7, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 3778.8811, + "serial_latency_p99": 0.0048, + "serial_latency_p95": 0.0042, + "recall": 0.9828, + "ndcg": 0.9851, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 275.1343, + 1220.1869, + 2080.6024, + 2544.5946, + 2934.4403, + 3222.4756, + 3520.6342, + 3778.8811 + ], + "conc_latency_p99_list": [ + 0.004224385871784762, + 0.006377705505292314, + 0.006628996630315663, + 0.01544506659847684, + 0.02085710293264129, + 0.02495263563963816, + 0.03397362035117112, + 0.0415226237432216 + ], + "conc_latency_p95_list": [ + 0.003899741142231505, + 0.004650917260732967, + 0.005831561920058448, + 0.011755315004847944, + 0.016476290803984734, + 0.020251012463995716, + 0.027687024004990225, + 0.03372844383848132 + ], + "conc_latency_avg_list": [ + 0.0036302553438832055, + 0.004091130600148307, + 0.0047987246661642296, + 0.007844535865702365, + 0.010192746767866709, + 0.012344825366337173, + 0.016888610032228992, + 0.020908843243531986 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 6, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 3974.8218, + "serial_latency_p99": 0.0048, + "serial_latency_p95": 0.0043, + "recall": 0.9396, + "ndcg": 0.9428, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 295.7118, + 1186.5002, + 1797.187, + 2449.5408, + 2857.3282, + 3164.2684, + 3660.3166, + 3974.8218 + ], + "conc_latency_p99_list": [ + 0.004793434205930709, + 0.005700148111791336, + 0.007426547166251105, + 0.014722349808434964, + 0.019732102921698247, + 0.02347602234221995, + 0.03035390855744481, + 0.03731970520602769 + ], + "conc_latency_p95_list": [ + 0.004169186984654516, + 0.004717364211683161, + 0.006335475675587076, + 0.01105007229198236, + 0.01624748620088212, + 0.019896189187420532, + 0.025662759994156657, + 0.031171760102733943 + ], + "conc_latency_avg_list": [ + 0.003377046875928882, + 0.004208154278338283, + 0.005554828192323107, + 0.00814848752272782, + 0.010461408703728187, + 0.012581798692238582, + 0.016239422653327416, + 0.019879490524355867 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 1, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 2971.1402, + "serial_latency_p99": 0.0116, + "serial_latency_p95": 0.0045, + "recall": 0.9729, + "ndcg": 0.9752, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 278.0971, + 1117.1061, + 1626.1624, + 2126.1398, + 2425.3155, + 2521.9392, + 2793.1709, + 2971.1402 + ], + "conc_latency_p99_list": [ + 0.004647255002055317, + 0.00566180575755425, + 0.011257058603223423, + 0.015976536156085786, + 0.023499950004043067, + 0.03039303767494857, + 0.04078626602888109, + 0.04914582587312907 + ], + "conc_latency_p95_list": [ + 0.004429338514455594, + 0.005089565094385761, + 0.007136018975870684, + 0.01292410076566739, + 0.019488130600075235, + 0.025593487400328737, + 0.03410469329974147, + 0.04188467259518802 + ], + "conc_latency_avg_list": [ + 0.003591256380123583, + 0.004470003286850064, + 0.00613923671987789, + 0.009389019339411041, + 0.012338699755027977, + 0.015788074921940728, + 0.02128022645956974, + 0.026573440291179154 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 4, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 8441.533, + "serial_latency_p99": 0.0069, + "serial_latency_p95": 0.0035, + "recall": 0.9785, + "ndcg": 0.9825, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 314.6659, + 1556.8466, + 3026.0164, + 4952.5941, + 6059.6492, + 6948.3916, + 7709.5146, + 8441.533 + ], + "conc_latency_p99_list": [ + 0.003662585185375065, + 0.003881094480166214, + 0.005496532852703231, + 0.012801527802657801, + 0.01098183460126168, + 0.011906764237210155, + 0.017680786546843585, + 0.02030978908645921 + ], + "conc_latency_p95_list": [ + 0.0033931825950276107, + 0.0034469130958314055, + 0.003594076508306898, + 0.00537703381414758, + 0.00756983550672885, + 0.009184699498291593, + 0.013430504295683926, + 0.015904919797321764 + ], + "conc_latency_avg_list": [ + 0.003173945048460723, + 0.003206842876612295, + 0.003299035623981205, + 0.004028295482036381, + 0.004934835427053102, + 0.005724545223453835, + 0.007703497705718975, + 0.009334748886407937 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 4, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 2703.5422, + "serial_latency_p99": 0.0129, + "serial_latency_p95": 0.0049, + "recall": 0.9903, + "ndcg": 0.992, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 230.8474, + 1033.6996, + 1653.7251, + 2021.6415, + 2247.5951, + 2433.7674, + 2579.1371, + 2703.5422 + ], + "conc_latency_p99_list": [ + 0.005364620329928583, + 0.008247331644815845, + 0.009261744469986306, + 0.019847353509976556, + 0.027152295518899337, + 0.0318573336204281, + 0.04668155071558431, + 0.053025491506559774 + ], + "conc_latency_p95_list": [ + 0.004779104294721037, + 0.005717401996662375, + 0.007679672696394844, + 0.015277760991011746, + 0.021049827802926295, + 0.025551227995310906, + 0.03599990590882953, + 0.045174946513725445 + ], + "conc_latency_avg_list": [ + 0.004327150692358546, + 0.004830543605956322, + 0.006038381999227765, + 0.009872808423519352, + 0.013310825758863387, + 0.016344593851929733, + 0.023038225918050447, + 0.02921974749485067 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 9, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 1628.2736, + "serial_latency_p99": 0.0056, + "serial_latency_p95": 0.0051, + "recall": 0.9913, + "ndcg": 0.9928, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 224.1263, + 854.2908, + 1191.923, + 1306.8314, + 1408.4071, + 1492.5651, + 1572.8756, + 1628.2736 + ], + "conc_latency_p99_list": [ + 0.005614027456322219, + 0.01122624498093481, + 0.013540994446957512, + 0.026513429321930727, + 0.0360522977943765, + 0.04583110647072318, + 0.06417124239553232, + 0.08320635374402624 + ], + "conc_latency_p95_list": [ + 0.0053050212460220795, + 0.006976215375470927, + 0.011254654199001379, + 0.022096098412293937, + 0.03048573801643215, + 0.037569552293280135, + 0.05408634888590313, + 0.06856599940219893 + ], + "conc_latency_avg_list": [ + 0.004456437869882418, + 0.005844962813409522, + 0.008378709673770517, + 0.01528081619265322, + 0.021239153420731547, + 0.026694518371955574, + 0.03784726188423443, + 0.04857833790231276 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 9, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 5019.5973, + "serial_latency_p99": 0.0057, + "serial_latency_p95": 0.0054, + "recall": 0.994, + "ndcg": 0.9954, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 222.5871, + 1150.5338, + 2195.8726, + 3676.709, + 4412.1372, + 4833.1625, + 5019.5973, + 5015.8037 + ], + "conc_latency_p99_list": [ + 0.005366289358644281, + 0.006616018440399769, + 0.0064373466832330474, + 0.010222766738734191, + 0.013315525980142408, + 0.016972876400686822, + 0.024728685971349477, + 0.031319626237964276 + ], + "conc_latency_p95_list": [ + 0.00513816498714732, + 0.004792370549694169, + 0.005205206610844471, + 0.007212611548311542, + 0.010267410500091497, + 0.013238379993708808, + 0.01984079669928178, + 0.02578976150834933 + ], + "conc_latency_avg_list": [ + 0.00448758533933214, + 0.0043394632993677415, + 0.0045465428246546005, + 0.0054275771136581205, + 0.006782299052349466, + 0.008242341170482134, + 0.011840508222988514, + 0.015742522812380217 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 9, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 7364.0396, + "serial_latency_p99": 0.004, + "serial_latency_p95": 0.0037, + "recall": 0.9893, + "ndcg": 0.9915, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 287.8688, + 1384.2897, + 2700.4491, + 4735.4478, + 5614.1579, + 6134.6648, + 6819.5071, + 7364.0396 + ], + "conc_latency_p99_list": [ + 0.00390698241040809, + 0.00731195724976711, + 0.00495785990206058, + 0.00734390948899089, + 0.012288954856630872, + 0.014986682942835616, + 0.01849594444676764, + 0.022832338945008815 + ], + "conc_latency_p95_list": [ + 0.0037214207914075814, + 0.003863213058502879, + 0.004057778001879342, + 0.005324349994771183, + 0.008072146945050915, + 0.010810172616038463, + 0.01463127658644225, + 0.01830196708324365 + ], + "conc_latency_avg_list": [ + 0.0034694395949774774, + 0.003606888217707231, + 0.003696606332343799, + 0.004213763141467857, + 0.0053274846319189195, + 0.006490250213165499, + 0.008702013864526974, + 0.010708946041863319 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 6, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 2724.9239, + "serial_latency_p99": 0.0124, + "serial_latency_p95": 0.0048, + "recall": 0.9811, + "ndcg": 0.9832, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 269.3651, + 1047.0123, + 1498.6024, + 1847.6803, + 2114.199, + 2284.3302, + 2537.2552, + 2724.9239 + ], + "conc_latency_p99_list": [ + 0.004742077292175964, + 0.00691745357704349, + 0.010720830453210511, + 0.01960868470487181, + 0.026337993171764537, + 0.032754160480981225, + 0.04255042473436333, + 0.05312397440138735 + ], + "conc_latency_p95_list": [ + 0.004494865804736036, + 0.0055081502971006556, + 0.007977579892030908, + 0.015719183007604443, + 0.022288659910555, + 0.02759612778027076, + 0.03666536140372045, + 0.04536134131485597 + ], + "conc_latency_avg_list": [ + 0.0037082374061278584, + 0.004769060159688272, + 0.0066635344327552045, + 0.010804306792214618, + 0.014153154028666374, + 0.017440448752945776, + 0.02343149585687298, + 0.029001097442020812 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 5, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 5514.815, + "serial_latency_p99": 0.0042, + "serial_latency_p95": 0.0036, + "recall": 0.9286, + "ndcg": 0.9355, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 313.799, + 1354.9874, + 2301.8011, + 3314.3036, + 3843.1259, + 4355.6117, + 5048.842, + 5514.815 + ], + "conc_latency_p99_list": [ + 0.0038773450409644284, + 0.008138228004099801, + 0.010878445263078868, + 0.011540972657094244, + 0.016388589342823227, + 0.01858296279708156, + 0.023569957185536613, + 0.02886486930365208 + ], + "conc_latency_p95_list": [ + 0.003412947052856907, + 0.004246345022693276, + 0.005220197608286979, + 0.008349375608668195, + 0.012340355810010787, + 0.0147909961087862, + 0.019336943980306387, + 0.023638575003133155 + ], + "conc_latency_avg_list": [ + 0.003182328343818464, + 0.003684439145729171, + 0.004337632718561462, + 0.006021014814140765, + 0.007777482876747058, + 0.009140085848755274, + 0.011767476800046897, + 0.014305157378590626 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 1, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 9901.1114, + "serial_latency_p99": 0.0039, + "serial_latency_p95": 0.0037, + "recall": 0.9385, + "ndcg": 0.9486, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 321.3357, + 1639.9924, + 3195.3018, + 5673.0951, + 6808.8856, + 7560.1348, + 9087.2499, + 9901.1114 + ], + "conc_latency_p99_list": [ + 0.00394074416602971, + 0.0041782407171558535, + 0.004392879804945551, + 0.010109323544893349, + 0.008123845955124116, + 0.014126944777672188, + 0.014278251143987291, + 0.01781155434960963 + ], + "conc_latency_p95_list": [ + 0.0034882987965829666, + 0.0035166182147804642, + 0.0035458351092529476, + 0.004310413988423532, + 0.006210639532946515, + 0.008767930739850271, + 0.010931958199944346, + 0.013749658003507645 + ], + "conc_latency_avg_list": [ + 0.0031001682930926035, + 0.0030430930397005885, + 0.0031233191875315565, + 0.0035155999207049896, + 0.004389041533959679, + 0.005263586471416786, + 0.006530556363972054, + 0.007958968690030266 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 1, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 3258.8508, + "serial_latency_p99": 0.0117, + "serial_latency_p95": 0.0045, + "recall": 0.9888, + "ndcg": 0.9906, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 244.7738, + 1130.2346, + 1809.9431, + 2312.2265, + 2588.2339, + 2788.0239, + 3069.518, + 3258.8508 + ], + "conc_latency_p99_list": [ + 0.005137932703364641, + 0.00555925140972249, + 0.008915685693500566, + 0.016561121140257453, + 0.022867804747074828, + 0.02806737951235845, + 0.036439670615363844, + 0.04337102690333264 + ], + "conc_latency_p95_list": [ + 0.004440846845682244, + 0.005053811200195923, + 0.0068235130165703595, + 0.01288393942813854, + 0.018162307806778695, + 0.02236156941507943, + 0.0296233788743848, + 0.03584610429388702 + ], + "conc_latency_avg_list": [ + 0.004080840300378187, + 0.004417934321996714, + 0.005516924373879563, + 0.008634555842136784, + 0.011560621919641356, + 0.014278915373578093, + 0.019378262406752788, + 0.024243179517198447 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 8, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 5907.441, + "serial_latency_p99": 0.0057, + "serial_latency_p95": 0.0054, + "recall": 0.9931, + "ndcg": 0.9946, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 214.7436, + 1124.1591, + 2263.1224, + 3695.1435, + 4635.8817, + 5035.3873, + 5614.0659, + 5907.441 + ], + "conc_latency_p99_list": [ + 0.005838954218779679, + 0.009590839385055032, + 0.0072934928326867585, + 0.009990225688670754, + 0.013466239573317574, + 0.01589230435347418, + 0.021308113009436094, + 0.026848825681372538 + ], + "conc_latency_p95_list": [ + 0.005094571305380669, + 0.0048950490017887205, + 0.00495702201151289, + 0.007446101757523138, + 0.009852354813483545, + 0.01273485799174523, + 0.017310541804181415, + 0.021899056984693743 + ], + "conc_latency_avg_list": [ + 0.004651503297485146, + 0.004441934131452666, + 0.004411218431154767, + 0.005397841813638808, + 0.00645052096323231, + 0.007903104976167643, + 0.010583279479885773, + 0.013354355155792764 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 8, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 5064.6982, + "serial_latency_p99": 0.0043, + "serial_latency_p95": 0.0036, + "recall": 0.9558, + "ndcg": 0.9606, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 306.5362, + 1368.7188, + 2310.9627, + 3255.5776, + 3791.6067, + 4073.62, + 4572.4511, + 5064.6982 + ], + "conc_latency_p99_list": [ + 0.0042355662427144124, + 0.0046483404963510114, + 0.009778717628214485, + 0.010217842382844546, + 0.015881566194002526, + 0.01989235390967224, + 0.025938013812992735, + 0.031185232510324568 + ], + "conc_latency_p95_list": [ + 0.0034556519065517934, + 0.004072950512636453, + 0.005186801168019884, + 0.008326760004274545, + 0.01228064175666077, + 0.016202768517541696, + 0.021469047002028674, + 0.025539752503391355 + ], + "conc_latency_avg_list": [ + 0.003258234978368169, + 0.0036474964905585366, + 0.004320538177772956, + 0.006130977769455615, + 0.007887396918013946, + 0.009779561744223083, + 0.013001510996033932, + 0.015590046073741476 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 3, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 3468.5933, + "serial_latency_p99": 0.0045, + "serial_latency_p95": 0.0043, + "recall": 0.9634, + "ndcg": 0.9661, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 274.7753, + 1086.6444, + 1657.8469, + 2230.3556, + 2576.5151, + 2840.9102, + 3207.9495, + 3468.5933 + ], + "conc_latency_p99_list": [ + 0.007525063498178497, + 0.00871218444284751, + 0.009115133894665613, + 0.01669143169856398, + 0.021790961647639086, + 0.026465490460395806, + 0.03498110753978835, + 0.042080024706956466 + ], + "conc_latency_p95_list": [ + 0.004381819497211836, + 0.00529403816035483, + 0.007012556104746182, + 0.01246311155118746, + 0.018361228803405537, + 0.022542869101744148, + 0.029249506900669076, + 0.03583758800959913 + ], + "conc_latency_avg_list": [ + 0.003635002963577244, + 0.0045950829257419635, + 0.006023093199217258, + 0.008949508158072033, + 0.011605007790743322, + 0.014003128689967868, + 0.018541925579661857, + 0.02278122412115334 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 3, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 2401.3204, + "serial_latency_p99": 0.0107, + "serial_latency_p95": 0.0047, + "recall": 0.9866, + "ndcg": 0.9884, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 261.8215, + 985.1764, + 1468.0087, + 1749.415, + 1931.5307, + 2090.0742, + 2278.9704, + 2401.3204 + ], + "conc_latency_p99_list": [ + 0.004836413672601339, + 0.009077261193306102, + 0.010354967679304536, + 0.02026946535654133, + 0.029030042108206543, + 0.03499512374750339, + 0.046797644338803394, + 0.057348916197370266 + ], + "conc_latency_p95_list": [ + 0.004629465389007237, + 0.0058674284955486655, + 0.0082442566199461, + 0.01657543244800763, + 0.02401039500546176, + 0.029254749882966277, + 0.03907370918313973, + 0.04936321394779952 + ], + "conc_latency_avg_list": [ + 0.0038149669483554033, + 0.00506845096702024, + 0.006802651714890116, + 0.01140782706843602, + 0.015489598411640663, + 0.019035290587860968, + 0.02608886756739919, + 0.03292382999700411 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 6, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 3568.396, + "serial_latency_p99": 0.0048, + "serial_latency_p95": 0.0043, + "recall": 0.9863, + "ndcg": 0.9883, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 249.5646, + 1116.5066, + 1828.4446, + 2389.4823, + 2645.8957, + 2879.4104, + 3356.3991, + 3568.396 + ], + "conc_latency_p99_list": [ + 0.004949838446336795, + 0.010793446648749472, + 0.011542035994352773, + 0.015714827887131834, + 0.023098365282639866, + 0.028270509486901574, + 0.03385155899741221, + 0.0405952908913605 + ], + "conc_latency_p95_list": [ + 0.004326387906621675, + 0.005205007104086689, + 0.0069263952391338535, + 0.012388522701803593, + 0.01838434826204321, + 0.022779258753871545, + 0.02818173549894709, + 0.034205201656732236 + ], + "conc_latency_avg_list": [ + 0.004002170059550175, + 0.0044719544014582705, + 0.005461118931996274, + 0.008354930265743018, + 0.011307457646718058, + 0.013821107916470069, + 0.01770926681597559, + 0.022161158532467397 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 7, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 4674.1861, + "serial_latency_p99": 0.0045, + "serial_latency_p95": 0.0038, + "recall": 0.967, + "ndcg": 0.9705, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 299.441, + 1293.8246, + 2204.8398, + 2998.8928, + 3416.6029, + 3814.1905, + 4403.9058, + 4674.1861 + ], + "conc_latency_p99_list": [ + 0.00402290889178403, + 0.007907517027342683, + 0.006291847964457707, + 0.013353673194069417, + 0.017164103323593728, + 0.021020031592343, + 0.027193114216788664, + 0.033575675918255006 + ], + "conc_latency_p95_list": [ + 0.00356991050648503, + 0.004438751634734216, + 0.005453104941989295, + 0.00961559300776571, + 0.013857692398596555, + 0.017230113997356966, + 0.02220476679212879, + 0.02798210658947937 + ], + "conc_latency_avg_list": [ + 0.003335578729612294, + 0.0038557001362392825, + 0.0045284253144977004, + 0.006655331635103421, + 0.008756953741340391, + 0.010436090944337196, + 0.013487455343736422, + 0.016883584811310505 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 4, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1770595200.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index c463463fc..566ce4285 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -1,3062 +1,2142 @@ [ - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 597.3641, - "latency": 12.1, - "recall": 0.9221, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 551.1757, - "latency": 10.7, - "recall": 0.9327, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 492.9765, - "latency": 13.4, - "recall": 0.9443, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 431.0155, - "latency": 15.2, - "recall": 0.954, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 377.3634, - "latency": 16.2, - "recall": 0.9615, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 337.5819, - "latency": 15.0, - "recall": 0.9666, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 279.7257, - "latency": 18.1, - "recall": 0.9729, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3033.786, - "latency": 8.7, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3019.2416, - "latency": 9.5, - "recall": 0.9765, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2890.9523, - "latency": 9.4, - "recall": 0.9625, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2789.7212, - "latency": 8.2, - "recall": 0.9538, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2457.2628, - "latency": 9.0, - "recall": 0.9378, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2209.4973, - "latency": 13.7, - "recall": 0.9228, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1960.388, - "latency": 11.0, - "recall": 0.9076, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1725.092, - "latency": 11.7, - "recall": 0.8969, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1307.419, - "latency": 12.3, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1520.4145, - "latency": 12.5, - "recall": 0.9028, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1273.3452, - "latency": 12.3, - "recall": 0.9242, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1011.7943, - "latency": 15.2, - "recall": 0.945, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 824.5097, - "latency": 15.5, - "recall": 0.9558, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 350.0132, - "latency": 29.7, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 179.5204, - "latency": 51.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 72.99, - "latency": 111.4, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 42.9877, - "latency": 201.9, - "recall": 0.9912, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 96.4987, - "latency": 113.1, - "recall": 0.9296, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 189.3789, - "latency": 58.8, - "recall": 0.9149, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 246.7071, - "latency": 45.1, - "recall": 0.9018, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 229.0379, - "latency": 43.0, - "recall": 0.8908, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 125.6164, - "latency": 69.8, - "recall": 0.8746, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 376.3752, - "latency": 14.5, - "recall": 0.9039, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 341.2325, - "latency": 13.4, - "recall": 0.9136, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 300.7678, - "latency": 12.8, - "recall": 0.922, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 257.0398, - "latency": 15.4, - "recall": 0.9303, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 228.7734, - "latency": 16.2, - "recall": 0.9374, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 204.3654, - "latency": 18.2, - "recall": 0.9424, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 167.5075, - "latency": 18.0, - "recall": 0.9501, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 146.339, - "latency": 20.9, - "recall": 0.9557, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 129.0705, - "latency": 24.3, - "recall": 0.96, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2095.7067, - "latency": 12.4, - "recall": 0.8961, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1925.3019, - "latency": 11.3, - "recall": 0.9141, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1707.8841, - "latency": 10.0, - "recall": 0.9314, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1442.0638, - "latency": 10.1, - "recall": 0.9482, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1115.106, - "latency": 13.1, - "recall": 0.9662, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 910.4322, - "latency": 14.2, - "recall": 0.9748, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2175.2694, - "latency": 9.8, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1430.0244, - "latency": 12.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 692.5751, - "latency": 18.7, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 364.3516, - "latency": 26.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 190.3777, - "latency": 47.9, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 249.3519, - "latency": 44.7, - "recall": 0.9446, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 437.8735, - "latency": 27.1, - "recall": 0.9364, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 669.9441, - "latency": 19.1, - "recall": 0.9227, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 899.3114, - "latency": 14.9, - "recall": 0.9072, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3465.1696, - "latency": 2.2, - "recall": 0.9528, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3102.1518, - "latency": 2.3, - "recall": 0.9608, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2681.2848, - "latency": 2.5, - "recall": 0.9681, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2232.9105, - "latency": 2.9, - "recall": 0.9757, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1913.17, - "latency": 3.2, - "recall": 0.9797, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1575.8137, - "latency": 3.5, - "recall": 0.9822, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1344.5652, - "latency": 4.2, - "recall": 0.9849, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1148.7813, - "latency": 4.8, - "recall": 0.9861, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3680.6045, - "latency": 2.2, - "recall": 0.9954, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3407.9972, - "latency": 2.2, - "recall": 0.994, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3062.6755, - "latency": 2.4, - "recall": 0.9932, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2568.3371, - "latency": 2.7, - "recall": 0.9927, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1886.2891, - "latency": 3.3, - "recall": 0.992, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 923.8685, - "latency": 6.7, - "recall": 0.9919, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1442.1076, - "latency": 4.1, - "recall": 0.9517, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2002.9865, - "latency": 3.2, - "recall": 0.9467, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3201.9438, - "latency": 2.3, - "recall": 0.922, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11763.5538, - "latency": 1.5, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11803.1944, - "latency": 1.5, - "recall": 0.9778, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11520.9234, - "latency": 1.5, - "recall": 0.9634, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11280.0849, - "latency": 1.6, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10671.8925, - "latency": 1.7, - "recall": 0.9339, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10258.2661, - "latency": 1.7, - "recall": 0.9139, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9681.5656, - "latency": 1.9, - "recall": 0.9008, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8945.4041, - "latency": 1.9, - "recall": 0.8894, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5436.8907, - "latency": 2.0, - "recall": 0.929, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 437.1695, - "latency": 5.1, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 400.0053, - "latency": 5.6, - "recall": 0.9558, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 343.938, - "latency": 6.5, - "recall": 0.9605, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 275.2271, - "latency": 7.7, - "recall": 0.9649, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 234.9937, - "latency": 8.7, - "recall": 0.9677, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 202.6975, - "latency": 9.9, - "recall": 0.9696, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 164.6073, - "latency": 12.1, - "recall": 0.972, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 135.9624, - "latency": 13.9, - "recall": 0.9733, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 432.8374, - "latency": 5.0, - "recall": 0.9865, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 368.6042, - "latency": 5.4, - "recall": 0.9843, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 318.8159, - "latency": 7.1, - "recall": 0.9836, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 217.0963, - "latency": 9.1, - "recall": 0.9822, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 150.9989, - "latency": 12.5, - "recall": 0.9814, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 76.0341, - "latency": 23.1, - "recall": 0.9797, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 166.5611, - "latency": 11.5, - "recall": 0.9675, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 248.9625, - "latency": 8.3, - "recall": 0.9608, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 435.3358, - "latency": 8.2, - "recall": 0.9417, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11397.7043, - "latency": 1.6, - "recall": 0.9597, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10891.7531, - "latency": 1.7, - "recall": 0.9408, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10276.7451, - "latency": 1.7, - "recall": 0.9159, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9664.2855, - "latency": 1.8, - "recall": 0.899, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8936.7962, - "latency": 2.0, - "recall": 0.8835, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5671.2562, - "latency": 2.1, - "recall": 0.903, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 3157.707, - "latency": 2.3, - "recall": 0.9347, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 1985.8124, - "latency": 2.6, - "recall": 0.9407, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 920.9627, - "latency": 3.4, - "recall": 0.9488, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1146.5286, - "latency": 13.7, - "recall": 0.9262, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1148.1735, - "latency": 8.9, - "recall": 0.9801, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1149.1219, - "latency": 10.3, - "recall": 0.9764, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1140.4099, - "latency": 13.5, - "recall": 0.9716, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1123.5147, - "latency": 18.5, - "recall": 0.9688, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 487.8343, - "latency": 25.4, - "recall": 0.9668, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 264.9324, - "latency": 49.6, - "recall": 0.936, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 492.4887, - "latency": 29.6, - "recall": 0.9269, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 823.1775, - "latency": 20.5, - "recall": 0.9148, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1147.1977, - "latency": 13.3, - "recall": 0.8999, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1131.3087, - "latency": 14.1, - "recall": 0.9024, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1114.952, - "latency": 12.7, - "recall": 0.97, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 583.5009, - "latency": 23.0, - "recall": 0.9668, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 31.4779, - "latency": 351.0, - "recall": 0.9414, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 57.8988, - "latency": 200.1, - "recall": 0.9332, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 101.1774, - "latency": 116.1, - "recall": 0.9241, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 212.7466, - "latency": 58.7, - "recall": 0.9099, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 372.2462, - "latency": 35.9, - "recall": 0.8977, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 617.0881, - "latency": 22.4, - "recall": 0.8844, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1094.5967, - "latency": 14.3, - "recall": 0.8659, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4318.9697, - "latency": 4.3, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4250.2894, - "latency": 4.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 2997.4391, - "latency": 6.1, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1494.5334, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1108.6473, - "latency": 7.4, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1289.5164, - "latency": 6.4, - "recall": 0.9906, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1059.3394, - "latency": 7.8, - "recall": 0.9856, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 987.0795, - "latency": 7.1, - "recall": 0.9804, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1591.7055, - "latency": 7.8, - "recall": 0.8506, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1202.8677, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 639.3991, - "latency": 7.3, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 274.8559, - "latency": 9.9, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 441.4152, - "latency": 8.3, - "recall": 0.997, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 358.8949, - "latency": 9.5, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 325.2245, - "latency": 10.3, - "recall": 0.9909, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 273.4174, - "latency": 13.3, - "recall": 0.9789, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 262.8314, - "latency": 11.3, - "recall": 0.9808, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 434.5481, - "latency": 8.5, - "recall": 0.7237, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 446.9116, - "latency": 9.2, - "recall": 0.9357, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 388.3028, - "latency": 9.6, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 323.3964, - "latency": 9.8, - "recall": 0.9507, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 256.4668, - "latency": 11.3, - "recall": 0.9588, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 145.5316, - "latency": 18.4, - "recall": 0.9726, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1242.428, - "latency": 6.4, - "recall": 0.9474, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1111.3633, - "latency": 7.0, - "recall": 0.955, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 955.4701, - "latency": 7.2, - "recall": 0.9629, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 783.5207, - "latency": 7.7, - "recall": 0.971, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 470.8546, - "latency": 9.5, - "recall": 0.9835, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 950.6332, - "latency": 13.2, - "recall": 0.914, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 823.2224, - "latency": 13.5, - "recall": 0.9434, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 743.9815, - "latency": 14.8, - "recall": 0.9583, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 683.1873, - "latency": 15.7, - "recall": 0.9677, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 619.7468, - "latency": 17.2, - "recall": 0.9738, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 537.4082, - "latency": 18.8, - "recall": 0.9809, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 474.9941, - "latency": 20.9, - "recall": 0.9848, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 505.7458, - "latency": 20.7, - "recall": 0.9068, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 433.9034, - "latency": 23.1, - "recall": 0.931, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 381.7737, - "latency": 25.7, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 342.1123, - "latency": 29.0, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 308.2216, - "latency": 31.3, - "recall": 0.9561, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 257.7928, - "latency": 36.4, - "recall": 0.9626, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 223.8166, - "latency": 42.1, - "recall": 0.9666, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3055.0123, - "latency": 7.2, - "recall": 0.9066, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3013.4439, - "latency": 6.9, - "recall": 0.9268, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2801.7241, - "latency": 7.4, - "recall": 0.9476, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2590.3809, - "latency": 8.6, - "recall": 0.9679, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2291.2159, - "latency": 8.9, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3099.4124, - "latency": 6.2, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3014.2483, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2073.2153, - "latency": 11.0, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1507.6899, - "latency": 12.8, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 942.2296, - "latency": 18.2, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 677.1414, - "latency": 33.5, - "recall": 0.7655, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2685.6654, - "latency": 7.6, - "recall": 0.4914, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2604.4444, - "latency": 7.8, - "recall": 0.63, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2159.051, - "latency": 9.4, - "recall": 0.801, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2251.1274, - "latency": 8.7, - "recall": 0.8848, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3103.0539, - "latency": 5.6, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3086.1957, - "latency": 6.7, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3090.0478, - "latency": 6.4, - "recall": 0.9628, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3064.6288, - "latency": 6.5, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3065.6134, - "latency": 6.2, - "recall": 0.9328, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3028.858, - "latency": 6.7, - "recall": 0.9133, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2935.9403, - "latency": 6.8, - "recall": 0.8992, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2771.2009, - "latency": 7.6, - "recall": 0.889, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1610.9496, - "latency": 10.8, - "recall": 0.9, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1557.3623, - "latency": 10.8, - "recall": 0.9244, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1473.9256, - "latency": 11.7, - "recall": 0.9484, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1388.5547, - "latency": 12.5, - "recall": 0.9597, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1022.2696, - "latency": 17.9, - "recall": 0.936, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 696.9777, - "latency": 24.6, - "recall": 0.997, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 353.7862, - "latency": 45.2, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 210.3227, - "latency": 71.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 114.8061, - "latency": 126.6, - "recall": 0.9985, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 504.9179, - "latency": 272.6, - "recall": 0.4664, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1053.1495, - "latency": 17.7, - "recall": 0.5673, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 808.3294, - "latency": 22.2, - "recall": 0.7016, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 8.0584, - "latency": 1757.9, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3033.5491, - "latency": 6.4, - "recall": 0.9844, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2988.4205, - "latency": 7.6, - "recall": 0.9741, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2950.717, - "latency": 6.9, - "recall": 0.9558, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2782.0274, - "latency": 7.4, - "recall": 0.9466, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2708.6752, - "latency": 8.4, - "recall": 0.9337, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2275.2854, - "latency": 9.1, - "recall": 0.917, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1844.8918, - "latency": 10.6, - "recall": 0.9085, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1301.4102, - "latency": 14.7, - "recall": 0.9011, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 13.0379, - "latency": 1063.5, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9704.4214, - "latency": 2.5, - "recall": 0.9169, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9463.4991, - "latency": 2.6, - "recall": 0.9393, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9194.9922, - "latency": 2.7, - "recall": 0.9543, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8779.8779, - "latency": 2.9, - "recall": 0.9685, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8153.4648, - "latency": 3.0, - "recall": 0.9757, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6848.5254, - "latency": 4.4, - "recall": 0.9835, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6124.4431, - "latency": 3.8, - "recall": 0.9873, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5186.8135, - "latency": 5.3, - "recall": 0.9893, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 4898.2048, - "latency": 5.6, - "recall": 0.9904, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9773.6593, - "latency": 3.7, - "recall": 0.9955, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9081.1518, - "latency": 3.0, - "recall": 0.9943, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8455.2896, - "latency": 4.0, - "recall": 0.9921, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7610.0519, - "latency": 3.3, - "recall": 0.9903, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7589.664, - "latency": 3.8, - "recall": 0.9235, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6750.2495, - "latency": 4.4, - "recall": 0.9105, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5506.1808, - "latency": 5.5, - "recall": 0.9193, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6860.8577, - "latency": 4.7, - "recall": 0.9226, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8468.4611, - "latency": 3.1, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10089.4308, - "latency": 2.6, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10557.4373, - "latency": 2.7, - "recall": 0.9393, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9805.0401, - "latency": 2.6, - "recall": 0.9257, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10020.5299, - "latency": 2.6, - "recall": 0.9788, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10041.0338, - "latency": 2.7, - "recall": 0.9693, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9861.9686, - "latency": 2.6, - "recall": 0.955, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9507.9991, - "latency": 2.8, - "recall": 0.9453, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9428.4531, - "latency": 2.6, - "recall": 0.9331, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9048.6431, - "latency": 3.9, - "recall": 0.9216, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8695.2765, - "latency": 4.3, - "recall": 0.9603, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9244.1135, - "latency": 4.2, - "recall": 0.9724, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9289.0118, - "latency": 4.2, - "recall": 0.9574, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9374.8941, - "latency": 4.2, - "recall": 0.9425, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9368.1325, - "latency": 3.8, - "recall": 0.9292, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9220.3627, - "latency": 3.8, - "recall": 0.9081, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8633.8949, - "latency": 4.1, - "recall": 0.8928, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 6820.6863, - "latency": 3.2, - "recall": 0.9159, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 3938.6004, - "latency": 3.7, - "recall": 0.9196, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3957.0757, - "latency": 2.7, - "recall": 0.932, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3539.2869, - "latency": 4.3, - "recall": 0.9471, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3154.6501, - "latency": 3.9, - "recall": 0.9565, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2743.4561, - "latency": 4.3, - "recall": 0.9681, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2318.7835, - "latency": 3.1, - "recall": 0.9763, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1763.2054, - "latency": 5.0, - "recall": 0.9829, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1454.0462, - "latency": 4.0, - "recall": 0.9863, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1251.1255, - "latency": 4.5, - "recall": 0.9884, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1076.8329, - "latency": 4.4, - "recall": 0.9897, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3411.0934, - "latency": 3.3, - "recall": 0.995, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2838.356, - "latency": 3.8, - "recall": 0.9946, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1826.0672, - "latency": 5.3, - "recall": 0.9938, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1234.6534, - "latency": 6.4, - "recall": 0.9942, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1773.0919, - "latency": 5.3, - "recall": 0.9699, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1454.8382, - "latency": 4.6, - "recall": 0.9659, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1373.0307, - "latency": 5.7, - "recall": 0.9716, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2039.8673, - "latency": 3.8, - "recall": 0.9559, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2950.8165, - "latency": 3.3, - "recall": 0.9147, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.4972, - "latency": 337.1, - "recall": 0.8717, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1164, - "latency": 345.6, - "recall": 0.4276, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 197.4455, - "latency": 349.3, - "recall": 0.5314, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 196.9391, - "latency": 263.4, - "recall": 0.6549, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.5401, - "latency": 282.4, - "recall": 0.7086, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.2424, - "latency": 301.7, - "recall": 0.7592, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.599, - "latency": 358.8, - "recall": 0.8085, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.0349, - "latency": 275.3, - "recall": 0.8325, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.1405, - "latency": 282.6, - "recall": 0.8492, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.1282, - "latency": 269.2, - "recall": 0.8637, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 194.8021, - "latency": 559.8, - "recall": 0.86, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 187.4268, - "latency": 453.7, - "recall": 0.4692, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.397, - "latency": 506.9, - "recall": 0.5409, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 174.3549, - "latency": 496.9, - "recall": 0.6279, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 172.95, - "latency": 515.6, - "recall": 0.7004, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 190.9747, - "latency": 517.4, - "recall": 0.7398, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 186.0237, - "latency": 474.0, - "recall": 0.7847, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1458, - "latency": 480.5, - "recall": 0.8103, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 179.4203, - "latency": 497.5, - "recall": 0.8273, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.5444, - "latency": 463.9, - "recall": 0.8478, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 346.5847, - "latency": 42.7, - "recall": 0.9631, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 369.4921, - "latency": 41.6, - "recall": 0.779, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 310.957, - "latency": 49.4, - "recall": 0.9698, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 798.328, - "latency": 56.7, - "recall": 0.8993, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 649.8781, - "latency": 55.2, - "recall": 0.8352, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 370.7241, - "latency": 49.6, - "recall": 0.7177, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 100.0554, - "latency": 69.3, - "recall": 0.9638, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 284.6367, - "latency": 47.6, - "recall": 0.9788, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 81.8678, - "latency": 105.6, - "recall": 0.8751, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 260.4031, - "latency": 48.3, - "recall": 0.9828, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 365.2505, - "latency": 34.9, - "recall": 0.8251, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 471.553, - "latency": 44.1, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 91.8612, - "latency": 85.7, - "recall": 0.8799, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 206.0934, - "latency": 56.7, - "recall": 0.9795, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 351.7114, - "latency": 46.7, - "recall": 0.8735, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 96.592, - "latency": 76.9, - "recall": 0.9178, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 802.6923, - "latency": 48.1, - "recall": 0.935, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 184.5363, - "latency": 53.4, - "recall": 0.9681, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 323.0238, - "latency": 50.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 382.5332, - "latency": 54.7, - "recall": 0.6135, - "filter_ratio": 0.98 - } + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1146.5286, + "latency": 13.7, + "recall": 0.9262, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1148.1735, + "latency": 8.9, + "recall": 0.9801, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1149.1219, + "latency": 10.3, + "recall": 0.9764, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1140.4099, + "latency": 13.5, + "recall": 0.9716, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1123.5147, + "latency": 18.5, + "recall": 0.9688, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 487.8343, + "latency": 25.4, + "recall": 0.9668, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 264.9324, + "latency": 49.6, + "recall": 0.936, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 492.4887, + "latency": 29.6, + "recall": 0.9269, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 823.1775, + "latency": 20.5, + "recall": 0.9148, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1147.1977, + "latency": 13.3, + "recall": 0.8999, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1131.3087, + "latency": 14.1, + "recall": 0.9024, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1114.952, + "latency": 12.7, + "recall": 0.97, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 583.5009, + "latency": 23.0, + "recall": 0.9668, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 31.4779, + "latency": 351.0, + "recall": 0.9414, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 57.8988, + "latency": 200.1, + "recall": 0.9332, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 101.1774, + "latency": 116.1, + "recall": 0.9241, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 212.7466, + "latency": 58.7, + "recall": 0.9099, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 372.2462, + "latency": 35.9, + "recall": 0.8977, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 617.0881, + "latency": 22.4, + "recall": 0.8844, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1094.5967, + "latency": 14.3, + "recall": 0.8659, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4318.9697, + "latency": 4.3, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4250.2894, + "latency": 4.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 2997.4391, + "latency": 6.1, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1494.5334, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1108.6473, + "latency": 7.4, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1289.5164, + "latency": 6.4, + "recall": 0.9906, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1059.3394, + "latency": 7.8, + "recall": 0.9856, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 987.0795, + "latency": 7.1, + "recall": 0.9804, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1591.7055, + "latency": 7.8, + "recall": 0.8506, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1202.8677, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 639.3991, + "latency": 7.3, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 274.8559, + "latency": 9.9, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 441.4152, + "latency": 8.3, + "recall": 0.997, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 358.8949, + "latency": 9.5, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 325.2245, + "latency": 10.3, + "recall": 0.9909, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 273.4174, + "latency": 13.3, + "recall": 0.9789, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 262.8314, + "latency": 11.3, + "recall": 0.9808, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 434.5481, + "latency": 8.5, + "recall": 0.7237, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 446.9116, + "latency": 9.2, + "recall": 0.9357, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 388.3028, + "latency": 9.6, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 323.3964, + "latency": 9.8, + "recall": 0.9507, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 256.4668, + "latency": 11.3, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 145.5316, + "latency": 18.4, + "recall": 0.9726, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1242.428, + "latency": 6.4, + "recall": 0.9474, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1111.3633, + "latency": 7.0, + "recall": 0.955, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 955.4701, + "latency": 7.2, + "recall": 0.9629, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 783.5207, + "latency": 7.7, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 470.8546, + "latency": 9.5, + "recall": 0.9835, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 950.6332, + "latency": 13.2, + "recall": 0.914, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 823.2224, + "latency": 13.5, + "recall": 0.9434, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 743.9815, + "latency": 14.8, + "recall": 0.9583, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 683.1873, + "latency": 15.7, + "recall": 0.9677, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 619.7468, + "latency": 17.2, + "recall": 0.9738, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 537.4082, + "latency": 18.8, + "recall": 0.9809, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 474.9941, + "latency": 20.9, + "recall": 0.9848, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 505.7458, + "latency": 20.7, + "recall": 0.9068, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 433.9034, + "latency": 23.1, + "recall": 0.931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 381.7737, + "latency": 25.7, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 342.1123, + "latency": 29.0, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 308.2216, + "latency": 31.3, + "recall": 0.9561, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 257.7928, + "latency": 36.4, + "recall": 0.9626, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 223.8166, + "latency": 42.1, + "recall": 0.9666, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3055.0123, + "latency": 7.2, + "recall": 0.9066, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3013.4439, + "latency": 6.9, + "recall": 0.9268, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2801.7241, + "latency": 7.4, + "recall": 0.9476, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2590.3809, + "latency": 8.6, + "recall": 0.9679, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2291.2159, + "latency": 8.9, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3099.4124, + "latency": 6.2, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3014.2483, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2073.2153, + "latency": 11.0, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1507.6899, + "latency": 12.8, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 942.2296, + "latency": 18.2, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 677.1414, + "latency": 33.5, + "recall": 0.7655, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2685.6654, + "latency": 7.6, + "recall": 0.4914, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2604.4444, + "latency": 7.8, + "recall": 0.63, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2159.051, + "latency": 9.4, + "recall": 0.801, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2251.1274, + "latency": 8.7, + "recall": 0.8848, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3103.0539, + "latency": 5.6, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3086.1957, + "latency": 6.7, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3090.0478, + "latency": 6.4, + "recall": 0.9628, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3064.6288, + "latency": 6.5, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3065.6134, + "latency": 6.2, + "recall": 0.9328, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3028.858, + "latency": 6.7, + "recall": 0.9133, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2935.9403, + "latency": 6.8, + "recall": 0.8992, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2771.2009, + "latency": 7.6, + "recall": 0.889, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1610.9496, + "latency": 10.8, + "recall": 0.9, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1557.3623, + "latency": 10.8, + "recall": 0.9244, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1473.9256, + "latency": 11.7, + "recall": 0.9484, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1388.5547, + "latency": 12.5, + "recall": 0.9597, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1022.2696, + "latency": 17.9, + "recall": 0.936, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 696.9777, + "latency": 24.6, + "recall": 0.997, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 353.7862, + "latency": 45.2, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 210.3227, + "latency": 71.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 114.8061, + "latency": 126.6, + "recall": 0.9985, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 504.9179, + "latency": 272.6, + "recall": 0.4664, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1053.1495, + "latency": 17.7, + "recall": 0.5673, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 808.3294, + "latency": 22.2, + "recall": 0.7016, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 8.0584, + "latency": 1757.9, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3033.5491, + "latency": 6.4, + "recall": 0.9844, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2988.4205, + "latency": 7.6, + "recall": 0.9741, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2950.717, + "latency": 6.9, + "recall": 0.9558, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2782.0274, + "latency": 7.4, + "recall": 0.9466, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2708.6752, + "latency": 8.4, + "recall": 0.9337, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2275.2854, + "latency": 9.1, + "recall": 0.917, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1844.8918, + "latency": 10.6, + "recall": 0.9085, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1301.4102, + "latency": 14.7, + "recall": 0.9011, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 13.0379, + "latency": 1063.5, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.4972, + "latency": 337.1, + "recall": 0.8717, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1164, + "latency": 345.6, + "recall": 0.4276, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 197.4455, + "latency": 349.3, + "recall": 0.5314, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 196.9391, + "latency": 263.4, + "recall": 0.6549, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.5401, + "latency": 282.4, + "recall": 0.7086, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.2424, + "latency": 301.7, + "recall": 0.7592, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.599, + "latency": 358.8, + "recall": 0.8085, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.0349, + "latency": 275.3, + "recall": 0.8325, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.1405, + "latency": 282.6, + "recall": 0.8492, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.1282, + "latency": 269.2, + "recall": 0.8637, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 194.8021, + "latency": 559.8, + "recall": 0.86, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 187.4268, + "latency": 453.7, + "recall": 0.4692, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.397, + "latency": 506.9, + "recall": 0.5409, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 174.3549, + "latency": 496.9, + "recall": 0.6279, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 172.95, + "latency": 515.6, + "recall": 0.7004, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 190.9747, + "latency": 517.4, + "recall": 0.7398, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 186.0237, + "latency": 474.0, + "recall": 0.7847, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1458, + "latency": 480.5, + "recall": 0.8103, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 179.4203, + "latency": 497.5, + "recall": 0.8273, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.5444, + "latency": 463.9, + "recall": 0.8478, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 346.5847, + "latency": 42.7, + "recall": 0.9631, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 369.4921, + "latency": 41.6, + "recall": 0.779, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 310.957, + "latency": 49.4, + "recall": 0.9698, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 798.328, + "latency": 56.7, + "recall": 0.8993, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 649.8781, + "latency": 55.2, + "recall": 0.8352, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 370.7241, + "latency": 49.6, + "recall": 0.7177, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 100.0554, + "latency": 69.3, + "recall": 0.9638, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 284.6367, + "latency": 47.6, + "recall": 0.9788, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 81.8678, + "latency": 105.6, + "recall": 0.8751, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 260.4031, + "latency": 48.3, + "recall": 0.9828, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 365.2505, + "latency": 34.9, + "recall": 0.8251, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 471.553, + "latency": 44.1, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 91.8612, + "latency": 85.7, + "recall": 0.8799, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 206.0934, + "latency": 56.7, + "recall": 0.9795, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 351.7114, + "latency": 46.7, + "recall": 0.8735, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 96.592, + "latency": 76.9, + "recall": 0.9178, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 802.6923, + "latency": 48.1, + "recall": 0.935, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 184.5363, + "latency": 53.4, + "recall": 0.9681, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 323.0238, + "latency": 50.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 382.5332, + "latency": 54.7, + "recall": 0.6135, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2030.4249, + "latency": 10.6, + "recall": 0.925, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1804.8996, + "latency": 12.3, + "recall": 0.9365, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2353.8935, + "latency": 17.1, + "recall": 0.9056, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1623.8421, + "latency": 11.8, + "recall": 0.945, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2808.2421, + "latency": 9.5, + "recall": 0.8674, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1482.3772, + "latency": 12.1, + "recall": 0.9523, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1721.5416, + "latency": 9.6, + "recall": 0.876, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1032.9696, + "latency": 14.8, + "recall": 0.9299, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1150.4393, + "latency": 13.4, + "recall": 0.9225, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1452.1536, + "latency": 10.8, + "recall": 0.8973, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2181.3939, + "latency": 9.4, + "recall": 0.8353, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1295.5543, + "latency": 11.2, + "recall": 0.9126, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9441.1235, + "latency": 5.2, + "recall": 0.9589, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6125.6146, + "latency": 4.9, + "recall": 0.9919, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5502.1797, + "latency": 3.8, + "recall": 0.9452, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1827.5849, + "latency": 5.4, + "recall": 0.9903, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1938.1932, + "latency": 5.6, + "recall": 0.989, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3778.8811, + "latency": 4.8, + "recall": 0.9828, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3974.8218, + "latency": 4.8, + "recall": 0.9396, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2971.1402, + "latency": 11.6, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8441.533, + "latency": 6.9, + "recall": 0.9785, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 2703.5422, + "latency": 12.9, + "recall": 0.9903, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1628.2736, + "latency": 5.6, + "recall": 0.9913, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5019.5973, + "latency": 5.7, + "recall": 0.994, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7364.0396, + "latency": 4.0, + "recall": 0.9893, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2724.9239, + "latency": 12.4, + "recall": 0.9811, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5514.815, + "latency": 4.2, + "recall": 0.9286, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9901.1114, + "latency": 3.9, + "recall": 0.9385, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3258.8508, + "latency": 11.7, + "recall": 0.9888, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5907.441, + "latency": 5.7, + "recall": 0.9931, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5064.6982, + "latency": 4.3, + "recall": 0.9558, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3468.5933, + "latency": 4.5, + "recall": 0.9634, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2401.3204, + "latency": 10.7, + "recall": 0.9866, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3568.396, + "latency": 4.8, + "recall": 0.9863, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 4674.1861, + "latency": 4.5, + "recall": 0.967, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3917.2035, + "latency": 2.4, + "recall": 0.9203, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3628.8527, + "latency": 2.6, + "recall": 0.9318, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3250.1112, + "latency": 2.7, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2762.4144, + "latency": 3.1, + "recall": 0.9556, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2384.6245, + "latency": 3.2, + "recall": 0.9627, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2134.1717, + "latency": 3.8, + "recall": 0.9671, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1641.3478, + "latency": 4.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1488.5841, + "latency": 4.7, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2747.3167, + "latency": 3.3, + "recall": 0.9204, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2514.4481, + "latency": 3.2, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2177.2345, + "latency": 3.4, + "recall": 0.9408, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1833.2575, + "latency": 3.9, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1552.4803, + "latency": 4.0, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1355.3121, + "latency": 4.4, + "recall": 0.9602, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1079.2123, + "latency": 5.3, + "recall": 0.9648, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 876.5772, + "latency": 6.3, + "recall": 0.9676, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10663.1231, + "latency": 2.0, + "recall": 0.8405, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10333.9072, + "latency": 2.0, + "recall": 0.889, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 9575.6863, + "latency": 2.3, + "recall": 0.9189, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 8596.7694, + "latency": 2.4, + "recall": 0.9416, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7704.3625, + "latency": 2.7, + "recall": 0.9541, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7023.6735, + "latency": 3.0, + "recall": 0.962, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 6031.3725, + "latency": 3.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 5258.1868, + "latency": 3.6, + "recall": 0.9768, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5973.0024, + "latency": 2.4, + "recall": 0.9192, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5416.5758, + "latency": 2.6, + "recall": 0.9334, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4771.4324, + "latency": 2.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4006.3994, + "latency": 3.2, + "recall": 0.9609, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3441.7597, + "latency": 3.5, + "recall": 0.9682, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3040.6216, + "latency": 3.7, + "recall": 0.9734, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2446.7373, + "latency": 4.3, + "recall": 0.9791, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2084.6245, + "latency": 5.0, + "recall": 0.9819, + "filter_ratio": 0.0 + } ] \ No newline at end of file From 1c771cc17a3927ca13a0309a4beda8316b2e5e6b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 09:26:14 +0000 Subject: [PATCH 11/38] Unify result timestamps to standard_20260403 Rename ElasticCloud and ZillizCloud result files from 20260209 to 20260403 and update task_label to standard_20260403 for consistency with Milvus results. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...ticcloud.json => result_20260403_standard_elasticcloud.json} | 2 +- ...llizcloud.json => result_20260403_standard_zillizcloud.json} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename vectordb_bench/results/ElasticCloud/{result_20260209_standard_elasticcloud.json => result_20260403_standard_elasticcloud.json} (99%) rename vectordb_bench/results/ZillizCloud/{result_20260209_standard_zillizcloud.json => result_20260403_standard_zillizcloud.json} (99%) diff --git a/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json similarity index 99% rename from vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json rename to vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json index 213c2d374..54f9065d0 100644 --- a/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json +++ b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json @@ -1,6 +1,6 @@ { "run_id": "80696b60e39749b295273db3cdba1b69", - "task_label": "standard_20260209", + "task_label": "standard_20260403", "results": [ { "metrics": { diff --git a/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json similarity index 99% rename from vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json rename to vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index 619b5fbf4..e118d1f1c 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -1,6 +1,6 @@ { "run_id": "80696b60e39749b295273db3cdba1b69", - "task_label": "standard_20260209", + "task_label": "standard_20260403", "results": [ { "metrics": { From 8f7d6bb5ea1e41bf05a6746437222cda0d32c628 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 09:34:44 +0000 Subject: [PATCH 12/38] fix: unify run_id across all result files Co-Authored-By: Claude Opus 4.6 (1M context) --- ...result_20260403_standard_elasticcloud.json | 2 +- .../result_20260403_standard_zillizcloud.json | 2 +- vectordb_bench/results/leaderboard_v2.json | 1626 ++++++++++++----- 3 files changed, 1220 insertions(+), 410 deletions(-) diff --git a/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json index 54f9065d0..2ec557926 100644 --- a/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json +++ b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json @@ -1,5 +1,5 @@ { - "run_id": "80696b60e39749b295273db3cdba1b69", + "run_id": "c11e83b51ff14060a08f06d58f801214", "task_label": "standard_20260403", "results": [ { diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index e118d1f1c..75760ff71 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -1,5 +1,5 @@ { - "run_id": "80696b60e39749b295273db3cdba1b69", + "run_id": "c11e83b51ff14060a08f06d58f801214", "task_label": "standard_20260403", "results": [ { diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index 566ce4285..bbc1e918b 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -1469,6 +1469,776 @@ "recall": 0.6135, "filter_ratio": 0.98 }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3917.2035, + "latency": 2.4, + "recall": 0.9203, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3628.8527, + "latency": 2.6, + "recall": 0.9318, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3250.1112, + "latency": 2.7, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2762.4144, + "latency": 3.1, + "recall": 0.9556, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2384.6245, + "latency": 3.2, + "recall": 0.9627, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2134.1717, + "latency": 3.8, + "recall": 0.9671, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1641.3478, + "latency": 4.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1488.5841, + "latency": 4.7, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2747.3167, + "latency": 3.3, + "recall": 0.9204, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2514.4481, + "latency": 3.2, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2177.2345, + "latency": 3.4, + "recall": 0.9408, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1833.2575, + "latency": 3.9, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1552.4803, + "latency": 4.0, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1355.3121, + "latency": 4.4, + "recall": 0.9602, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1079.2123, + "latency": 5.3, + "recall": 0.9648, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 876.5772, + "latency": 6.3, + "recall": 0.9676, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10663.1231, + "latency": 2.0, + "recall": 0.8405, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10333.9072, + "latency": 2.0, + "recall": 0.889, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 9575.6863, + "latency": 2.3, + "recall": 0.9189, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 8596.7694, + "latency": 2.4, + "recall": 0.9416, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7704.3625, + "latency": 2.7, + "recall": 0.9541, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7023.6735, + "latency": 3.0, + "recall": 0.962, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 6031.3725, + "latency": 3.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 5258.1868, + "latency": 3.6, + "recall": 0.9768, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5973.0024, + "latency": 2.4, + "recall": 0.9192, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5416.5758, + "latency": 2.6, + "recall": 0.9334, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4771.4324, + "latency": 2.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4006.3994, + "latency": 3.2, + "recall": 0.9609, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3441.7597, + "latency": 3.5, + "recall": 0.9682, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3040.6216, + "latency": 3.7, + "recall": 0.9734, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2446.7373, + "latency": 4.3, + "recall": 0.9791, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2084.6245, + "latency": 5.0, + "recall": 0.9819, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11763.5538, + "latency": 1.5, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11803.1944, + "latency": 1.5, + "recall": 0.9778, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11520.9234, + "latency": 1.5, + "recall": 0.9634, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11280.0849, + "latency": 1.6, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10671.8925, + "latency": 1.7, + "recall": 0.9339, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10258.2661, + "latency": 1.7, + "recall": 0.9139, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9681.5656, + "latency": 1.9, + "recall": 0.9008, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8945.4041, + "latency": 1.9, + "recall": 0.8894, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5436.8907, + "latency": 2.0, + "recall": 0.929, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11397.7043, + "latency": 1.6, + "recall": 0.9597, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10891.7531, + "latency": 1.7, + "recall": 0.9408, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10276.7451, + "latency": 1.7, + "recall": 0.9159, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9664.2855, + "latency": 1.8, + "recall": 0.899, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8936.7962, + "latency": 2.0, + "recall": 0.8835, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5671.2562, + "latency": 2.1, + "recall": 0.903, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 3157.707, + "latency": 2.3, + "recall": 0.9347, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 1985.8124, + "latency": 2.6, + "recall": 0.9407, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 920.9627, + "latency": 3.4, + "recall": 0.9488, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3033.786, + "latency": 8.7, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3019.2416, + "latency": 9.5, + "recall": 0.9765, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2890.9523, + "latency": 9.4, + "recall": 0.9625, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2789.7212, + "latency": 8.2, + "recall": 0.9538, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2457.2628, + "latency": 9.0, + "recall": 0.9378, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2209.4973, + "latency": 13.7, + "recall": 0.9228, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1960.388, + "latency": 11.0, + "recall": 0.9076, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1725.092, + "latency": 11.7, + "recall": 0.8969, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1307.419, + "latency": 12.3, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 350.0132, + "latency": 29.7, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 179.5204, + "latency": 51.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 72.99, + "latency": 111.4, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 42.9877, + "latency": 201.9, + "recall": 0.9912, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 96.4987, + "latency": 113.1, + "recall": 0.9296, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 189.3789, + "latency": 58.8, + "recall": 0.9149, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 246.7071, + "latency": 45.1, + "recall": 0.9018, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 229.0379, + "latency": 43.0, + "recall": 0.8908, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 125.6164, + "latency": 69.8, + "recall": 0.8746, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2175.2694, + "latency": 9.8, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1430.0244, + "latency": 12.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 692.5751, + "latency": 18.7, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 364.3516, + "latency": 26.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 190.3777, + "latency": 47.9, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 249.3519, + "latency": 44.7, + "recall": 0.9446, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 437.8735, + "latency": 27.1, + "recall": 0.9364, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 669.9441, + "latency": 19.1, + "recall": 0.9227, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 899.3114, + "latency": 14.9, + "recall": 0.9072, + "filter_ratio": 0.5 + }, { "dataset": "Cohere (Medium)", "db": "ElasticCloud", @@ -1476,7 +2246,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2030.4249, "latency": 10.6, - "recall": 0.925, + "recall": 0.9306, "filter_ratio": 0.0 }, { @@ -1486,7 +2256,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1804.8996, "latency": 12.3, - "recall": 0.9365, + "recall": 0.9405, "filter_ratio": 0.0 }, { @@ -1496,7 +2266,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2353.8935, "latency": 17.1, - "recall": 0.9056, + "recall": 0.9143, "filter_ratio": 0.0 }, { @@ -1506,7 +2276,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1623.8421, "latency": 11.8, - "recall": 0.945, + "recall": 0.9479, "filter_ratio": 0.0 }, { @@ -1516,7 +2286,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2808.2421, "latency": 9.5, - "recall": 0.8674, + "recall": 0.8815, "filter_ratio": 0.0 }, { @@ -1526,67 +2296,67 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1482.3772, "latency": 12.1, - "recall": 0.9523, + "recall": 0.9546, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1721.5416, "latency": 9.6, - "recall": 0.876, + "recall": 0.8855, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1032.9696, "latency": 14.8, - "recall": 0.9299, + "recall": 0.933, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1150.4393, "latency": 13.4, - "recall": 0.9225, + "recall": 0.9265, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1452.1536, "latency": 10.8, - "recall": 0.8973, + "recall": 0.9042, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2181.3939, "latency": 9.4, - "recall": 0.8353, + "recall": 0.8501, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1295.5543, "latency": 11.2, - "recall": 0.9126, + "recall": 0.9176, "filter_ratio": 0.0 }, { @@ -1594,549 +2364,589 @@ "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 9441.1235, - "latency": 5.2, - "recall": 0.9589, - "filter_ratio": 0.0 + "qps": 9773.6593, + "latency": 3.7, + "recall": 0.9955, + "filter_ratio": 0.999 }, { "dataset": "Cohere (Medium)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 6125.6146, - "latency": 4.9, - "recall": 0.9919, - "filter_ratio": 0.0 + "qps": 9081.1518, + "latency": 3.0, + "recall": 0.9943, + "filter_ratio": 0.998 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5502.1797, - "latency": 3.8, - "recall": 0.9452, - "filter_ratio": 0.0 + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8455.2896, + "latency": 4.0, + "recall": 0.9921, + "filter_ratio": 0.995 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 1827.5849, - "latency": 5.4, + "qps": 7610.0519, + "latency": 3.3, "recall": 0.9903, - "filter_ratio": 0.0 + "filter_ratio": 0.99 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 1938.1932, - "latency": 5.6, - "recall": 0.989, - "filter_ratio": 0.0 + "qps": 7589.664, + "latency": 3.8, + "recall": 0.9235, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6750.2495, + "latency": 4.4, + "recall": 0.9105, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5506.1808, + "latency": 5.5, + "recall": 0.9193, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6860.8577, + "latency": 4.7, + "recall": 0.9226, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8468.4611, + "latency": 3.1, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10089.4308, + "latency": 2.6, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10557.4373, + "latency": 2.7, + "recall": 0.9393, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9805.0401, + "latency": 2.6, + "recall": 0.9257, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10020.5299, + "latency": 2.6, + "recall": 0.9788, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10041.0338, + "latency": 2.7, + "recall": 0.9693, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9861.9686, + "latency": 2.6, + "recall": 0.955, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9507.9991, + "latency": 2.8, + "recall": 0.9453, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9428.4531, + "latency": 2.6, + "recall": 0.9331, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9048.6431, + "latency": 3.9, + "recall": 0.9216, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8695.2765, + "latency": 4.3, + "recall": 0.9603, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9244.1135, + "latency": 4.2, + "recall": 0.9724, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9289.0118, + "latency": 4.2, + "recall": 0.9574, + "filter_ratio": 0.995 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3778.8811, - "latency": 4.8, - "recall": 0.9828, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9374.8941, + "latency": 4.2, + "recall": 0.9425, + "filter_ratio": 0.99 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3974.8218, - "latency": 4.8, - "recall": 0.9396, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9368.1325, + "latency": 3.8, + "recall": 0.9292, + "filter_ratio": 0.98 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2971.1402, - "latency": 11.6, - "recall": 0.9729, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9220.3627, + "latency": 3.8, + "recall": 0.9081, + "filter_ratio": 0.95 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8441.533, - "latency": 6.9, - "recall": 0.9785, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8633.8949, + "latency": 4.1, + "recall": 0.8928, + "filter_ratio": 0.9 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 2703.5422, - "latency": 12.9, - "recall": 0.9903, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 6820.6863, + "latency": 3.2, + "recall": 0.9159, + "filter_ratio": 0.8 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1628.2736, - "latency": 5.6, - "recall": 0.9913, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 3938.6004, + "latency": 3.7, + "recall": 0.9196, + "filter_ratio": 0.5 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 5019.5973, - "latency": 5.7, - "recall": 0.994, - "filter_ratio": 0.0 + "qps": 3411.0934, + "latency": 3.3, + "recall": 0.995, + "filter_ratio": 0.999 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 7364.0396, - "latency": 4.0, - "recall": 0.9893, - "filter_ratio": 0.0 + "qps": 2838.356, + "latency": 3.8, + "recall": 0.9946, + "filter_ratio": 0.998 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 2724.9239, - "latency": 12.4, - "recall": 0.9811, - "filter_ratio": 0.0 + "qps": 1826.0672, + "latency": 5.3, + "recall": 0.9938, + "filter_ratio": 0.995 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5514.815, - "latency": 4.2, - "recall": 0.9286, - "filter_ratio": 0.0 + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1234.6534, + "latency": 6.4, + "recall": 0.9942, + "filter_ratio": 0.99 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 9901.1114, - "latency": 3.9, - "recall": 0.9385, - "filter_ratio": 0.0 + "qps": 1773.0919, + "latency": 5.3, + "recall": 0.9699, + "filter_ratio": 0.98 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3258.8508, - "latency": 11.7, - "recall": 0.9888, - "filter_ratio": 0.0 + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1454.8382, + "latency": 4.6, + "recall": 0.9659, + "filter_ratio": 0.95 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 5907.441, + "qps": 1373.0307, "latency": 5.7, - "recall": 0.9931, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5064.6982, - "latency": 4.3, - "recall": 0.9558, - "filter_ratio": 0.0 + "recall": 0.9716, + "filter_ratio": 0.9 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 3468.5933, - "latency": 4.5, - "recall": 0.9634, - "filter_ratio": 0.0 + "qps": 2039.8673, + "latency": 3.8, + "recall": 0.9559, + "filter_ratio": 0.8 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 2401.3204, - "latency": 10.7, - "recall": 0.9866, - "filter_ratio": 0.0 + "qps": 2950.8165, + "latency": 3.3, + "recall": 0.9147, + "filter_ratio": 0.5 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3568.396, - "latency": 4.8, - "recall": 0.9863, + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9441.1235, + "latency": 5.2, + "recall": 0.9658, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 4674.1861, - "latency": 4.5, - "recall": 0.967, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3917.2035, - "latency": 2.4, - "recall": 0.9203, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3628.8527, - "latency": 2.6, - "recall": 0.9318, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3250.1112, - "latency": 2.7, - "recall": 0.9443, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2762.4144, - "latency": 3.1, - "recall": 0.9556, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2384.6245, - "latency": 3.2, - "recall": 0.9627, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2134.1717, - "latency": 3.8, - "recall": 0.9671, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1641.3478, - "latency": 4.1, - "recall": 0.9729, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1488.5841, - "latency": 4.7, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2747.3167, - "latency": 3.3, - "recall": 0.9204, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2514.4481, - "latency": 3.2, - "recall": 0.9303, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2177.2345, - "latency": 3.4, - "recall": 0.9408, + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6125.6146, + "latency": 4.9, + "recall": 0.9936, "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1833.2575, - "latency": 3.9, - "recall": 0.951, + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5502.1797, + "latency": 3.8, + "recall": 0.9509, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1552.4803, - "latency": 4.0, - "recall": 0.9565, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1827.5849, + "latency": 5.4, + "recall": 0.9918, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1355.3121, - "latency": 4.4, - "recall": 0.9602, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1938.1932, + "latency": 5.6, + "recall": 0.9906, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1079.2123, - "latency": 5.3, - "recall": 0.9648, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3778.8811, + "latency": 4.8, + "recall": 0.9851, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 876.5772, - "latency": 6.3, - "recall": 0.9676, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3974.8218, + "latency": 4.8, + "recall": 0.9428, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10663.1231, - "latency": 2.0, - "recall": 0.8405, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2971.1402, + "latency": 11.6, + "recall": 0.9752, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10333.9072, - "latency": 2.0, - "recall": 0.889, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8441.533, + "latency": 6.9, + "recall": 0.9825, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 9575.6863, - "latency": 2.3, - "recall": 0.9189, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 2703.5422, + "latency": 12.9, + "recall": 0.992, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 8596.7694, - "latency": 2.4, - "recall": 0.9416, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1628.2736, + "latency": 5.6, + "recall": 0.9928, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7704.3625, - "latency": 2.7, - "recall": 0.9541, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5019.5973, + "latency": 5.7, + "recall": 0.9954, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7023.6735, - "latency": 3.0, - "recall": 0.962, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7364.0396, + "latency": 4.0, + "recall": 0.9915, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 6031.3725, - "latency": 3.3, - "recall": 0.971, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2724.9239, + "latency": 12.4, + "recall": 0.9832, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 5258.1868, - "latency": 3.6, - "recall": 0.9768, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5514.815, + "latency": 4.2, + "recall": 0.9355, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5973.0024, - "latency": 2.4, - "recall": 0.9192, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9901.1114, + "latency": 3.9, + "recall": 0.9486, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5416.5758, - "latency": 2.6, - "recall": 0.9334, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3258.8508, + "latency": 11.7, + "recall": 0.9906, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4771.4324, - "latency": 2.8, - "recall": 0.9479, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5907.441, + "latency": 5.7, + "recall": 0.9946, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4006.3994, - "latency": 3.2, - "recall": 0.9609, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5064.6982, + "latency": 4.3, + "recall": 0.9606, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3441.7597, - "latency": 3.5, - "recall": 0.9682, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3468.5933, + "latency": 4.5, + "recall": 0.9661, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3040.6216, - "latency": 3.7, - "recall": 0.9734, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2401.3204, + "latency": 10.7, + "recall": 0.9884, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2446.7373, - "latency": 4.3, - "recall": 0.9791, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3568.396, + "latency": 4.8, + "recall": 0.9883, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2084.6245, - "latency": 5.0, - "recall": 0.9819, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 4674.1861, + "latency": 4.5, + "recall": 0.9705, "filter_ratio": 0.0 } ] \ No newline at end of file From 7c2a4b79734420b97e9b6e0625a2f50994bf6122 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 13:09:09 +0000 Subject: [PATCH 13/38] fix: align streaming leaderboard labels with vector search results Update db_name/label in leaderboard_v2_streaming.json to match leaderboard_v2.json after force_merge became the default: - Milvus: 16c64g-sq8 -> 16c64g-sq8-force_merge - ElasticCloud: 8c60g -> 8c60g-force_merge This fixes the website failing to associate streaming and vector search results due to mismatched db_name keys. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../results/leaderboard_v2_streaming.json | 288 +++++++++--------- 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/vectordb_bench/results/leaderboard_v2_streaming.json b/vectordb_bench/results/leaderboard_v2_streaming.json index 222c68bf7..73075e8a8 100644 --- a/vectordb_bench/results/leaderboard_v2_streaming.json +++ b/vectordb_bench/results/leaderboard_v2_streaming.json @@ -1,146 +1,146 @@ [ - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "insert_rate": 500, - "streaming_qps": 61.6708, - "streaming_latency": 0.0794 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "insert_rate": 1000, - "streaming_qps": 61.8172, - "streaming_latency": 0.2223 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "insert_rate": 500, - "streaming_qps": 305.9971, - "streaming_latency": 0.005 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "insert_rate": 1000, - "streaming_qps": 155.9613, - "streaming_latency": 0.0203 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "insert_rate": 500, - "streaming_qps": 367.4299, - "streaming_latency": 1.8286 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "insert_rate": 1000, - "streaming_qps": 369.6771, - "streaming_latency": 5.992 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "insert_rate": 500, - "streaming_qps": 393.753, - "streaming_latency": 0.0162 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "insert_rate": 1000, - "streaming_qps": 347.5774, - "streaming_latency": 0.0118 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "insert_rate": 1000, - "streaming_qps": 149.7168, - "streaming_latency": 0.098 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "insert_rate": 500, - "streaming_qps": 161.6694, - "streaming_latency": 0.052 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "insert_rate": 500, - "streaming_qps": 2118.7516, - "streaming_latency": 0.0068 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "insert_rate": 1000, - "streaming_qps": 1860.2575, - "streaming_latency": 0.0101 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "insert_rate": 500, - "streaming_qps": 180.9549, - "streaming_latency": 0.4204 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "insert_rate": 1000, - "streaming_qps": 167.2689, - "streaming_latency": 0.5048 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "insert_rate": 500, - "streaming_qps": 536.0198, - "streaming_latency": 0.3132 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "insert_rate": 1000, - "streaming_qps": 442.5824, - "streaming_latency": 0.0724 - } + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "insert_rate": 500, + "streaming_qps": 61.6708, + "streaming_latency": 0.0794 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "insert_rate": 1000, + "streaming_qps": 61.8172, + "streaming_latency": 0.2223 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "insert_rate": 500, + "streaming_qps": 305.9971, + "streaming_latency": 0.005 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "insert_rate": 1000, + "streaming_qps": 155.9613, + "streaming_latency": 0.0203 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "insert_rate": 500, + "streaming_qps": 367.4299, + "streaming_latency": 1.8286 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "insert_rate": 1000, + "streaming_qps": 369.6771, + "streaming_latency": 5.992 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "insert_rate": 500, + "streaming_qps": 393.753, + "streaming_latency": 0.0162 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "insert_rate": 1000, + "streaming_qps": 347.5774, + "streaming_latency": 0.0118 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "insert_rate": 1000, + "streaming_qps": 149.7168, + "streaming_latency": 0.098 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "insert_rate": 500, + "streaming_qps": 161.6694, + "streaming_latency": 0.052 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "insert_rate": 500, + "streaming_qps": 2118.7516, + "streaming_latency": 0.0068 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "insert_rate": 1000, + "streaming_qps": 1860.2575, + "streaming_latency": 0.0101 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "insert_rate": 500, + "streaming_qps": 180.9549, + "streaming_latency": 0.4204 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "insert_rate": 1000, + "streaming_qps": 167.2689, + "streaming_latency": 0.5048 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 500, + "streaming_qps": 536.0198, + "streaming_latency": 0.3132 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 1000, + "streaming_qps": 442.5824, + "streaming_latency": 0.0724 + } ] \ No newline at end of file From 51c5158506cd09e6a19dbcb6029f60aa8071cda6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 8 Apr 2026 02:41:36 +0000 Subject: [PATCH 14/38] fix: update ZillizCloud benchmark with Cardinal backend results Replace ZillizCloud-8cu-perf case_id=4/5 data with new Cardinal backend benchmark results (level 1-9, 1M and 10M datasets, v2026.4). Remove force_merge entries as Cardinal uses unified 4-segment architecture for 10M. New results show significant QPS improvement: - 1M: 13,316 QPS (was 9,704) at recall 0.938 - 10M: 7,385 QPS (was 3,957) at recall 0.938 Sort all leaderboard entries by (db_name, dataset, filter_ratio, qps DESC) to fix line chart rendering. Remove one SQ4U 1M outlier (recall=0.84). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../result_20260403_standard_zillizcloud.json | 2264 +++---- vectordb_bench/results/leaderboard_v2.json | 5840 ++++++++--------- 2 files changed, 3690 insertions(+), 4414 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index 75760ff71..e02463ebd 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,14 +5,14 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 9441.1235, - "serial_latency_p99": 0.0052, - "serial_latency_p95": 0.0039, - "recall": 0.9589, - "ndcg": 0.9658, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 13316.2336, + "serial_latency_p99": 0.002, + "serial_latency_p95": 0.0019, + "recall": 0.9383, + "ndcg": 0.9484, "conc_num_list": [ 1, 5, @@ -24,44 +24,44 @@ 80 ], "conc_qps_list": [ - 306.3773, - 1519.6034, - 3129.1309, - 5457.1507, - 6585.7082, - 7420.0391, - 8468.6183, - 9441.1235 + 557.5439, + 2665.8166, + 5047.6131, + 8707.1886, + 10381.4448, + 11435.3821, + 12833.6289, + 13316.2336 ], "conc_latency_p99_list": [ - 0.004086580532602967, - 0.00566743115196005, - 0.005243223664583638, - 0.0099503791576717, - 0.009588507658627355, - 0.011322382240905426, - 0.01665949933376398, - 0.018269318575912616 + 0.001960459982510656, + 0.002204382496420288, + 0.0025926707847975195, + 0.003785052003804594, + 0.005270074445288626, + 0.006814960413612425, + 0.009530203816248103, + 0.01240227443340701 ], "conc_latency_p95_list": [ - 0.0036428007049835284, - 0.003802539707976394, - 0.003850769612472504, - 0.004741756187286226, - 0.006738374719861894, - 0.00850677301059477, - 0.01233988689491525, - 0.01412305135163478 + 0.0018868701998144388, + 0.0020324485929450018, + 0.0021866516792215405, + 0.002869376807939261, + 0.004040546333999372, + 0.0052645970426965505, + 0.007324896720820107, + 0.009507914245477877 ], "conc_latency_avg_list": [ - 0.0032598009878015348, - 0.003285434242748037, - 0.0031902432390411135, - 0.0036557288568108254, - 0.004539560740384104, - 0.005359882091644495, - 0.007008278288237143, - 0.008346175450269581 + 0.0017909009081054225, + 0.0018719543108110278, + 0.001976279835245286, + 0.002288533358936022, + 0.0028752306497382865, + 0.0034729409658481964, + 0.004612408416183334, + 0.0058844483816759795 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -82,10 +82,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -93,7 +93,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 2, + "level": 1, "num_shards": 1 }, "case_config": { @@ -116,25 +116,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 6125.6146, - "serial_latency_p99": 0.0049, - "serial_latency_p95": 0.0047, - "recall": 0.9919, - "ndcg": 0.9936, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 12837.5287, + "serial_latency_p99": 0.0021, + "serial_latency_p95": 0.002, + "recall": 0.9588, + "ndcg": 0.9657, "conc_num_list": [ 1, 5, @@ -146,44 +143,44 @@ 80 ], "conc_qps_list": [ - 238.8712, - 1196.2193, - 2440.8391, - 4133.8486, - 4858.2331, - 5446.0047, - 6000.1652, - 6125.6146 + 539.7039, + 2582.4122, + 4917.6589, + 8479.8846, + 10116.1707, + 11068.2155, + 12229.2208, + 12837.5287 ], "conc_latency_p99_list": [ - 0.004919703922932965, - 0.010525701696751709, - 0.006511175064661075, - 0.011745981796411804, - 0.01397663167037535, - 0.014958455199375772, - 0.02016973898542343, - 0.026595940839324612 + 0.0020551326422719287, + 0.002271793531253935, + 0.0026150090881856096, + 0.00380311052547768, + 0.005449252330581657, + 0.0069998929975554295, + 0.010041847208049142, + 0.012852289543952794 ], "conc_latency_p95_list": [ - 0.0046619078202638775, - 0.004602618556236848, - 0.004652375826844946, - 0.006574279977940023, - 0.009700259550299961, - 0.011811009392840788, - 0.016332678495382422, - 0.02156840759853367 + 0.0019641239661723374, + 0.0021013408317230643, + 0.002235009270953014, + 0.002924549448653124, + 0.004163405022700318, + 0.0054554910166189075, + 0.007754770980682224, + 0.009949398809112607 ], "conc_latency_avg_list": [ - 0.004181373075740294, - 0.004174027799032302, - 0.004089987135078621, - 0.004828215031413884, - 0.006158122789762511, - 0.007310655337680553, - 0.009892293927032914, - 0.012884722355037423 + 0.0018501577211713547, + 0.001932346768512776, + 0.0020285184508130197, + 0.0023497830070767353, + 0.002947098563192882, + 0.003586949439268913, + 0.0048461200425519, + 0.006100907381850077 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -204,132 +201,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 7, - "num_shards": 1 - }, - "case_config": { - "case_id": 5, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 5502.1797, - "serial_latency_p99": 0.0038, - "serial_latency_p95": 0.0035, - "recall": 0.9452, - "ndcg": 0.9509, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 309.5267, - 1357.6945, - 2213.2564, - 3349.9157, - 3910.9875, - 4385.5899, - 5039.0299, - 5502.1797 - ], - "conc_latency_p99_list": [ - 0.004089714956353409, - 0.0056684831657912264, - 0.012667869632714424, - 0.01000201591959918, - 0.015246839204337462, - 0.01789945445081684, - 0.02318606456159614, - 0.02852195684099569 - ], - "conc_latency_p95_list": [ - 0.003446204590727575, - 0.00420360880671069, - 0.0055909310030983735, - 0.008043184356938578, - 0.011835442011943087, - 0.014750372216803953, - 0.01907540229440201, - 0.023490797984413794 - ], - "conc_latency_avg_list": [ - 0.0032266616589272513, - 0.0036776357130507064, - 0.004511172191326527, - 0.0059575964382477965, - 0.007649578970207299, - 0.009076414605964197, - 0.011790618147195132, - 0.01435249249351469 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -341,495 +216,7 @@ "num_shards": 1 }, "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 1827.5849, - "serial_latency_p99": 0.0054, - "serial_latency_p95": 0.0052, - "recall": 0.9903, - "ndcg": 0.9918, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 218.8679, - 861.6609, - 1195.6337, - 1339.6438, - 1475.742, - 1546.5953, - 1747.9077, - 1827.5849 - ], - "conc_latency_p99_list": [ - 0.006139998821017798, - 0.008449624522763766, - 0.014799785086070211, - 0.02650444130937102, - 0.03739181953598747, - 0.04722500271425814, - 0.05959104605717584, - 0.0715404962032335 - ], - "conc_latency_p95_list": [ - 0.005324425446451642, - 0.006872303040290717, - 0.011261535996163731, - 0.021800653115496966, - 0.03018094879225827, - 0.038646315991354645, - 0.04888677580165675, - 0.06127824939903802 - ], - "conc_latency_avg_list": [ - 0.004563713319136163, - 0.0057948937011398916, - 0.00835191496328548, - 0.014905808578982384, - 0.020272801297484298, - 0.025762202275047597, - 0.034040973878552726, - 0.043269632825280596 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 8, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 1938.1932, - "serial_latency_p99": 0.0056, - "serial_latency_p95": 0.0053, - "recall": 0.989, - "ndcg": 0.9906, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 252.1234, - 954.3561, - 1355.554, - 1490.2007, - 1619.1811, - 1720.5715, - 1860.9305, - 1938.1932 - ], - "conc_latency_p99_list": [ - 0.0050690684028086245, - 0.008166075175395238, - 0.011429371475242079, - 0.024431421170011097, - 0.03405256026599093, - 0.043166847461543534, - 0.05706310785026292, - 0.07251818811346307 - ], - "conc_latency_p95_list": [ - 0.004786731592321303, - 0.006139315699692815, - 0.009358109103050082, - 0.019912595101050083, - 0.028057811519829556, - 0.035778356752416585, - 0.04874245898681693, - 0.061324251000769436 - ], - "conc_latency_avg_list": [ - 0.003961802805693023, - 0.005232545384579334, - 0.007366828470110833, - 0.01339630708221878, - 0.018484241519041742, - 0.023149088452180534, - 0.03196686416892556, - 0.04073973314917765 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 7, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 3778.8811, - "serial_latency_p99": 0.0048, - "serial_latency_p95": 0.0042, - "recall": 0.9828, - "ndcg": 0.9851, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 275.1343, - 1220.1869, - 2080.6024, - 2544.5946, - 2934.4403, - 3222.4756, - 3520.6342, - 3778.8811 - ], - "conc_latency_p99_list": [ - 0.004224385871784762, - 0.006377705505292314, - 0.006628996630315663, - 0.01544506659847684, - 0.02085710293264129, - 0.02495263563963816, - 0.03397362035117112, - 0.0415226237432216 - ], - "conc_latency_p95_list": [ - 0.003899741142231505, - 0.004650917260732967, - 0.005831561920058448, - 0.011755315004847944, - 0.016476290803984734, - 0.020251012463995716, - 0.027687024004990225, - 0.03372844383848132 - ], - "conc_latency_avg_list": [ - 0.0036302553438832055, - 0.004091130600148307, - 0.0047987246661642296, - 0.007844535865702365, - 0.010192746767866709, - 0.012344825366337173, - 0.016888610032228992, - 0.020908843243531986 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 6, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 3974.8218, - "serial_latency_p99": 0.0048, - "serial_latency_p95": 0.0043, - "recall": 0.9396, - "ndcg": 0.9428, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 295.7118, - 1186.5002, - 1797.187, - 2449.5408, - 2857.3282, - 3164.2684, - 3660.3166, - 3974.8218 - ], - "conc_latency_p99_list": [ - 0.004793434205930709, - 0.005700148111791336, - 0.007426547166251105, - 0.014722349808434964, - 0.019732102921698247, - 0.02347602234221995, - 0.03035390855744481, - 0.03731970520602769 - ], - "conc_latency_p95_list": [ - 0.004169186984654516, - 0.004717364211683161, - 0.006335475675587076, - 0.01105007229198236, - 0.01624748620088212, - 0.019896189187420532, - 0.025662759994156657, - 0.031171760102733943 - ], - "conc_latency_avg_list": [ - 0.003377046875928882, - 0.004208154278338283, - 0.005554828192323107, - 0.00814848752272782, - 0.010461408703728187, - 0.012581798692238582, - 0.016239422653327416, - 0.019879490524355867 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 1, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -848,25 +235,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 2971.1402, - "serial_latency_p99": 0.0116, - "serial_latency_p95": 0.0045, - "recall": 0.9729, - "ndcg": 0.9752, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 12248.9154, + "serial_latency_p99": 0.0022, + "serial_latency_p95": 0.0021, + "recall": 0.9687, + "ndcg": 0.9742, "conc_num_list": [ 1, 5, @@ -878,44 +262,44 @@ 80 ], "conc_qps_list": [ - 278.0971, - 1117.1061, - 1626.1624, - 2126.1398, - 2425.3155, - 2521.9392, - 2793.1709, - 2971.1402 + 526.2995, + 2512.1902, + 4765.4856, + 8188.3548, + 9485.6147, + 10432.6026, + 11820.0344, + 12248.9154 ], "conc_latency_p99_list": [ - 0.004647255002055317, - 0.00566180575755425, - 0.011257058603223423, - 0.015976536156085786, - 0.023499950004043067, - 0.03039303767494857, - 0.04078626602888109, - 0.04914582587312907 + 0.0021030055452138188, + 0.0023121346672996877, + 0.0026887326262658466, + 0.003929865991231055, + 0.005846867947839196, + 0.007567329368903298, + 0.01027507190126926, + 0.013403068128973255 ], "conc_latency_p95_list": [ - 0.004429338514455594, - 0.005089565094385761, - 0.007136018975870684, - 0.01292410076566739, - 0.019488130600075235, - 0.025593487400328737, - 0.03410469329974147, - 0.04188467259518802 + 0.002018616924760863, + 0.00216002133092843, + 0.0023071649571647867, + 0.003043863424682058, + 0.004536616246332414, + 0.005861608259147033, + 0.008004442710080184, + 0.010495775798335672 ], "conc_latency_avg_list": [ - 0.003591256380123583, - 0.004470003286850064, - 0.00613923671987789, - 0.009389019339411041, - 0.012338699755027977, - 0.015788074921940728, - 0.02128022645956974, - 0.026573440291179154 + 0.0018973754593796914, + 0.001986612150848518, + 0.0020934129605979786, + 0.0024318237152467248, + 0.0031476610878461374, + 0.00380841888066139, + 0.005011055927151683, + 0.006389479412319173 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -936,10 +320,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -947,11 +331,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 4, + "level": 3, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -970,23 +354,20 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 8441.533, - "serial_latency_p99": 0.0069, - "serial_latency_p95": 0.0035, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 11501.6652, + "serial_latency_p99": 0.0022, + "serial_latency_p95": 0.0022, "recall": 0.9785, "ndcg": 0.9825, "conc_num_list": [ @@ -1000,44 +381,44 @@ 80 ], "conc_qps_list": [ - 314.6659, - 1556.8466, - 3026.0164, - 4952.5941, - 6059.6492, - 6948.3916, - 7709.5146, - 8441.533 + 498.953, + 2391.8074, + 4531.3051, + 7692.4049, + 8962.935, + 9790.7556, + 10733.1679, + 11501.6652 ], "conc_latency_p99_list": [ - 0.003662585185375065, - 0.003881094480166214, - 0.005496532852703231, - 0.012801527802657801, - 0.01098183460126168, - 0.011906764237210155, - 0.017680786546843585, - 0.02030978908645921 + 0.002246916518197395, + 0.002433281935518607, + 0.0027782795106759276, + 0.004184300593915395, + 0.006221411311416887, + 0.007918057614006104, + 0.011295880685211153, + 0.013962703556753681 ], "conc_latency_p95_list": [ - 0.0033931825950276107, - 0.0034469130958314055, - 0.003594076508306898, - 0.00537703381414758, - 0.00756983550672885, - 0.009184699498291593, - 0.013430504295683926, - 0.015904919797321764 + 0.002148064094944857, + 0.002272934094071388, + 0.00242809351766482, + 0.0032858662889339025, + 0.0048346503434004255, + 0.006268709182040765, + 0.008884398700320158, + 0.011115554475691165 ], "conc_latency_avg_list": [ - 0.003173945048460723, - 0.003206842876612295, - 0.003299035623981205, - 0.004028295482036381, - 0.004934835427053102, - 0.005724545223453835, - 0.007703497705718975, - 0.009334748886407937 + 0.0020014255442295315, + 0.002086812448327015, + 0.0022017174912128874, + 0.0025910515942768643, + 0.0033292378315385594, + 0.0040559394841899535, + 0.005528263390600377, + 0.006816899394463919 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1058,10 +439,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1092,25 +473,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 2703.5422, - "serial_latency_p99": 0.0129, - "serial_latency_p95": 0.0049, - "recall": 0.9903, - "ndcg": 0.992, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 10566.6823, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9838, + "ndcg": 0.9868, "conc_num_list": [ 1, 5, @@ -1122,44 +500,44 @@ 80 ], "conc_qps_list": [ - 230.8474, - 1033.6996, - 1653.7251, - 2021.6415, - 2247.5951, - 2433.7674, - 2579.1371, - 2703.5422 + 473.2503, + 2266.5174, + 4325.3899, + 7263.7246, + 8352.1369, + 9198.3778, + 10210.3429, + 10566.6823 ], "conc_latency_p99_list": [ - 0.005364620329928583, - 0.008247331644815845, - 0.009261744469986306, - 0.019847353509976556, - 0.027152295518899337, - 0.0318573336204281, - 0.04668155071558431, - 0.053025491506559774 + 0.0023700519639533015, + 0.0025644300674321128, + 0.0029225554369622857, + 0.0044386597932316255, + 0.00657977487426251, + 0.008452395589556558, + 0.011632206621579828, + 0.015279923828202287 ], "conc_latency_p95_list": [ - 0.004779104294721037, - 0.005717401996662375, - 0.007679672696394844, - 0.015277760991011746, - 0.021049827802926295, - 0.025551227995310906, - 0.03599990590882953, - 0.045174946513725445 + 0.0022803591098636386, + 0.002403592297923751, + 0.002544367348309606, + 0.0034909876121673724, + 0.005244998395210129, + 0.0067171422066167, + 0.009315501491073517, + 0.01212728061946109 ], "conc_latency_avg_list": [ - 0.004327150692358546, - 0.004830543605956322, - 0.006038381999227765, - 0.009872808423519352, - 0.013310825758863387, - 0.016344593851929733, - 0.023038225918050447, - 0.02921974749485067 + 0.0021102672976180386, + 0.0022023083714165855, + 0.0023067257051930207, + 0.0027442442136128803, + 0.003573842319898506, + 0.004316896363141895, + 0.005797124719790328, + 0.007427947098361652 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1179,11 +557,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1191,11 +569,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 9, + "level": 5, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1214,25 +592,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 1628.2736, - "serial_latency_p99": 0.0056, - "serial_latency_p95": 0.0051, - "recall": 0.9913, - "ndcg": 0.9928, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 9227.318, + "serial_latency_p99": 0.0027, + "serial_latency_p95": 0.0026, + "recall": 0.9893, + "ndcg": 0.9915, "conc_num_list": [ 1, 5, @@ -1244,44 +619,44 @@ 80 ], "conc_qps_list": [ - 224.1263, - 854.2908, - 1191.923, - 1306.8314, - 1408.4071, - 1492.5651, - 1572.8756, - 1628.2736 + 427.7206, + 2082.4408, + 3978.3343, + 6561.5406, + 7528.8901, + 8016.929, + 8877.3559, + 9227.318 ], "conc_latency_p99_list": [ - 0.005614027456322219, - 0.01122624498093481, - 0.013540994446957512, - 0.026513429321930727, - 0.0360522977943765, - 0.04583110647072318, - 0.06417124239553232, - 0.08320635374402624 + 0.002635034592822194, + 0.002782075599534437, + 0.003112639953033066, + 0.0048944263125304125, + 0.00728108051931485, + 0.009557482128147962, + 0.013117193853249772, + 0.017160092100966726 ], "conc_latency_p95_list": [ - 0.0053050212460220795, - 0.006976215375470927, - 0.011254654199001379, - 0.022096098412293937, - 0.03048573801643215, - 0.037569552293280135, - 0.05408634888590313, - 0.06856599940219893 + 0.0025485639926046133, + 0.0026276092685293406, + 0.002766418919782154, + 0.003919153648894278, + 0.005883905396331093, + 0.007771545994910409, + 0.010690590104786672, + 0.013842354586813595 ], "conc_latency_avg_list": [ - 0.004456437869882418, - 0.005844962813409522, - 0.008378709673770517, - 0.01528081619265322, - 0.021239153420731547, - 0.026694518371955574, - 0.03784726188423443, - 0.04857833790231276 + 0.002334789800170702, + 0.0023967380922842478, + 0.002508067757169752, + 0.0030384944076291527, + 0.003962180586705604, + 0.004954530890871444, + 0.006679939886193658, + 0.00850275890671233 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1302,10 +677,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1313,11 +688,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 9, + "level": 6, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1336,25 +711,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 5019.5973, - "serial_latency_p99": 0.0057, - "serial_latency_p95": 0.0054, - "recall": 0.994, - "ndcg": 0.9954, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 8320.4606, + "serial_latency_p99": 0.0029, + "serial_latency_p95": 0.0028, + "recall": 0.9919, + "ndcg": 0.9936, "conc_num_list": [ 1, 5, @@ -1366,44 +738,44 @@ 80 ], "conc_qps_list": [ - 222.5871, - 1150.5338, - 2195.8726, - 3676.709, - 4412.1372, - 4833.1625, - 5019.5973, - 5015.8037 + 396.8576, + 1911.4443, + 3643.8766, + 5864.3359, + 6725.025, + 7313.0633, + 7887.1755, + 8320.4606 ], "conc_latency_p99_list": [ - 0.005366289358644281, - 0.006616018440399769, - 0.0064373466832330474, - 0.010222766738734191, - 0.013315525980142408, - 0.016972876400686822, - 0.024728685971349477, - 0.031319626237964276 + 0.002857399091590195, + 0.0030318665952654557, + 0.003405487750424072, + 0.00554735084413551, + 0.00816060005221516, + 0.010382328977575536, + 0.014569980294909328, + 0.019027257570996884 ], "conc_latency_p95_list": [ - 0.00513816498714732, - 0.004792370549694169, - 0.005205206610844471, - 0.007212611548311542, - 0.010267410500091497, - 0.013238379993708808, - 0.01984079669928178, - 0.02578976150834933 + 0.002767942799255252, + 0.0028832201816840096, + 0.0030418253620155154, + 0.004497910931240768, + 0.006637068011332303, + 0.008489592577097936, + 0.011977570212911815, + 0.015408052015118301 ], "conc_latency_avg_list": [ - 0.00448758533933214, - 0.0043394632993677415, - 0.0045465428246546005, - 0.0054275771136581205, - 0.006782299052349466, - 0.008242341170482134, - 0.011840508222988514, - 0.015742522812380217 + 0.0025167798504936094, + 0.0026116185368739234, + 0.0027388090118584786, + 0.003400314036561817, + 0.0044390573829028, + 0.005437121205254614, + 0.0075142181236358555, + 0.009432022524963333 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1424,10 +796,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1435,7 +807,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 9, + "level": 7, "num_shards": 1 }, "case_config": { @@ -1458,25 +830,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 7364.0396, - "serial_latency_p99": 0.004, - "serial_latency_p95": 0.0037, - "recall": 0.9893, - "ndcg": 0.9915, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7524.9879, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.0031, + "recall": 0.9931, + "ndcg": 0.9946, "conc_num_list": [ 1, 5, @@ -1488,44 +857,44 @@ 80 ], "conc_qps_list": [ - 287.8688, - 1384.2897, - 2700.4491, - 4735.4478, - 5614.1579, - 6134.6648, - 6819.5071, - 7364.0396 + 363.7387, + 1761.1002, + 3372.7075, + 5421.2445, + 6091.3075, + 6579.0629, + 7061.7035, + 7524.9879 ], "conc_latency_p99_list": [ - 0.00390698241040809, - 0.00731195724976711, - 0.00495785990206058, - 0.00734390948899089, - 0.012288954856630872, - 0.014986682942835616, - 0.01849594444676764, - 0.022832338945008815 + 0.0031524766457732764, + 0.0032982631359482185, + 0.0036804132821271207, + 0.006010826830170116, + 0.008973520016297695, + 0.011436859995592378, + 0.016333586145192387, + 0.020569344094838005 ], "conc_latency_p95_list": [ - 0.0037214207914075814, - 0.003863213058502879, - 0.004057778001879342, - 0.005324349994771183, - 0.008072146945050915, - 0.010810172616038463, - 0.01463127658644225, - 0.01830196708324365 + 0.0030544937413651495, + 0.0031513541325693946, + 0.0033083077374612907, + 0.004909411718836053, + 0.007369707978796214, + 0.009455612045712769, + 0.013346620369702576, + 0.01678576849226374 ], "conc_latency_avg_list": [ - 0.0034694395949774774, - 0.003606888217707231, - 0.003696606332343799, - 0.004213763141467857, - 0.0053274846319189195, - 0.006490250213165499, - 0.008702013864526974, - 0.010708946041863319 + 0.0027460860977463145, + 0.0028345832649254296, + 0.0029595516874138823, + 0.003678043908615719, + 0.004902225199747265, + 0.006042755015168581, + 0.008396028243715591, + 0.010450248464720832 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1546,10 +915,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1557,7 +926,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 6, + "level": 8, "num_shards": 1 }, "case_config": { @@ -1580,25 +949,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 2724.9239, - "serial_latency_p99": 0.0124, - "serial_latency_p95": 0.0048, - "recall": 0.9811, - "ndcg": 0.9832, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6813.2439, + "serial_latency_p99": 0.0035, + "serial_latency_p95": 0.0034, + "recall": 0.9939, + "ndcg": 0.9953, "conc_num_list": [ 1, 5, @@ -1610,44 +976,44 @@ 80 ], "conc_qps_list": [ - 269.3651, - 1047.0123, - 1498.6024, - 1847.6803, - 2114.199, - 2284.3302, - 2537.2552, - 2724.9239 + 336.5978, + 1642.6462, + 3138.7921, + 4969.8163, + 5673.7771, + 6090.0396, + 6493.0362, + 6813.2439 ], "conc_latency_p99_list": [ - 0.004742077292175964, - 0.00691745357704349, - 0.010720830453210511, - 0.01960868470487181, - 0.026337993171764537, - 0.032754160480981225, - 0.04255042473436333, - 0.05312397440138735 + 0.003431524743209593, + 0.003548497949959711, + 0.0039453561976552, + 0.006659876625053583, + 0.009535451157134956, + 0.012163680926314556, + 0.017726751818554484, + 0.022734694816172126 ], "conc_latency_p95_list": [ - 0.004494865804736036, - 0.0055081502971006556, - 0.007977579892030908, - 0.015719183007604443, - 0.022288659910555, - 0.02759612778027076, - 0.03666536140372045, - 0.04536134131485597 + 0.0033281981566688048, + 0.0034026612091111017, + 0.0035765988053753973, + 0.005437539250124246, + 0.007899306231411173, + 0.0101619676919654, + 0.014541133219609037, + 0.01846120802219957 ], "conc_latency_avg_list": [ - 0.0037082374061278584, - 0.004769060159688272, - 0.0066635344327552045, - 0.010804306792214618, - 0.014153154028666374, - 0.017440448752945776, - 0.02343149585687298, - 0.029001097442020812 + 0.0029676403704890024, + 0.0030391423281907995, + 0.003179950204961118, + 0.0040130850174019085, + 0.005263782041679775, + 0.006520409861910488, + 0.009132446429719516, + 0.011525998827399965 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1668,10 +1034,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1679,11 +1045,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 5, + "level": 9, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1702,25 +1068,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 5514.815, - "serial_latency_p99": 0.0042, - "serial_latency_p95": 0.0036, - "recall": 0.9286, - "ndcg": 0.9355, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7385.2066, + "serial_latency_p99": 0.0021, + "serial_latency_p95": 0.002, + "recall": 0.9384, + "ndcg": 0.9441, "conc_num_list": [ 1, 5, @@ -1732,44 +1095,44 @@ 80 ], "conc_qps_list": [ - 313.799, - 1354.9874, - 2301.8011, - 3314.3036, - 3843.1259, - 4355.6117, - 5048.842, - 5514.815 + 505.8686, + 2189.7181, + 3241.0071, + 4661.5936, + 5380.3716, + 5932.9765, + 6767.3029, + 7385.2066 ], "conc_latency_p99_list": [ - 0.0038773450409644284, - 0.008138228004099801, - 0.010878445263078868, - 0.011540972657094244, - 0.016388589342823227, - 0.01858296279708156, - 0.023569957185536613, - 0.02886486930365208 + 0.0022006261744536458, + 0.002760201054625213, + 0.0037472858920227733, + 0.0067906589363701635, + 0.010014167174231262, + 0.012363597416551783, + 0.016403084830380974, + 0.01993567283730954 ], "conc_latency_p95_list": [ - 0.003412947052856907, - 0.004246345022693276, - 0.005220197608286979, - 0.008349375608668195, - 0.012340355810010787, - 0.0147909961087862, - 0.019336943980306387, - 0.023638575003133155 + 0.002098581724567339, + 0.002593909669667482, + 0.0035126500617479904, + 0.005520994003745727, + 0.008371637808158994, + 0.010467902664095164, + 0.01375423202989623, + 0.016782393981702625 ], "conc_latency_avg_list": [ - 0.003182328343818464, - 0.003684439145729171, - 0.004337632718561462, - 0.006021014814140765, - 0.007777482876747058, - 0.009140085848755274, - 0.011767476800046897, - 0.014305157378590626 + 0.001973905573358728, + 0.0022795015833306665, + 0.0030794282993742857, + 0.004278176924915009, + 0.005550848216261312, + 0.0066920424895898075, + 0.008768764746552707, + 0.010642102641863554 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1789,11 +1152,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1824,25 +1187,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 9901.1114, - "serial_latency_p99": 0.0039, - "serial_latency_p95": 0.0037, - "recall": 0.9385, - "ndcg": 0.9486, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6793.8443, + "serial_latency_p99": 0.0022, + "serial_latency_p95": 0.0021, + "recall": 0.9522, + "ndcg": 0.9568, "conc_num_list": [ 1, 5, @@ -1854,44 +1214,44 @@ 80 ], "conc_qps_list": [ - 321.3357, - 1639.9924, - 3195.3018, - 5673.0951, - 6808.8856, - 7560.1348, - 9087.2499, - 9901.1114 + 503.9478, + 2114.1911, + 3099.2869, + 4416.7094, + 5150.037, + 5691.2226, + 6289.7121, + 6793.8443 ], "conc_latency_p99_list": [ - 0.00394074416602971, - 0.0041782407171558535, - 0.004392879804945551, - 0.010109323544893349, - 0.008123845955124116, - 0.014126944777672188, - 0.014278251143987291, - 0.01781155434960963 + 0.00215859985910356, + 0.002847761964658275, + 0.003923368623363787, + 0.007356263077235778, + 0.010528954989276825, + 0.01292447085143066, + 0.01769668879453093, + 0.021927826073952026 ], "conc_latency_p95_list": [ - 0.0034882987965829666, - 0.0035166182147804642, - 0.0035458351092529476, - 0.004310413988423532, - 0.006210639532946515, - 0.008767930739850271, - 0.010931958199944346, - 0.013749658003507645 + 0.002082229100051336, + 0.0026818424608791246, + 0.0036669230728875847, + 0.005921932504861616, + 0.00885509350337088, + 0.010938107722904525, + 0.014863643678836526, + 0.018132617010269306 ], "conc_latency_avg_list": [ - 0.0031001682930926035, - 0.0030430930397005885, - 0.0031233191875315565, - 0.0035155999207049896, - 0.004389041533959679, - 0.005263586471416786, - 0.006530556363972054, - 0.007958968690030266 + 0.001981333490019148, + 0.002360769086835183, + 0.003220095266754697, + 0.004516460962995695, + 0.005803644141310085, + 0.006980248662568344, + 0.009430996778203294, + 0.011560318047614999 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1912,10 +1272,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1923,11 +1283,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 1, + "level": 2, "num_shards": 1 }, "case_config": { - "case_id": 5, + "case_id": 4, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1946,25 +1306,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 3258.8508, - "serial_latency_p99": 0.0117, - "serial_latency_p95": 0.0045, - "recall": 0.9888, - "ndcg": 0.9906, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6242.5346, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.0022, + "recall": 0.961, + "ndcg": 0.965, "conc_num_list": [ 1, 5, @@ -1976,44 +1333,44 @@ 80 ], "conc_qps_list": [ - 244.7738, - 1130.2346, - 1809.9431, - 2312.2265, - 2588.2339, - 2788.0239, - 3069.518, - 3258.8508 + 446.4866, + 2002.9326, + 2953.3925, + 4164.4285, + 4805.5397, + 5240.789, + 5870.8363, + 6242.5346 ], "conc_latency_p99_list": [ - 0.005137932703364641, - 0.00555925140972249, - 0.008915685693500566, - 0.016561121140257453, - 0.022867804747074828, - 0.02806737951235845, - 0.036439670615363844, - 0.04337102690333264 + 0.002856350458459926, + 0.0029789366439217715, + 0.004114359063096344, + 0.007887981488602236, + 0.011494717993773522, + 0.014308837521821265, + 0.019093695696210486, + 0.023705490870634095 ], "conc_latency_p95_list": [ - 0.004440846845682244, - 0.005053811200195923, - 0.0068235130165703595, - 0.01288393942813854, - 0.018162307806778695, - 0.02236156941507943, - 0.0296233788743848, - 0.03584610429388702 + 0.0025365250825416293, + 0.0028055029979441315, + 0.0038476928253658115, + 0.0064046091894852, + 0.009612656582612544, + 0.012057665473548695, + 0.01607741924817674, + 0.019808521203231066 ], "conc_latency_avg_list": [ - 0.004080840300378187, - 0.004417934321996714, - 0.005516924373879563, - 0.008634555842136784, - 0.011560621919641356, - 0.014278915373578093, - 0.019378262406752788, - 0.024243179517198447 + 0.0022367817378751067, + 0.0024923140599496687, + 0.003379658321121232, + 0.0047897960603506306, + 0.0062167658101995584, + 0.007583969576561268, + 0.010102243992110408, + 0.012617357519441232 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2033,11 +1390,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2045,7 +1402,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 8, + "level": 3, "num_shards": 1 }, "case_config": { @@ -2068,25 +1425,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 5907.441, - "serial_latency_p99": 0.0057, - "serial_latency_p95": 0.0054, - "recall": 0.9931, - "ndcg": 0.9946, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5779.119, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.0023, + "recall": 0.971, + "ndcg": 0.9742, "conc_num_list": [ 1, 5, @@ -2098,44 +1452,44 @@ 80 ], "conc_qps_list": [ - 214.7436, - 1124.1591, - 2263.1224, - 3695.1435, - 4635.8817, - 5035.3873, - 5614.0659, - 5907.441 + 476.6126, + 1920.6379, + 2828.6202, + 3905.7911, + 4495.2986, + 4903.9634, + 5352.3634, + 5779.119 ], "conc_latency_p99_list": [ - 0.005838954218779679, - 0.009590839385055032, - 0.0072934928326867585, - 0.009990225688670754, - 0.013466239573317574, - 0.01589230435347418, - 0.021308113009436094, - 0.026848825681372538 + 0.002284108918393031, + 0.003114189200568944, + 0.004329544047359377, + 0.008613102727103977, + 0.012333529423922296, + 0.015281949284835726, + 0.020651346329832433, + 0.025012934936676175 ], "conc_latency_p95_list": [ - 0.005094571305380669, - 0.0048950490017887205, - 0.00495702201151289, - 0.007446101757523138, - 0.009852354813483545, - 0.01273485799174523, - 0.017310541804181415, - 0.021899056984693743 + 0.0022121331770904363, + 0.0029337720014154913, + 0.004045496415346861, + 0.007010379375424236, + 0.010350381198804826, + 0.012867126709898001, + 0.01739263068011496, + 0.021097359794657676 ], "conc_latency_avg_list": [ - 0.004651503297485146, - 0.004441934131452666, - 0.004411218431154767, - 0.005397841813638808, - 0.00645052096323231, - 0.007903104976167643, - 0.010583279479885773, - 0.013354355155792764 + 0.002095217670483641, + 0.0025990775656036295, + 0.003529053754031682, + 0.005105319901460065, + 0.006648317818211828, + 0.008106390568584667, + 0.011082872888195554, + 0.01359851345251022 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2156,10 +1510,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2167,11 +1521,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 8, + "level": 4, "num_shards": 1 }, "case_config": { - "case_id": 5, + "case_id": 4, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -2190,25 +1544,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 5064.6982, - "serial_latency_p99": 0.0043, - "serial_latency_p95": 0.0036, - "recall": 0.9558, - "ndcg": 0.9606, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5183.5843, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9784, + "ndcg": 0.981, "conc_num_list": [ 1, 5, @@ -2220,44 +1571,44 @@ 80 ], "conc_qps_list": [ - 306.5362, - 1368.7188, - 2310.9627, - 3255.5776, - 3791.6067, - 4073.62, - 4572.4511, - 5064.6982 + 456.5329, + 1821.263, + 2677.2938, + 3617.961, + 4097.4679, + 4447.7515, + 4863.2316, + 5183.5843 ], "conc_latency_p99_list": [ - 0.0042355662427144124, - 0.0046483404963510114, - 0.009778717628214485, - 0.010217842382844546, - 0.015881566194002526, - 0.01989235390967224, - 0.025938013812992735, - 0.031185232510324568 + 0.002395828692242503, + 0.0033142844232497736, + 0.004636971018044278, + 0.009431733234669086, + 0.013644934630719942, + 0.01654091664124283, + 0.022408935175044466, + 0.027210572804324325 ], "conc_latency_p95_list": [ - 0.0034556519065517934, - 0.004072950512636453, - 0.005186801168019884, - 0.008326760004274545, - 0.01228064175666077, - 0.016202768517541696, - 0.021469047002028674, - 0.025539752503391355 + 0.00232212619157508, + 0.0031105756846955047, + 0.0043217132915742695, + 0.007749295464600434, + 0.011374025803525001, + 0.01404695180244743, + 0.01901522108237258, + 0.022979849949479103 ], "conc_latency_avg_list": [ - 0.003258234978368169, - 0.0036474964905585366, - 0.004320538177772956, - 0.006130977769455615, - 0.007887396918013946, - 0.009779561744223083, - 0.013001510996033932, - 0.015590046073741476 + 0.0021874608272499383, + 0.0027410038616561545, + 0.003728350102575746, + 0.005515110815448404, + 0.0072920914670729624, + 0.008931282929280606, + 0.012187039287903855, + 0.015173242426486785 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2277,11 +1628,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2289,7 +1640,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 3, + "level": 5, "num_shards": 1 }, "case_config": { @@ -2312,25 +1663,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 3468.5933, - "serial_latency_p99": 0.0045, - "serial_latency_p95": 0.0043, - "recall": 0.9634, - "ndcg": 0.9661, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4259.5572, + "serial_latency_p99": 0.0026, + "serial_latency_p95": 0.0026, + "recall": 0.9845, + "ndcg": 0.9866, "conc_num_list": [ 1, 5, @@ -2342,44 +1690,44 @@ 80 ], "conc_qps_list": [ - 274.7753, - 1086.6444, - 1657.8469, - 2230.3556, - 2576.5151, - 2840.9102, - 3207.9495, - 3468.5933 + 424.077, + 1676.9478, + 2434.6516, + 3184.2011, + 3571.9199, + 3767.5544, + 4061.1698, + 4259.5572 ], "conc_latency_p99_list": [ - 0.007525063498178497, - 0.00871218444284751, - 0.009115133894665613, - 0.01669143169856398, - 0.021790961647639086, - 0.026465490460395806, - 0.03498110753978835, - 0.042080024706956466 + 0.002595295326318592, + 0.003621646617539228, + 0.005367588647641237, + 0.011171487247338519, + 0.0155316284415312, + 0.019644083841703817, + 0.02703630503267048, + 0.03310978163208347 ], "conc_latency_p95_list": [ - 0.004381819497211836, - 0.00529403816035483, - 0.007012556104746182, - 0.01246311155118746, - 0.018361228803405537, - 0.022542869101744148, - 0.029249506900669076, - 0.03583758800959913 + 0.002518506458727643, + 0.0034135113819502294, + 0.004873151995707303, + 0.009142277223872952, + 0.013012332818470895, + 0.016382047021761527, + 0.022542749025160444, + 0.027544378107995725 ], "conc_latency_avg_list": [ - 0.003635002963577244, - 0.0045950829257419635, - 0.006023093199217258, - 0.008949508158072033, - 0.011605007790743322, - 0.014003128689967868, - 0.018541925579661857, - 0.02278122412115334 + 0.00235500177392571, + 0.0029770619258879757, + 0.00410082317131261, + 0.006266042699984648, + 0.008367804647869575, + 0.010561791085210772, + 0.01460435324245386, + 0.018471206884305532 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2400,10 +1748,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2411,7 +1759,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 3, + "level": 6, "num_shards": 1 }, "case_config": { @@ -2434,25 +1782,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 2401.3204, - "serial_latency_p99": 0.0107, - "serial_latency_p95": 0.0047, - "recall": 0.9866, - "ndcg": 0.9884, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3614.3118, + "serial_latency_p99": 0.0029, + "serial_latency_p95": 0.0028, + "recall": 0.9877, + "ndcg": 0.9894, "conc_num_list": [ 1, 5, @@ -2464,44 +1809,44 @@ 80 ], "conc_qps_list": [ - 261.8215, - 985.1764, - 1468.0087, - 1749.415, - 1931.5307, - 2090.0742, - 2278.9704, - 2401.3204 + 397.2298, + 1557.1023, + 2236.3239, + 2845.1058, + 3146.0016, + 3289.0762, + 3520.8116, + 3614.3118 ], "conc_latency_p99_list": [ - 0.004836413672601339, - 0.009077261193306102, - 0.010354967679304536, - 0.02026946535654133, - 0.029030042108206543, - 0.03499512374750339, - 0.046797644338803394, - 0.057348916197370266 + 0.0027946092444472016, + 0.003967489110655152, + 0.00615017178060953, + 0.0128463427053066, + 0.017453623053152116, + 0.022870610732934445, + 0.029518100013956432, + 0.03869990034552756 ], "conc_latency_p95_list": [ - 0.004629465389007237, - 0.0058674284955486655, - 0.0082442566199461, - 0.01657543244800763, - 0.02401039500546176, - 0.029254749882966277, - 0.03907370918313973, - 0.04936321394779952 + 0.0027128490241011606, + 0.003724151346250437, + 0.0054281810007523745, + 0.01044328572170343, + 0.01436160156154074, + 0.01821845376980491, + 0.024413908016867936, + 0.031903167749987915 ], "conc_latency_avg_list": [ - 0.0038149669483554033, - 0.00506845096702024, - 0.006802651714890116, - 0.01140782706843602, - 0.015489598411640663, - 0.019035290587860968, - 0.02608886756739919, - 0.03292382999700411 + 0.0025143936207342324, + 0.003206745084255346, + 0.004464954355791834, + 0.007013879446545796, + 0.009503234791544602, + 0.012096279585844553, + 0.01685092675726034, + 0.021781507836163158 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2522,10 +1867,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2533,7 +1878,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 6, + "level": 7, "num_shards": 1 }, "case_config": { @@ -2556,25 +1901,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 3568.396, - "serial_latency_p99": 0.0048, - "serial_latency_p95": 0.0043, - "recall": 0.9863, - "ndcg": 0.9883, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3181.0537, + "serial_latency_p99": 0.003, + "serial_latency_p95": 0.0029, + "recall": 0.9892, + "ndcg": 0.9908, "conc_num_list": [ 1, 5, @@ -2586,44 +1928,44 @@ 80 ], "conc_qps_list": [ - 249.5646, - 1116.5066, - 1828.4446, - 2389.4823, - 2645.8957, - 2879.4104, - 3356.3991, - 3568.396 + 373.2168, + 1461.7612, + 2063.4765, + 2583.7979, + 2781.1167, + 2908.4205, + 3073.9858, + 3181.0537 ], "conc_latency_p99_list": [ - 0.004949838446336795, - 0.010793446648749472, - 0.011542035994352773, - 0.015714827887131834, - 0.023098365282639866, - 0.028270509486901574, - 0.03385155899741221, - 0.0405952908913605 + 0.003002641891944222, + 0.004314229479641651, + 0.007060362810152582, + 0.01453221907839178, + 0.01971446685201954, + 0.024677592392545203, + 0.032532848112750784, + 0.04069755139702464 ], "conc_latency_p95_list": [ - 0.004326387906621675, - 0.005205007104086689, - 0.0069263952391338535, - 0.012388522701803593, - 0.01838434826204321, - 0.022779258753871545, - 0.02818173549894709, - 0.034205201656732236 + 0.0029138700163457544, + 0.00403262600011658, + 0.0060169674427015705, + 0.011576333804987366, + 0.01587381720310077, + 0.019931912398897106, + 0.027404746599495412, + 0.03406435703218449 ], "conc_latency_avg_list": [ - 0.004002170059550175, - 0.0044719544014582705, - 0.005461118931996274, - 0.008354930265743018, - 0.011307457646718058, - 0.013821107916470069, - 0.01770926681597559, - 0.022161158532467397 + 0.002676176823972926, + 0.0034160777606854604, + 0.004838368703871575, + 0.0077237947842559935, + 0.010749740541723023, + 0.013688260722494459, + 0.01930447747175536, + 0.024747977200758154 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2643,11 +1985,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2655,7 +1997,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 7, + "level": 8, "num_shards": 1 }, "case_config": { @@ -2678,25 +2020,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 4674.1861, - "serial_latency_p99": 0.0045, - "serial_latency_p95": 0.0038, - "recall": 0.967, - "ndcg": 0.9705, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2804.8357, + "serial_latency_p99": 0.0033, + "serial_latency_p95": 0.0032, + "recall": 0.9903, + "ndcg": 0.9918, "conc_num_list": [ 1, 5, @@ -2708,44 +2047,44 @@ 80 ], "conc_qps_list": [ - 299.441, - 1293.8246, - 2204.8398, - 2998.8928, - 3416.6029, - 3814.1905, - 4403.9058, - 4674.1861 + 352.7729, + 1367.945, + 1917.0391, + 2338.0313, + 2510.7421, + 2617.9646, + 2727.8716, + 2804.8357 ], "conc_latency_p99_list": [ - 0.00402290889178403, - 0.007907517027342683, - 0.006291847964457707, - 0.013353673194069417, - 0.017164103323593728, - 0.021020031592343, - 0.027193114216788664, - 0.033575675918255006 + 0.00323100520123262, + 0.004680905669229106, + 0.00799012375646271, + 0.01590309410821647, + 0.021047461275593433, + 0.025625948917586353, + 0.036803203278686886, + 0.0467973963287659 ], "conc_latency_p95_list": [ - 0.00356991050648503, - 0.004438751634734216, - 0.005453104941989295, - 0.00961559300776571, - 0.013857692398596555, - 0.017230113997356966, - 0.02220476679212879, - 0.02798210658947937 + 0.0031083543435670435, + 0.0043611170200165365, + 0.0067188970482675355, + 0.0125031745119486, + 0.017026124306721607, + 0.021185590000823137, + 0.03006786260521039, + 0.03819325279910117 ], "conc_latency_avg_list": [ - 0.003335578729612294, - 0.0038557001362392825, - 0.0045284253144977004, - 0.006655331635103421, - 0.008756953741340391, - 0.010436090944337196, - 0.013487455343736422, - 0.016883584811310505 + 0.002831514758067566, + 0.003649909442997153, + 0.00520858526610239, + 0.00853725560951756, + 0.011906267087115343, + 0.015190065559178532, + 0.021789805178996344, + 0.0280490530042168 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2765,11 +2104,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2777,7 +2116,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 4, + "level": 9, "num_shards": 1 }, "case_config": { @@ -2800,13 +2139,10 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } } ], "file_fmt": "result_{}_{}_{}.json", diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index bbc1e918b..c987ac9e7 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -1,2952 +1,2892 @@ [ - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1146.5286, - "latency": 13.7, - "recall": 0.9262, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1148.1735, - "latency": 8.9, - "recall": 0.9801, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1149.1219, - "latency": 10.3, - "recall": 0.9764, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1140.4099, - "latency": 13.5, - "recall": 0.9716, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1123.5147, - "latency": 18.5, - "recall": 0.9688, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 487.8343, - "latency": 25.4, - "recall": 0.9668, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 264.9324, - "latency": 49.6, - "recall": 0.936, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 492.4887, - "latency": 29.6, - "recall": 0.9269, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 823.1775, - "latency": 20.5, - "recall": 0.9148, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1147.1977, - "latency": 13.3, - "recall": 0.8999, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1131.3087, - "latency": 14.1, - "recall": 0.9024, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1114.952, - "latency": 12.7, - "recall": 0.97, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 583.5009, - "latency": 23.0, - "recall": 0.9668, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 31.4779, - "latency": 351.0, - "recall": 0.9414, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 57.8988, - "latency": 200.1, - "recall": 0.9332, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 101.1774, - "latency": 116.1, - "recall": 0.9241, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 212.7466, - "latency": 58.7, - "recall": 0.9099, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 372.2462, - "latency": 35.9, - "recall": 0.8977, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 617.0881, - "latency": 22.4, - "recall": 0.8844, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1094.5967, - "latency": 14.3, - "recall": 0.8659, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4318.9697, - "latency": 4.3, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4250.2894, - "latency": 4.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 2997.4391, - "latency": 6.1, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1494.5334, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1108.6473, - "latency": 7.4, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1289.5164, - "latency": 6.4, - "recall": 0.9906, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1059.3394, - "latency": 7.8, - "recall": 0.9856, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 987.0795, - "latency": 7.1, - "recall": 0.9804, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1591.7055, - "latency": 7.8, - "recall": 0.8506, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1202.8677, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 639.3991, - "latency": 7.3, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 274.8559, - "latency": 9.9, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 441.4152, - "latency": 8.3, - "recall": 0.997, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 358.8949, - "latency": 9.5, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 325.2245, - "latency": 10.3, - "recall": 0.9909, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 273.4174, - "latency": 13.3, - "recall": 0.9789, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 262.8314, - "latency": 11.3, - "recall": 0.9808, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 434.5481, - "latency": 8.5, - "recall": 0.7237, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 446.9116, - "latency": 9.2, - "recall": 0.9357, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 388.3028, - "latency": 9.6, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 323.3964, - "latency": 9.8, - "recall": 0.9507, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 256.4668, - "latency": 11.3, - "recall": 0.9588, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 145.5316, - "latency": 18.4, - "recall": 0.9726, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1242.428, - "latency": 6.4, - "recall": 0.9474, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1111.3633, - "latency": 7.0, - "recall": 0.955, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 955.4701, - "latency": 7.2, - "recall": 0.9629, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 783.5207, - "latency": 7.7, - "recall": 0.971, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 470.8546, - "latency": 9.5, - "recall": 0.9835, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 950.6332, - "latency": 13.2, - "recall": 0.914, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 823.2224, - "latency": 13.5, - "recall": 0.9434, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 743.9815, - "latency": 14.8, - "recall": 0.9583, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 683.1873, - "latency": 15.7, - "recall": 0.9677, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 619.7468, - "latency": 17.2, - "recall": 0.9738, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 537.4082, - "latency": 18.8, - "recall": 0.9809, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 474.9941, - "latency": 20.9, - "recall": 0.9848, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 505.7458, - "latency": 20.7, - "recall": 0.9068, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 433.9034, - "latency": 23.1, - "recall": 0.931, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 381.7737, - "latency": 25.7, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 342.1123, - "latency": 29.0, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 308.2216, - "latency": 31.3, - "recall": 0.9561, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 257.7928, - "latency": 36.4, - "recall": 0.9626, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 223.8166, - "latency": 42.1, - "recall": 0.9666, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3055.0123, - "latency": 7.2, - "recall": 0.9066, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3013.4439, - "latency": 6.9, - "recall": 0.9268, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2801.7241, - "latency": 7.4, - "recall": 0.9476, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2590.3809, - "latency": 8.6, - "recall": 0.9679, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2291.2159, - "latency": 8.9, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3099.4124, - "latency": 6.2, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3014.2483, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2073.2153, - "latency": 11.0, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1507.6899, - "latency": 12.8, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 942.2296, - "latency": 18.2, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 677.1414, - "latency": 33.5, - "recall": 0.7655, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2685.6654, - "latency": 7.6, - "recall": 0.4914, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2604.4444, - "latency": 7.8, - "recall": 0.63, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2159.051, - "latency": 9.4, - "recall": 0.801, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2251.1274, - "latency": 8.7, - "recall": 0.8848, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3103.0539, - "latency": 5.6, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3086.1957, - "latency": 6.7, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3090.0478, - "latency": 6.4, - "recall": 0.9628, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3064.6288, - "latency": 6.5, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3065.6134, - "latency": 6.2, - "recall": 0.9328, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3028.858, - "latency": 6.7, - "recall": 0.9133, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2935.9403, - "latency": 6.8, - "recall": 0.8992, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2771.2009, - "latency": 7.6, - "recall": 0.889, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1610.9496, - "latency": 10.8, - "recall": 0.9, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1557.3623, - "latency": 10.8, - "recall": 0.9244, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1473.9256, - "latency": 11.7, - "recall": 0.9484, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1388.5547, - "latency": 12.5, - "recall": 0.9597, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1022.2696, - "latency": 17.9, - "recall": 0.936, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 696.9777, - "latency": 24.6, - "recall": 0.997, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 353.7862, - "latency": 45.2, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 210.3227, - "latency": 71.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 114.8061, - "latency": 126.6, - "recall": 0.9985, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 504.9179, - "latency": 272.6, - "recall": 0.4664, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1053.1495, - "latency": 17.7, - "recall": 0.5673, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 808.3294, - "latency": 22.2, - "recall": 0.7016, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 8.0584, - "latency": 1757.9, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3033.5491, - "latency": 6.4, - "recall": 0.9844, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2988.4205, - "latency": 7.6, - "recall": 0.9741, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2950.717, - "latency": 6.9, - "recall": 0.9558, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2782.0274, - "latency": 7.4, - "recall": 0.9466, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2708.6752, - "latency": 8.4, - "recall": 0.9337, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2275.2854, - "latency": 9.1, - "recall": 0.917, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1844.8918, - "latency": 10.6, - "recall": 0.9085, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1301.4102, - "latency": 14.7, - "recall": 0.9011, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 13.0379, - "latency": 1063.5, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.4972, - "latency": 337.1, - "recall": 0.8717, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1164, - "latency": 345.6, - "recall": 0.4276, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 197.4455, - "latency": 349.3, - "recall": 0.5314, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 196.9391, - "latency": 263.4, - "recall": 0.6549, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.5401, - "latency": 282.4, - "recall": 0.7086, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.2424, - "latency": 301.7, - "recall": 0.7592, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.599, - "latency": 358.8, - "recall": 0.8085, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.0349, - "latency": 275.3, - "recall": 0.8325, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.1405, - "latency": 282.6, - "recall": 0.8492, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.1282, - "latency": 269.2, - "recall": 0.8637, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 194.8021, - "latency": 559.8, - "recall": 0.86, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 187.4268, - "latency": 453.7, - "recall": 0.4692, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.397, - "latency": 506.9, - "recall": 0.5409, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 174.3549, - "latency": 496.9, - "recall": 0.6279, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 172.95, - "latency": 515.6, - "recall": 0.7004, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 190.9747, - "latency": 517.4, - "recall": 0.7398, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 186.0237, - "latency": 474.0, - "recall": 0.7847, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1458, - "latency": 480.5, - "recall": 0.8103, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 179.4203, - "latency": 497.5, - "recall": 0.8273, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.5444, - "latency": 463.9, - "recall": 0.8478, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 346.5847, - "latency": 42.7, - "recall": 0.9631, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 369.4921, - "latency": 41.6, - "recall": 0.779, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 310.957, - "latency": 49.4, - "recall": 0.9698, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 798.328, - "latency": 56.7, - "recall": 0.8993, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 649.8781, - "latency": 55.2, - "recall": 0.8352, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 370.7241, - "latency": 49.6, - "recall": 0.7177, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 100.0554, - "latency": 69.3, - "recall": 0.9638, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 284.6367, - "latency": 47.6, - "recall": 0.9788, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 81.8678, - "latency": 105.6, - "recall": 0.8751, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 260.4031, - "latency": 48.3, - "recall": 0.9828, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 365.2505, - "latency": 34.9, - "recall": 0.8251, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 471.553, - "latency": 44.1, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 91.8612, - "latency": 85.7, - "recall": 0.8799, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 206.0934, - "latency": 56.7, - "recall": 0.9795, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 351.7114, - "latency": 46.7, - "recall": 0.8735, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 96.592, - "latency": 76.9, - "recall": 0.9178, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 802.6923, - "latency": 48.1, - "recall": 0.935, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 184.5363, - "latency": 53.4, - "recall": 0.9681, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 323.0238, - "latency": 50.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 382.5332, - "latency": 54.7, - "recall": 0.6135, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3917.2035, - "latency": 2.4, - "recall": 0.9203, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3628.8527, - "latency": 2.6, - "recall": 0.9318, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3250.1112, - "latency": 2.7, - "recall": 0.9443, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2762.4144, - "latency": 3.1, - "recall": 0.9556, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2384.6245, - "latency": 3.2, - "recall": 0.9627, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2134.1717, - "latency": 3.8, - "recall": 0.9671, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1641.3478, - "latency": 4.1, - "recall": 0.9729, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1488.5841, - "latency": 4.7, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2747.3167, - "latency": 3.3, - "recall": 0.9204, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2514.4481, - "latency": 3.2, - "recall": 0.9303, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2177.2345, - "latency": 3.4, - "recall": 0.9408, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1833.2575, - "latency": 3.9, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1552.4803, - "latency": 4.0, - "recall": 0.9565, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1355.3121, - "latency": 4.4, - "recall": 0.9602, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1079.2123, - "latency": 5.3, - "recall": 0.9648, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 876.5772, - "latency": 6.3, - "recall": 0.9676, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10663.1231, - "latency": 2.0, - "recall": 0.8405, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10333.9072, - "latency": 2.0, - "recall": 0.889, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 9575.6863, - "latency": 2.3, - "recall": 0.9189, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 8596.7694, - "latency": 2.4, - "recall": 0.9416, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7704.3625, - "latency": 2.7, - "recall": 0.9541, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7023.6735, - "latency": 3.0, - "recall": 0.962, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 6031.3725, - "latency": 3.3, - "recall": 0.971, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 5258.1868, - "latency": 3.6, - "recall": 0.9768, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5973.0024, - "latency": 2.4, - "recall": 0.9192, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5416.5758, - "latency": 2.6, - "recall": 0.9334, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4771.4324, - "latency": 2.8, - "recall": 0.9479, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4006.3994, - "latency": 3.2, - "recall": 0.9609, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3441.7597, - "latency": 3.5, - "recall": 0.9682, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3040.6216, - "latency": 3.7, - "recall": 0.9734, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2446.7373, - "latency": 4.3, - "recall": 0.9791, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2084.6245, - "latency": 5.0, - "recall": 0.9819, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11763.5538, - "latency": 1.5, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11803.1944, - "latency": 1.5, - "recall": 0.9778, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11520.9234, - "latency": 1.5, - "recall": 0.9634, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11280.0849, - "latency": 1.6, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10671.8925, - "latency": 1.7, - "recall": 0.9339, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10258.2661, - "latency": 1.7, - "recall": 0.9139, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9681.5656, - "latency": 1.9, - "recall": 0.9008, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8945.4041, - "latency": 1.9, - "recall": 0.8894, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5436.8907, - "latency": 2.0, - "recall": 0.929, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11397.7043, - "latency": 1.6, - "recall": 0.9597, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10891.7531, - "latency": 1.7, - "recall": 0.9408, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10276.7451, - "latency": 1.7, - "recall": 0.9159, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9664.2855, - "latency": 1.8, - "recall": 0.899, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8936.7962, - "latency": 2.0, - "recall": 0.8835, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5671.2562, - "latency": 2.1, - "recall": 0.903, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 3157.707, - "latency": 2.3, - "recall": 0.9347, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 1985.8124, - "latency": 2.6, - "recall": 0.9407, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 920.9627, - "latency": 3.4, - "recall": 0.9488, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3033.786, - "latency": 8.7, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3019.2416, - "latency": 9.5, - "recall": 0.9765, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2890.9523, - "latency": 9.4, - "recall": 0.9625, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2789.7212, - "latency": 8.2, - "recall": 0.9538, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2457.2628, - "latency": 9.0, - "recall": 0.9378, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2209.4973, - "latency": 13.7, - "recall": 0.9228, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1960.388, - "latency": 11.0, - "recall": 0.9076, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1725.092, - "latency": 11.7, - "recall": 0.8969, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1307.419, - "latency": 12.3, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 350.0132, - "latency": 29.7, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 179.5204, - "latency": 51.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 72.99, - "latency": 111.4, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 42.9877, - "latency": 201.9, - "recall": 0.9912, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 96.4987, - "latency": 113.1, - "recall": 0.9296, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 189.3789, - "latency": 58.8, - "recall": 0.9149, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 246.7071, - "latency": 45.1, - "recall": 0.9018, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 229.0379, - "latency": 43.0, - "recall": 0.8908, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 125.6164, - "latency": 69.8, - "recall": 0.8746, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2175.2694, - "latency": 9.8, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1430.0244, - "latency": 12.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 692.5751, - "latency": 18.7, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 364.3516, - "latency": 26.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 190.3777, - "latency": 47.9, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 249.3519, - "latency": 44.7, - "recall": 0.9446, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 437.8735, - "latency": 27.1, - "recall": 0.9364, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 669.9441, - "latency": 19.1, - "recall": 0.9227, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 899.3114, - "latency": 14.9, - "recall": 0.9072, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2030.4249, - "latency": 10.6, - "recall": 0.9306, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1804.8996, - "latency": 12.3, - "recall": 0.9405, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2353.8935, - "latency": 17.1, - "recall": 0.9143, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1623.8421, - "latency": 11.8, - "recall": 0.9479, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2808.2421, - "latency": 9.5, - "recall": 0.8815, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1482.3772, - "latency": 12.1, - "recall": 0.9546, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1721.5416, - "latency": 9.6, - "recall": 0.8855, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1032.9696, - "latency": 14.8, - "recall": 0.933, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1150.4393, - "latency": 13.4, - "recall": 0.9265, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1452.1536, - "latency": 10.8, - "recall": 0.9042, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2181.3939, - "latency": 9.4, - "recall": 0.8501, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1295.5543, - "latency": 11.2, - "recall": 0.9176, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9773.6593, - "latency": 3.7, - "recall": 0.9955, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9081.1518, - "latency": 3.0, - "recall": 0.9943, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8455.2896, - "latency": 4.0, - "recall": 0.9921, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7610.0519, - "latency": 3.3, - "recall": 0.9903, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7589.664, - "latency": 3.8, - "recall": 0.9235, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6750.2495, - "latency": 4.4, - "recall": 0.9105, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5506.1808, - "latency": 5.5, - "recall": 0.9193, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6860.8577, - "latency": 4.7, - "recall": 0.9226, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8468.4611, - "latency": 3.1, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10089.4308, - "latency": 2.6, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10557.4373, - "latency": 2.7, - "recall": 0.9393, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9805.0401, - "latency": 2.6, - "recall": 0.9257, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10020.5299, - "latency": 2.6, - "recall": 0.9788, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10041.0338, - "latency": 2.7, - "recall": 0.9693, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9861.9686, - "latency": 2.6, - "recall": 0.955, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9507.9991, - "latency": 2.8, - "recall": 0.9453, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9428.4531, - "latency": 2.6, - "recall": 0.9331, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9048.6431, - "latency": 3.9, - "recall": 0.9216, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8695.2765, - "latency": 4.3, - "recall": 0.9603, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9244.1135, - "latency": 4.2, - "recall": 0.9724, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9289.0118, - "latency": 4.2, - "recall": 0.9574, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9374.8941, - "latency": 4.2, - "recall": 0.9425, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9368.1325, - "latency": 3.8, - "recall": 0.9292, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9220.3627, - "latency": 3.8, - "recall": 0.9081, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8633.8949, - "latency": 4.1, - "recall": 0.8928, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 6820.6863, - "latency": 3.2, - "recall": 0.9159, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 3938.6004, - "latency": 3.7, - "recall": 0.9196, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3411.0934, - "latency": 3.3, - "recall": 0.995, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2838.356, - "latency": 3.8, - "recall": 0.9946, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1826.0672, - "latency": 5.3, - "recall": 0.9938, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1234.6534, - "latency": 6.4, - "recall": 0.9942, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1773.0919, - "latency": 5.3, - "recall": 0.9699, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1454.8382, - "latency": 4.6, - "recall": 0.9659, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1373.0307, - "latency": 5.7, - "recall": 0.9716, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2039.8673, - "latency": 3.8, - "recall": 0.9559, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2950.8165, - "latency": 3.3, - "recall": 0.9147, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9441.1235, - "latency": 5.2, - "recall": 0.9658, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6125.6146, - "latency": 4.9, - "recall": 0.9936, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5502.1797, - "latency": 3.8, - "recall": 0.9509, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1827.5849, - "latency": 5.4, - "recall": 0.9918, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1938.1932, - "latency": 5.6, - "recall": 0.9906, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3778.8811, - "latency": 4.8, - "recall": 0.9851, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3974.8218, - "latency": 4.8, - "recall": 0.9428, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2971.1402, - "latency": 11.6, - "recall": 0.9752, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8441.533, - "latency": 6.9, - "recall": 0.9825, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 2703.5422, - "latency": 12.9, - "recall": 0.992, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1628.2736, - "latency": 5.6, - "recall": 0.9928, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5019.5973, - "latency": 5.7, - "recall": 0.9954, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7364.0396, - "latency": 4.0, - "recall": 0.9915, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2724.9239, - "latency": 12.4, - "recall": 0.9832, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5514.815, - "latency": 4.2, - "recall": 0.9355, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9901.1114, - "latency": 3.9, - "recall": 0.9486, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3258.8508, - "latency": 11.7, - "recall": 0.9906, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5907.441, - "latency": 5.7, - "recall": 0.9946, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5064.6982, - "latency": 4.3, - "recall": 0.9606, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3468.5933, - "latency": 4.5, - "recall": 0.9661, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2401.3204, - "latency": 10.7, - "recall": 0.9884, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3568.396, - "latency": 4.8, - "recall": 0.9883, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 4674.1861, - "latency": 4.5, - "recall": 0.9705, - "filter_ratio": 0.0 - } + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2181.3939, + "latency": 9.4, + "recall": 0.8501, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1721.5416, + "latency": 9.6, + "recall": 0.8855, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1452.1536, + "latency": 10.8, + "recall": 0.9042, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1295.5543, + "latency": 11.2, + "recall": 0.9176, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1150.4393, + "latency": 13.4, + "recall": 0.9265, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1032.9696, + "latency": 14.8, + "recall": 0.933, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 125.6164, + "latency": 69.8, + "recall": 0.8746, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 229.0379, + "latency": 43.0, + "recall": 0.8908, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 246.7071, + "latency": 45.1, + "recall": 0.9018, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 189.3789, + "latency": 58.8, + "recall": 0.9149, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 96.4987, + "latency": 113.1, + "recall": 0.9296, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 42.9877, + "latency": 201.9, + "recall": 0.9912, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 72.99, + "latency": 111.4, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 179.5204, + "latency": 51.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 350.0132, + "latency": 29.7, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2808.2421, + "latency": 9.5, + "recall": 0.8815, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2353.8935, + "latency": 17.1, + "recall": 0.9143, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2030.4249, + "latency": 10.6, + "recall": 0.9306, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1804.8996, + "latency": 12.3, + "recall": 0.9405, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1623.8421, + "latency": 11.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1482.3772, + "latency": 12.1, + "recall": 0.9546, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 899.3114, + "latency": 14.9, + "recall": 0.9072, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 669.9441, + "latency": 19.1, + "recall": 0.9227, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 437.8735, + "latency": 27.1, + "recall": 0.9364, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 249.3519, + "latency": 44.7, + "recall": 0.9446, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 190.3777, + "latency": 47.9, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 364.3516, + "latency": 26.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 692.5751, + "latency": 18.7, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1430.0244, + "latency": 12.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2175.2694, + "latency": 9.8, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1307.419, + "latency": 12.3, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1725.092, + "latency": 11.7, + "recall": 0.8969, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1960.388, + "latency": 11.0, + "recall": 0.9076, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2209.4973, + "latency": 13.7, + "recall": 0.9228, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2457.2628, + "latency": 9.0, + "recall": 0.9378, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2789.7212, + "latency": 8.2, + "recall": 0.9538, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2890.9523, + "latency": 9.4, + "recall": 0.9625, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3019.2416, + "latency": 9.5, + "recall": 0.9765, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3033.786, + "latency": 8.7, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3917.2035, + "latency": 2.4, + "recall": 0.9203, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3628.8527, + "latency": 2.6, + "recall": 0.9318, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3250.1112, + "latency": 2.7, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2762.4144, + "latency": 3.1, + "recall": 0.9556, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2384.6245, + "latency": 3.2, + "recall": 0.9627, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2134.1717, + "latency": 3.8, + "recall": 0.9671, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1641.3478, + "latency": 4.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1488.5841, + "latency": 4.7, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10333.9072, + "latency": 2.0, + "recall": 0.889, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 9575.6863, + "latency": 2.3, + "recall": 0.9189, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 8596.7694, + "latency": 2.4, + "recall": 0.9416, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7704.3625, + "latency": 2.7, + "recall": 0.9541, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7023.6735, + "latency": 3.0, + "recall": 0.962, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 6031.3725, + "latency": 3.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 5258.1868, + "latency": 3.6, + "recall": 0.9768, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2747.3167, + "latency": 3.3, + "recall": 0.9204, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2514.4481, + "latency": 3.2, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2177.2345, + "latency": 3.4, + "recall": 0.9408, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1833.2575, + "latency": 3.9, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1552.4803, + "latency": 4.0, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1355.3121, + "latency": 4.4, + "recall": 0.9602, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1079.2123, + "latency": 5.3, + "recall": 0.9648, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 876.5772, + "latency": 6.3, + "recall": 0.9676, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5973.0024, + "latency": 2.4, + "recall": 0.9192, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5416.5758, + "latency": 2.6, + "recall": 0.9334, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4771.4324, + "latency": 2.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4006.3994, + "latency": 3.2, + "recall": 0.9609, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3441.7597, + "latency": 3.5, + "recall": 0.9682, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3040.6216, + "latency": 3.7, + "recall": 0.9734, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2446.7373, + "latency": 4.3, + "recall": 0.9791, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2084.6245, + "latency": 5.0, + "recall": 0.9819, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 920.9627, + "latency": 3.4, + "recall": 0.9488, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 1985.8124, + "latency": 2.6, + "recall": 0.9407, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 3157.707, + "latency": 2.3, + "recall": 0.9347, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5671.2562, + "latency": 2.1, + "recall": 0.903, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8936.7962, + "latency": 2.0, + "recall": 0.8835, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9664.2855, + "latency": 1.8, + "recall": 0.899, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10276.7451, + "latency": 1.7, + "recall": 0.9159, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10891.7531, + "latency": 1.7, + "recall": 0.9408, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11397.7043, + "latency": 1.6, + "recall": 0.9597, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5436.8907, + "latency": 2.0, + "recall": 0.929, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8945.4041, + "latency": 1.9, + "recall": 0.8894, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9681.5656, + "latency": 1.9, + "recall": 0.9008, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10258.2661, + "latency": 1.7, + "recall": 0.9139, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10671.8925, + "latency": 1.7, + "recall": 0.9339, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11280.0849, + "latency": 1.6, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11520.9234, + "latency": 1.5, + "recall": 0.9634, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11803.1944, + "latency": 1.5, + "recall": 0.9778, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11763.5538, + "latency": 1.5, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 505.7458, + "latency": 20.7, + "recall": 0.9068, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 433.9034, + "latency": 23.1, + "recall": 0.931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 381.7737, + "latency": 25.7, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 342.1123, + "latency": 29.0, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 308.2216, + "latency": 31.3, + "recall": 0.9561, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 257.7928, + "latency": 36.4, + "recall": 0.9626, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 223.8166, + "latency": 42.1, + "recall": 0.9666, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 950.6332, + "latency": 13.2, + "recall": 0.914, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 823.2224, + "latency": 13.5, + "recall": 0.9434, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 743.9815, + "latency": 14.8, + "recall": 0.9583, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 683.1873, + "latency": 15.7, + "recall": 0.9677, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 619.7468, + "latency": 17.2, + "recall": 0.9738, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 537.4082, + "latency": 18.8, + "recall": 0.9809, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 474.9941, + "latency": 20.9, + "recall": 0.9848, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1610.9496, + "latency": 10.8, + "recall": 0.9, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1557.3623, + "latency": 10.8, + "recall": 0.9244, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1473.9256, + "latency": 11.7, + "recall": 0.9484, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1388.5547, + "latency": 12.5, + "recall": 0.9597, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 8.0584, + "latency": 1757.9, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 808.3294, + "latency": 22.2, + "recall": 0.7016, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1053.1495, + "latency": 17.7, + "recall": 0.5673, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 504.9179, + "latency": 272.6, + "recall": 0.4664, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 114.8061, + "latency": 126.6, + "recall": 0.9985, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 210.3227, + "latency": 71.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 353.7862, + "latency": 45.2, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 696.9777, + "latency": 24.6, + "recall": 0.997, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1022.2696, + "latency": 17.9, + "recall": 0.936, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3055.0123, + "latency": 7.2, + "recall": 0.9066, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3013.4439, + "latency": 6.9, + "recall": 0.9268, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2801.7241, + "latency": 7.4, + "recall": 0.9476, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2590.3809, + "latency": 8.6, + "recall": 0.9679, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2291.2159, + "latency": 8.9, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2159.051, + "latency": 9.4, + "recall": 0.801, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2604.4444, + "latency": 7.8, + "recall": 0.63, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2685.6654, + "latency": 7.6, + "recall": 0.4914, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 677.1414, + "latency": 33.5, + "recall": 0.7655, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 942.2296, + "latency": 18.2, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1507.6899, + "latency": 12.8, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2073.2153, + "latency": 11.0, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3014.2483, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3099.4124, + "latency": 6.2, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 13.0379, + "latency": 1063.5, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1301.4102, + "latency": 14.7, + "recall": 0.9011, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1844.8918, + "latency": 10.6, + "recall": 0.9085, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2275.2854, + "latency": 9.1, + "recall": 0.917, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2708.6752, + "latency": 8.4, + "recall": 0.9337, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2782.0274, + "latency": 7.4, + "recall": 0.9466, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2950.717, + "latency": 6.9, + "recall": 0.9558, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2988.4205, + "latency": 7.6, + "recall": 0.9741, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3033.5491, + "latency": 6.4, + "recall": 0.9844, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2251.1274, + "latency": 8.7, + "recall": 0.8848, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2771.2009, + "latency": 7.6, + "recall": 0.889, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2935.9403, + "latency": 6.8, + "recall": 0.8992, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3028.858, + "latency": 6.7, + "recall": 0.9133, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3065.6134, + "latency": 6.2, + "recall": 0.9328, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3064.6288, + "latency": 6.5, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3090.0478, + "latency": 6.4, + "recall": 0.9628, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3086.1957, + "latency": 6.7, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3103.0539, + "latency": 5.6, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1131.3087, + "latency": 14.1, + "recall": 0.9024, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1094.5967, + "latency": 14.3, + "recall": 0.8659, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 617.0881, + "latency": 22.4, + "recall": 0.8844, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 372.2462, + "latency": 35.9, + "recall": 0.8977, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 212.7466, + "latency": 58.7, + "recall": 0.9099, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 101.1774, + "latency": 116.1, + "recall": 0.9241, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 57.8988, + "latency": 200.1, + "recall": 0.9332, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 31.4779, + "latency": 351.0, + "recall": 0.9414, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 583.5009, + "latency": 23.0, + "recall": 0.9668, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1114.952, + "latency": 12.7, + "recall": 0.97, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1146.5286, + "latency": 13.7, + "recall": 0.9262, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1147.1977, + "latency": 13.3, + "recall": 0.8999, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 823.1775, + "latency": 20.5, + "recall": 0.9148, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 492.4887, + "latency": 29.6, + "recall": 0.9269, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 264.9324, + "latency": 49.6, + "recall": 0.936, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 487.8343, + "latency": 25.4, + "recall": 0.9668, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1123.5147, + "latency": 18.5, + "recall": 0.9688, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1140.4099, + "latency": 13.5, + "recall": 0.9716, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1149.1219, + "latency": 10.3, + "recall": 0.9764, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1148.1735, + "latency": 8.9, + "recall": 0.9801, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 446.9116, + "latency": 9.2, + "recall": 0.9357, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 388.3028, + "latency": 9.6, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 323.3964, + "latency": 9.8, + "recall": 0.9507, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 256.4668, + "latency": 11.3, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 145.5316, + "latency": 18.4, + "recall": 0.9726, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1242.428, + "latency": 6.4, + "recall": 0.9474, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1111.3633, + "latency": 7.0, + "recall": 0.955, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 955.4701, + "latency": 7.2, + "recall": 0.9629, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 783.5207, + "latency": 7.7, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 470.8546, + "latency": 9.5, + "recall": 0.9835, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 434.5481, + "latency": 8.5, + "recall": 0.7237, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 262.8314, + "latency": 11.3, + "recall": 0.9808, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 273.4174, + "latency": 13.3, + "recall": 0.9789, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 325.2245, + "latency": 10.3, + "recall": 0.9909, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 358.8949, + "latency": 9.5, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 441.4152, + "latency": 8.3, + "recall": 0.997, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 274.8559, + "latency": 9.9, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 639.3991, + "latency": 7.3, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1202.8677, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1591.7055, + "latency": 7.8, + "recall": 0.8506, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 987.0795, + "latency": 7.1, + "recall": 0.9804, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1059.3394, + "latency": 7.8, + "recall": 0.9856, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1289.5164, + "latency": 6.4, + "recall": 0.9906, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1108.6473, + "latency": 7.4, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1494.5334, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 2997.4391, + "latency": 6.1, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4250.2894, + "latency": 4.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4318.9697, + "latency": 4.3, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 194.8021, + "latency": 559.8, + "recall": 0.86, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.5444, + "latency": 463.9, + "recall": 0.8478, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 179.4203, + "latency": 497.5, + "recall": 0.8273, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1458, + "latency": 480.5, + "recall": 0.8103, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 186.0237, + "latency": 474.0, + "recall": 0.7847, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 190.9747, + "latency": 517.4, + "recall": 0.7398, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 172.95, + "latency": 515.6, + "recall": 0.7004, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 174.3549, + "latency": 496.9, + "recall": 0.6279, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.397, + "latency": 506.9, + "recall": 0.5409, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 187.4268, + "latency": 453.7, + "recall": 0.4692, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.4972, + "latency": 337.1, + "recall": 0.8717, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.1282, + "latency": 269.2, + "recall": 0.8637, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.1405, + "latency": 282.6, + "recall": 0.8492, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.0349, + "latency": 275.3, + "recall": 0.8325, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.599, + "latency": 358.8, + "recall": 0.8085, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.2424, + "latency": 301.7, + "recall": 0.7592, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.5401, + "latency": 282.4, + "recall": 0.7086, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 196.9391, + "latency": 263.4, + "recall": 0.6549, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 197.4455, + "latency": 349.3, + "recall": 0.5314, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1164, + "latency": 345.6, + "recall": 0.4276, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 649.8781, + "latency": 55.2, + "recall": 0.8352, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 351.7114, + "latency": 46.7, + "recall": 0.8735, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 365.2505, + "latency": 34.9, + "recall": 0.8251, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 369.4921, + "latency": 41.6, + "recall": 0.779, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 370.7241, + "latency": 49.6, + "recall": 0.7177, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 382.5332, + "latency": 54.7, + "recall": 0.6135, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 81.8678, + "latency": 105.6, + "recall": 0.8751, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 91.8612, + "latency": 85.7, + "recall": 0.8799, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 96.592, + "latency": 76.9, + "recall": 0.9178, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 100.0554, + "latency": 69.3, + "recall": 0.9638, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 798.328, + "latency": 56.7, + "recall": 0.8993, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 802.6923, + "latency": 48.1, + "recall": 0.935, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 310.957, + "latency": 49.4, + "recall": 0.9698, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 260.4031, + "latency": 48.3, + "recall": 0.9828, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 206.0934, + "latency": 56.7, + "recall": 0.9795, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 184.5363, + "latency": 53.4, + "recall": 0.9681, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 346.5847, + "latency": 42.7, + "recall": 0.9631, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 284.6367, + "latency": 47.6, + "recall": 0.9788, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 323.0238, + "latency": 50.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 471.553, + "latency": 44.1, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7385.2066, + "latency": 2.1, + "recall": 0.9384, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6793.8443, + "latency": 2.2, + "recall": 0.9522, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6242.5346, + "latency": 2.3, + "recall": 0.961, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5779.119, + "latency": 2.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5183.5843, + "latency": 2.4, + "recall": 0.9784, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 4259.5572, + "latency": 2.6, + "recall": 0.9845, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3614.3118, + "latency": 2.9, + "recall": 0.9877, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3181.0537, + "latency": 3.0, + "recall": 0.9892, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2804.8357, + "latency": 3.3, + "recall": 0.9903, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 13316.2336, + "latency": 2.0, + "recall": 0.9383, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 12837.5287, + "latency": 2.1, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 12248.9154, + "latency": 2.2, + "recall": 0.9687, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 11501.6652, + "latency": 2.2, + "recall": 0.9785, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 10566.6823, + "latency": 2.4, + "recall": 0.9838, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9227.318, + "latency": 2.7, + "recall": 0.9893, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8320.4606, + "latency": 2.9, + "recall": 0.9919, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7524.9879, + "latency": 3.2, + "recall": 0.9931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6813.2439, + "latency": 3.5, + "recall": 0.9939, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2950.8165, + "latency": 3.3, + "recall": 0.9147, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2039.8673, + "latency": 3.8, + "recall": 0.9559, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1373.0307, + "latency": 5.7, + "recall": 0.9716, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1454.8382, + "latency": 4.6, + "recall": 0.9659, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1773.0919, + "latency": 5.3, + "recall": 0.9699, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1234.6534, + "latency": 6.4, + "recall": 0.9942, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1826.0672, + "latency": 5.3, + "recall": 0.9938, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2838.356, + "latency": 3.8, + "recall": 0.9946, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3411.0934, + "latency": 3.3, + "recall": 0.995, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8468.4611, + "latency": 3.1, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6860.8577, + "latency": 4.7, + "recall": 0.9226, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5506.1808, + "latency": 5.5, + "recall": 0.9193, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6750.2495, + "latency": 4.4, + "recall": 0.9105, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7589.664, + "latency": 3.8, + "recall": 0.9235, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7610.0519, + "latency": 3.3, + "recall": 0.9903, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8455.2896, + "latency": 4.0, + "recall": 0.9921, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9081.1518, + "latency": 3.0, + "recall": 0.9943, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9773.6593, + "latency": 3.7, + "recall": 0.9955, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 3938.6004, + "latency": 3.7, + "recall": 0.9196, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 6820.6863, + "latency": 3.2, + "recall": 0.9159, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8633.8949, + "latency": 4.1, + "recall": 0.8928, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9220.3627, + "latency": 3.8, + "recall": 0.9081, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9368.1325, + "latency": 3.8, + "recall": 0.9292, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9374.8941, + "latency": 4.2, + "recall": 0.9425, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9289.0118, + "latency": 4.2, + "recall": 0.9574, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9244.1135, + "latency": 4.2, + "recall": 0.9724, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8695.2765, + "latency": 4.3, + "recall": 0.9603, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9048.6431, + "latency": 3.9, + "recall": 0.9216, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9428.4531, + "latency": 2.6, + "recall": 0.9331, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9507.9991, + "latency": 2.8, + "recall": 0.9453, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9861.9686, + "latency": 2.6, + "recall": 0.955, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10041.0338, + "latency": 2.7, + "recall": 0.9693, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10020.5299, + "latency": 2.6, + "recall": 0.9788, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9805.0401, + "latency": 2.6, + "recall": 0.9257, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10557.4373, + "latency": 2.7, + "recall": 0.9393, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10089.4308, + "latency": 2.6, + "recall": 0.9934, + "filter_ratio": 0.999 + } ] \ No newline at end of file From dad3c3d7227d9810fe0172a1f117a46d18ed1a29 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 9 Apr 2026 10:16:52 +0800 Subject: [PATCH 15/38] feat: Upgrade pydantic to v2 (#750) 1. Upgrade pydantic to 2.x 2. Remove results/ from .gitignore, those files need to track 3. fix the coding styles in the results Signed-off-by: yangxuan --- .gitignore | 1 - install/requirements_py3.11.txt | 2 +- pyproject.toml | 2 +- .../backend/clients/alisql/config.py | 4 +- .../backend/clients/alloydb/config.py | 22 ++++----- vectordb_bench/backend/clients/api.py | 21 +++++--- .../backend/clients/aws_opensearch/config.py | 25 +++++----- .../backend/clients/chroma/config.py | 2 +- .../backend/clients/cockroachdb/config.py | 8 +-- .../backend/clients/doris/config.py | 9 ++-- .../backend/clients/lindorm/config.py | 28 +++++------ .../backend/clients/mariadb/config.py | 6 +-- .../backend/clients/milvus/config.py | 25 +++++----- .../backend/clients/oss_opensearch/config.py | 49 ++++++++++--------- .../backend/clients/pgdiskann/config.py | 12 ++--- .../backend/clients/pgvecto_rs/config.py | 6 +-- .../backend/clients/pgvector/config.py | 14 +++--- .../backend/clients/pgvectorscale/config.py | 16 +++--- .../backend/clients/polardb/config.py | 2 +- .../backend/clients/qdrant_cloud/config.py | 19 ++++--- vectordb_bench/backend/clients/tidb/config.py | 21 +++++--- vectordb_bench/backend/dataset.py | 26 +++++----- vectordb_bench/base.py | 5 +- .../components/custom/getCustomConfig.py | 6 ++- .../frontend/config/dbCaseConfigs.py | 6 +-- vectordb_bench/frontend/pages/qps_recall.py | 5 +- vectordb_bench/frontend/pages/results.py | 2 +- vectordb_bench/models.py | 6 +-- vectordb_bench/restful/format_res.py | 4 +- vectordb_bench/results/getLeaderboardData.py | 29 ++++++----- .../results/getLeaderboardDataV2.py | 15 +++--- 31 files changed, 213 insertions(+), 185 deletions(-) diff --git a/.gitignore b/.gitignore index 8985eeb4d..cea1306b0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ build/ venv/ .venv/ .idea/ -results/ logs/ # Worktrees diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 4214267a3..130745816 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -20,7 +20,7 @@ psutil polars plotly environs -pydantic=2.0,<3 scikit-learn pymilvus clickhouse_connect diff --git a/pyproject.toml b/pyproject.toml index 2baeb16e3..e72be9697 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "polars", "plotly", "environs", - "pydantic=2.0,<3", "scikit-learn", "pymilvus", # with pandas, numpy "hdrhistogram>=0.10.1", diff --git a/vectordb_bench/backend/clients/alisql/config.py b/vectordb_bench/backend/clients/alisql/config.py index f942c30f1..16c56e90f 100644 --- a/vectordb_bench/backend/clients/alisql/config.py +++ b/vectordb_bench/backend/clients/alisql/config.py @@ -49,8 +49,8 @@ def parse_metric(self) -> str: class AliSQLHNSWConfig(AliSQLIndexConfig, DBCaseConfig): - M: int | None - ef_search: int | None + M: int | None = None + ef_search: int | None = None index: IndexType = IndexType.HNSW def index_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/alloydb/config.py b/vectordb_bench/backend/clients/alloydb/config.py index d6e54e487..11e65084e 100644 --- a/vectordb_bench/backend/clients/alloydb/config.py +++ b/vectordb_bench/backend/clients/alloydb/config.py @@ -43,8 +43,8 @@ class AlloyDBIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None class AlloyDBSearchParam(TypedDict): @@ -120,15 +120,15 @@ def _optionally_build_set_options( class AlloyDBScaNNConfig(AlloyDBIndexConfig): index: IndexType = IndexType.SCANN - num_leaves: int | None - quantizer: str | None - enable_pca: str | None - max_num_levels: int | None - num_leaves_to_search: int | None - max_top_neighbors_buffer_size: int | None - pre_reordering_num_neighbors: int | None - num_search_threads: int | None - max_num_prefetch_datasets: int | None + num_leaves: int | None = None + quantizer: str | None = None + enable_pca: str | None = None + max_num_levels: int | None = None + num_leaves_to_search: int | None = None + max_top_neighbors_buffer_size: int | None = None + pre_reordering_num_neighbors: int | None = None + num_search_threads: int | None = None + max_num_prefetch_datasets: int | None = None maintenance_work_mem: str | None = None max_parallel_workers: int | None = None diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 5511f18db..0f8103597 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -2,7 +2,7 @@ from contextlib import contextmanager from enum import StrEnum -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, model_validator from vectordb_bench.backend.filter import Filter, FilterOp @@ -90,13 +90,18 @@ def common_long_configs() -> list[str]: def to_dict(self) -> dict: raise NotImplementedError - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in cls.common_short_configs() or field.name in cls.common_long_configs(): - return v - if not v and isinstance(v, str | SecretStr): - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class DBCaseConfig(ABC): diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 5ab63010d..7742d421d 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,7 +1,7 @@ import logging from enum import Enum -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -32,17 +32,18 @@ def to_dict(self) -> dict: "timeout": 600, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password", "host"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class AWSOS_Engine(Enum): diff --git a/vectordb_bench/backend/clients/chroma/config.py b/vectordb_bench/backend/clients/chroma/config.py index cd3e01ecc..a1d9903d1 100644 --- a/vectordb_bench/backend/clients/chroma/config.py +++ b/vectordb_bench/backend/clients/chroma/config.py @@ -6,7 +6,7 @@ class ChromaConfig(DBConfig): user: str | None = None - password: SecretStr | None + password: SecretStr | None = None host: SecretStr = "localhost" port: int = 8000 diff --git a/vectordb_bench/backend/clients/cockroachdb/config.py b/vectordb_bench/backend/clients/cockroachdb/config.py index 0d608da8f..88ec0e5ea 100644 --- a/vectordb_bench/backend/clients/cockroachdb/config.py +++ b/vectordb_bench/backend/clients/cockroachdb/config.py @@ -75,16 +75,16 @@ class CockroachDBIndexParam(TypedDict): metric: str index_creation_with_options: Sequence[dict[str, Any]] - min_partition_size: int | None - max_partition_size: int | None - build_beam_size: int | None + min_partition_size: int | None = None + max_partition_size: int | None = None + build_beam_size: int | None = None class CockroachDBSearchParam(TypedDict): """Search parameters for CockroachDB vector queries.""" metric_fun_op: LiteralString - vector_search_beam_size: int | None + vector_search_beam_size: int | None = None class CockroachDBSessionCommands(TypedDict): diff --git a/vectordb_bench/backend/clients/doris/config.py b/vectordb_bench/backend/clients/doris/config.py index a15309922..7c79ba728 100644 --- a/vectordb_bench/backend/clients/doris/config.py +++ b/vectordb_bench/backend/clients/doris/config.py @@ -1,6 +1,6 @@ import logging -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -17,9 +17,10 @@ class DorisConfig(DBConfig): db_name: str = "test" ssl: bool = False - @validator("*") - def not_empty_field(cls, v: any, field: any): - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + return data def to_dict(self) -> dict: pwd_str = self.password.get_secret_value() diff --git a/vectordb_bench/backend/clients/lindorm/config.py b/vectordb_bench/backend/clients/lindorm/config.py index 367f369e3..0e0d4aae6 100644 --- a/vectordb_bench/backend/clients/lindorm/config.py +++ b/vectordb_bench/backend/clients/lindorm/config.py @@ -43,9 +43,9 @@ def parse_metric(self) -> str: class HNSWConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.HNSW - M: int | None - efConstruction: int | None - efSearch: int | None + M: int | None = None + efConstruction: int | None = None + efSearch: int | None = None filter_type: str | None = "efficient_filter" k_expand_scope: int | None = 1000 @@ -72,12 +72,12 @@ def search_param(self, do_filter: bool = False) -> dict: # first layer searching for cluster centroids is hnsw class IVFPQConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.IVFPQ - nlist: int | None - nprobe: int | None + nlist: int | None = None + nprobe: int | None = None # search parameters - centroids_hnsw_M: int | None - centroids_hnsw_efConstruction: int | None - centroids_hnsw_efSearch: int | None + centroids_hnsw_M: int | None = None + centroids_hnsw_efConstruction: int | None = None + centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" reorder_factor: int | None = 10 @@ -116,13 +116,13 @@ def search_param(self, do_filter: bool = False) -> dict: class IVFBQConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.IVFBQ - nlist: int | None - exbits: int | None - nprobe: int | None + nlist: int | None = None + exbits: int | None = None + nprobe: int | None = None # search parameters - centroids_hnsw_M: int | None - centroids_hnsw_efConstruction: int | None - centroids_hnsw_efSearch: int | None + centroids_hnsw_M: int | None = None + centroids_hnsw_efConstruction: int | None = None + centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" reorder_factor: int | None = 10 diff --git a/vectordb_bench/backend/clients/mariadb/config.py b/vectordb_bench/backend/clients/mariadb/config.py index d183adc76..21ea9ac2e 100644 --- a/vectordb_bench/backend/clients/mariadb/config.py +++ b/vectordb_bench/backend/clients/mariadb/config.py @@ -46,11 +46,11 @@ def parse_metric(self) -> str: class MariaDBHNSWConfig(MariaDBIndexConfig, DBCaseConfig): - M: int | None - ef_search: int | None + M: int | None = None + ef_search: int | None = None index: IndexType = IndexType.HNSW storage_engine: str = "InnoDB" - max_cache_size: int | None + max_cache_size: int | None = None def index_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 9ffbdcece..98118c6df 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType @@ -19,17 +19,18 @@ def to_dict(self) -> dict: "replica_number": self.replica_number, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class MilvusIndexConfig(BaseModel): diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py index 83fed3d58..a5d69459a 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/config.py +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -1,7 +1,7 @@ import logging from enum import Enum -from pydantic import BaseModel, SecretStr, root_validator, validator +from pydantic import BaseModel, SecretStr, field_validator, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -32,17 +32,18 @@ def to_dict(self) -> dict: "timeout": 600, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password", "host"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class OSSOS_Engine(Enum): @@ -111,7 +112,8 @@ class OSSOpenSearchIndexConfig(BaseModel, DBCaseConfig): compression_level: str = CompressionLevel.LEVEL_32X oversample_factor: float = 1.0 - @validator("quantization_type", pre=True, always=True) + @field_validator("quantization_type", mode="before") + @classmethod def validate_quantization_type(cls, value: any): """Convert string values to enum""" if not value: @@ -128,19 +130,22 @@ def validate_quantization_type(cls, value: any): return mapping.get(value, OSSOpenSearchQuantization.NONE) - @root_validator - def validate_engine_name(cls, values: dict): - """Map engine_name string from UI to engine enum""" - if values.get("engine_name"): - engine_name = values["engine_name"].lower() + @model_validator(mode="before") + @classmethod + def validate_engine_name(cls, data: any) -> any: + if not isinstance(data, dict): + return data + # Map engine_name to engine enum + if data.get("engine_name"): + engine_name = data["engine_name"].lower() if engine_name == "faiss": - values["engine"] = OSSOS_Engine.faiss + data["engine"] = OSSOS_Engine.faiss elif engine_name == "lucene": - values["engine"] = OSSOS_Engine.lucene + data["engine"] = OSSOS_Engine.lucene else: log.warning(f"Unknown engine_name: {engine_name}, defaulting to faiss") - values["engine"] = OSSOS_Engine.faiss - return values + data["engine"] = OSSOS_Engine.faiss + return data def __eq__(self, obj: any): return ( diff --git a/vectordb_bench/backend/clients/pgdiskann/config.py b/vectordb_bench/backend/clients/pgdiskann/config.py index 7f83a05c8..8715b1e42 100644 --- a/vectordb_bench/backend/clients/pgdiskann/config.py +++ b/vectordb_bench/backend/clients/pgdiskann/config.py @@ -43,8 +43,8 @@ class PgDiskANNIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None create_index_before_load: bool = False create_index_after_load: bool = True - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -120,10 +120,10 @@ def _optionally_build_set_options( class PgDiskANNImplConfig(PgDiskANNIndexConfig): index: IndexType = IndexType.DISKANN - max_neighbors: int | None - l_value_ib: int | None - pq_param_num_chunks: int | None - l_value_is: float | None + max_neighbors: int | None = None + l_value_ib: int | None = None + pq_param_num_chunks: int | None = None + l_value_is: float | None = None reranking: bool | None = None reranking_metric: str | None = None quantized_fetch_limit: int | None = None diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py index fbb7c5d81..73c2573a0 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/config.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -78,7 +78,7 @@ def session_param(self) -> dict[str, str | int]: ... class PgVectoRSHNSWConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.HNSW m: int | None = None - ef_search: int | None + ef_search: int | None = None ef_construction: int | None = None def index_param(self) -> dict[str, str]: @@ -106,8 +106,8 @@ def session_param(self) -> dict[str, str | int]: class PgVectoRSIVFFlatConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.IVFFlat - probes: int | None - lists: int | None + probes: int | None = None + lists: int | None = None def index_param(self) -> dict[str, str]: if self.quantization_type is None: diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 98e82f1c2..7da238a4b 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -47,8 +47,8 @@ class PgVectorIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None class PgVectorSearchParam(TypedDict): @@ -175,13 +175,13 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): a good place to start is sqrt(lists) """ - lists: int | None - probes: int | None + lists: int | None = None + probes: int | None = None index: IndexType = IndexType.ES_IVFFlat maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None - table_quantization_type: str | None + table_quantization_type: str | None = None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None @@ -226,12 +226,12 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): m: int | None # DETAIL: Valid values are between "2" and "100". ef_construction: int | None # ef_construction must be greater than or equal to 2 * m - ef_search: int | None + ef_search: int | None = None index: IndexType = IndexType.ES_HNSW maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None - table_quantization_type: str | None + table_quantization_type: str | None = None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None diff --git a/vectordb_bench/backend/clients/pgvectorscale/config.py b/vectordb_bench/backend/clients/pgvectorscale/config.py index e22c45c8d..07750cffb 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/config.py +++ b/vectordb_bench/backend/clients/pgvectorscale/config.py @@ -70,14 +70,14 @@ def session_param(self) -> dict: ... class PgVectorScaleStreamingDiskANNConfig(PgVectorScaleIndexConfig): index: IndexType = IndexType.STREAMING_DISKANN - storage_layout: str | None - num_neighbors: int | None - search_list_size: int | None - max_alpha: float | None - num_dimensions: int | None - num_bits_per_dimension: int | None - query_search_list_size: int | None - query_rescore: int | None + storage_layout: str | None = None + num_neighbors: int | None = None + search_list_size: int | None = None + max_alpha: float | None = None + num_dimensions: int | None = None + num_bits_per_dimension: int | None = None + query_search_list_size: int | None = None + query_rescore: int | None = None def index_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/polardb/config.py b/vectordb_bench/backend/clients/polardb/config.py index c75448c49..5121c1e19 100644 --- a/vectordb_bench/backend/clients/polardb/config.py +++ b/vectordb_bench/backend/clients/polardb/config.py @@ -11,7 +11,7 @@ class PolarDBConfigDict(TypedDict): host: str port: int database: str - unix_socket: str | None + unix_socket: str | None = None class PolarDBConfig(DBConfig): diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index b2eeb2ce6..06543aaab 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,6 +1,6 @@ from typing import TypeVar -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -25,11 +25,18 @@ def to_dict(self) -> dict: "url": self.url.get_secret_value(), } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["api_key"]: - return v - return super().not_empty_field(v, field) + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"api_key"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class QdrantIndexConfig(BaseModel, DBCaseConfig): diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 71fdbad66..93098ede1 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,6 +1,6 @@ from typing import TypedDict -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -35,13 +35,18 @@ def to_dict(self) -> TiDBConfigDict: "ssl_verify_identity": self.ssl, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["password", "db_label"]: - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"password"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class TiDBIndexConfig(BaseModel, DBCaseConfig): diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index d1de9e328..94216532f 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -7,12 +7,12 @@ import logging import pathlib from enum import Enum -from typing import Any, NamedTuple +from typing import Any, ClassVar, NamedTuple import pandas as pd import polars as pl from pyarrow.parquet import ParquetFile -from pydantic import PrivateAttr, validator +from pydantic import field_validator from vectordb_bench import config from vectordb_bench.base import BaseModel @@ -38,7 +38,7 @@ class BaseDataset(BaseModel): metric_type: MetricType use_shuffled: bool with_gt: bool = False - _size_label: dict[int, SizeLabel] = PrivateAttr() + _size_label: ClassVar[dict[int, SizeLabel]] is_custom: bool = False with_remote_resource: bool = True # for label filter cases @@ -57,7 +57,8 @@ class BaseDataset(BaseModel): gt_id_field: str = "id" gt_neighbors_field: str = "neighbors_id" - @validator("size") + @field_validator("size") + @classmethod def verify_size(cls, v: int): if v not in cls._size_label: msg = f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}" @@ -102,7 +103,8 @@ class CustomDataset(BaseDataset): scalar_labels_file: str = "scalar_labels.parquet" label_percentages: list[float] = [] - @validator("size") + @field_validator("size") + @classmethod def verify_size(cls, v: int): return v @@ -136,7 +138,7 @@ class LAION(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -146,7 +148,7 @@ class GIST(BaseDataset): dim: int = 960 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), } @@ -158,7 +160,7 @@ class Cohere(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), @@ -196,7 +198,7 @@ class Bioasq(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } @@ -232,7 +234,7 @@ class Glove(BaseDataset): dim: int = 200 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = False - _size_label: dict = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} + _size_label: ClassVar[dict] = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} class SIFT(BaseDataset): @@ -240,7 +242,7 @@ class SIFT(BaseDataset): dim: int = 128 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = { + _size_label: ClassVar[dict] = { 500_000: SizeLabel( 500_000, "SMALL", @@ -257,7 +259,7 @@ class OpenAI(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), 5_000_000: SizeLabel(5_000_000, "LARGE", 10), diff --git a/vectordb_bench/base.py b/vectordb_bench/base.py index 502d5fa49..401d2086d 100644 --- a/vectordb_bench/base.py +++ b/vectordb_bench/base.py @@ -1,5 +1,6 @@ from pydantic import BaseModel as PydanticBaseModel +from pydantic import ConfigDict -class BaseModel(PydanticBaseModel, arbitrary_types_allowed=True): - pass +class BaseModel(PydanticBaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/vectordb_bench/frontend/components/custom/getCustomConfig.py b/vectordb_bench/frontend/components/custom/getCustomConfig.py index a1ddfb737..03831a83c 100644 --- a/vectordb_bench/frontend/components/custom/getCustomConfig.py +++ b/vectordb_bench/frontend/components/custom/getCustomConfig.py @@ -62,14 +62,16 @@ def get_custom_streaming_configs(): def save_custom_configs(custom_configs: list[CustomDatasetConfig]): with open(config.CUSTOM_CONFIG_DIR, "w") as f: - json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4) + json.dump([custom_config.model_dump() for custom_config in custom_configs], f, indent=4) def save_all_custom_configs( performance_configs: list[CustomCaseConfig], streaming_configs: list[CustomStreamingCaseConfig] ): """Save both performance and streaming configs to the same JSON file""" - all_configs = [config.dict() for config in performance_configs] + [config.dict() for config in streaming_configs] + all_configs = [config.model_dump() for config in performance_configs] + [ + config.model_dump() for config in streaming_configs + ] with open(config.CUSTOM_CONFIG_DIR, "w") as f: json.dump(all_configs, f, indent=4) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 387f7fb4a..d15c4e7ee 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -119,7 +119,7 @@ def get_custom_case_items() -> list[UICaseItem]: CaseConfig( case_id=CaseType.PerformanceCustomDataset, custom_case={ - **custom_config.dict(), + **custom_config.model_dump(), "use_filter": False, }, ) @@ -140,7 +140,7 @@ def get_custom_case_items() -> list[UICaseItem]: CaseConfig( case_id=CaseType.PerformanceCustomDataset, custom_case={ - **custom_config.dict(), + **custom_config.model_dump(), "use_filter": True, "label_percentage": label_percentage, }, @@ -174,7 +174,7 @@ def get_custom_streaming_case_items() -> list[UICaseItem]: case_id=CaseType.StreamingCustomDataset, custom_case={ "description": custom_config.description, - "dataset_config": custom_config.dataset_config.dict(), + "dataset_config": custom_config.dataset_config.model_dump(), }, ) ], diff --git a/vectordb_bench/frontend/pages/qps_recall.py b/vectordb_bench/frontend/pages/qps_recall.py index 27f9c4691..fb8f680c5 100644 --- a/vectordb_bench/frontend/pages/qps_recall.py +++ b/vectordb_bench/frontend/pages/qps_recall.py @@ -43,7 +43,10 @@ def case_results_filter(case_result: CaseResult) -> bool: case = case_result.task_config.case_config.case return case.label == CaseLabel.Performance and case.filters.type == FilterOp.NonFilter - default_selected_task_labels = ["standard_2025"] + default_selected_task_labels = ["standard_20260403", "standard_20250519"] + # Filter defaults to only include labels that exist in results + available_labels = {r.task_label for r in allResults} + default_selected_task_labels = [l for l in default_selected_task_labels if l in available_labels] shownData, failedTasks, showCaseNames = getshownData( resultSelectorContainer, allResults, diff --git a/vectordb_bench/frontend/pages/results.py b/vectordb_bench/frontend/pages/results.py index a146f2fdc..216029bb1 100644 --- a/vectordb_bench/frontend/pages/results.py +++ b/vectordb_bench/frontend/pages/results.py @@ -32,7 +32,7 @@ def main(): st.caption( "Choose your desired test results to display from the sidebar. " "For your reference, we've included two standard benchmarks tested by our team. " - "Note that `standard_2025` was tested in 2025; the others in 2023. " + "Note that `standard_20260403` is the latest benchmark; the others were tested in 2023-2025. " "Unless explicitly labeled as distributed multi-node, test with single-node mode by default." ) st.caption("We welcome community contributions for better results, parameter configurations, and optimizations.") diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index cdc64b9d7..a7e7c09f1 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -205,7 +205,7 @@ def k(self, value): ''' def __hash__(self) -> int: - return hash(self.json()) + return hash(self.model_dump_json()) @property def case(self) -> Case: @@ -314,7 +314,7 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"write results to disk {result_file}") with pathlib.Path(result_file).open("w") as f: - b = partial.json(exclude={"db_config": {"password", "api_key"}}) + b = partial.model_dump_json(exclude={"db_config": {"password", "api_key"}}) f.write(b) def get_case_config(case_config: CaseConfig) -> dict[CaseConfig]: @@ -381,7 +381,7 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: else: # Default to 0 for older result files that don't have P95 data case_result["metrics"]["serial_latency_p95"] = 0.0 - return TestResult.validate(test_result) + return TestResult.model_validate(test_result) def display(self, dbs: list[DB] | None = None): filter_list = dbs if dbs and isinstance(dbs, list) else None diff --git a/vectordb_bench/restful/format_res.py b/vectordb_bench/restful/format_res.py index 2e289ec3b..326986319 100644 --- a/vectordb_bench/restful/format_res.py +++ b/vectordb_bench/restful/format_res.py @@ -63,7 +63,7 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict db_label=task_config.db_config.db_label, version=task_config.db_config.version, note=task_config.db_config.note, - params=task_config.db_case_config.dict(), + params=task_config.db_case_config.model_dump(), case_name=case.name, dataset=dataset.full_name, dim=dataset.dim, @@ -71,6 +71,6 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict filter_rate=filter_.filter_rate, k=task_config.case_config.k, **metrics, - ).dict() + ).model_dump() ) return results diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index aef024bdc..1650a6f87 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -1,14 +1,16 @@ -from vectordb_bench import config -import ujson import pathlib +from datetime import datetime + +import ujson + +from vectordb_bench import config from vectordb_bench.backend.cases import CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.frontend.config.dbPrices import DB_DBLABEL_TO_PRICE from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import ResultLabel, TestResult -from datetime import datetime -taskLabelToCode = { +task_label_to_code = { ResultLabel.FAILED: -1, ResultLabel.OUTOFRANGE: -2, ResultLabel.NORMAL: 1, @@ -18,15 +20,14 @@ def format_time(ts: float) -> str: default_standard_test_time = datetime(2023, 8, 1) t = datetime.fromtimestamp(ts) - if t < default_standard_test_time: - t = default_standard_test_time + t = max(t, default_standard_test_time) return t.strftime("%Y-%m") def main(): - allResults: list[TestResult] = benchMarkRunner.get_results() + all_results: list[TestResult] = benchMarkRunner.get_results() - if allResults is not None: + if all_results is not None: data = [ { "db": d.task_config.db.value, @@ -36,18 +37,16 @@ def main(): "qps": d.metrics.qps, "latency": d.metrics.serial_latency_p99, "recall": d.metrics.recall, - "label": taskLabelToCode[d.label], + "label": task_label_to_code[d.label], "note": d.task_config.db_config.note, "version": d.task_config.db_config.version, "test_time": format_time(test_result.timestamp), } - for test_result in allResults + for test_result in all_results if "standard" in test_result.task_label for d in test_result.results - if d.task_config.case_config.case_id != CaseType.CapacityDim128 - and d.task_config.case_config.case_id != CaseType.CapacityDim960 - if d.task_config.db != DB.ZillizCloud - or test_result.timestamp >= datetime(2024, 1, 1).timestamp() + if d.task_config.case_config.case_id not in {CaseType.CapacityDim128, CaseType.CapacityDim960} + if d.task_config.db != DB.ZillizCloud or test_result.timestamp >= datetime(2024, 1, 1).timestamp() ] # compute qp$ @@ -58,7 +57,7 @@ def main(): price = DB_DBLABEL_TO_PRICE.get(db, {}).get(db_label, 0) d["qp$"] = (qps / price * 3600) if price > 0 else 0.0 - with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json"), "w") as f: + with pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json").open("w") as f: ujson.dump(data, f) diff --git a/vectordb_bench/results/getLeaderboardDataV2.py b/vectordb_bench/results/getLeaderboardDataV2.py index 62440886f..188d9876f 100644 --- a/vectordb_bench/results/getLeaderboardDataV2.py +++ b/vectordb_bench/results/getLeaderboardDataV2.py @@ -1,17 +1,14 @@ import json import logging +import pathlib - +from vectordb_bench import config from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase -from vectordb_bench.backend.clients import DB +from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseResult -from vectordb_bench import config -import numpy as np logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s") -from vectordb_bench.interface import BenchMarkRunner - def get_standard_2025_results() -> list[CaseResult]: all_results = BenchMarkRunner.get_results() @@ -23,7 +20,7 @@ def get_standard_2025_results() -> list[CaseResult]: def save_to_json(data: list[dict], file_name: str): - with open(file_name, "w") as f: + with pathlib.Path(file_name).open("w") as f: json.dump(data, f, indent=4) @@ -56,11 +53,11 @@ def main(): } ) else: - case: StreamingPerformanceCase = case + streaming_case: StreamingPerformanceCase = case # use 90p search stage results to represent streaming performance qps_90p = metrics.st_max_qps_list_list[metrics.st_search_stage_list.index(90)] latency_90p = metrics.st_serial_latency_p99_list[metrics.st_search_stage_list.index(90)] - insert_rate = case.insert_rate + insert_rate = streaming_case.insert_rate streaming_data.append( { "dataset": dataset, From cf09d634edafcd0379a707b72873b7a8879e30dc Mon Sep 17 00:00:00 2001 From: ChenLiqing Date: Thu, 9 Apr 2026 11:01:38 +0800 Subject: [PATCH 16/38] fix: fill missing build durations in Milvus and ZillizCloud results (#751) Populate insert_duration, optimize_duration, load_duration for all entries in result_20260403 files. Previously only the first entry per index had values while the rest were 0.0. Milvus (re-measured on 2.6-opt-v2): - 1M SQ4U: insert=129.8s, optimize=152.2s, load=282.0s - 1M SQ8: insert=119.5s, optimize=235.9s, load=355.4s - 10M SQ4U/SQ8: copied from existing first-entry values ZillizCloud (from prior build runs): - 1M: insert=246.7s, optimize=101.2s, load=347.9s - 10M: insert=2450.8s, optimize=136.9s, load=2587.8s Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.6 (1M context) --- .../result_20260403_standard_milvus.json | 180 +++++++++--------- .../result_20260403_standard_zillizcloud.json | 108 +++++------ 2 files changed, 144 insertions(+), 144 deletions(-) diff --git a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json index e10404530..56219eb1d 100644 --- a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json +++ b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json @@ -134,9 +134,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 3628.8527, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0024, @@ -261,9 +261,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 3250.1112, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0025, @@ -388,9 +388,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2762.4144, "serial_latency_p99": 0.0031, "serial_latency_p95": 0.0029, @@ -515,9 +515,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2384.6245, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -642,9 +642,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2134.1717, "serial_latency_p99": 0.0038, "serial_latency_p95": 0.0036, @@ -769,9 +769,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 1641.3478, "serial_latency_p99": 0.0041, "serial_latency_p95": 0.0039, @@ -896,9 +896,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 1488.5841, "serial_latency_p99": 0.0047, "serial_latency_p95": 0.0043, @@ -1152,9 +1152,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 2514.4481, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -1279,9 +1279,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 2177.2345, "serial_latency_p99": 0.0034, "serial_latency_p95": 0.0031, @@ -1406,9 +1406,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1833.2575, "serial_latency_p99": 0.0039, "serial_latency_p95": 0.0035, @@ -1533,9 +1533,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1552.4803, "serial_latency_p99": 0.004, "serial_latency_p95": 0.0037, @@ -1660,9 +1660,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1355.3121, "serial_latency_p99": 0.0044, "serial_latency_p95": 0.0042, @@ -1787,9 +1787,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1079.2123, "serial_latency_p99": 0.0053, "serial_latency_p95": 0.0049, @@ -1914,9 +1914,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 876.5772, "serial_latency_p99": 0.0063, "serial_latency_p95": 0.0059, @@ -2041,9 +2041,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 10663.1231, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0018, @@ -2168,9 +2168,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 10333.9072, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -2295,9 +2295,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 9575.6863, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0021, @@ -2422,9 +2422,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 8596.7694, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0022, @@ -2549,9 +2549,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 7704.3625, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0025, @@ -2676,9 +2676,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 7023.6735, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0028, @@ -2803,9 +2803,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 6031.3725, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.003, @@ -2930,9 +2930,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 5258.1868, "serial_latency_p99": 0.0036, "serial_latency_p95": 0.0033, @@ -3057,9 +3057,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 5973.0024, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -3184,9 +3184,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 5416.5758, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0024, @@ -3311,9 +3311,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 4771.4324, "serial_latency_p99": 0.0028, "serial_latency_p95": 0.0025, @@ -3438,9 +3438,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 4006.3994, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -3565,9 +3565,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 3441.7597, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0032, @@ -3692,9 +3692,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 3040.6216, "serial_latency_p99": 0.0037, "serial_latency_p95": 0.0035, @@ -3819,9 +3819,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 2446.7373, "serial_latency_p99": 0.0043, "serial_latency_p95": 0.004, @@ -3946,9 +3946,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 2084.6245, "serial_latency_p99": 0.005, "serial_latency_p95": 0.0046, diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index e02463ebd..fe81ffe05 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,9 +5,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 13316.2336, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -124,9 +124,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 12837.5287, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -243,9 +243,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 12248.9154, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -362,9 +362,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 11501.6652, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0022, @@ -481,9 +481,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 10566.6823, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -600,9 +600,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 9227.318, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0026, @@ -719,9 +719,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 8320.4606, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -838,9 +838,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 7524.9879, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.0031, @@ -957,9 +957,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 6813.2439, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0034, @@ -1076,9 +1076,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 7385.2066, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -1195,9 +1195,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 6793.8443, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -1314,9 +1314,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 6242.5346, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0022, @@ -1433,9 +1433,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 5779.119, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0023, @@ -1552,9 +1552,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 5183.5843, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -1671,9 +1671,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 4259.5572, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0026, @@ -1790,9 +1790,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 3614.3118, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -1909,9 +1909,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 3181.0537, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0029, @@ -2028,9 +2028,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 2804.8357, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.0032, From 268e7ab3d8e282af3f422c772f1db81f12c3d16b Mon Sep 17 00:00:00 2001 From: Alexander Guzhva Date: Thu, 9 Apr 2026 07:03:18 +0000 Subject: [PATCH 17/38] Introduce Intel SVS (#749) Signed-off-by: Alexandr Guzhva --- vectordb_bench/backend/clients/api.py | 3 + vectordb_bench/backend/clients/milvus/cli.py | 163 ++++++++++++++++++ .../backend/clients/milvus/config.py | 62 +++++++ 3 files changed, 228 insertions(+) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 0f8103597..ecbddef39 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -43,6 +43,9 @@ class IndexType(StrEnum): GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" SCANN_MILVUS = "SCANN_MILVUS" + SVS_VAMANA = "SVS_VAMANA" + SVS_VAMANA_LVQ = "SVS_VAMANA_LVQ" + SVS_VAMANA_LEANVEC = "SVS_VAMANA_LEANVEC" Hologres_HGraph = "HGraph" Hologres_Graph = "Graph" NONE = "NONE" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 2f2a286be..ae7269801 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -485,6 +485,169 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): ) +class MilvusSVSVamanaTypedDict(CommonTypedDict, MilvusTypedDict): + svs_graph_max_degree: Annotated[ + int, + click.option( + "--svs-graph-max-degree", + type=int, + help="Maximum degree of the Vamana graph (4-256).", + required=True, + ), + ] + svs_construction_window_size: Annotated[ + int, + click.option( + "--svs-construction-window-size", + type=int, + help="Window size for graph construction.", + required=False, + default=40, + show_default=True, + ), + ] + svs_alpha: Annotated[ + float | None, + click.option( + "--svs-alpha", + type=float, + help="Pruning parameter (default: 1.2 for L2, 0.95 for IP/COSINE).", + required=False, + default=None, + ), + ] + svs_storage_kind: Annotated[ + str, + click.option( + "--svs-storage-kind", + type=click.Choice( + ["fp32", "fp16", "sqi8", "lvq4x0", "lvq4x4", "lvq4x8", "leanvec4x4", "leanvec4x8", "leanvec8x8"], + case_sensitive=False, + ), + help="Data storage format.", + required=False, + default="fp32", + show_default=True, + ), + ] + svs_search_window_size: Annotated[ + int | None, + click.option( + "--svs-search-window-size", + type=int, + help="Window size for search (1-10000).", + required=False, + default=None, + ), + ] + svs_search_buffer_capacity: Annotated[ + int | None, + click.option( + "--svs-search-buffer-capacity", + type=int, + help="Buffer capacity for search priority queue (1-10000).", + required=False, + default=None, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) +def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): + from .config import MilvusConfig, SVSVamanaConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) +def MilvusSVSVamanaLVQ(**parameters: Unpack[MilvusSVSVamanaTypedDict]): + from .config import MilvusConfig, SVSVamanaLVQConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaLVQConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + **parameters, + ) + + +class MilvusSVSVamanaLeanVecTypedDict(MilvusSVSVamanaTypedDict): + svs_leanvec_dim: Annotated[ + int, + click.option( + "--svs-leanvec-dim", + type=int, + help="Dimensionality for LeanVec compression (0 = d/2).", + required=False, + default=0, + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaLeanVecTypedDict) +def MilvusSVSVamanaLeanVec(**parameters: Unpack[MilvusSVSVamanaLeanVecTypedDict]): + from .config import MilvusConfig, SVSVamanaLeanVecConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaLeanVecConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + svs_leanvec_dim=parameters["svs_leanvec_dim"], + ), + **parameters, + ) + + class MilvusGPUIVFPQTypedDict( CommonTypedDict, MilvusTypedDict, diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 98118c6df..620a6b484 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -442,6 +442,65 @@ def search_param(self) -> dict: } +class SVSVamanaConfig(MilvusIndexConfig, DBCaseConfig): + svs_graph_max_degree: int + svs_construction_window_size: int = 40 + svs_alpha: float | None = None + svs_storage_kind: str = "fp32" + svs_search_window_size: int | None = None + svs_search_buffer_capacity: int | None = None + index: IndexType = IndexType.SVS_VAMANA + + def index_param(self) -> dict: + params = { + "svs_graph_max_degree": self.svs_graph_max_degree, + "svs_construction_window_size": self.svs_construction_window_size, + "svs_storage_kind": self.svs_storage_kind, + } + if self.svs_alpha is not None: + params["svs_alpha"] = self.svs_alpha + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": params, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": { + "svs_search_window_size": self.svs_search_window_size, + "svs_search_buffer_capacity": self.svs_search_buffer_capacity, + }, + } + + +class SVSVamanaLVQConfig(SVSVamanaConfig): + svs_storage_kind: str = "lvq4x4" + index: IndexType = IndexType.SVS_VAMANA_LVQ + + +class SVSVamanaLeanVecConfig(SVSVamanaConfig): + svs_storage_kind: str = "leanvec4x4" + svs_leanvec_dim: int = 0 + index: IndexType = IndexType.SVS_VAMANA_LEANVEC + + def index_param(self) -> dict: + params = { + "svs_graph_max_degree": self.svs_graph_max_degree, + "svs_construction_window_size": self.svs_construction_window_size, + "svs_storage_kind": self.svs_storage_kind, + "svs_leanvec_dim": self.svs_leanvec_dim, + } + if self.svs_alpha is not None: + params["svs_alpha"] = self.svs_alpha + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": params, + } + + _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, IndexType.HNSW: HNSWConfig, @@ -459,4 +518,7 @@ def search_param(self) -> dict: IndexType.GPU_CAGRA: GPUCAGRAConfig, IndexType.GPU_BRUTE_FORCE: GPUBruteForceConfig, IndexType.SCANN_MILVUS: SCANNConfig, + IndexType.SVS_VAMANA: SVSVamanaConfig, + IndexType.SVS_VAMANA_LVQ: SVSVamanaLVQConfig, + IndexType.SVS_VAMANA_LEANVEC: SVSVamanaLeanVecConfig, } From 619ce1bb82bec5aa05956f45ec0c70ba59c8366c Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 9 Apr 2026 15:37:04 +0800 Subject: [PATCH 18/38] fix: Skip compaction when encouters permission error (#753) Signed-off-by: yangxuan --- vectordb_bench/backend/clients/milvus/milvus.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 9e9dfb7f9..d36a15c24 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -165,23 +165,27 @@ def _optimize(self): log.info(f"{self.name} optimizing before search") try: self.client.flush(self.collection_name) - self._wait_for_segments_sorted() - self._wait_for_index() + if self.case_config.is_gpu_index: log.debug("skip force merge compaction for gpu index type.") else: try: + # wait for sort, index, compact + self._wait_for_segments_sorted() + self._wait_for_index() compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) if compaction_id > 0: self._wait_for_compaction(compaction_id) log.info(f"{self.name} force merge compaction completed.") - self._wait_for_index() except Exception as e: - log.warning(f"{self.name} compact error: {e}") + log.warning(f"{self.name} compact or list segments error: {e}") if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": - log.warning("Skip compact due to permission denied.") + log.warning("Skip compact due to list segments or compact permission denied.") else: raise e from None + + # wait for index no matter what + self._wait_for_index() self.client.refresh_load(self.collection_name) except Exception as e: log.warning(f"{self.name} optimize error: {e}") From f0a8d031ecd3b9692617a7f20a13f83d8a742882 Mon Sep 17 00:00:00 2001 From: ChenLiqing Date: Tue, 14 Apr 2026 10:55:36 +0800 Subject: [PATCH 19/38] fix: refresh ZillizCloud build durations for 1M and 10M baselines (#754) Update result_20260403_standard_zillizcloud.json to use the latest validated build timings from recent reruns for case_id=5 (1M) and case_id=4 (10M), including insert_duration, optimize_duration, and load_duration. Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- .../result_20260403_standard_zillizcloud.json | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index fe81ffe05..1f18a729a 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,9 +5,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 13316.2336, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -124,9 +124,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 12837.5287, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -243,9 +243,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 12248.9154, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -362,9 +362,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 11501.6652, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0022, @@ -481,9 +481,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 10566.6823, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -600,9 +600,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 9227.318, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0026, @@ -719,9 +719,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 8320.4606, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -838,9 +838,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 7524.9879, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.0031, @@ -957,9 +957,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 6813.2439, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0034, @@ -1076,9 +1076,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 7385.2066, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -1195,9 +1195,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 6793.8443, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -1314,9 +1314,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 6242.5346, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0022, @@ -1433,9 +1433,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 5779.119, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0023, @@ -1552,9 +1552,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 5183.5843, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -1671,9 +1671,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 4259.5572, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0026, @@ -1790,9 +1790,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 3614.3118, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -1909,9 +1909,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 3181.0537, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0029, @@ -2028,9 +2028,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 2804.8357, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.0032, From 77d76ab9d7c612737c5e22bcf2698ccb8cc48893 Mon Sep 17 00:00:00 2001 From: SongYoungUk Date: Tue, 14 Apr 2026 18:24:22 +0900 Subject: [PATCH 20/38] feat: add VectorChord benchmark support (#745) * feat: add VectorChord support and VCHORDRQ index type * feat: add VectorChordRQ command to CLI * feat: add VectorChord support to README * feat: add VectorChordGraph support and configuration * feat: add max_scan_tuples parameter to VectorChordGraph * feat: enhance VectorChord with improved type safety and search functionality * feat: add vectorchord extension creation on connection Co-authored-by: edgar-p --- README.md | 33 +- vectordb_bench/backend/clients/__init__.py | 14 + vectordb_bench/backend/clients/api.py | 2 + .../backend/clients/vectorchord/__init__.py | 0 .../backend/clients/vectorchord/cli.py | 267 ++++++++++++++ .../backend/clients/vectorchord/config.py | 196 +++++++++++ .../clients/vectorchord/vectorchord.py | 325 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 3 + 8 files changed, 838 insertions(+), 2 deletions(-) create mode 100644 vectordb_bench/backend/clients/vectorchord/__init__.py create mode 100644 vectordb_bench/backend/clients/vectorchord/cli.py create mode 100644 vectordb_bench/backend/clients/vectorchord/config.py create mode 100644 vectordb_bench/backend/clients/vectorchord/vectorchord.py diff --git a/README.md b/README.md index 1b0f46309..685e37a47 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ All the database client supported | pinecone | `pip install vectordb-bench[pinecone]` | | weaviate | `pip install vectordb-bench[weaviate]` | | elastic, aliyun_elasticsearch| `pip install vectordb-bench[elastic]` | -| pgvector, pgvectorscale, pgdiskann, alloydb | `pip install vectordb-bench[pgvector]` | +| pgvector, pgvectorscale, pgdiskann, alloydb, vectorchord | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | @@ -86,6 +86,7 @@ Options: Commands: pgvectorhnsw pgvectorivfflat + vectorchordrq test weaviate ``` @@ -179,6 +180,34 @@ Options: --help Show this message and exit. ``` +### Run VectorChord (vchordrq) from command line + +VectorChord is a PostgreSQL extension for scalable vector similarity search using IVF + RaBitQ indexing. +It is fully compatible with pgvector data types and provides faster queries and index builds. + +```shell +vectordbbench vectorchordrq \ + --user-name postgres --password '' \ + --host localhost --port 5432 --db-name vectordb \ + --case-type Performance1536D50K \ + --lists 1000 --probes 10 --epsilon 1.9 \ + --spherical-centroids --build-threads 8 \ + --max-parallel-workers 15 +``` + +Key VectorChord-specific options: +| Option | Description | +|--------|-------------| +| `--lists` | Number of IVF lists for vchordrq index | +| `--probes` | Number of probes during search (default: 10) | +| `--epsilon` | Reranking precision factor, 0.0-4.0 (default: 1.9) | +| `--residual-quantization` | Enable residual quantization | +| `--spherical-centroids` | L2-normalize centroids (recommended for cosine/IP) | +| `--build-threads` | Number of threads for index building (1-255) | +| `--degree-of-parallelism` | Degree of parallelism for index build (1-256) | +| `--max-parallel-workers` | Sets max_parallel_workers & max_parallel_maintenance_workers | +| `--max-scan-tuples` | Max tuples to scan before stopping (-1 for unlimited) | + ### Run awsopensearch from command line ```shell @@ -756,7 +785,7 @@ Now we can only run one task at the same time. ### Code Structure ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639) ### Client -Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. +Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, VectorChord, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into four main types: #### Capacity Case diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 214d85e96..8437a3458 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -59,6 +59,7 @@ class DB(Enum): Zvec = "Zvec" Endee = "Endee" Lindorm = "Lindorm" + VectorChord = "VectorChord" PolarDB = "PolarDB" @property @@ -247,6 +248,10 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormVector + if self == DB.VectorChord: + from .vectorchord.vectorchord import VectorChord + + return VectorChord if self == DB.PolarDB: from .polardb.polardb import PolarDB @@ -441,6 +446,10 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormConfig + if self == DB.VectorChord: + from .vectorchord.config import VectorChordConfig + + return VectorChordConfig if self == DB.PolarDB: from .polardb.config import PolarDBConfig @@ -617,6 +626,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _lindorm_vector_case_config.get(index_type) + if self == DB.VectorChord: + from .vectorchord.config import _vectorchord_case_config + + return _vectorchord_case_config.get(index_type) + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index ecbddef39..f507abe33 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -42,6 +42,8 @@ class IndexType(StrEnum): GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" + VCHORDRQ = "vchordrq" + VCHORDG = "vchordg" SCANN_MILVUS = "SCANN_MILVUS" SVS_VAMANA = "SVS_VAMANA" SVS_VAMANA_LVQ = "SVS_VAMANA_LVQ" diff --git a/vectordb_bench/backend/clients/vectorchord/__init__.py b/vectordb_bench/backend/clients/vectorchord/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/vectorchord/cli.py b/vectordb_bench/backend/clients/vectorchord/cli.py new file mode 100644 index 000000000..2b3e3862b --- /dev/null +++ b/vectordb_bench/backend/clients/vectorchord/cli.py @@ -0,0 +1,267 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class VectorChordTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option("--user-name", type=str, help="Db username", required=True), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + port: Annotated[ + int, + click.option( + "--port", + type=int, + help="Postgres database port", + default=5432, + show_default=True, + required=False, + ), + ] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] + max_parallel_workers: Annotated[ + int | None, + click.option( + "--max-parallel-workers", + type=int, + help="Sets the maximum number of parallel workers for index creation", + required=False, + ), + ] + quantization_type: Annotated[ + str | None, + click.option( + "--quantization-type", + type=click.Choice(["vector", "halfvec", "rabitq8", "rabitq4"]), + help="Quantization type for vectors", + default="vector", + show_default=True, + ), + ] + + +class VectorChordRQTypedDict(VectorChordTypedDict): + lists: Annotated[ + int | None, + click.option( + "--lists", + type=int, + help="Number of IVF lists for vchordrq index", + ), + ] + probes: Annotated[ + int | None, + click.option( + "--probes", + type=int, + help="Number of probes during search", + default=10, + show_default=True, + ), + ] + epsilon: Annotated[ + float | None, + click.option( + "--epsilon", + type=float, + help="Reranking precision factor (0.0-4.0, higher is more accurate but slower)", + default=1.9, + show_default=True, + ), + ] + residual_quantization: Annotated[ + bool, + click.option( + "--residual-quantization/--no-residual-quantization", + type=bool, + help="Enable residual quantization for improved accuracy", + default=False, + show_default=True, + ), + ] + rerank_in_table: Annotated[ + bool, + click.option( + "--rerank-in-table/--no-rerank-in-table", + type=bool, + help="Read vectors from table instead of storing in index (saves storage, degrades query performance)", + default=False, + show_default=True, + ), + ] + spherical_centroids: Annotated[ + bool, + click.option( + "--spherical-centroids/--no-spherical-centroids", + type=bool, + help="L2-normalize centroids during K-means (recommended for cosine/IP)", + default=False, + show_default=True, + ), + ] + build_threads: Annotated[ + int | None, + click.option( + "--build-threads", + type=int, + help="Number of threads for index building (range: 1-255)", + ), + ] + degree_of_parallelism: Annotated[ + int | None, + click.option( + "--degree-of-parallelism", + type=int, + help="Degree of parallelism for index build (range: 1-256, default: 32)", + ), + ] + max_scan_tuples: Annotated[ + int | None, + click.option( + "--max-scan-tuples", + type=int, + help="Max tuples to scan before stopping (-1 for unlimited)", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(VectorChordRQTypedDict) +def VectorChordRQ( + **parameters: Unpack[VectorChordRQTypedDict], +): + from .config import VectorChordConfig, VectorChordRQConfig + + run( + db=DB.VectorChord, + db_config=VectorChordConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + ), + db_case_config=VectorChordRQConfig( + quantization_type=parameters["quantization_type"], + lists=parameters["lists"], + probes=parameters["probes"], + epsilon=parameters["epsilon"], + residual_quantization=parameters["residual_quantization"], + rerank_in_table=parameters["rerank_in_table"], + spherical_centroids=parameters["spherical_centroids"], + build_threads=parameters["build_threads"], + degree_of_parallelism=parameters["degree_of_parallelism"], + max_scan_tuples=parameters["max_scan_tuples"], + max_parallel_workers=parameters["max_parallel_workers"], + ), + **parameters, + ) + + +class VectorChordGraphTypedDict(VectorChordTypedDict): + m: Annotated[ + int | None, + click.option( + "--m", + type=int, + help="Max neighbors per vertex (default: 32)", + ), + ] + ef_construction: Annotated[ + int | None, + click.option( + "--ef-construction", + type=int, + help="Dynamic list size during insertion (default: 64)", + ), + ] + bits: Annotated[ + int | None, + click.option( + "--bits", + type=int, + help="RaBitQ quantization ratio (1 or 2, default: 2)", + ), + ] + ef_search: Annotated[ + int | None, + click.option( + "--ef-search", + type=int, + help="Dynamic list size for search (default: 64)", + default=64, + show_default=True, + ), + ] + beam_search: Annotated[ + int | None, + click.option( + "--beam-search", + type=int, + help="Batch vertex access width during search (default: 1)", + ), + ] + max_scan_tuples: Annotated[ + int | None, + click.option( + "--max-scan-tuples", + type=int, + help="Max tuples to scan before stopping (-1 for unlimited)", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(VectorChordGraphTypedDict) +def VectorChordGraph( + **parameters: Unpack[VectorChordGraphTypedDict], +): + from .config import VectorChordConfig, VectorChordGraphConfig + + run( + db=DB.VectorChord, + db_config=VectorChordConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + ), + db_case_config=VectorChordGraphConfig( + quantization_type=parameters["quantization_type"], + m=parameters["m"], + ef_construction=parameters["ef_construction"], + bits=parameters["bits"], + ef_search=parameters["ef_search"], + beam_search=parameters["beam_search"], + max_parallel_workers=parameters["max_parallel_workers"], + max_scan_tuples=parameters["max_scan_tuples"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/vectorchord/config.py b/vectordb_bench/backend/clients/vectorchord/config.py new file mode 100644 index 000000000..95916eb06 --- /dev/null +++ b/vectordb_bench/backend/clients/vectorchord/config.py @@ -0,0 +1,196 @@ +from abc import abstractmethod +from typing import Literal, LiteralString, TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class VectorChordConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + +class VectorChordConfig(DBConfig): + user_name: SecretStr = SecretStr("postgres") + password: SecretStr + host: str = "localhost" + port: int = 5432 + db_name: str = "vectordb" + + def to_dict(self) -> VectorChordConfigDict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +_METRIC_OPS = { + "vector": { + MetricType.L2: "vector_l2_ops", + MetricType.IP: "vector_ip_ops", + MetricType.COSINE: "vector_cosine_ops", + }, + "halfvec": { + MetricType.L2: "halfvec_l2_ops", + MetricType.IP: "halfvec_ip_ops", + MetricType.COSINE: "halfvec_cosine_ops", + }, + "rabitq8": { + MetricType.L2: "rabitq8_l2_ops", + MetricType.IP: "rabitq8_ip_ops", + MetricType.COSINE: "rabitq8_cosine_ops", + }, + "rabitq4": { + MetricType.L2: "rabitq4_l2_ops", + MetricType.IP: "rabitq4_ip_ops", + MetricType.COSINE: "rabitq4_cosine_ops", + }, +} + + +class VectorChordIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + quantization_type: Literal["vector", "halfvec", "rabitq8", "rabitq4"] = "vector" + + def parse_metric(self) -> str: + ops = _METRIC_OPS.get(self.quantization_type, _METRIC_OPS["vector"]) + return ops.get(self.metric_type, ops[MetricType.COSINE]) + + def parse_metric_fun_op(self) -> LiteralString: + if self.metric_type == MetricType.L2: + return "<->" + if self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + + @abstractmethod + def index_param(self) -> dict: ... + + @abstractmethod + def search_param(self) -> dict: ... + + @abstractmethod + def session_param(self) -> dict: ... + + +class VectorChordRQConfig(VectorChordIndexConfig): + index: IndexType = IndexType.VCHORDRQ + # Build parameters (top-level options) + residual_quantization: bool = False + rerank_in_table: bool = False + degree_of_parallelism: int | None = None # default 32, range [1, 256] + # Build parameters ([build.internal] section) + lists: int | None = None + spherical_centroids: bool = False + build_threads: int | None = None # range [1, 255] + # PostgreSQL tuning parameter + max_parallel_workers: int | None = None # sets max_parallel_workers & max_parallel_maintenance_workers + # Search parameters (GUCs) + probes: int | None = 10 + epsilon: float | None = 1.9 # range [0.0, 4.0] + max_scan_tuples: int | None = None # default -1, range [-1, 2147483647] + + def index_param(self) -> dict: + options_parts = [] + if self.rerank_in_table: + options_parts.append("rerank_in_table = true") + if self.residual_quantization: + options_parts.append("residual_quantization = true") + if self.degree_of_parallelism is not None: + options_parts.append(f"degree_of_parallelism = {self.degree_of_parallelism}") + options_parts.append("[build.internal]") + if self.lists is not None: + options_parts.append(f"lists = [{self.lists}]") + if self.spherical_centroids: + options_parts.append("spherical_centroids = true") + if self.build_threads is not None: + options_parts.append(f"build_threads = {self.build_threads}") + + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "quantization_type": self.quantization_type, + "options": "\n".join(options_parts), + "max_parallel_workers": self.max_parallel_workers, + } + + def search_param(self) -> dict: + return { + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> dict: + params = {} + if self.probes is not None: + params["vchordrq.probes"] = str(self.probes) + if self.epsilon is not None: + params["vchordrq.epsilon"] = str(self.epsilon) + if self.max_scan_tuples is not None: + params["vchordrq.max_scan_tuples"] = str(self.max_scan_tuples) + return params + + +class VectorChordGraphConfig(VectorChordIndexConfig): + index: IndexType = IndexType.VCHORDG + # Build parameters + m: int | None = None # default 32, max neighbors per vertex + ef_construction: int | None = None # default 64 + bits: int | None = None # default 2, quantization ratio (1 or 2) + # PostgreSQL tuning parameter + max_parallel_workers: int | None = None + # Search parameters (GUCs) + ef_search: int | None = 64 # range [1, 65535] + beam_search: int | None = None # default 1 + max_scan_tuples: int | None = None # default -1, range [-1, 2147483647] + + def index_param(self) -> dict: + options_parts = [] + if self.m is not None: + options_parts.append(f"m = {self.m}") + if self.ef_construction is not None: + options_parts.append(f"ef_construction = {self.ef_construction}") + if self.bits is not None: + options_parts.append(f"bits = {self.bits}") + + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "quantization_type": self.quantization_type, + "options": "\n".join(options_parts), + "max_parallel_workers": self.max_parallel_workers, + } + + def search_param(self) -> dict: + return { + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> dict: + params = {} + if self.ef_search is not None: + params["vchordg.ef_search"] = str(self.ef_search) + if self.beam_search is not None: + params["vchordg.beam_search"] = str(self.beam_search) + if self.max_scan_tuples is not None: + params["vchordg.max_scan_tuples"] = str(self.max_scan_tuples) + return params + + +_vectorchord_case_config = { + IndexType.VCHORDRQ: VectorChordRQConfig, + IndexType.VCHORDG: VectorChordGraphConfig, +} diff --git a/vectordb_bench/backend/clients/vectorchord/vectorchord.py b/vectordb_bench/backend/clients/vectorchord/vectorchord.py new file mode 100644 index 000000000..77a462e75 --- /dev/null +++ b/vectordb_bench/backend/clients/vectorchord/vectorchord.py @@ -0,0 +1,325 @@ +"""Wrapper around the VectorChord vector database over VectorDB""" + +import logging +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from ...filter import Filter, FilterOp +from ..api import VectorDB +from .config import VectorChordConfigDict, VectorChordIndexConfig + +log = logging.getLogger(__name__) + + +class VectorChord(VectorDB): + """Use psycopg instructions""" + + thread_safe: bool = False + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + + _search: sql.Composed + where_clause: str = "" + + def __init__( + self, + dim: int, + db_config: VectorChordConfigDict, + db_case_config: VectorChordIndexConfig, + collection_name: str = "vectorchord_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "VectorChord" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "vectorchord_index" + self._primary_field = "id" + self._vector_field = "embedding" + + index_param = self.case_config.index_param() + self._quantization_type = index_param["quantization_type"] + self._index_method = index_param["index_type"] + + self.conn, self.cursor = self._create_connection(**self.db_config) + + # create vectorchord extension if not exists + self.cursor.execute("CREATE EXTENSION IF NOT EXISTS vchord CASCADE") + self.conn.commit() + + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ), + ): + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" + ) + log.error(msg) + raise RuntimeError(msg) + + if drop_old: + self._drop_index() + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + @contextmanager + def init(self) -> Generator[None, None, None]: + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options: dict[str, Any] = self.case_config.session_param() + + if len(session_options) > 0: + for setting_name, setting_val in session_options.items(): + command = sql.SQL("SET {setting_name} " + "= {setting_val};").format( + setting_name=sql.Identifier(setting_name), + setting_val=sql.Literal(str(setting_val)), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") + + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name), + ), + ) + self.conn.commit() + + def optimize(self, data_size: int | None = None): + self._post_insert() + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") + + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name), + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) + self.conn.commit() + + def _set_parallel_index_build_param(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + if index_param["max_parallel_workers"] is not None: + self.cursor.execute( + sql.SQL("SET max_parallel_workers TO '{}';").format( + index_param["max_parallel_workers"], + ), + ) + self.cursor.execute( + sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( + index_param["max_parallel_workers"], + ), + ) + self.cursor.execute( + sql.SQL("ALTER TABLE {} SET (parallel_workers = {});").format( + sql.Identifier(self.table_name), + index_param["max_parallel_workers"], + ), + ) + self.conn.commit() + + results = self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() + results.extend(self.cursor.execute(sql.SQL("SHOW max_parallel_maintenance_workers;")).fetchall()) + log.info(f"{self.name} parallel index creation parameters: {results}") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") + + index_param: dict[str, Any] = self.case_config.index_param() + self._set_parallel_index_build_param() + + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_method} (embedding {embedding_metric}) + """, + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_method=sql.SQL(self._index_method), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + + options_str = index_param.get("options", "") + if options_str: + with_clause = sql.SQL( + "WITH (options = $vchord$\n{options}\n$vchord$);", + ).format(options=sql.SQL(options_str)) + else: + with_clause = sql.SQL(";") + + full_sql = index_create_sql + sql.SQL(" ") + with_clause + log.debug(full_sql.as_string(self.cursor)) + self.cursor.execute(full_sql) + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create table : {self.table_name}") + + col_type = self._quantization_type + if col_type in ("rabitq8", "rabitq4"): + # rabitq types need vector column + quantization during insert + col_type = "vector" + + self.cursor.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS public.{table_name} " + "(id BIGINT PRIMARY KEY, embedding {col_type}({dim}));", + ).format( + table_name=sql.Identifier(self.table_name), + col_type=sql.SQL(col_type), + dim=dim, + ), + ) + self.conn.commit() + except Exception as e: + log.warning(f"Failed to create vectorchord table: {self.table_name} error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + if self._quantization_type == "halfvec": + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name), + ), + ) as copy: + copy.set_types(["bigint", "halfvec"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, np.float16(embeddings_arr[i]))) + else: + # vector, rabitq8, rabitq4 all store as vector column + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name), + ), + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) + self.conn.commit() + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into vectorchord table ({self.table_name}), error: {e}") + return 0, e + + def _generate_search_query(self) -> sql.Composed: + # Search query cast type: rabitq8/rabitq4 queries still accept ::vector input + cast_type = "vector" + return sql.Composed( + [ + sql.SQL("SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ").format( + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(self.where_clause), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(f" %s::{cast_type} LIMIT %s::int"), + ], + ) + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + self.where_clause = f"WHERE {self._primary_field} >= {filters.int_value}" + else: + msg = f"Not support Filter for VectorChord - {filters}" + raise ValueError(msg) + + self._search = self._generate_search_query() + + def search_embedding( + self, + query: list[float], + k: int = 100, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + q = np.asarray(query) + result = self.cursor.execute(self._search, (q, k), prepare=True, binary=True) + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index b48d5900c..3e21746aa 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -38,6 +38,7 @@ from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB from ..backend.clients.turbopuffer.cli import TurboPuffer +from ..backend.clients.vectorchord.cli import VectorChordGraph, VectorChordRQ from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex @@ -87,6 +88,8 @@ cli.add_command(LindormHNSW) cli.add_command(LindormIVFBQ) cli.add_command(Pinecone) +cli.add_command(VectorChordRQ) +cli.add_command(VectorChordGraph) cli.add_command(PolarDBHNSWFlat) cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) From 5a9173e90d1cd9b010580ffc5a6e507ff792e5eb Mon Sep 17 00:00:00 2001 From: shaohuasong-fang Date: Mon, 20 Apr 2026 10:37:12 +0800 Subject: [PATCH 21/38] =?UTF-8?q?fix(pgvector):=20normalize=20index=5Ftype?= =?UTF-8?q?=20to=20lowercase=20in=20=5Fcreate=5Findex=20to=20=E2=80=A6=20(?= =?UTF-8?q?#760)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(pgvector): normalize index_type to lowercase in _create_index to match PostgreSQL access method names PostgreSQL pgvector extension registers index access methods in lowercase (e.g. "hnsw", "ivfflat"), but the frontend passes IndexType.HNSW.value which is uppercase "HNSW", causing "access method HNSW does not exist" error. * Fix index type usage in pgvector.py Replaced index_param['index_type'] with index_type_lower for consistency. * add comment sign '#' I have added the # before [FIX] --- .../backend/clients/pgvector/pgvector.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 30c797c38..0b0750172 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -335,6 +335,12 @@ def _create_index(self): index_param = self.case_config.index_param() self._set_parallel_index_build_param() + # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in lowercase (e.g., "hnsw", "ivfflat"), + # but the index type passed from the frontend UI is uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" + # to fail with error "access method HNSW does not exist". Here we uniformly convert it to lowercase to match PostgreSQL's access method name. + index_type_lower = index_param["index_type"].lower() + log.info(f"index_type (original={index_param['index_type']}, normalized={index_type_lower})") + options = [] for option in index_param["index_creation_with_options"]: if option["val"] is not None: @@ -360,7 +366,9 @@ def _create_index(self): if index_param["quantization_type"] == "bit" else sql.Identifier("embedding") ), - index_type=sql.Identifier(index_param["index_type"]), + # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] + index_type=sql.Identifier(index_type_lower), + # index_type=sql.Identifier(index_param["index_type"]), # This assumes that the quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), dim=self.dim, @@ -375,7 +383,9 @@ def _create_index(self): ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), - index_type=sql.Identifier(index_param["index_type"]), + # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] + index_type=sql.Identifier(index_type_lower), + # index_type=sql.Identifier(index_param["index_type"]), embedding_metric=sql.Identifier(index_param["metric"]), ) From c4083f31d37ea3d73af5e2bdef8b223d418fe889 Mon Sep 17 00:00:00 2001 From: Xiang Fu Date: Sun, 19 Apr 2026 23:18:51 -0700 Subject: [PATCH 22/38] feat: add Apache Pinot vector search client (#757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a complete Apache Pinot client for VectorDBBench. Index types: HNSW (Lucene), IVF_FLAT, IVF_PQ, IVF_ON_DISK Metrics: L2, IP, COSINE Filters: NumGE, StrEqual Optional dep: pip install "vectordb-bench[pinot]" Parallel loading: thread_safe=True — each worker thread maintains its own row buffer and flushes to Pinot via a fresh HTTP session. Since Pinot's ingestFromFile is synchronous (blocks until HNSW index is built, ~6 min per 100K×768D segment), concurrent flushes across threads reduce load time significantly vs sequential flushing. Benchmark results: Small dataset (OpenAI 50K, 768D, L2): HNSW: 798 QPS, recall=1.000 IVF_FLAT: 800 QPS, recall=1.000 IVF_PQ: 795 QPS, recall=1.000 IVF_ON_DISK: 691 QPS, recall=1.000 Large dataset (Cohere 1M, 768D, COSINE): HNSW m=16: 74 QPS, recall=0.982 Filter benchmark (Cohere 1M, COSINE, HNSW m=32): 1% NumGE: 71 QPS, recall=0.977 99% NumGE: 97 QPS, recall=0.649 Co-authored-by: Claude Sonnet 4.6 --- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 20 + .../backend/clients/pinot/__init__.py | 0 vectordb_bench/backend/clients/pinot/cli.py | 202 ++++++++ .../backend/clients/pinot/config.py | 94 ++++ vectordb_bench/backend/clients/pinot/pinot.py | 449 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + 7 files changed, 768 insertions(+) create mode 100644 vectordb_bench/backend/clients/pinot/__init__.py create mode 100644 vectordb_bench/backend/clients/pinot/cli.py create mode 100644 vectordb_bench/backend/clients/pinot/config.py create mode 100644 vectordb_bench/backend/clients/pinot/pinot.py diff --git a/pyproject.toml b/pyproject.toml index e72be9697..2ed18b115 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] +pinot = [ "requests" ] [project.urls] Repository = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 8437a3458..9029be05f 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -61,6 +61,7 @@ class DB(Enum): Lindorm = "Lindorm" VectorChord = "VectorChord" PolarDB = "PolarDB" + Pinot = "Pinot" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -257,6 +258,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return PolarDB + if self == DB.Pinot: + from .pinot.pinot import Pinot + + return Pinot + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -455,6 +461,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return PolarDBConfig + if self == DB.Pinot: + from .pinot.config import PinotConfig + + return PinotConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -631,6 +642,15 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _vectorchord_case_config.get(index_type) + if self == DB.Pinot: + from .pinot.config import PinotHNSWConfig, PinotIVFFlatConfig, PinotIVFPQConfig + + return { + IndexType.HNSW: PinotHNSWConfig, + IndexType.IVFFlat: PinotIVFFlatConfig, + IndexType.IVFPQ: PinotIVFPQConfig, + }.get(index_type, PinotHNSWConfig) + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/pinot/__init__.py b/vectordb_bench/backend/clients/pinot/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/pinot/cli.py b/vectordb_bench/backend/clients/pinot/cli.py new file mode 100644 index 000000000..e51160b05 --- /dev/null +++ b/vectordb_bench/backend/clients/pinot/cli.py @@ -0,0 +1,202 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor2, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class PinotTypedDict(TypedDict): + controller_host: Annotated[ + str, + click.option("--controller-host", type=str, default="localhost", help="Pinot Controller host"), + ] + controller_port: Annotated[ + int, + click.option("--controller-port", type=int, default=9000, help="Pinot Controller port"), + ] + broker_host: Annotated[ + str, + click.option("--broker-host", type=str, default="localhost", help="Pinot Broker host"), + ] + broker_port: Annotated[ + int, + click.option("--broker-port", type=int, default=8099, help="Pinot Broker port"), + ] + username: Annotated[ + str, + click.option("--username", type=str, default=None, help="Pinot username (optional)"), + ] + password: Annotated[ + str, + click.option("--password", type=str, default=None, help="Pinot password (optional)"), + ] + ingest_batch_size: Annotated[ + int, + click.option( + "--ingest-batch-size", + type=int, + default=100_000, + show_default=True, + help=( + "Rows buffered before flushing one Pinot segment (one ingestFromFile call). " + "Larger values mean fewer segments and better IVF training / query performance. " + "Reduce if memory is constrained (100K x 768-dim float32 ~= 300 MB)." + ), + ), + ] + + +def _pinot_db_config(parameters: dict): + from .config import PinotConfig + + return PinotConfig( + db_label=parameters["db_label"], + controller_host=parameters["controller_host"], + controller_port=parameters["controller_port"], + broker_host=parameters["broker_host"], + broker_port=parameters["broker_port"], + username=parameters.get("username"), + password=SecretStr(parameters["password"]) if parameters.get("password") else None, + ingest_batch_size=parameters["ingest_batch_size"], + ) + + +@click.group() +def Pinot(): + """Apache Pinot vector search benchmarks.""" + + +# --------------------------------------------------------------------------- +# HNSW +# --------------------------------------------------------------------------- + + +class PinotHNSWTypedDict(CommonTypedDict, PinotTypedDict, HNSWFlavor2): ... + + +@Pinot.command("hnsw") +@click_parameter_decorators_from_typed_dict(PinotHNSWTypedDict) +def pinot_hnsw(**parameters: Unpack[PinotHNSWTypedDict]): + from .config import PinotHNSWConfig + + run( + db=DB.Pinot, + db_config=_pinot_db_config(parameters), + db_case_config=PinotHNSWConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef=parameters["ef_runtime"], + ), + **parameters, + ) + + +# --------------------------------------------------------------------------- +# IVF_FLAT +# --------------------------------------------------------------------------- + + +class PinotIVFFlatTypedDict(CommonTypedDict, PinotTypedDict): + nlist: Annotated[ + int, + click.option("--nlist", type=int, default=128, help="Number of Voronoi cells (IVF nlist)"), + ] + quantizer: Annotated[ + str, + click.option( + "--quantizer", + type=click.Choice(["FLAT", "SQ8", "SQ4"]), + default="FLAT", + help="Quantizer type for IVF_FLAT", + ), + ] + nprobe: Annotated[ + int, + click.option("--nprobe", type=int, default=8, help="Number of cells to probe at query time"), + ] + train_sample_size: Annotated[ + int, + click.option( + "--train-sample-size", + type=int, + default=None, + help="Training sample size (defaults to max(nlist*50, 1000) if not set)", + ), + ] + + +@Pinot.command("ivf-flat") +@click_parameter_decorators_from_typed_dict(PinotIVFFlatTypedDict) +def pinot_ivf_flat(**parameters: Unpack[PinotIVFFlatTypedDict]): + from .config import PinotIVFFlatConfig + + run( + db=DB.Pinot, + db_config=_pinot_db_config(parameters), + db_case_config=PinotIVFFlatConfig( + nlist=parameters["nlist"], + quantizer=parameters["quantizer"], + nprobe=parameters["nprobe"], + train_sample_size=parameters.get("train_sample_size"), + ), + **parameters, + ) + + +# --------------------------------------------------------------------------- +# IVF_PQ +# --------------------------------------------------------------------------- + + +class PinotIVFPQTypedDict(CommonTypedDict, PinotTypedDict): + nlist: Annotated[ + int, + click.option("--nlist", type=int, default=128, help="Number of Voronoi cells (IVF nlist)"), + ] + pq_m: Annotated[ + int, + click.option("--pq-m", type=int, default=8, help="Number of PQ sub-quantizers (must divide dimension)"), + ] + pq_nbits: Annotated[ + int, + click.option( + "--pq-nbits", + type=click.Choice(["4", "6", "8"]), + default="8", + help="Bits per PQ code (4, 6, or 8)", + ), + ] + train_sample_size: Annotated[ + int, + click.option("--train-sample-size", type=int, default=6400, help="Training sample size (must be >= nlist)"), + ] + nprobe: Annotated[ + int, + click.option("--nprobe", type=int, default=8, help="Number of cells to probe at query time"), + ] + + +@Pinot.command("ivf-pq") +@click_parameter_decorators_from_typed_dict(PinotIVFPQTypedDict) +def pinot_ivf_pq(**parameters: Unpack[PinotIVFPQTypedDict]): + from .config import PinotIVFPQConfig + + run( + db=DB.Pinot, + db_config=_pinot_db_config(parameters), + db_case_config=PinotIVFPQConfig( + nlist=parameters["nlist"], + pq_m=parameters["pq_m"], + pq_nbits=int(parameters["pq_nbits"]), + train_sample_size=parameters["train_sample_size"], + nprobe=parameters["nprobe"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/pinot/config.py b/vectordb_bench/backend/clients/pinot/config.py new file mode 100644 index 000000000..e29db0bec --- /dev/null +++ b/vectordb_bench/backend/clients/pinot/config.py @@ -0,0 +1,94 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class PinotConfig(DBConfig): + controller_host: str = "localhost" + controller_port: int = 9000 + broker_host: str = "localhost" + broker_port: int = 8099 + username: str | None = None + password: SecretStr | None = None + # Rows buffered before flushing one Pinot segment (one ingestFromFile call). + # Larger values → fewer segments → better IVF training & query perf. + # 100_000 rows x 768-dim float32 ~= 300 MB in-memory. + ingest_batch_size: int = 100_000 + + def to_dict(self) -> dict: + return { + "controller_host": self.controller_host, + "controller_port": self.controller_port, + "broker_host": self.broker_host, + "broker_port": self.broker_port, + "username": self.username, + "password": self.password.get_secret_value() if self.password else None, + "ingest_batch_size": self.ingest_batch_size, + } + + +class PinotHNSWConfig(BaseModel, DBCaseConfig): + """HNSW vector index config for Apache Pinot (Lucene-based).""" + + metric_type: MetricType | None = None + m: int = 16 # maxCon: max connections per node + ef_construction: int = 100 # beamWidth: construction beam width + ef: int | None = None # ef_search: HNSW candidate list size at query time (default=k) + + def index_param(self) -> dict: + return { + "vectorIndexType": "HNSW", + "maxCon": str(self.m), + "beamWidth": str(self.ef_construction), + } + + def search_param(self) -> dict: + # ef controls the HNSW candidate list during search via vectorSimilarity(col, q, ef). + # Larger ef → better recall, slightly higher latency. Defaults to k if not set. + return {"ef": self.ef} if self.ef is not None else {} + + +class PinotIVFFlatConfig(BaseModel, DBCaseConfig): + """IVF_FLAT vector index config for Apache Pinot.""" + + metric_type: MetricType | None = None + nlist: int = 128 # number of Voronoi cells (centroids) + quantizer: str = "FLAT" # FLAT, SQ8, or SQ4 + train_sample_size: int | None = None # defaults to max(nlist*50, 1000) if None + nprobe: int = 8 # number of cells to probe at query time + + def index_param(self) -> dict: + params: dict = { + "vectorIndexType": "IVF_FLAT", + "nlist": str(self.nlist), + "quantizer": self.quantizer, + } + if self.train_sample_size is not None: + params["trainSampleSize"] = str(self.train_sample_size) + return params + + def search_param(self) -> dict: + return {"nprobe": self.nprobe} + + +class PinotIVFPQConfig(BaseModel, DBCaseConfig): + """IVF_PQ vector index config for Apache Pinot (residual product quantization).""" + + metric_type: MetricType | None = None + nlist: int = 128 # number of Voronoi cells (centroids) + pq_m: int = 8 # number of sub-quantizers (must divide vectorDimension) + pq_nbits: int = 8 # bits per sub-quantizer code: 4, 6, or 8 + train_sample_size: int = 6400 # training sample size (must be >= nlist) + nprobe: int = 8 # number of cells to probe at query time + + def index_param(self) -> dict: + return { + "vectorIndexType": "IVF_PQ", + "nlist": str(self.nlist), + "pqM": str(self.pq_m), + "pqNbits": str(self.pq_nbits), + "trainSampleSize": str(self.train_sample_size), + } + + def search_param(self) -> dict: + return {"nprobe": self.nprobe} diff --git a/vectordb_bench/backend/clients/pinot/pinot.py b/vectordb_bench/backend/clients/pinot/pinot.py new file mode 100644 index 000000000..bdae58d64 --- /dev/null +++ b/vectordb_bench/backend/clients/pinot/pinot.py @@ -0,0 +1,449 @@ +"""Wrapper around Apache Pinot vector database over VectorDB""" + +import json +import logging +import tempfile +import threading +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import requests + +from ...filter import Filter, FilterOp +from ..api import DBCaseConfig, MetricType, VectorDB + +log = logging.getLogger(__name__) + +# Rows accumulated before flushing a segment to Pinot. +# Large enough to avoid thousands of tiny segments (which breaks IVF training +# and hurts query performance), yet small enough to keep memory use bounded. +# For 768-dim float32 vectors: 100K rows ≈ 300 MB in-memory. +DEFAULT_INGEST_BATCH_SIZE = 100_000 + + +class Pinot(VectorDB): + """Apache Pinot vector database client for VectorDBBench.""" + + name = "Pinot" + # thread_safe=True: each flush uses a fresh requests.Session (not a shared one), + # and each worker thread has its own row buffer via threading.local(). + # This lets the framework spawn multiple load workers that flush segments in parallel. + thread_safe: bool = True + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "VectorBenchCollection", + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + self.dim = dim + self.case_config = db_case_config + self.table_name = collection_name + self._primary_field = "id" + self._vector_field = "embedding" + self._label_field = "labels" + self.with_scalar_labels = with_scalar_labels + self._filter_where: str = "" # set by prepare_filter(); applied in search_embedding() + + controller_host = db_config["controller_host"] + controller_port = db_config["controller_port"] + broker_host = db_config["broker_host"] + broker_port = db_config["broker_port"] + self._controller_url = f"http://{controller_host}:{controller_port}" + self._broker_url = f"http://{broker_host}:{broker_port}" + + self._auth = None + if db_config.get("username") and db_config.get("password"): + self._auth = (db_config["username"], db_config["password"]) + + # Per-thread row buffers — each worker thread accumulates rows independently + # and flushes its own segment when the threshold is reached. + # _registered_buffers tracks all thread-local objects so init() teardown + # can flush any remaining rows from every worker thread. + self._thread_local = threading.local() + self._registered_buffers: list = [] + self._buffers_lock = threading.Lock() + self._ingest_batch_size: int = db_config.get("ingest_batch_size", DEFAULT_INGEST_BATCH_SIZE) + + self.session = None + + with requests.Session() as setup_session: + if self._auth: + setup_session.auth = self._auth + + if drop_old: + self._delete_table(setup_session) + self._delete_schema(setup_session) + + if not self._schema_exists(setup_session): + self._create_schema(setup_session) + + if not self._table_exists(setup_session): + self._create_table(setup_session) + + def _schema_exists(self, session: requests.Session) -> bool: + resp = session.get(f"{self._controller_url}/schemas/{self.table_name}") + return resp.status_code == 200 + + def _table_exists(self, session: requests.Session) -> bool: + resp = session.get(f"{self._controller_url}/tables/{self.table_name}") + if resp.status_code != 200: + return False + data = resp.json() + return bool(data.get("OFFLINE") or data.get("tables")) + + def _delete_table(self, session: requests.Session): + resp = session.delete(f"{self._controller_url}/tables/{self.table_name}?type=offline") + if resp.status_code not in (200, 404): + log.warning(f"Failed to delete Pinot table {self.table_name}: {resp.text}") + else: + log.info(f"Deleted Pinot table: {self.table_name}") + # Wait for Pinot to finish cleaning up the external view + for _ in range(30): + check = session.get(f"{self._controller_url}/tables/{self.table_name}/externalview") + if check.status_code == 404 or not check.json(): + break + log.info(f"Waiting for Pinot external view cleanup for {self.table_name}...") + time.sleep(2) + else: + log.warning(f"External view for {self.table_name} did not clear within 60s") + + def _delete_schema(self, session: requests.Session): + resp = session.delete(f"{self._controller_url}/schemas/{self.table_name}") + if resp.status_code not in (200, 404): + log.warning(f"Failed to delete Pinot schema {self.table_name}: {resp.text}") + else: + log.info(f"Deleted Pinot schema: {self.table_name}") + + def _create_schema(self, session: requests.Session): + dimension_fields = [ + {"name": self._primary_field, "dataType": "INT"}, + {"name": self._vector_field, "dataType": "FLOAT", "singleValueField": False}, + ] + if self.with_scalar_labels: + dimension_fields.append({"name": self._label_field, "dataType": "STRING"}) + + schema = { + "schemaName": self.table_name, + "dimensionFieldSpecs": dimension_fields, + } + resp = session.post( + f"{self._controller_url}/schemas", + json=schema, + headers={"Content-Type": "application/json"}, + ) + if not resp.ok: + log.error(f"Failed to create Pinot schema: {resp.text}") + resp.raise_for_status() + log.info(f"Created Pinot schema: {self.table_name}") + + def _create_table(self, session: requests.Session): + metric_str = self._get_index_metric_str() + index_params = self.case_config.index_param() + + # Pull vectorIndexType out of index_params; remaining entries are type-specific properties. + vector_index_type = index_params.pop("vectorIndexType", "HNSW") + properties: dict = { + "vectorIndexType": vector_index_type, + "vectorDimension": str(self.dim), + "vectorDistanceFunction": metric_str, + "version": "1", + } + properties.update(index_params) + + table_config = { + "tableName": self.table_name, + "tableType": "OFFLINE", + "segmentsConfig": { + "replication": "1", + "schemaName": self.table_name, + }, + "tenants": {}, + "tableIndexConfig": { + "loadMode": "MMAP", + # Inverted index on id for fast equality lookups; range index for >=/<= filters. + "invertedIndexColumns": [self._primary_field], + "rangeIndexColumns": [self._primary_field], + }, + "fieldConfigList": [ + { + "encodingType": "RAW", + "indexType": "VECTOR", + "name": self._vector_field, + "properties": properties, + } + ], + "ingestionConfig": { + "batchIngestionConfig": { + "segmentIngestionType": "APPEND", + "segmentIngestionFrequency": "DAILY", + } + }, + "metadata": {}, + } + resp = session.post( + f"{self._controller_url}/tables", + json=table_config, + headers={"Content-Type": "application/json"}, + ) + if not resp.ok: + log.error(f"Failed to create Pinot table: {resp.text}") + resp.raise_for_status() + log.info(f"Created Pinot table: {self.table_name}") + + def _get_index_metric_str(self) -> str: + if self.case_config.metric_type == MetricType.COSINE: + return "COSINE" + if self.case_config.metric_type == MetricType.IP: + return "INNER_PRODUCT" + return "L2" + + def _get_query_distance_fn(self) -> tuple[str, str]: + """Returns (sql_function_name, sort_order) for vector search.""" + if self.case_config.metric_type == MetricType.COSINE: + return "cosineDistance", "ASC" + if self.case_config.metric_type == MetricType.IP: + return "innerProduct", "DESC" + return "l2Distance", "ASC" + + def prepare_filter(self, filters: Filter): + """Pre-compute the SQL WHERE fragment for the given filter condition.""" + if filters.type == FilterOp.NonFilter: + self._filter_where = "" + elif filters.type == FilterOp.NumGE: + self._filter_where = f"{filters.int_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self._filter_where = f"{self._label_field} = '{filters.label_value}'" + + def __getstate__(self): + # threading.local and Lock cannot be pickled; the framework pickles the DB + # instance to send it to the load subprocess, so we must exclude them here + # and recreate them in __setstate__ after unpickling. + state = self.__dict__.copy() + state.pop("_thread_local", None) + state.pop("_buffers_lock", None) + state.pop("_registered_buffers", None) + return state + + def __setstate__(self, state: dict) -> None: + self.__dict__.update(state) + self._thread_local = threading.local() + self._buffers_lock = threading.Lock() + self._registered_buffers = [] + + def _get_thread_buffer(self) -> list: + """Return this thread's row buffer, creating and registering it on first access. + + The list itself (not the threading.local container) is registered so that the + teardown in init() can access buffered rows from the main thread, which has its + own (empty) thread-local slot and cannot see other threads' data through the + threading.local object. + """ + if not hasattr(self._thread_local, "pending_rows"): + self._thread_local.pending_rows = [] + with self._buffers_lock: + # Register the list itself, not self._thread_local, so teardown can + # read the contents from any thread (main thread included). + self._registered_buffers.append(self._thread_local.pending_rows) + return self._thread_local.pending_rows + + @contextmanager + def init(self): + self.session = requests.Session() + if self._auth: + self.session.auth = self._auth + # Reset buffer registry so teardown only flushes buffers from this init() scope. + self._registered_buffers = [] + try: + yield + finally: + # Flush any rows that were buffered but not yet sent to Pinot. + # Each worker thread may have its own partially-filled buffer. + # This must happen here — not in optimize() — because optimize() runs + # in a separate subprocess where the buffers are always empty. + for pending in self._registered_buffers: + if pending: + log.info(f"Pinot init teardown: flushing {len(pending)} remaining buffered rows") + _, err = self._flush_rows(pending) + if err: + log.warning(f"Pinot init teardown: flush error: {err}") + self.session.close() + self.session = None + + def _flush_rows(self, rows: list) -> tuple[int, Exception | None]: + """Flush the given row list to Pinot as one segment using a fresh HTTP session. + + Using a fresh session (not self.session) makes this method safe to call + from multiple threads concurrently. On success the list is cleared in-place. + Returns (rows_flushed, error). On error the list is left intact so the caller + can decide whether to retry. + """ + if not rows: + return 0, None + + n = len(rows) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, prefix="pinot_ingest_") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + tmp_path = f.name + + try: + batch_config = json.dumps({"inputFormat": "json"}) + params = { + "tableNameWithType": f"{self.table_name}_OFFLINE", + "batchConfigMapStr": batch_config, + } + last_err = None + with requests.Session() as session: + if self._auth: + session.auth = self._auth + for attempt in range(3): + try: + with Path(tmp_path).open("rb") as f: + resp = session.post( + f"{self._controller_url}/ingestFromFile", + params=params, + files={"file": (Path(tmp_path).name, f, "application/json")}, + timeout=1800, # HNSW index building for 100K x 768D can take 10+ min + ) + if resp.ok: + rows.clear() + log.debug(f"Pinot: flushed segment with {n} rows") + return n, None + last_err = Exception(f"HTTP {resp.status_code}: {resp.text[:200]}") + log.warning(f"Pinot flush attempt {attempt + 1} failed: {last_err}") + time.sleep(1 + attempt) + except Exception as e: + last_err = e + log.warning(f"Pinot flush attempt {attempt + 1} error: {e}") + time.sleep(1 + attempt) + return 0, last_err + finally: + Path(tmp_path).unlink() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs: Any, + ) -> tuple[int, Exception]: + # Each thread has its own buffer; no locking needed here. + pending = self._get_thread_buffer() + + for i, (emb, meta) in enumerate(zip(embeddings, metadata, strict=False)): + row = {self._primary_field: meta, self._vector_field: list(emb)} + if self.with_scalar_labels and labels_data is not None: + row[self._label_field] = labels_data[i] + pending.append(row) + + if len(pending) >= self._ingest_batch_size: + _flushed, err = self._flush_rows(pending) + if err: + log.warning(f"Failed to flush Pinot buffer: {err}") + return 0, err + + return len(embeddings), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.session is not None, "Session not initialized" + + query_arr = ",".join(str(v) for v in query) + dist_fn, order = self._get_query_distance_fn() + + search_params = self.case_config.search_param() + nprobe = search_params.get("nprobe") + + filter_clause = self._filter_where + + if nprobe is not None: + # IVF-based index: set probe count via session option, use ORDER BY for top-k + where = f"WHERE {filter_clause} " if filter_clause else "" + sql = ( + f"set vectorNprobe={nprobe}; " + f"SELECT {self._primary_field} " + f"FROM {self.table_name} " + f"{where}" + f"ORDER BY {dist_fn}({self._vector_field}, ARRAY[{query_arr}]) {order} " + f"LIMIT {k}" + ) + else: + # HNSW index: WHERE vectorSimilarity(..., ef) triggers Lucene HNSW graph search + # with a candidate list of size ef (defaults to k). Larger ef → better recall. + # ORDER BY dist re-ranks the ANN candidates for correct final recall. + ef = search_params.get("ef") or k + extra = f" AND {filter_clause}" if filter_clause else "" + sql = ( + f"SELECT {self._primary_field} " + f"FROM {self.table_name} " + f"WHERE vectorSimilarity({self._vector_field}, ARRAY[{query_arr}], {ef}){extra} " + f"ORDER BY {dist_fn}({self._vector_field}, ARRAY[{query_arr}]) {order} " + f"LIMIT {k}" + ) + + resp = self.session.post( + f"{self._broker_url}/query/sql", + json={"sql": sql}, + headers={"Content-Type": "application/json"}, + timeout=timeout, + ) + resp.raise_for_status() + result = resp.json() + + rows = result.get("resultTable", {}).get("rows", []) + return [row[0] for row in rows] + + def optimize(self, data_size: int | None = None): + """Wait for all ingested data to be queryable in Pinot. + + Remaining buffered rows are flushed on init() teardown (in the insert + subprocess), so by the time optimize() runs they are already in Pinot. + """ + if self.session is None: + return + + if data_size is None: + time.sleep(5) + return + + max_wait = 600 + check_interval = 10 + start = time.time() + + while time.time() - start < max_wait: + try: + resp = self.session.post( + f"{self._broker_url}/query/sql", + json={"sql": f"SELECT COUNT(*) FROM {self.table_name}"}, + headers={"Content-Type": "application/json"}, + ) + if resp.status_code == 200: + rows = resp.json().get("resultTable", {}).get("rows", []) + current_count = rows[0][0] if rows else 0 + if current_count >= data_size: + log.info(f"Pinot: all {data_size} rows are queryable") + return + log.info(f"Pinot: {current_count}/{data_size} rows queryable, waiting...") + except Exception as e: + log.warning(f"Pinot optimize check error: {e}") + + time.sleep(check_interval) + + log.warning(f"Pinot optimize timed out after {max_wait}s") diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 3e21746aa..42048d57f 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -25,6 +25,7 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.pinecone.cli import Pinecone +from ..backend.clients.pinot.cli import Pinot from ..backend.clients.polardb.cli import ( PolarDBHNSWFlat, PolarDBHNSWPQ, @@ -90,6 +91,7 @@ cli.add_command(Pinecone) cli.add_command(VectorChordRQ) cli.add_command(VectorChordGraph) +cli.add_command(Pinot) cli.add_command(PolarDBHNSWFlat) cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) From 0c20701725a84fbcd2a14b5d628c77cac2beb071 Mon Sep 17 00:00:00 2001 From: James <83447078+xiaofan-luan@users.noreply.github.com> Date: Sun, 19 Apr 2026 23:21:55 -0700 Subject: [PATCH 23/38] fix: support self-hosted Elasticsearch via --host/--port in elasticcloud commands (#761) * fix: support self-hosted Elasticsearch via --host/--port in elasticcloud commands ElasticCloudConfig previously required cloud_id, so the elasticcloudhnsw* subcommands could only target Elastic Cloud. Users benchmarking self-hosted stock Elasticsearch had no working path: tencentelasticsearch accepts host/port but forces Tencent's vsearch index_options type, which stock ES rejects with "Unknown vector index options type [vsearch]". Extend ElasticCloudConfig with scheme/host/port/user fields (mutually exclusive with cloud_id) and expose them on all four ElasticCloudHNSW* CLI subcommands. Existing cloud_id callers are unchanged. Refs #758 Co-Authored-By: Claude Opus 4.7 (1M context) * style: apply black formatting to elastic_cloud/config.py Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../backend/clients/elastic_cloud/cli.py | 77 +++++++++++++++++-- .../backend/clients/elastic_cloud/config.py | 42 +++++++++- 2 files changed, 109 insertions(+), 10 deletions(-) diff --git a/vectordb_bench/backend/clients/elastic_cloud/cli.py b/vectordb_bench/backend/clients/elastic_cloud/cli.py index 55e521d8f..f277a6ff2 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/cli.py +++ b/vectordb_bench/backend/clients/elastic_cloud/cli.py @@ -17,11 +17,60 @@ class ElasticCloudTypedDict(TypedDict): cloud_id: Annotated[ str, - click.option("--cloud-id", type=str, help="Elastic Cloud ID", required=True), + click.option( + "--cloud-id", + type=str, + help="Elastic Cloud ID. Omit when connecting to a self-hosted ES via --host.", + required=False, + default="", + ), + ] + scheme: Annotated[ + str, + click.option( + "--scheme", + type=click.Choice(["http", "https"], case_sensitive=False), + help="Scheme for host-based connection.", + required=False, + default="https", + show_default=True, + ), + ] + host: Annotated[ + str, + click.option( + "--host", + type=str, + help="Elasticsearch host (for self-hosted ES; alternative to --cloud-id).", + required=False, + default="", + ), + ] + port: Annotated[ + int, + click.option( + "--port", + type=int, + help="Elasticsearch port (for host-based connection).", + required=False, + default=9200, + show_default=True, + ), + ] + user: Annotated[ + str, + click.option( + "--user", + type=str, + help="Elasticsearch user.", + required=False, + default="elastic", + show_default=True, + ), ] password: Annotated[ str, - click.option("--password", type=str, help="Elastic Cloud password", required=True), + click.option("--password", type=str, help="Elasticsearch password", required=True), ] number_of_shards: Annotated[ int, @@ -170,7 +219,11 @@ def ElasticCloudHNSW(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( @@ -202,7 +255,11 @@ def ElasticCloudHNSWInt8(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( @@ -234,7 +291,11 @@ def ElasticCloudHNSWInt4(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( @@ -266,7 +327,11 @@ def ElasticCloudHNSWBBQ(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index 7ee54b49c..bdae177f3 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -1,18 +1,52 @@ from enum import StrEnum -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, IndexType, MetricType class ElasticCloudConfig(DBConfig, BaseModel): - cloud_id: SecretStr + # Elastic Cloud connection. Takes precedence when set. + cloud_id: SecretStr | None = None + # Self-hosted / host-based connection (used when cloud_id is not provided). + scheme: str = "https" + host: str = "" + port: int = 9200 + user: str = "elastic" password: SecretStr + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"cloud_id", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + msg = "Empty string!" + raise ValueError(msg) + return data + + @model_validator(mode="after") + def _check_connection_target(self) -> "ElasticCloudConfig": + has_cloud_id = bool(self.cloud_id and self.cloud_id.get_secret_value()) + if not has_cloud_id and not self.host: + msg = "ElasticCloudConfig requires either cloud_id or host to be set." + raise ValueError(msg) + return self + def to_dict(self) -> dict: + auth = (self.user, self.password.get_secret_value()) + if self.cloud_id and self.cloud_id.get_secret_value(): + return { + "cloud_id": self.cloud_id.get_secret_value(), + "basic_auth": auth, + } return { - "cloud_id": self.cloud_id.get_secret_value(), - "basic_auth": ("elastic", self.password.get_secret_value()), + "hosts": [{"scheme": self.scheme, "host": self.host, "port": self.port}], + "basic_auth": auth, } From b3613ff6befcc5c802b77617cd880750917c1c51 Mon Sep 17 00:00:00 2001 From: B Nagaraju Reddy <107165377+NagarajuReddyBoggala@users.noreply.github.com> Date: Tue, 21 Apr 2026 09:33:17 +0530 Subject: [PATCH 24/38] Fix: Map "ivf_flat" to "ivfflat" for pgvector index access method (#763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix: Map "ivf_flat" to "ivfflat" for pgvector index access method - IndexType.IVFFlat.value="IVF_FLAT" → .lower()="ivf_flat" caused SQL to fail with "access method 'ivf_flat' does not exist" - pgvector PostgreSQL extension expects "ivfflat" (no underscore), not "ivf_flat" - Added explicit mapping after lowercase normalization: if index_type_lower == "ivf_flat": index_type_lower = "ivfflat" * style(pgvector): fix comment wrapping and remove commented code --------- Co-authored-by: rnagaraju --- .../backend/clients/pgvector/pgvector.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 0b0750172..630758e1b 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -335,12 +335,20 @@ def _create_index(self): index_param = self.case_config.index_param() self._set_parallel_index_build_param() - # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in lowercase (e.g., "hnsw", "ivfflat"), - # but the index type passed from the frontend UI is uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" - # to fail with error "access method HNSW does not exist". Here we uniformly convert it to lowercase to match PostgreSQL's access method name. + # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in + # lowercase (e.g., "hnsw", "ivfflat"), but the index type passed from the frontend UI is + # uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" to fail + # with error "access method HNSW does not exist". Here we uniformly convert it to lowercase + # to match PostgreSQL's access method name. index_type_lower = index_param["index_type"].lower() + # [FIX] The pgvector access method name is "ivfflat" (no underscore), but IndexType.IVFFlat.value + # produces "IVF_FLAT" which becomes "ivf_flat" after lowercase conversion, causing SQL syntax + # "USING 'ivf_flat'" to fail with error "access method 'ivf_flat' does not exist". + # Here we map "ivf_flat" → "ivfflat" to match PostgreSQL pgvector's registered access method name. + if index_type_lower == "ivf_flat": + index_type_lower = "ivfflat" log.info(f"index_type (original={index_param['index_type']}, normalized={index_type_lower})") - + options = [] for option in index_param["index_creation_with_options"]: if option["val"] is not None: @@ -368,7 +376,6 @@ def _create_index(self): ), # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), - # index_type=sql.Identifier(index_param["index_type"]), # This assumes that the quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), dim=self.dim, @@ -385,7 +392,6 @@ def _create_index(self): table_name=sql.Identifier(self.table_name), # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), - # index_type=sql.Identifier(index_param["index_type"]), embedding_metric=sql.Identifier(index_param["metric"]), ) From 02e5d33df8b2de83a08b7fd61b66529344865781 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Tue, 21 Apr 2026 15:01:56 +0800 Subject: [PATCH 25/38] fix(pgvector): fix ConcurrentInsertRunner for non-thread-safe DBs (#764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For non-thread-safe DBs (e.g. PgVector), ConcurrentInsertRunner clamps max_workers to 1, so there is always exactly one worker thread. There is no need to deepcopy self.db per thread — the single worker can use self.db directly via the connection already opened by task()'s `with self.db.init():`. The original code called deepcopy(self.db) inside _get_thread_db() after task() had already opened a live psycopg C-extension Connection on self.db. C-extension objects cannot be deep-copied, causing: TypeError: no default __reduce__ due to non-trivial __cinit__ Fix: remove the deepcopy branch entirely. All workers (thread-safe or not) now use self.db directly; thread-safety is guaranteed for non-thread-safe DBs by the max_workers=1 clamp. Also clean up stale comments in pgvector.py left over from #760/#763. Adds tests/test_pgvector.py with: - unit test that reproduces the bug (fails on original, passes on fix) - e2e regression test via ConcurrentInsertRunner + OpenAI 50K dataset See also: #756 Signed-off-by: yangxuan --- tests/pytest.ini | 3 + tests/test_pgvector.py | 180 ++++++++++++++++++ .../backend/clients/pgvector/pgvector.py | 15 +- .../backend/runner/concurrent_runner.py | 90 +++------ 4 files changed, 211 insertions(+), 77 deletions(-) create mode 100644 tests/test_pgvector.py diff --git a/tests/pytest.ini b/tests/pytest.ini index e5915e89e..9f5751ee3 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -3,3 +3,6 @@ filterwarnings = ignore::UserWarning ignore::DeprecationWarning + +markers = + integration: tests that require external services or network access (deselect with -m "not integration") diff --git a/tests/test_pgvector.py b/tests/test_pgvector.py new file mode 100644 index 000000000..cdd9461a9 --- /dev/null +++ b/tests/test_pgvector.py @@ -0,0 +1,180 @@ +"""Tests for PgVector client and ConcurrentInsertRunner. + +Reproduces issue #756: insert fails with + TypeError: no default __reduce__ due to non-trivial __cinit__ +when ConcurrentInsertRunner deep-copies a PgVector instance that has a live +psycopg connection open (the connection is opened by `with self.db.init():` +inside task() before the deepcopy in _get_thread_db()). + +Requires: + docker run -d --name pgvector-test \ + -e POSTGRES_USER=vectordb -e POSTGRES_PASSWORD=vectordb \ + -e POSTGRES_DB=vectordb -p 5432:5432 \ + pgvector/pgvector:pg17 + +Usage: + pytest tests/test_pgvector.py -v -s +""" + +from __future__ import annotations + +import logging +import pickle +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.pgvector.config import PgVectorHNSWConfig +from vectordb_bench.backend.dataset import Dataset, DatasetSource +from vectordb_bench.backend.filter import Filter, FilterOp, non_filter +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner + +log = logging.getLogger(__name__) + +# ── Connection config ──────────────────────────────────────────────────────── + +DB_CONFIG = { + "connect_config": { + "host": "localhost", + "port": 5432, + "dbname": "vectordb", + "user": "vectordb", + "password": "vectordb", + }, + "table_name": "test_pgvector", +} + +DIM = 128 +COUNT = 500 +RNG = np.random.default_rng(42) + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def make_hnsw_config(**kwargs) -> PgVectorHNSWConfig: + return PgVectorHNSWConfig( + metric_type="COSINE", + m=16, + ef_construction=64, + ef_search=64, + **kwargs, + ) + + +def make_db(table_name: str = "test_pgvector", drop_old: bool = True) -> DB.PgVector.init_cls: + cfg = dict(DB_CONFIG) + cfg["table_name"] = table_name + return DB.PgVector.init_cls( + dim=DIM, + db_config=cfg, + db_case_config=make_hnsw_config(), + drop_old=drop_old, + ) + + +def random_embeddings(n: int = COUNT, d: int = DIM) -> list[list[float]]: + return RNG.random((n, d)).tolist() + + +# ── Basic client tests ──────────────────────────────────────────────────────── + + +class TestPgVectorBasic: + """Unit tests for the PgVector client (no subprocess).""" + + def test_insert_and_search(self): + db = make_db("test_basic") + embeddings = random_embeddings() + metadata = list(range(COUNT)) + + with db.init(): + count, err = db.insert_embeddings(embeddings=embeddings, metadata=metadata) + assert err is None, f"Insert error: {err}" + assert count == COUNT + + with db.init(): + db.optimize() + + with db.init(): + db.prepare_filter(Filter(type=FilterOp.NonFilter)) + results = db.search_embedding(query=embeddings[0], k=10) + assert len(results) > 0 + + def test_db_is_not_thread_safe(self): + db = make_db("test_thread_safe") + assert db.thread_safe is False + + def test_db_picklable_after_init(self): + """PgVector instance must be picklable after __init__ (conn/cursor are None). + + This is required for ConcurrentInsertRunner which spawns a subprocess + and pickles self (which includes self.db). + """ + db = make_db("test_pickle") + data = pickle.dumps(db) + db2 = pickle.loads(data) # noqa: S301 + assert db2.dim == DIM + + def test_get_thread_db_with_open_connection(self): + """Regression test for issue #756. + + ConcurrentInsertRunner.task() opens `with self.db.init()` before calling + workers. For non-thread-safe DBs the original _get_thread_db() then called + deepcopy(self.db) — but the live psycopg C-extension Connection is not + deep-copyable, causing TypeError. + + Fixed code returns self.db directly (no deepcopy), so this test must pass + without raising. + """ + db = make_db("test_get_thread_db") + runner = ConcurrentInsertRunner(db=db, dataset=MagicMock(), normalize=False) + + with db.init(): + assert db.conn is not None + result = runner._get_thread_db() # TypeError here on original code + + assert result is db + + +# ── ConcurrentInsertRunner tests ────────────────────────────────────────────── + + +class TestPgVectorConcurrentInsert: + """Tests for ConcurrentInsertRunner with PgVector (reproduces issue #756).""" + + @pytest.mark.integration + def test_concurrent_insert_e2e(self): + """E2E regression test for issue #756 using the OpenAI 50K dataset. + + Exercises the full pipeline: + ProcessPoolExecutor(spawn) → pickle runner → subprocess task() + → with self.db.init() → worker _get_thread_db() → insert batches + + FAILS on original code (TypeError: deepcopy of live psycopg connection). + PASSES on fixed code. + """ + dataset = Dataset.OPENAI.manager(50_000) + dataset.prepare(DatasetSource.AliyunOSS) + + cfg = dict(DB_CONFIG) + cfg["table_name"] = "test_e2e_insert" + db = DB.PgVector.init_cls( + dim=dataset.data.dim, + db_config=cfg, + db_case_config=PgVectorHNSWConfig( + metric_type="COSINE", + m=16, + ef_construction=64, + ef_search=64, + ), + drop_old=True, + ) + + runner = ConcurrentInsertRunner(db=db, dataset=dataset, normalize=True, filters=non_filter) + count = runner.run() + + assert count == 50_000, f"Expected 50000 rows, got {count}" + log.info(f"E2E insert completed: {count} rows") diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 630758e1b..41060af27 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -335,16 +335,9 @@ def _create_index(self): index_param = self.case_config.index_param() self._set_parallel_index_build_param() - # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in - # lowercase (e.g., "hnsw", "ivfflat"), but the index type passed from the frontend UI is - # uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" to fail - # with error "access method HNSW does not exist". Here we uniformly convert it to lowercase - # to match PostgreSQL's access method name. + # pgvector registers access methods in lowercase ("hnsw", "ivfflat") but + # IndexType enum values are uppercase; also IVFFlat maps to "ivfflat" (no underscore). index_type_lower = index_param["index_type"].lower() - # [FIX] The pgvector access method name is "ivfflat" (no underscore), but IndexType.IVFFlat.value - # produces "IVF_FLAT" which becomes "ivf_flat" after lowercase conversion, causing SQL syntax - # "USING 'ivf_flat'" to fail with error "access method 'ivf_flat' does not exist". - # Here we map "ivf_flat" → "ivfflat" to match PostgreSQL pgvector's registered access method name. if index_type_lower == "ivf_flat": index_type_lower = "ivfflat" log.info(f"index_type (original={index_param['index_type']}, normalized={index_type_lower})") @@ -374,9 +367,8 @@ def _create_index(self): if index_param["quantization_type"] == "bit" else sql.Identifier("embedding") ), - # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), - # This assumes that the quantization_type value matches the quantization function name + # quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), dim=self.dim, embedding_metric=sql.Identifier(index_param["metric"]), @@ -390,7 +382,6 @@ def _create_index(self): ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), - # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), embedding_metric=sql.Identifier(index_param["metric"]), ) diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 6ed8e39fb..37201f88e 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -13,7 +13,6 @@ import multiprocessing as mp import threading import time -from copy import deepcopy from enum import StrEnum from typing import TYPE_CHECKING @@ -44,7 +43,7 @@ class ConcurrentInsertRunner: """Concurrent insert runner with pluggable executor backend. Thread-safety: If db.thread_safe is False, max_workers is clamped to 1 - and each worker thread gets a deep-copied DB instance with its own connection. + so the single worker thread uses self.db directly (no deepcopy needed). Args: db: VectorDB instance. @@ -78,57 +77,31 @@ def __init__( log.info(f"DB {db.name} is not thread-safe, falling back to max_workers=1") effective_workers = 1 self.max_workers = effective_workers + assert db.thread_safe or self.max_workers == 1, ( + "Non-thread-safe DBs must use max_workers=1 — " + "_get_thread_db() relies on this to avoid concurrent access to self.db" + ) def __getstate__(self): """Exclude unpicklable thread-local state for ProcessPoolExecutor(spawn).""" state = self.__dict__.copy() - state.pop("_local", None) - state.pop("_ctx_lock", None) - state.pop("_thread_contexts", None) state.pop("_iter_lock", None) state.pop("_dataset_iter", None) return state - def __setstate__(self, state: dict): - self.__dict__.update(state) - self._local = threading.local() - self._ctx_lock = threading.Lock() - self._thread_contexts = [] - def _create_executor(self) -> TaskExecutor: if self.backend == ExecutorBackend.ASYNC: return AsyncExecutor(max_workers=self.max_workers) return ThreadExecutor(max_workers=self.max_workers) def _get_thread_db(self) -> api.VectorDB: - """Get or create a per-thread DB instance. + """Return self.db. - Thread-safe DBs reuse self.db (connection opened in task()). - Non-thread-safe DBs get a deep-copied instance with its own connection, - cached in thread-local storage so it is created once per thread. + All workers share the connection opened by task()'s `with self.db.init()`. + Thread-safe DBs share it across multiple workers. Non-thread-safe DBs are + clamped to max_workers=1, so there is never concurrent access. """ - if not hasattr(self._local, "db"): - if self.db.thread_safe: - self._local.db = self.db - else: - db = deepcopy(self.db) - # Manual __enter__/__exit__ because enter and exit happen in - # different scopes (here vs _cleanup_thread_contexts). - ctx = db.init() - ctx.__enter__() - self._local.db = db - with self._ctx_lock: - self._thread_contexts.append(ctx) - return self._local.db - - def _cleanup_thread_contexts(self) -> None: - """Close per-thread DB connections opened for non-thread-safe clients.""" - for ctx in self._thread_contexts: - try: - ctx.__exit__(None, None, None) - except Exception: - log.warning("Failed to close per-thread DB connection", exc_info=True) - self._thread_contexts.clear() + return self.db def _insert_batch_with_retry( self, @@ -160,14 +133,7 @@ def _worker_insert( metadata: list[int], labels_data: list[str] | None = None, ) -> int: - """Worker function: insert a batch with retry. - - Thread-safe DBs: reuse self.db whose connection is already open - via task()'s `with self.db.init()` — all threads share it safely. - - Non-thread-safe DBs: use a per-thread deep-copied instance with - its own connection, cached via threading.local. - """ + """Worker function: insert a batch with retry.""" db = self._get_thread_db() return self._insert_batch_with_retry(db, embeddings, metadata, labels_data) @@ -214,9 +180,6 @@ def _worker_loop(self) -> int: def task(self) -> int: """Insert entire dataset using concurrent executor. Runs in subprocess.""" count = 0 - self._local = threading.local() - self._ctx_lock = threading.Lock() - self._thread_contexts = [] self._iter_lock = threading.Lock() self._dataset_iter = iter(self.dataset) @@ -227,23 +190,20 @@ def task(self) -> int: ) start = time.perf_counter() - try: - with self._create_executor() as executor: - for _ in range(self.max_workers): - executor.submit(self._worker_loop) - - batch_results = executor.wait_all() - - # Log all errors, then raise the first one - errors = [r.error for r in batch_results if r.error is not None] - if errors: - for err in errors: - log.warning(f"Batch insert error: {err}") - raise errors[0] - - count = sum(r.value for r in batch_results) - finally: - self._cleanup_thread_contexts() + with self._create_executor() as executor: + for _ in range(self.max_workers): + executor.submit(self._worker_loop) + + batch_results = executor.wait_all() + + # Log all errors, then raise the first one + errors = [r.error for r in batch_results if r.error is not None] + if errors: + for err in errors: + log.warning(f"Batch insert error: {err}") + raise errors[0] + + count = sum(r.value for r in batch_results) log.info( f"({mp.current_process().name:16}) Finish concurrent insert, " From 63cc50a02d9e4e9e576a7fd640448fa93d9a7341 Mon Sep 17 00:00:00 2001 From: EeshaaKhan <170761203+EeshaaKhan@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:44:32 +0500 Subject: [PATCH 26/38] Feat: Add label filter support in pgdiskann client (#724) * Add label filtering support to pgdiskann client * Refactor pgdiskann filtering logic * Refactor: remove unrelated function * style: apply black formatting to pgdiskann.py * fix: remove trailing whitespace and fix import sorting * docs: add comments for label naming and vector storage optimization * Revert "docs: add comments for label naming and vector storage optimization" This reverts commit d10b296b612f6568f5c2abf043b0003d2f4ca8b4. --------- Co-authored-by: Eesha Faisal --- .../backend/clients/pgdiskann/pgdiskann.py | 204 +++++++++--------- 1 file changed, 106 insertions(+), 98 deletions(-) diff --git a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py index 5f069ace5..46e8fabd4 100644 --- a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +++ b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py @@ -10,6 +10,8 @@ from pgvector.psycopg import register_vector from psycopg import Connection, Cursor, sql +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import PgDiskANNConfigDict, PgDiskANNIndexConfig @@ -19,11 +21,16 @@ class PgDiskANN(VectorDB): """Use psycopg instructions""" + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + conn: psycopg.Connection[Any] | None = None - coursor: psycopg.Cursor[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None - _filtered_search: sql.Composed - _unfiltered_search: sql.Composed + _search: sql.Composed def __init__( self, @@ -32,6 +39,7 @@ def __init__( db_case_config: PgDiskANNIndexConfig, collection_name: str = "pg_diskann_collection", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.name = "PgDiskANN" @@ -39,6 +47,9 @@ def __init__( self.case_config = db_case_config self.table_name = collection_name self.dim = dim + self.with_scalar_labels = with_scalar_labels + self._scalar_label_field = "label" + self.where_clause = "" self._index_name = "pgdiskann_index" self._primary_field = "id" @@ -86,83 +97,58 @@ def _create_connection(**kwargs) -> tuple[Connection, Cursor]: return conn, cursor - @contextmanager - def init(self) -> Generator[None, None, None]: - self.conn, self.cursor = self._create_connection(**self.db_config) - - session_options: dict[str, Any] = self.case_config.session_param() - - if len(session_options) > 0: - for setting_name, setting_val in session_options.items(): - command = sql.SQL("SET {setting_name} = {setting_val};").format( - setting_name=sql.Identifier(setting_name), setting_val=sql.Literal(setting_val) - ) - log.debug(command.as_string(self.cursor)) - self.cursor.execute(command) - self.conn.commit() - + def _generate_search_query(self) -> sql.Composed: + """Generate search query with where_clause placeholder""" search_params = self.case_config.search_param() if search_params.get("reranking"): - # Reranking-enabled queries - self._filtered_search = sql.SQL(""" + search_query = sql.SQL(""" SELECT i.id FROM ( SELECT id, embedding FROM public.{table_name} - WHERE id >= %s + {where_clause} ORDER BY embedding {metric_fun_op} %s::vector LIMIT {quantized_fetch_limit}::int ) i ORDER BY i.embedding {reranking_metric_fun_op} %s::vector LIMIT %s::int - """).format( + """).format( table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(self.where_clause), metric_fun_op=sql.SQL(search_params["metric_fun_op"]), reranking_metric_fun_op=sql.SQL(search_params["reranking_metric_fun_op"]), quantized_fetch_limit=sql.Literal(search_params["quantized_fetch_limit"]), ) - - self._unfiltered_search = sql.SQL(""" - SELECT i.id - FROM ( - SELECT id, embedding - FROM public.{table_name} - ORDER BY embedding {metric_fun_op} %s::vector - LIMIT {quantized_fetch_limit}::int - ) i - ORDER BY i.embedding {reranking_metric_fun_op} %s::vector - LIMIT %s::int - """).format( - table_name=sql.Identifier(self.table_name), - metric_fun_op=sql.SQL(search_params["metric_fun_op"]), - reranking_metric_fun_op=sql.SQL(search_params["reranking_metric_fun_op"]), - quantized_fetch_limit=sql.Literal(search_params["quantized_fetch_limit"]), - ) - else: - self._filtered_search = sql.Composed( + search_query = sql.Composed( [ - sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding ", - ).format(table_name=sql.Identifier(self.table_name)), - sql.SQL(search_params["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ] - ) - - self._unfiltered_search = sql.Composed( - [ - sql.SQL("SELECT id FROM public.{table_name} ORDER BY embedding ").format( - table_name=sql.Identifier(self.table_name) + sql.SQL("SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ").format( + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(self.where_clause), ), sql.SQL(search_params["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), ] ) - log.debug(f"Unfiltered search query={self._unfiltered_search.as_string(self.conn)}") - log.debug(f"Filtered search query={self._filtered_search.as_string(self.conn)}") + return search_query + + @contextmanager + def init(self) -> Generator[None, None, None]: + self.conn, self.cursor = self._create_connection(**self.db_config) + + session_options: dict[str, Any] = self.case_config.session_param() + + if len(session_options) > 0: + for setting_name, setting_val in session_options.items(): + command = sql.SQL("SET {setting_name} = {setting_val};").format( + setting_name=sql.Identifier(setting_name), + setting_val=sql.Literal(setting_val), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() try: yield @@ -281,12 +267,10 @@ def _create_index(self): with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) - index_create_sql = sql.SQL( - """ + index_create_sql = sql.SQL(""" CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric}) - """, - ).format( + """).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), index_type=sql.Identifier(index_param["index_type"].lower()), @@ -304,11 +288,36 @@ def _create_table(self, dim: int): try: log.info(f"{self.name} client create table : {self.table_name}") + if self.with_scalar_labels: + self.cursor.execute( + sql.SQL(""" + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding vector({dim}), {label_field} VARCHAR(64)); + """).format( + table_name=sql.Identifier(self.table_name), + dim=dim, + primary_field=sql.Identifier(self._primary_field), + label_field=sql.Identifier(self._scalar_label_field), + ), + ) + else: + self.cursor.execute( + sql.SQL(""" + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding vector({dim})); + """).format( + table_name=sql.Identifier(self.table_name), + dim=dim, + primary_field=sql.Identifier(self._primary_field), + ), + ) + self.cursor.execute( - sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", - ).format(table_name=sql.Identifier(self.table_name), dim=dim), + sql.SQL("ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;").format( + table_name=sql.Identifier(self.table_name) + ), ) + self.conn.commit() except Exception as e: log.warning(f"Failed to create pgdiskann table: {self.table_name} error: {e}") @@ -318,11 +327,15 @@ def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs: Any, ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + if self.with_scalar_labels: + assert labels_data is not None, "labels_data should be provided if with_scalar_labels is set to True" + try: metadata_arr = np.array(metadata) embeddings_arr = np.array(embeddings) @@ -332,9 +345,14 @@ def insert_embeddings( table_name=sql.Identifier(self.table_name), ), ) as copy: - copy.set_types(["bigint", "vector"]) - for i, row in enumerate(metadata_arr): - copy.write_row((row, embeddings_arr[i])) + if self.with_scalar_labels: + copy.set_types(["bigint", "vector", "varchar"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i], labels_data[i])) + else: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) self.conn.commit() if kwargs.get("last_batch"): @@ -345,49 +363,39 @@ def insert_embeddings( log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") return 0, e + def prepare_filter(self, filters: Filter): + """Prepare filter - builds where_clause""" + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + self.where_clause = f"WHERE {self._primary_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.where_clause = f"WHERE {self._scalar_label_field} = '{filters.label_value}'" + else: + msg = f"Not support Filter for PgDiskANN - {filters}" + raise ValueError(msg) + + self._search = self._generate_search_query() + log.debug(f"Search query={self._search.as_string(self.conn)}") + def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, timeout: int | None = None, + **kwargs: Any, ) -> list[int]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" search_params = self.case_config.search_param() - is_reranking = search_params.get("reranking", False) - q = np.asarray(query) - if filters: - gt = filters.get("id") - if is_reranking: - result = self.cursor.execute( - self._filtered_search, - (gt, q, q, k), - prepare=True, - binary=True, - ) - else: - result = self.cursor.execute( - self._filtered_search, - (gt, q, k), - prepare=True, - binary=True, - ) - elif is_reranking: - result = self.cursor.execute( - self._unfiltered_search, - (q, q, k), - prepare=True, - binary=True, - ) - else: - result = self.cursor.execute( - self._unfiltered_search, - (q, k), - prepare=True, - binary=True, - ) + + result = self.cursor.execute( + self._search, + (q, q, k) if search_params.get("reranking", False) else (q, k), + prepare=True, + binary=True, + ) return [int(i[0]) for i in result.fetchall()] From 4082eff8ff602a245abd14d915724d718bcbe2f2 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 24 Apr 2026 17:21:00 +0800 Subject: [PATCH 27/38] fix(ui): Run Test page error surfacing and streamlit upgrade (#766) - Migrate DB config validators to pydantic v2; list all empty fields instead of raising on first; consolidate via `_extra_empty_skip`. - Surface missing client modules at config render time as `{DB} needs `{module}` but it is not installed.` - Replace streamlit-autorefresh with native `@st.fragment(run_every)` so live progress does not block UI. - Bump streamlit to 1.47+ (picks up streamlit#11890 fragment fix); switch to native `st.switch_page`, drop `streamlit_extras`. - Migrate deprecated `use_container_width=True` to `width="stretch"`. - Patch tornado `write_message` to consume expected `WebSocketClosedError` on tab-close races (streamlit#9787). - Add contract test: each DB enum resolves config_cls/init_cls or raises ModuleNotFoundError. See also: #446 Signed-off-by: yangxuan --- pyproject.toml | 4 +- tests/test_db_client_resolution.py | 17 ++ vectordb_bench/backend/clients/api.py | 15 +- .../backend/clients/aws_opensearch/config.py | 18 +-- .../backend/clients/elastic_cloud/config.py | 17 +- .../backend/clients/milvus/config.py | 19 +-- .../backend/clients/oss_opensearch/config.py | 16 +- .../backend/clients/qdrant_cloud/config.py | 19 +-- vectordb_bench/backend/clients/tidb/config.py | 19 +-- .../components/check_results/charts.py | 2 +- .../frontend/components/check_results/nav.py | 9 +- .../components/check_results/priceTable.py | 2 +- .../frontend/components/concurrent/charts.py | 2 +- .../frontend/components/int_filter/charts.py | 2 +- .../components/label_filter/charts.py | 2 +- .../frontend/components/qps_recall/charts.py | 2 +- .../components/run_test/autoRefresh.py | 10 -- .../components/run_test/dbConfigSetting.py | 17 +- .../components/run_test/submitTask.py | 153 +++++++++--------- .../frontend/components/streaming/charts.py | 4 +- .../components/streaming/concurrent_detail.py | 4 +- vectordb_bench/frontend/pages/run_test.py | 4 - vectordb_bench/frontend/vdbbench.py | 23 +++ 23 files changed, 171 insertions(+), 209 deletions(-) create mode 100644 tests/test_db_client_resolution.py delete mode 100644 vectordb_bench/frontend/components/run_test/autoRefresh.py diff --git a/pyproject.toml b/pyproject.toml index 2ed18b115..8bd5de2ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,7 @@ classifiers = [ dependencies = [ "click", "pytz", - "streamlit-autorefresh", - "streamlit<1.44,!=1.34.0", # There is a breaking change in 1.44 related to get_page https://discuss.streamlit.io/t/from-streamlit-source-util-import-get-pages-gone-in-v-1-44-0-need-urgent-help/98399 - "streamlit_extras", + "streamlit>=1.47,<2", # 1.47 fixes streamlit#11660 "tqdm", "s3fs", "oss2", diff --git a/tests/test_db_client_resolution.py b/tests/test_db_client_resolution.py new file mode 100644 index 000000000..ce0278eb0 --- /dev/null +++ b/tests/test_db_client_resolution.py @@ -0,0 +1,17 @@ +"""Every DB must resolve config_cls/init_cls or raise ModuleNotFoundError. +Anything else breaks the Run Test page's missing-optional-dep hint path. +""" + +import pytest + +from vectordb_bench.backend.clients import DB + + +@pytest.mark.parametrize("db", list(DB), ids=lambda d: d.name) +def test_db_resolves_or_missing_module(db): + for attr in ("config_cls", "init_cls"): + try: + getattr(db, attr) + except ModuleNotFoundError as e: + assert e.name, f"{db.name}.{attr}: ModuleNotFoundError has no .name" + return diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index f507abe33..118c505ff 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from contextlib import contextmanager from enum import StrEnum +from typing import ClassVar from pydantic import BaseModel, model_validator @@ -77,6 +78,9 @@ class DBConfig(ABC, BaseModel): version: str = "" note: str = "" + # Field names subclasses allow to be empty (optional creds, alt-route fields). + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset() + @staticmethod def common_short_configs() -> list[str]: """ @@ -100,12 +104,11 @@ def to_dict(self) -> dict: def not_empty_field(cls, data: any) -> any: if not isinstance(data, dict): return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | cls._extra_empty_skip + empty = [k for k, v in data.items() if k not in skip and isinstance(v, str) and not v] + if empty: + msg = f"Empty field(s): {', '.join(empty)}" + raise ValueError(msg) return data diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 7742d421d..62c284317 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,7 +1,8 @@ import logging from enum import Enum +from typing import ClassVar -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -9,6 +10,8 @@ class AWSOpenSearchConfig(DBConfig, BaseModel): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password", "host"}) + host: str = "" port: int = 80 user: str | None = None @@ -32,19 +35,6 @@ def to_dict(self) -> dict: "timeout": 600, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class AWSOS_Engine(Enum): faiss = "faiss" diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index bdae177f3..be2c2dce8 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -1,4 +1,5 @@ from enum import StrEnum +from typing import ClassVar from pydantic import BaseModel, SecretStr, model_validator @@ -6,6 +7,8 @@ class ElasticCloudConfig(DBConfig, BaseModel): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"cloud_id", "host"}) + # Elastic Cloud connection. Takes precedence when set. cloud_id: SecretStr | None = None # Self-hosted / host-based connection (used when cloud_id is not provided). @@ -15,20 +18,6 @@ class ElasticCloudConfig(DBConfig, BaseModel): user: str = "elastic" password: SecretStr - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"cloud_id", "host"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - msg = "Empty string!" - raise ValueError(msg) - return data - @model_validator(mode="after") def _check_connection_target(self) -> "ElasticCloudConfig": has_cloud_id = bool(self.cloud_id and self.cloud_id.get_secret_value()) diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 620a6b484..054a3fddb 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,9 +1,13 @@ -from pydantic import BaseModel, SecretStr, model_validator +from typing import ClassVar + +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType class MilvusConfig(DBConfig): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password"}) + uri: SecretStr = "http://localhost:19530" user: str | None = None password: SecretStr | None = None @@ -19,19 +23,6 @@ def to_dict(self) -> dict: "replica_number": self.replica_number, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class MilvusIndexConfig(BaseModel): """Base config for milvus""" diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py index a5d69459a..7a8b1d98e 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/config.py +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -1,5 +1,6 @@ import logging from enum import Enum +from typing import ClassVar from pydantic import BaseModel, SecretStr, field_validator, model_validator @@ -9,6 +10,8 @@ class OSSOpenSearchConfig(DBConfig, BaseModel): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password", "host"}) + host: str = "" port: int = 80 user: str | None = None @@ -32,19 +35,6 @@ def to_dict(self) -> dict: "timeout": 600, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class OSSOS_Engine(Enum): faiss = "faiss" diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index 06543aaab..c4466dc17 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,6 +1,6 @@ -from typing import TypeVar +from typing import ClassVar, TypeVar -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -10,6 +10,8 @@ # Allowing `api_key` to be left empty, to ensure compatibility with the open-source Qdrant. class QdrantConfig(DBConfig): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"api_key"}) + url: SecretStr api_key: SecretStr | None = None @@ -25,19 +27,6 @@ def to_dict(self) -> dict: "url": self.url.get_secret_value(), } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"api_key"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class QdrantIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 93098ede1..5e2f032d4 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,6 +1,6 @@ -from typing import TypedDict +from typing import ClassVar, TypedDict -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -16,6 +16,8 @@ class TiDBConfigDict(TypedDict): class TiDBConfig(DBConfig): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"password"}) + user_name: str = "root" password: SecretStr host: str = "127.0.0.1" @@ -35,19 +37,6 @@ def to_dict(self) -> TiDBConfigDict: "ssl_verify_identity": self.ssl, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"password"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class TiDBIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/frontend/components/check_results/charts.py b/vectordb_bench/frontend/components/check_results/charts.py index 0e74d2752..d36cc5fc6 100644 --- a/vectordb_bench/frontend/components/check_results/charts.py +++ b/vectordb_bench/frontend/components/check_results/charts.py @@ -153,4 +153,4 @@ def drawMetricChart(data, metric, st, key: str): ), ) - chart.plotly_chart(fig, use_container_width=True, key=key) + chart.plotly_chart(fig, width="stretch", key=key) diff --git a/vectordb_bench/frontend/components/check_results/nav.py b/vectordb_bench/frontend/components/check_results/nav.py index ba4fa99c7..2267024bb 100644 --- a/vectordb_bench/frontend/components/check_results/nav.py +++ b/vectordb_bench/frontend/components/check_results/nav.py @@ -1,25 +1,22 @@ -from streamlit_extras.switch_page_button import switch_page - - def NavToRunTest(st): st.subheader("Run your test") st.write("You can set the configs and run your own test.") navClick = st.button("Run Your Test   >") if navClick: - switch_page("run test") + st.switch_page("pages/run_test.py") def NavToQuriesPerDollar(st): st.subheader("Compare qps with price.") navClick = st.button("QP$ (Quries per Dollar)   >") if navClick: - switch_page("quries_per_dollar") + st.switch_page("pages/quries_per_dollar.py") def NavToResults(st, key="nav-to-results"): navClick = st.button("<   Back to Results", key=key) if navClick: - switch_page("results") + st.switch_page("pages/results.py") def NavToPages(st): diff --git a/vectordb_bench/frontend/components/check_results/priceTable.py b/vectordb_bench/frontend/components/check_results/priceTable.py index f2c0ae001..f34f70872 100644 --- a/vectordb_bench/frontend/components/check_results/priceTable.py +++ b/vectordb_bench/frontend/components/check_results/priceTable.py @@ -27,7 +27,7 @@ def priceTable(container, data): expander = container.expander("Price List (Editable).") editTable = expander.data_editor( table, - use_container_width=True, + width="stretch", hide_index=True, height=height, disabled=("DB", "Label"), diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py index 004fcb261..5369d5912 100644 --- a/vectordb_bench/frontend/components/concurrent/charts.py +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -94,4 +94,4 @@ def drawChart(data, st, key: str, x_metric: str = "latency_p99", y_metric: str = fig.update_yaxes(range=yrange, title_text=gen_title(y_metric)) fig.update_traces(textposition="bottom right", texttemplate="conc-%{text:,.4~r}") - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) diff --git a/vectordb_bench/frontend/components/int_filter/charts.py b/vectordb_bench/frontend/components/int_filter/charts.py index 881681031..5c32a089e 100644 --- a/vectordb_bench/frontend/components/int_filter/charts.py +++ b/vectordb_bench/frontend/components/int_filter/charts.py @@ -57,4 +57,4 @@ def drawChart(st, data: list[object], metric): margin=dict(l=0, r=0, t=40, b=0, pad=8), legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), ) - st.plotly_chart(fig, use_container_width=True) + st.plotly_chart(fig, width="stretch") diff --git a/vectordb_bench/frontend/components/label_filter/charts.py b/vectordb_bench/frontend/components/label_filter/charts.py index 881681031..5c32a089e 100644 --- a/vectordb_bench/frontend/components/label_filter/charts.py +++ b/vectordb_bench/frontend/components/label_filter/charts.py @@ -57,4 +57,4 @@ def drawChart(st, data: list[object], metric): margin=dict(l=0, r=0, t=40, b=0, pad=8), legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), ) - st.plotly_chart(fig, use_container_width=True) + st.plotly_chart(fig, width="stretch") diff --git a/vectordb_bench/frontend/components/qps_recall/charts.py b/vectordb_bench/frontend/components/qps_recall/charts.py index ab57dd0ce..a44c51a36 100644 --- a/vectordb_bench/frontend/components/qps_recall/charts.py +++ b/vectordb_bench/frontend/components/qps_recall/charts.py @@ -115,4 +115,4 @@ def drawlinechart(st, data: list[object], metric, key: str): margin=dict(l=0, r=0, t=40, b=0, pad=8), legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), ) - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) diff --git a/vectordb_bench/frontend/components/run_test/autoRefresh.py b/vectordb_bench/frontend/components/run_test/autoRefresh.py deleted file mode 100644 index 034ab5017..000000000 --- a/vectordb_bench/frontend/components/run_test/autoRefresh.py +++ /dev/null @@ -1,10 +0,0 @@ -from streamlit_autorefresh import st_autorefresh -from vectordb_bench.frontend.config.styles import * - - -def autoRefresh(): - auto_refresh_count = st_autorefresh( - interval=MAX_AUTO_REFRESH_INTERVAL, - limit=MAX_AUTO_REFRESH_COUNT, - key="streamlit-auto-refresh", - ) diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index a2d2de77f..85167fb66 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -11,19 +11,18 @@ def dbConfigSettings(st, activedDbList: list[DB]): isAllValid = True for activeDb in activedDbList: dbConfigSettingItemContainer = expander.container() - dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb) try: + dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb) dbConfigs[activeDb] = activeDb.config_cls(**dbConfig) + # Probe client module so missing optional deps surface now, not on Run. + _ = activeDb.init_cls + except ModuleNotFoundError as e: + isAllValid = False + dbConfigSettingItemContainer.error(f"{activeDb.value} needs `{e.name}` but it is not installed.") except ValidationError as e: isAllValid = False - errTexts = [] - for err in e.raw_errors: - errLocs = err.loc_tuple() - errInfo = err.exc - errText = f"{', '.join(errLocs)} - {errInfo}" - errTexts.append(errText) - - dbConfigSettingItemContainer.error(f"{'; '.join(errTexts)}") + errTexts = [f"{', '.join(str(x) for x in err['loc'])} - {err['msg']}" for err in e.errors()] + dbConfigSettingItemContainer.error("; ".join(errTexts)) return dbConfigs, isAllValid diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index e5c2a1e42..93fd280c3 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -1,77 +1,76 @@ from datetime import datetime + +import streamlit as st + from vectordb_bench import config from vectordb_bench.frontend.config import styles from vectordb_bench.interface import benchmark_runner from vectordb_bench.models import TaskConfig -def submitTask(st, tasks, isAllValid): - st.markdown( +def submitTask(container, tasks, isAllValid): + container.markdown( "
", unsafe_allow_html=True, ) - st.subheader("STEP 3: Task Label") - st.markdown( + container.subheader("STEP 3: Task Label") + container.markdown( "
This description is used to mark the result.
", unsafe_allow_html=True, ) - taskLabel = taskLabelInput(st) + taskLabel = taskLabelInput(container) - st.markdown( + container.markdown( "
", unsafe_allow_html=True, ) - controlPanelContainer = st.container() - controlPanel(controlPanelContainer, tasks, taskLabel, isAllValid) + controlPanel(container.container(), tasks, taskLabel, isAllValid) -def taskLabelInput(st): +def taskLabelInput(container): defaultTaskLabel = datetime.now().strftime("%Y%m%d%H") - columns = st.columns(styles.TASK_LABEL_INPUT_COLUMNS) - taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed") - return taskLabel + cols = container.columns(styles.TASK_LABEL_INPUT_COLUMNS) + return cols[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed") -def advancedSettings(st): - container = st.columns([1, 2]) - index_already_exists = container[0].checkbox("Index already exists", value=False) - container[1].caption("if selected, inserting and building will be skipped.") +def advancedSettings(container): + cols = container.columns([1, 2]) + index_already_exists = cols[0].checkbox("Index already exists", value=False) + cols[1].caption("if selected, inserting and building will be skipped.") - container = st.columns([1, 2]) - use_aliyun = container[0].checkbox("Dataset from Aliyun (Shanghai)", value=False) - container[1].caption( - "if selected, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west." - ) + cols = container.columns([1, 2]) + use_aliyun = cols[0].checkbox("Dataset from Aliyun (Shanghai)", value=False) + cols[1].caption("if selected, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west.") - container = st.columns([1, 2]) - k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed") - container[1].caption("K value for number of nearest neighbors to search") + cols = container.columns([1, 2]) + k = cols[0].number_input("k", min_value=1, value=100, label_visibility="collapsed") + cols[1].caption("K value for number of nearest neighbors to search") - container = st.columns([1, 2]) + cols = container.columns([1, 2]) defaultconcurrentInput = ",".join(map(str, config.NUM_CONCURRENCY)) - concurrentInput = container[0].text_input( - "Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed" - ) - container[1].caption("num of concurrencies for search tests to get max-qps") + concurrentInput = cols[0].text_input("Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed") + cols[1].caption("num of concurrencies for search tests to get max-qps") - container = st.columns([1, 2]) - concurrency_duration = container[0].number_input( + cols = container.columns([1, 2]) + concurrency_duration = cols[0].number_input( "Concurrency Duration", value=config.CONCURRENCY_DURATION, label_visibility="collapsed" ) - container[1].caption("concurrency duration for each concurrency search test") + cols[1].caption("concurrency duration for each concurrency search test") - container = st.columns([1, 2]) - load_concurrency = container[0].number_input( + cols = container.columns([1, 2]) + load_concurrency = cols[0].number_input( "Load Concurrency", min_value=0, value=config.LOAD_CONCURRENCY, label_visibility="collapsed" ) - container[1].caption("number of concurrent workers for data loading in performance cases (0 = cpu_count)") + cols[1].caption("number of concurrent workers for data loading in performance cases (0 = cpu_count)") return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency -def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid): - index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency = advancedSettings(st) +def controlPanel(container, tasks: list[TaskConfig], taskLabel, isAllValid): + index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency = advancedSettings( + container + ) def runHandler(): benchmark_runner.set_drop_old(not index_already_exists) @@ -79,7 +78,7 @@ def runHandler(): try: concurrentInput_list = [int(item.strip()) for item in concurrentInput.split(",")] except ValueError: - st.write("please input correct number") + container.write("please input correct number") return None for task in tasks: @@ -93,41 +92,43 @@ def runHandler(): def stopHandler(): benchmark_runner.stop_running() - isRunning = benchmark_runner.has_running() - - if isRunning: - currentTaskId = benchmark_runner.get_current_task_id() - tasksCount = benchmark_runner.get_tasks_count() - text = f":running: Running Task {currentTaskId} / {tasksCount}" - - if tasksCount > 0: - st.progress(currentTaskId / tasksCount, text=text) - - columns = st.columns(6) - columns[0].button( - "Run Your Test", - disabled=True, - on_click=runHandler, - type="primary", - ) - columns[1].button( - "Stop", - on_click=stopHandler, - type="primary", - ) - - else: - errorText = benchmark_runner.latest_error or "" - if len(errorText) > 0: - st.error(errorText) - disabled = True if len(tasks) == 0 or not isAllValid else False - if not isAllValid: - st.error("Make sure all config is valid.") - elif len(tasks) == 0: - st.warning("No tests to run.") - st.button( - "Run Your Test", - disabled=disabled, - on_click=runHandler, - type="primary", - ) + @st.fragment(run_every=f"{styles.MAX_AUTO_REFRESH_INTERVAL / 1000}s") + def _renderLiveStatus(): + if benchmark_runner.has_running(): + currentTaskId = benchmark_runner.get_current_task_id() + tasksCount = benchmark_runner.get_tasks_count() + text = f":running: Running Task {currentTaskId} / {tasksCount}" + if tasksCount > 0: + st.progress(currentTaskId / tasksCount, text=text) + cols = st.columns(6) + cols[0].button( + "Run Your Test", + disabled=True, + on_click=runHandler, + type="primary", + key="run-disabled", + ) + cols[1].button( + "Stop", + on_click=stopHandler, + type="primary", + key="stop-btn", + ) + else: + errorText = benchmark_runner.latest_error or "" + if len(errorText) > 0: + st.error(errorText) + disabled = len(tasks) == 0 or not isAllValid + if not isAllValid: + st.error("Make sure all config is valid.") + elif len(tasks) == 0: + st.warning("No tests to run.") + st.button( + "Run Your Test", + disabled=disabled, + on_click=runHandler, + type="primary", + key="run-btn", + ) + + _renderLiveStatus() diff --git a/vectordb_bench/frontend/components/streaming/charts.py b/vectordb_bench/frontend/components/streaming/charts.py index a05da9b25..09357cf44 100644 --- a/vectordb_bench/frontend/components/streaming/charts.py +++ b/vectordb_bench/frontend/components/streaming/charts.py @@ -119,7 +119,7 @@ def drawLineChart( if x_metric == DisplayedMetric.search_time: x_title = "Actual Time (s)" fig.update_layout(xaxis_title=x_title) - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) def get_normal_scatter( @@ -234,7 +234,7 @@ def drawBarChart( fig.update_layout(xaxis_title="time (s)") fig.update_layout(barmode="stack") fig.update_traces(width=0.15) - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) def get_bar( diff --git a/vectordb_bench/frontend/components/streaming/concurrent_detail.py b/vectordb_bench/frontend/components/streaming/concurrent_detail.py index 0580bad14..37852a7ab 100644 --- a/vectordb_bench/frontend/components/streaming/concurrent_detail.py +++ b/vectordb_bench/frontend/components/streaming/concurrent_detail.py @@ -124,7 +124,7 @@ def drawQPSLatencyChart(container, qps_values, latencies_ms, stage, metric_name, showlegend=False, ) - container.plotly_chart(fig, use_container_width=True, key=f"{case_name}-chart-{stage}") + container.plotly_chart(fig, width="stretch", key=f"{case_name}-chart-{stage}") def drawMetricsTable(container, qps_values, conc_nums, p99_list, p95_list, avg_list, stage): @@ -267,7 +267,7 @@ def drawComparisonChart(container, case_data, selected_stages, metric_name, case legend=dict(yanchor="top", y=0.99, xanchor="right", x=0.99), ) - container.plotly_chart(fig, use_container_width=True, key=f"{case_name}-compare-chart") + container.plotly_chart(fig, width="stretch", key=f"{case_name}-compare-chart") # Add insight container.info("**Insight:** Compare curves across stages to understand how performance scales with data growth.") diff --git a/vectordb_bench/frontend/pages/run_test.py b/vectordb_bench/frontend/pages/run_test.py index e4472e767..64115ff17 100644 --- a/vectordb_bench/frontend/pages/run_test.py +++ b/vectordb_bench/frontend/pages/run_test.py @@ -1,5 +1,4 @@ import streamlit as st -from vectordb_bench.frontend.components.run_test.autoRefresh import autoRefresh from vectordb_bench.frontend.components.run_test.caseSelector import caseSelector from vectordb_bench.frontend.components.run_test.dbConfigSetting import dbConfigSettings from vectordb_bench.frontend.components.run_test.dbSelector import dbSelector @@ -57,9 +56,6 @@ def main(): # nav to results NavToResults(st, key="footer-nav-to-results") - # autofresh - autoRefresh() - if __name__ == "__main__": main() diff --git a/vectordb_bench/frontend/vdbbench.py b/vectordb_bench/frontend/vdbbench.py index 860467734..90769019c 100644 --- a/vectordb_bench/frontend/vdbbench.py +++ b/vectordb_bench/frontend/vdbbench.py @@ -1,10 +1,33 @@ import streamlit as st +from tornado.iostream import StreamClosedError +from tornado.websocket import WebSocketClosedError, WebSocketProtocol13 + from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.custom.initStyle import initStyle from vectordb_bench.frontend.components.welcome.explainPrams import explainPrams from vectordb_bench.frontend.components.welcome.welcomePrams import welcomePrams from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE +# Consume expected WS-close errors on streamlit's fire-and-forget writes +# (streamlit#9787, unfixed upstream). +_orig_write_message = WebSocketProtocol13.write_message + + +def _write_message_with_consumer(self, message, binary=False): + task = _orig_write_message(self, message, binary=binary) + + def _consume(t): + exc = t.exception() + if exc is None or isinstance(exc, (WebSocketClosedError, StreamClosedError)): + return + t.get_loop().call_exception_handler({"message": "websocket write failed", "exception": exc, "task": t}) + + task.add_done_callback(_consume) + return task + + +WebSocketProtocol13.write_message = _write_message_with_consumer + def main(): st.set_page_config( From a424b025f00e246d617826b9bd793d123c927a88 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 24 Apr 2026 19:32:06 +0800 Subject: [PATCH 28/38] feat(loader): cap default insert workers to min(cpu, 4) (#767) ConcurrentInsertRunner previously defaulted to mp.cpu_count(), spawning one worker per CPU when load_concurrency was unset. On high-core hosts this opens many parallel client connections, saturating modest DBs / network paths and yielding worse load throughput than a smaller, steadier worker count. Cap the unset default to min(cpu_count, 4). Explicit load_concurrency from CLI / config / submitTask still wins. Signed-off-by: yangxuan --- vectordb_bench/backend/runner/concurrent_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 37201f88e..7c8aeb24f 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -51,7 +51,7 @@ class ConcurrentInsertRunner: normalize: Whether to L2-normalize embeddings. filters: Filter configuration. timeout: Timeout in seconds for the overall operation. - max_workers: Number of concurrent workers (default: cpu_count). + max_workers: Number of concurrent workers (default: min(cpu_count, 4)). backend: Executor backend to use ('threading' or 'async'). """ @@ -72,7 +72,7 @@ def __init__( self.filters = filters self.backend = backend - effective_workers = max_workers or mp.cpu_count() + effective_workers = max_workers or min(mp.cpu_count(), 4) if not db.thread_safe: log.info(f"DB {db.name} is not thread-safe, falling back to max_workers=1") effective_workers = 1 From c2a6f859831d6d63ce2385d189809839b123a0ae Mon Sep 17 00:00:00 2001 From: liuhao6741 <157583880+liuhao6741@users.noreply.github.com> Date: Mon, 11 May 2026 11:45:54 +0800 Subject: [PATCH 29/38] feat(seekdb): add SeekDB backend with HNSW index support (#770) Add a new vector database backend for SeekDB, connecting via mysql-connector-python over the MySQL wire protocol. Key components: - seekdb.py: VectorDB implementation with heap-organized table, HNSW vector index, and version-aware optimize() that calls dbms_index_manager.refresh() on SeekDB >= 1.3.0 - config.py: DBConfig with host/port/user/password/database and SeekDBHNSWConfig with m/ef_construction/ef_search parameters - cli.py: Click command `SeekDBHNSW` for command-line benchmarks Registration: - Add SeekDB to the DB enum in backend/clients/__init__.py with lazy imports for init_cls, config_cls, and case_config_cls - Register SeekDBHNSW CLI command in cli/vectordbbench.py - Add seekdb optional dependency in pyproject.toml (pip install vectordb-bench[seekdb]) Filter support: - NonFilter and NumGE (id >= N) filters are supported - StrEqual (label filter) is intentionally excluded since the table schema only has id and embedding columns Thread safety: - mysql.connector is not thread-safe (thread_safe = False). ConcurrentInsertRunner uses max_workers=1 accordingly - rate_runner.py handles SeekDB specially: copies the db object, resets the connection, and calls init() per worker thread Co-authored-by: liuhao6741 Co-authored-by: Claude Opus 4.7 --- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 16 + vectordb_bench/backend/clients/seekdb/cli.py | 60 ++++ .../backend/clients/seekdb/config.py | 77 +++++ .../backend/clients/seekdb/seekdb.py | 282 ++++++++++++++++++ vectordb_bench/backend/runner/rate_runner.py | 13 +- vectordb_bench/cli/vectordbbench.py | 2 + 7 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/seekdb/cli.py create mode 100644 vectordb_bench/backend/clients/seekdb/config.py create mode 100644 vectordb_bench/backend/clients/seekdb/seekdb.py diff --git a/pyproject.toml b/pyproject.toml index 8bd5de2ad..ee57d8339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] +seekdb = [ "mysql-connector-python" ] pinot = [ "requests" ] [project.urls] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 9029be05f..4be8d0424 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -62,6 +62,7 @@ class DB(Enum): VectorChord = "VectorChord" PolarDB = "PolarDB" Pinot = "Pinot" + SeekDB = "SeekDB" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -263,6 +264,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return Pinot + if self == DB.SeekDB: + from .seekdb.seekdb import SeekDB + + return SeekDB + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -466,6 +472,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return PinotConfig + if self == DB.SeekDB: + from .seekdb.config import SeekDBConfig + + return SeekDBConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -651,6 +662,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 IndexType.IVFPQ: PinotIVFPQConfig, }.get(index_type, PinotHNSWConfig) + if self == DB.SeekDB: + from .seekdb.config import _seekdb_case_config + + return _seekdb_case_config.get(index_type) + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/seekdb/cli.py b/vectordb_bench/backend/clients/seekdb/cli.py new file mode 100644 index 000000000..9c135220f --- /dev/null +++ b/vectordb_bench/backend/clients/seekdb/cli.py @@ -0,0 +1,60 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor3, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class SeekDBTypedDict(CommonTypedDict): + host: Annotated[str, click.option("--host", type=str, help="SeekDB host", required=True)] + user: Annotated[str, click.option("--user", type=str, help="SeekDB username", required=True)] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="SeekDB password", + default=lambda: os.environ.get("SEEKDB_PASSWORD", ""), + ), + ] + database: Annotated[str, click.option("--database", type=str, help="Database name", required=True)] + port: Annotated[int, click.option("--port", type=int, help="SeekDB port", default=3306, show_default=True)] + + +class SeekDBHNSWTypedDict(SeekDBTypedDict, HNSWFlavor3): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(SeekDBHNSWTypedDict) +def SeekDBHNSW(**parameters: Unpack[SeekDBHNSWTypedDict]): + """Run VectorDBBench against SeekDB with an HNSW index.""" + from ..api import IndexType + from .config import SeekDBConfig, SeekDBHNSWConfig + + run( + DB.SeekDB, + SeekDBConfig( + db_label=parameters["db_label"], + user=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + SeekDBHNSWConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + index=IndexType.HNSW, + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/seekdb/config.py b/vectordb_bench/backend/clients/seekdb/config.py new file mode 100644 index 000000000..01a43ce4b --- /dev/null +++ b/vectordb_bench/backend/clients/seekdb/config.py @@ -0,0 +1,77 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class SeekDBConfigDict(TypedDict): + user: str + host: str + port: int + password: str + database: str + + +class SeekDBConfig(DBConfig): + user: SecretStr = SecretStr("root") + password: SecretStr + host: str + port: int = 3306 + database: str + + def to_dict(self) -> SeekDBConfigDict: + return { + "user": self.user.get_secret_value(), + "host": self.host, + "port": self.port, + "password": self.password.get_secret_value(), + "database": self.database, + } + + +class SeekDBIndexConfig(BaseModel): + index: IndexType + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + if self.metric_type == MetricType.IP: + return "inner_product" + return "cosine" + + def parse_metric_func_str(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_distance" + if self.metric_type == MetricType.IP: + return "negative_inner_product" + return "cosine_distance" + + +class SeekDBHNSWConfig(SeekDBIndexConfig, DBCaseConfig): + m: int + ef_construction: int + ef_search: int + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "m": self.m, + "ef_construction": self.ef_construction, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric_func_str(), + "params": {"ef_search": self.ef_search}, + } + + +_seekdb_case_config = { + IndexType.HNSW: SeekDBHNSWConfig, +} diff --git a/vectordb_bench/backend/clients/seekdb/seekdb.py b/vectordb_bench/backend/clients/seekdb/seekdb.py new file mode 100644 index 000000000..f4551a577 --- /dev/null +++ b/vectordb_bench/backend/clients/seekdb/seekdb.py @@ -0,0 +1,282 @@ +import logging +import re +import struct +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import mysql.connector as mysql + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import IndexType, VectorDB +from .config import SeekDBConfigDict, SeekDBHNSWConfig + +log = logging.getLogger(__name__) + +SEEKDB_DEFAULT_LOAD_BATCH_SIZE = 256 + +# Minimum SeekDB version for dbms_index_manager.refresh() after bulk load (see VERSION()). +_SEEKDB_REFRESH_MIN_VERSION = (1, 3, 0) +_SEEKDB_VERSION_IN_VERSION_STRING = re.compile(r"seekdb-v(\d+(?:\.\d+)*)", re.IGNORECASE) + + +def _seekdb_version_tuple(version_row: str | None) -> tuple[int, ...] | None: + if not version_row: + return None + m = _SEEKDB_VERSION_IN_VERSION_STRING.search(version_row.strip()) + if not m: + return None + return tuple(int(p) for p in m.group(1).split(".") if p.isdigit()) + + +def _version_tuple_ge(parsed: tuple[int, ...], minimum: tuple[int, ...]) -> bool: + n = max(len(parsed), len(minimum)) + for i in range(n): + p = parsed[i] if i < len(parsed) else 0 + m = minimum[i] if i < len(minimum) else 0 + if p != m: + return p > m + return True + + +class SeekDB(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + # mysql.connector is not thread-safe; ConcurrentInsertRunner uses max_workers=1 when False. + # Streaming fixed-rate inserts use rate_runner with per-thread deepcopy+init() instead. + thread_safe: bool = False + + def __init__( + self, + dim: int, + db_config: SeekDBConfigDict, + db_case_config: SeekDBHNSWConfig, + collection_name: str = "items", + drop_old: bool = False, + **kwargs, + ): + self.name = "SeekDB" + self.dim = dim + self.db_config = db_config + self.db_case_config = db_case_config + self.table_name = collection_name + self.load_batch_size = SEEKDB_DEFAULT_LOAD_BATCH_SIZE + self._index_name = "vidx" + self._primary_field = "id" + self._vector_field = "embedding" + self.expr = "" + + log.info( + f"{self.name} initialized with config:\nDatabase: {self.db_config}\nCase Config: {self.db_case_config}" + ) + + self._conn = None + self._cursor = None + + try: + self._connect() + self._apply_system_settings() + if drop_old: + self._drop_table() + self._create_table() + self._create_index() + finally: + self._disconnect() + + def _connect(self): + try: + self._conn = mysql.connect( + host=self.db_config["host"], + user=self.db_config["user"], + port=self.db_config["port"], + password=self.db_config["password"], + database=self.db_config["database"], + ) + self._cursor = self._conn.cursor() + except mysql.Error: + log.exception("Failed to connect to SeekDB") + raise + + def _disconnect(self): + if self._cursor: + self._cursor.close() + self._cursor = None + if self._conn: + self._conn.close() + self._conn = None + + def _apply_system_settings(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + self._cursor.execute('ALTER SYSTEM SET memory_limit = "0M"') + self._cursor.execute("ALTER SYSTEM SET cpu_count = 0") + + def _init_session_settings(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + self._cursor.execute("SET autocommit=1") + if self.db_case_config.index == IndexType.HNSW: + ef_search = self.db_case_config.search_param()["params"]["ef_search"] + # SeekDB uses OceanBase-style session vars (not plain hnsw_ef_search). + self._cursor.execute(f"SET ob_hnsw_ef_search={ef_search}") + + @contextmanager + def init(self) -> Generator[None, None, None]: + try: + self._connect() + self._init_session_settings() + yield + finally: + self._disconnect() + + def _drop_table(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + log.info(f"Dropping table {self.table_name}") + self._cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + def _create_table(self): + """Create a heap table with a vector column. + + ORGANIZATION HEAP specifies a heap-organized table (no clustered primary + key order), which is required by SeekDB for vector workloads. + """ + if not self._cursor: + raise ValueError("Cursor is not initialized") + + log.info(f"Creating heap table {self.table_name}") + create_table_query = f""" + CREATE TABLE {self.table_name} ( + id INT, + embedding VECTOR({self.dim}) + ) ORGANIZATION HEAP; + """ + self._cursor.execute(create_table_query) + + def _create_index(self): + """Create the HNSW vector index immediately after table creation. + + Following Milvus's approach: the index is built upfront so that + streaming inserts are indexed incrementally and searches can run + concurrently with writes (StreamingPerformanceCase). + """ + if not self._cursor: + raise ValueError("Cursor is not initialized") + + index_params = self.db_case_config.index_param() + params = index_params["params"] + index_args = ", ".join(f"{k}={v}" for k, v in params.items()) + + index_query = ( + f"CREATE VECTOR INDEX {self._index_name} " + f"ON {self.table_name}({self._vector_field}) " + f"WITH (distance={index_params['metric_type']}, " + f"type={index_params['index_type']}, {index_args})" + ) + + log.info("Creating HNSW index: %s", index_query) + try: + self._cursor.execute(index_query) + log.info("HNSW index created successfully") + except mysql.Error: + log.exception("Failed to create HNSW index") + raise + + def optimize(self, data_size: int | None = None): + """Post-load hook: refresh index metadata on SeekDB >= 1.3.0 when available. + + Older releases rely on incremental HNSW indexing only. From 1.3.0 onward, + ``CALL dbms_index_manager.refresh()`` aligns on-disk index state after bulk + load (VERSION() strings look like ``... seekdb-v1.3.0.0``). + """ + if not self._cursor: + raise ValueError("Cursor is not initialized") + + self._cursor.execute("SELECT VERSION()") + row = self._cursor.fetchone() + version_str = row[0] if row else None + parsed = _seekdb_version_tuple(version_str) + + if parsed is None or not _version_tuple_ge(parsed, _SEEKDB_REFRESH_MIN_VERSION): + log.info( + "%s optimize: skip dbms_index_manager.refresh (version=%r parsed=%s; need >= %s)", + self.name, + version_str, + parsed, + ".".join(map(str, _SEEKDB_REFRESH_MIN_VERSION)), + ) + return + + log.info( + "%s optimize: SeekDB %s >= %s, calling dbms_index_manager.refresh()", + self.name, + ".".join(map(str, parsed)), + ".".join(map(str, _SEEKDB_REFRESH_MIN_VERSION)), + ) + try: + self._cursor.execute("CALL dbms_index_manager.refresh()") + except mysql.Error: + log.exception("dbms_index_manager.refresh() failed") + raise + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + insert_count = 0 + try: + for batch_start in range(0, len(embeddings), self.load_batch_size): + batch_end = min(batch_start + self.load_batch_size, len(embeddings)) + batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] + values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) + self._cursor.execute(f"INSERT INTO {self.table_name} VALUES {values}") + insert_count += len(batch) + except mysql.Error: + log.exception("Failed to insert embeddings") + raise + + return insert_count, None + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.expr = "" + elif filters.type == FilterOp.NumGE: + self.expr = f"WHERE id >= {filters.int_value}" + else: + msg = f"Unsupported filter for SeekDB: {filters}" + raise ValueError(msg) + + def search_embedding( + self, + query: list[float], + k: int = 100, + ) -> list[int]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + packed = struct.pack(f"<{len(query)}f", *query) + hex_vec = packed.hex() + + query_str = ( + f"SELECT id FROM {self.table_name} " + f"{self.expr} ORDER BY " + f"{self.db_case_config.parse_metric_func_str()}({self._vector_field}, X'{hex_vec}') " + f"APPROXIMATE LIMIT {k}" + ) + + try: + self._cursor.execute(query_str) + return [row[0] for row in self._cursor.fetchall()] + except mysql.Error: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index c56c8ca61..2387abfcb 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -3,7 +3,7 @@ import multiprocessing as mp import time from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy +from copy import copy, deepcopy from vectordb_bench import config from vectordb_bench.backend.clients import api @@ -65,6 +65,17 @@ def _insert_embeddings(db: api.VectorDB, emb: list[list[float]], metadata: list[ log.debug("Failed to reset Doris client or table on thread-local copy", exc_info=True) with db_copy.init(): _insert_embeddings(db_copy, emb, metadata, retry_idx=0) + elif db.name == "SeekDB": + # mysql.connector is not thread-safe; do not share one connection across workers. + # deepcopy() fails on an open _conn (socket is not picklable / not copy-safe in spawn workers). + db_copy = copy(db) + try: + db_copy._conn = None + db_copy._cursor = None + except Exception: + log.debug("Failed to reset SeekDB connection on thread-local copy", exc_info=True) + with db_copy.init(): + _insert_embeddings(db_copy, emb, metadata, retry_idx=0) else: _insert_embeddings(db, emb, metadata, retry_idx=0) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 42048d57f..eca3dbc52 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -35,6 +35,7 @@ from ..backend.clients.qdrant_local.cli import QdrantLocal from ..backend.clients.redis.cli import Redis from ..backend.clients.s3_vectors.cli import S3Vectors +from ..backend.clients.seekdb.cli import SeekDBHNSW from ..backend.clients.tencent_elasticsearch.cli import TencentElasticsearch from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB @@ -95,6 +96,7 @@ cli.add_command(PolarDBHNSWFlat) cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) +cli.add_command(SeekDBHNSW) if __name__ == "__main__": From aaab64324d64290e84cb42312be51a1135396805 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 15 May 2026 18:05:31 +0800 Subject: [PATCH 30/38] fix: Require pymilvus<3.0.0 and fix the overflow size (#781) Signed-off-by: yangxuan --- install/requirements_py3.11.txt | 2 +- pyproject.toml | 3 +- tests/test_milvus.py | 51 ++++++++++++++++++- .../backend/clients/milvus/milvus.py | 7 ++- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 130745816..7208439bb 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -22,7 +22,7 @@ plotly environs pydantic>=2.0,<3 scikit-learn -pymilvus +pymilvus<3.0.0 clickhouse_connect pyvespa mysql-connector-python diff --git a/pyproject.toml b/pyproject.toml index ee57d8339..cd63e5a33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "environs", "pydantic>=2.0,<3", "scikit-learn", - "pymilvus", # with pandas, numpy + "pymilvus<3.0.0", # with pandas, numpy "hdrhistogram>=0.10.1", "ujson", ] @@ -211,4 +211,3 @@ builtins-ignorelist = [ "vectordb_bench/backend/clients/*" = ["PLC0415"] "vectordb_bench/cli/batch_cli.py" = ["PLC0415"] "vectordb_bench/backend/data_source.py" = ["PLC0415"] - diff --git a/tests/test_milvus.py b/tests/test_milvus.py index 8cc391acc..1c5de7ce0 100644 --- a/tests/test_milvus.py +++ b/tests/test_milvus.py @@ -4,20 +4,67 @@ """ import logging +from types import SimpleNamespace +from unittest.mock import MagicMock +import pytest from pydantic import SecretStr +from vectordb_bench.backend.cases import CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import IndexType from vectordb_bench.backend.clients.milvus.config import MilvusConfig -from vectordb_bench.backend.cases import CaseType +from vectordb_bench.backend.clients.milvus.milvus import MILVUS_FORCE_MERGE_TARGET_SIZE_MB, Milvus from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseConfig, TaskConfig - log = logging.getLogger(__name__) +class TestMilvusOptimize: + def _milvus(self, *, compact_side_effect: Exception | None = None): + milvus = Milvus.__new__(Milvus) + milvus.name = "Milvus" + milvus.collection_name = "test_collection" + milvus.case_config = SimpleNamespace(is_gpu_index=False) + milvus.client = MagicMock() + milvus.client.compact.side_effect = compact_side_effect + milvus.client.compact.return_value = 0 + milvus._wait_for_segments_sorted = MagicMock() + milvus._wait_for_index = MagicMock() + milvus._wait_for_compaction = MagicMock() + return milvus + + def test_optimize_compact_uses_safe_force_merge_target_size(self): + milvus = self._milvus() + + milvus._optimize() + + milvus.client.compact.assert_called_once_with("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) + milvus.client.refresh_load.assert_called_once_with("test_collection") + + def test_optimize_skips_property_style_permission_denied(self): + error = RuntimeError("permission denied") + error.code = SimpleNamespace(name="PERMISSION_DENIED") + milvus = self._milvus(compact_side_effect=error) + + milvus._optimize() + + milvus.client.refresh_load.assert_called_once_with("test_collection") + + def test_optimize_reraises_non_permission_error(self): + error = RuntimeError("boom") + error.code = SimpleNamespace(name="UNAVAILABLE") + milvus = self._milvus(compact_side_effect=error) + + with pytest.raises(RuntimeError, match="boom") as exc_info: + milvus._optimize() + + assert exc_info.value is error + milvus.client.refresh_load.assert_not_called() + + +@pytest.mark.integration class TestMilvus: """E2E test for Milvus using Performance1536D50K (OpenAI 50K dataset).""" diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index d36a15c24..740c89509 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -15,6 +15,7 @@ log = logging.getLogger(__name__) MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 * 1024 +MILVUS_FORCE_MERGE_TARGET_SIZE_MB = ((1 << 63) - 1) // (1024**2) class Milvus(VectorDB): @@ -173,13 +174,15 @@ def _optimize(self): # wait for sort, index, compact self._wait_for_segments_sorted() self._wait_for_index() - compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) + compaction_id = self.client.compact( + self.collection_name, target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB + ) if compaction_id > 0: self._wait_for_compaction(compaction_id) log.info(f"{self.name} force merge compaction completed.") except Exception as e: log.warning(f"{self.name} compact or list segments error: {e}") - if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": + if getattr(getattr(e, "code", None), "name", None) == "PERMISSION_DENIED": log.warning("Skip compact due to list segments or compact permission denied.") else: raise e from None From 191b7106a08a3e6f9f9ffe9bf5604d8f5daa8270 Mon Sep 17 00:00:00 2001 From: fan <37357096+wyfanxiao@users.noreply.github.com> Date: Fri, 15 May 2026 18:06:06 +0800 Subject: [PATCH 31/38] feat(oceanbase): configurable index params, KEY partitioning, HNSW_BQ cosine support (#776) * feat(oceanbase): configurable index params, KEY partitioning, HNSW_BQ cosine support - Add --create-index-parallel CLI option (default 16) - Add --extra-info-max-size CLI option (default 32, set 0 to omit) - Add --partitions CLI option for KEY partitioning (default 0, no partition) - HNSW_BQ: remove forced L2 for cosine, now supports cosine natively - need_normalize_cosine returns False for all index types - pyproject.toml: add pyyaml dependency, fix packages.find to include all subpackages * fix(oceanbase): declare thread_safe=False to prevent cursor sharing across threads * fix: restore seekdb dependency accidentally removed --- pyproject.toml | 3 +- .../backend/clients/oceanbase/cli.py | 38 ++++++++++++++++++- .../backend/clients/oceanbase/config.py | 11 +++--- .../backend/clients/oceanbase/oceanbase.py | 31 +++++++-------- 4 files changed, 58 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd63e5a33..3bbba8ac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["."] -include = ["vectordb_bench", "vectordb_bench.cli"] +include = ["vectordb_bench", "vectordb_bench.*"] [project] name = "vectordb-bench" @@ -26,6 +26,7 @@ classifiers = [ ] dependencies = [ "click", + "pyyaml", "pytz", "streamlit>=1.47,<2", # 1.47 fixes streamlit#11660 "tqdm", diff --git a/vectordb_bench/backend/clients/oceanbase/cli.py b/vectordb_bench/backend/clients/oceanbase/cli.py index 61583cc82..81ccacc79 100644 --- a/vectordb_bench/backend/clients/oceanbase/cli.py +++ b/vectordb_bench/backend/clients/oceanbase/cli.py @@ -31,9 +31,40 @@ class OceanBaseTypedDict(CommonTypedDict): ] database: Annotated[str, click.option("--database", type=str, help="DataBase name", required=True)] port: Annotated[int, click.option("--port", type=int, help="OceanBase port", required=True)] + create_index_parallel: Annotated[ + int, + click.option( + "--create-index-parallel", + type=int, + default=16, + show_default=True, + help="PARALLEL hint degree for CREATE VECTOR INDEX", + ), + ] + partitions: Annotated[ + int, + click.option( + "--partitions", + type=int, + default=0, + show_default=True, + help="Number of KEY partitions for the table. 0 or 1 means no partitioning.", + ), + ] -class OceanBaseHNSWTypedDict(CommonTypedDict, OceanBaseTypedDict, HNSWFlavor4): ... +class OceanBaseHNSWTypedDict(CommonTypedDict, OceanBaseTypedDict, HNSWFlavor4): + extra_info_max_size: Annotated[ + int | None, + click.option( + "--extra-info-max-size", + type=int, + default=32, + show_default=True, + help="extra_info_max_size for HNSW index. Set to 0 to omit.", + required=False, + ), + ] @cli.command() @@ -55,6 +86,9 @@ def OceanBaseHNSW(**parameters: Unpack[OceanBaseHNSWTypedDict]): m=parameters["m"], efConstruction=parameters["ef_construction"], ef_search=parameters["ef_search"], + extra_info_max_size=parameters["extra_info_max_size"] or None, + create_index_parallel=parameters["create_index_parallel"], + partitions=parameters["partitions"], index=parameters["index_type"], ), **parameters, @@ -94,6 +128,8 @@ def OceanBaseIVF(**parameters: Unpack[OceanBaseIVFTypedDict]): nlist=parameters["nlist"], sample_per_nlist=parameters["sample_per_nlist"], nbits=parameters["nbits"], + create_index_parallel=parameters["create_index_parallel"], + partitions=parameters["partitions"], index=input_index_type, ivf_nprobes=parameters["ivf_nprobes"], ), diff --git a/vectordb_bench/backend/clients/oceanbase/config.py b/vectordb_bench/backend/clients/oceanbase/config.py index 1f37cfc75..384a7c263 100644 --- a/vectordb_bench/backend/clients/oceanbase/config.py +++ b/vectordb_bench/backend/clients/oceanbase/config.py @@ -36,20 +36,18 @@ class OceanBaseIndexConfig(BaseModel): index: IndexType metric_type: MetricType | None = None lib: str = "vsag" + create_index_parallel: int = 16 + partitions: int = 0 def parse_metric(self) -> str: - if self.metric_type == MetricType.L2 or ( - self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE - ): + if self.metric_type == MetricType.L2: return "l2" if self.metric_type == MetricType.IP: return "inner_product" return "cosine" def parse_metric_func_str(self) -> str: - if self.metric_type == MetricType.L2 or ( - self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE - ): + if self.metric_type == MetricType.L2: return "l2_distance" if self.metric_type == MetricType.IP: return "negative_inner_product" @@ -60,6 +58,7 @@ class OceanBaseHNSWConfig(OceanBaseIndexConfig, DBCaseConfig): m: int efConstruction: int ef_search: int | None = None + extra_info_max_size: int | None = 32 index: IndexType def index_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/oceanbase/oceanbase.py b/vectordb_bench/backend/clients/oceanbase/oceanbase.py index bf615e4d0..a34161037 100644 --- a/vectordb_bench/backend/clients/oceanbase/oceanbase.py +++ b/vectordb_bench/backend/clients/oceanbase/oceanbase.py @@ -23,6 +23,8 @@ class OceanBase(VectorDB): FilterOp.NumGE, FilterOp.StrEqual, ] + # mysql-connector cursor cannot be shared across threads + thread_safe: bool = False def __init__( self, @@ -109,27 +111,28 @@ def _create_table(self): if not self._cursor: raise ValueError("Cursor is not initialized") - log.info(f"Creating table {self.table_name}") - create_table_query = f""" - CREATE TABLE {self.table_name} ( - id INT PRIMARY KEY, - embedding VECTOR({self.dim}) - ); - """ + partitions = getattr(self.db_case_config, "partitions", 0) + log.info(f"Creating table {self.table_name} (partitions={partitions})") + + create_table_query = f"CREATE TABLE {self.table_name} (id INT PRIMARY KEY, embedding VECTOR({self.dim}))" + if partitions > 1: + create_table_query += f" PARTITION BY KEY(id) PARTITIONS {partitions}" + create_table_query += ";" self._cursor.execute(create_table_query) def optimize(self, data_size: int): index_params = self.db_case_config.index_param() index_args = ", ".join(f"{k}={v}" for k, v in index_params["params"].items()) index_query = ( - f"CREATE /*+ PARALLEL(18) */ VECTOR INDEX idx1 " + f"CREATE /*+ PARALLEL({self.db_case_config.create_index_parallel}) */ VECTOR INDEX idx1 " f"ON {self.table_name}(embedding) " f"WITH (distance={self.db_case_config.parse_metric()}, " f"type={index_params['index_type']}, lib={index_params['lib']}, {index_args}" ) - if self.db_case_config.index in {IndexType.HNSW, IndexType.HNSW_SQ, IndexType.HNSW_BQ}: - index_query += ", extra_info_max_size=32" + extra_info = getattr(self.db_case_config, "extra_info_max_size", None) + if extra_info is not None: + index_query += f", extra_info_max_size={extra_info}" index_query += ")" @@ -153,10 +156,6 @@ def optimize(self, data_size: int): raise def need_normalize_cosine(self) -> bool: - if self.db_case_config.index == IndexType.HNSW_BQ: - log.info("current HNSW_BQ only supports L2, cosine dataset need normalize.") - return True - return False def _wait_for_major_compaction(self): @@ -185,9 +184,7 @@ def insert_embeddings( batch_end = min(batch_start + self.load_batch_size, len(embeddings)) batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) - self._cursor.execute( - f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" - ) + self._cursor.execute(f"INSERT INTO {self.table_name} VALUES {values}") insert_count += len(batch) except mysql.Error: log.exception("Failed to insert embeddings") From c6f96f7b9829b462d392db71cccbc55202ce3c27 Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Fri, 29 May 2026 16:43:57 +0800 Subject: [PATCH 32/38] feat: Add VectorDBBench Cloud Leaderboard benchmark cases and client support (#775) --- .gitignore | 1 + README.md | 11 + docs/release/2026-05-cloud-leaderboard.md | 170 +++ tests/test_case_runner_reuse.py | 171 +++ tests/test_cloud_cold_latency_case.py | 470 +++++++ tests/test_cloud_insert_case.py | 1108 +++++++++++++++++ tests/test_cloud_payload_case.py | 180 +++ tests/test_cloud_payload_search.py | 128 ++ tests/test_milvus.py | 162 +++ tests/test_milvus_zilliz_cli.py | 57 + tests/test_multitenant_case.py | 400 ++++++ tests/test_pinecone_multitenant.py | 214 ++++ tests/test_turbopuffer_cli.py | 293 +++++ vectordb_bench/__init__.py | 2 + vectordb_bench/backend/assembler.py | 7 +- vectordb_bench/backend/cases.py | 276 +++- vectordb_bench/backend/clients/api.py | 42 + vectordb_bench/backend/clients/milvus/cli.py | 257 ++-- .../backend/clients/milvus/milvus.py | 139 ++- .../backend/clients/pinecone/config.py | 2 + .../backend/clients/pinecone/pinecone.py | 236 +++- .../backend/clients/turbopuffer/cli.py | 221 +++- .../backend/clients/turbopuffer/config.py | 23 + .../clients/turbopuffer/turbopuffer.py | 285 ++++- .../backend/clients/zilliz_cloud/cli.py | 32 +- .../backend/clients/zilliz_cloud/config.py | 10 +- vectordb_bench/backend/dataset.py | 28 +- vectordb_bench/backend/payload.py | 21 + vectordb_bench/backend/runner/__init__.py | 2 + .../backend/runner/cold_warm_runner.py | 120 ++ .../backend/runner/concurrent_runner.py | 85 +- vectordb_bench/backend/runner/mp_runner.py | 27 +- .../backend/runner/serial_runner.py | 36 +- vectordb_bench/backend/task_runner.py | 263 +++- vectordb_bench/cli/cli.py | 178 ++- vectordb_bench/cli/vectordbbench.py | 3 +- vectordb_bench/interface.py | 17 +- vectordb_bench/metric.py | 9 + vectordb_bench/models.py | 112 +- 39 files changed, 5535 insertions(+), 263 deletions(-) create mode 100644 docs/release/2026-05-cloud-leaderboard.md create mode 100644 tests/test_case_runner_reuse.py create mode 100644 tests/test_cloud_cold_latency_case.py create mode 100644 tests/test_cloud_insert_case.py create mode 100644 tests/test_cloud_payload_case.py create mode 100644 tests/test_cloud_payload_search.py create mode 100644 tests/test_milvus_zilliz_cli.py create mode 100644 tests/test_multitenant_case.py create mode 100644 tests/test_pinecone_multitenant.py create mode 100644 tests/test_turbopuffer_cli.py create mode 100644 vectordb_bench/backend/payload.py create mode 100644 vectordb_bench/backend/runner/cold_warm_runner.py diff --git a/.gitignore b/.gitignore index cea1306b0..b33099105 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ venv/ .venv/ .idea/ logs/ +vectordb_bench/results/cloudleaderboard/ # Worktrees .worktrees/ diff --git a/README.md b/README.md index 685e37a47..3cdceddc0 100644 --- a/README.md +++ b/README.md @@ -708,6 +708,17 @@ vectordbbench batchcli --batch-config-file ### Introduction To facilitate the presentation of test results and provide a comprehensive performance analysis report, we offer a [leaderboard page](https://zilliz.com/benchmark). It allows us to choose from QPS, QP$, and latency metrics, and provides a comprehensive assessment of a system's performance based on the test results of various cases and a set of scoring mechanisms (to be introduced later). On this leaderboard, we can select the systems and models to be compared, and filter out cases we do not want to consider. Comprehensive scores are always ranked from best to worst, and the specific test results of each query will be presented in the list below. +### Cloud Leaderboard + +VectorDBBench now includes Cloud Leaderboard cases for production-oriented cloud vector database evaluation. These cases complement the original raw-performance leaderboard by measuring behaviors that matter for managed services: + +- `CloudInsertCase`: insert throughput plus searchable and indexed readiness delays. +- `CloudPayloadSearchCase`: search performance when responses return IDs only, scalar metadata, or vectors. +- `CloudMultiTenantSearchCase`: tenant-routed search for SaaS-shaped workloads. +- `CloudColdLatencyCase`: cold and warm serial latency for first-query and cache-sensitive serving paths. + +The May 2026 release note explains why the Cloud Leaderboard was added, what changed, which systems were tested this round, and how to run each new case: [docs/release/2026-05-cloud-leaderboard.md](docs/release/2026-05-cloud-leaderboard.md). + ### Scoring Rules 1. For each case, select a base value and score each system based on relative values. diff --git a/docs/release/2026-05-cloud-leaderboard.md b/docs/release/2026-05-cloud-leaderboard.md new file mode 100644 index 000000000..c9dc76a7b --- /dev/null +++ b/docs/release/2026-05-cloud-leaderboard.md @@ -0,0 +1,170 @@ +# VectorDBBench Cloud Leaderboard Release Note + +May 2026 + +The VectorDBBench Cloud Leaderboard moves beyond a single raw-throughput ranking. It evaluates managed vector databases around the behaviors production teams have to plan for: ingest readiness, payload-aware search, tenant-shaped workloads, cold latency, and cost at practical QPS targets. + +## Why we need a new leaderboard now + +The vector database market has moved past the "highest QPS wins" phase. Production teams choosing a managed vector database also care about budget, data freshness, tail latency, recall, metadata payloads, tenant isolation, and operational predictability. + +The existing VectorDBBench leaderboard remains useful for comparing baseline search performance across systems. But cloud buyers ask a wider set of questions: + +- When does newly inserted data become searchable? +- When is it fully indexed? +- What happens when search returns metadata or vectors instead of only IDs? +- What happens when traffic is split across many tenants? +- What does each reachable QPS tier cost? + +The Cloud Leaderboard is designed around those questions. It keeps performance visible, but puts it next to the readiness, payload, tenant, cold-start, and cost signals that determine what a customer can safely deploy. + +## What the Cloud Leaderboard changes + +The Cloud Leaderboard is a production cloud decision layer, not a replacement for the original raw-performance board. The main change is that benchmark cases now model cloud operating concerns directly instead of treating all products as simple warm search engines. + +The new cases add: + +- Insert readiness measurement, including client insert completion, searchable delay, and indexed delay. +- Explicit response payload profiles: IDs only, scalar label metadata, or vector values. +- Cloud cold-latency measurement for the first search path after idle or cache-cold conditions. +- Multi-tenant search, where data is split into deterministic tenant labels or namespaces and queries are routed by tenant. +- Cost-oriented interpretation, so raw QPS can be read together with monthly cost and readiness constraints. + +This matters because a top-line QPS table can hide important tradeoffs. A system can look strong on peak throughput while losing ground on recall, p99 latency, cold-start behavior, payload cost, or sustained cost at the same target QPS. + +## Who we tested this round + +This round focuses on three popular cloud vector databases: + +- Zilliz Cloud, including tiered and fixed-capacity configurations. +- turbopuffer, including normal, pinned, and backpressure-related configurations where applicable. +- Pinecone serverless. + +The tested matrix is intentionally cloud-oriented. It compares managed products and managed-service modes rather than only local or self-hosted engine behavior. + +## The new tests we added + +Version 2 adds four cloud-oriented cases in VectorDBBench. Each case is designed to expose a production behavior that a plain QPS benchmark can miss. + +### CloudInsertCase + +**Purpose.** CloudInsertCase measures write readiness, not just client-side insert speed. This is important for backfills, migrations, daily refreshes, and release workflows where a team needs to know when newly written vectors can safely take traffic. + +**How it works.** The case loads the dataset with `ConcurrentInsertRunner`, records insert completion time and rows per second, then polls the database until inserted data is fully searchable and fully indexed. The resulting metric separates: + +- `insert_completion_seconds` +- `insert_rows_per_second` +- `searchable_after_insert_seconds` +- `indexed_after_searchable_seconds` + +Example: run LAION 100M insert readiness on Zilliz Cloud with a 10k batch size. + +```bash +vectordbbench zillizautoindex \ + --case-type CloudInsertCase \ + --uri "$ZILLIZ_URI" \ + --token "$ZILLIZ_TOKEN" \ + --collection-name cloud_insert_laion100m_bs10k \ + --cloud-insert-batch-size 10000 \ + --load-concurrency 16 \ + --skip-search-serial \ + --skip-search-concurrent \ + --task-label cloud-insert-zilliz-12cu +``` + +### CloudPayloadSearchCase + +**Purpose.** CloudPayloadSearchCase measures search when the response body resembles production traffic. Many applications return more than vector IDs: they return scalar metadata, labels, or the vector values themselves. That response payload can change throughput, latency, and even product ranking. + +**How it works.** The case extends the normal performance case with an explicit `payload_profile`. Supported profiles are: + +- `ids_only` +- `scalar_label` +- `vector` + +The case can also run unfiltered search, integer-filter search through `--cloud-filter-rate`, or scalar-label filter search through `--cloud-label-percentage`. It records QPS, latency, recall where applicable, and estimated response payload bytes per query. + +When `payload_profile` is `scalar_label`, VectorDBBench materializes scalar label data even for unfiltered runs. This keeps the loaded schema aligned with the requested response payload instead of only loading labels for scalar-label filter runs. + +Example: run vector-payload search on Pinecone with a highly selective integer filter. + +```bash +vectordbbench pinecone \ + --case-type CloudPayloadSearchCase \ + --api-key "$PINECONE_API_KEY" \ + --index-name "$PINECONE_INDEX" \ + --payload-profile vector \ + --cloud-filter-rate 0.001 \ + --k 100 \ + --num-concurrency 60,80 \ + --concurrency-duration 30 \ + --task-label cloud-payload-pinecone-vector-int-filter-0-1p +``` + +### CloudMultiTenantSearchCase + +**Purpose.** CloudMultiTenantSearchCase models SaaS-shaped traffic. Instead of treating the dataset as one flat global collection, it splits records across many tenants and routes each query to a tenant. This highlights products whose namespace, partition-key, or tenant-filter paths behave differently from single-tenant search. + +**How it works.** The case defaults to the Cohere 10M dataset and assigns each row to a deterministic tenant by `row_id % tenant_count`. During search, queries are routed to the corresponding tenant label or namespace. The case supports the same payload profiles and optional filter modes as payload search. + +Tenant routing labels and scalar payload labels are separate concepts. A multi-tenant run can route by tenant while still storing and returning scalar-label payload data when `payload_profile` is `scalar_label`, and scalar-label filters continue to use the scalar label field rather than the tenant routing field. + +TurboPuffer tenant namespace cache warmup is explicit. By default, `CloudMultiTenantSearchCase` does not warm tenant namespaces during `optimize()`; use `--multitenant-warmup-policy all` only when the benchmark should model proactively warmed tenant namespaces. + +Example: run 1,000-tenant IDs-only search on turbopuffer. + +```bash +vectordbbench turbopuffer \ + --case-type CloudMultiTenantSearchCase \ + --dataset-with-size-type "Large Cohere (768dim, 10M)" \ + --api-key "$TURBOPUFFER_API_KEY" \ + --region aws-us-east-1 \ + --namespace vdbbench_mt_seed \ + --multitenant-namespace-prefix vdbbench_mt_ \ + --tenant-count 1000 \ + --tenant-prefix tenant_ \ + --tenant-id-width 4 \ + --payload-profile ids_only \ + --num-concurrency 40,60,80 \ + --concurrency-duration 30 \ + --task-label cloud-multitenant-turbopuffer-1000t +``` + +### CloudColdLatencyCase + +**Purpose.** CloudColdLatencyCase measures first-query and cold-path latency that warm benchmark loops can hide. This matters for serverless products, storage-tiered products, idle workloads, and customer-facing applications where the first query after an idle period is visible to users. + +**How it works.** The case is intentionally search-only and must run against an existing collection that has already become cold according to the product's cache and storage behavior. It rejects `drop_old` and `load` stages because insert-then-immediately-search runs can leave caches, indexing paths, or vendor warmup APIs in an ambiguous state. `ColdWarmSearchRunner` runs serial searches in cold and warm passes. It records cold-latency details in `additional_parameters["cold_latency"]` and also records payload profile and estimated payload bytes per query. + +Example: run a pinned turbopuffer cold-latency test with scalar-label payloads. + +```bash +vectordbbench turbopuffer \ + --case-type CloudColdLatencyCase \ + --skip-drop-old \ + --skip-load \ + --api-key "$TURBOPUFFER_API_KEY" \ + --region aws-us-east-1 \ + --namespace cloud_cold_latency_scalar_label \ + --pin-namespace \ + --pin-replicas 2 \ + --payload-profile scalar_label \ + --cloud-cold-query-count 1000 \ + --skip-search-concurrent \ + --task-label cloud-cold-latency-turbopuffer-pinned-scalar-label +``` + +## Caveats + +This release note introduces the new Cloud Leaderboard direction; it is not the full benchmark report. Detailed tables, raw JSON artifacts, pricing worksheets, and edge-case analysis should live in the benchmark report or external result artifact repository. + +Important caveats: + +- Pricing changes over time. Cost charts need a pricing date, region, and configuration assumptions. +- Managed-service configuration can materially change results, especially for serverless scaling, pinned replicas, capacity units, and storage-tiering modes. +- "Fully indexed" and "fully searchable" readiness may be exposed differently by each vendor, so the implementation must document how each status is detected or inferred. +- The current multi-tenant case uses deterministic tenant assignment and uniform tenant routing. It does not represent every SaaS tenant distribution. +- Multi-tenant routing labels or namespaces are not equivalent to scalar payload labels. Benchmark clients must keep those fields separate when a run combines tenant routing with scalar-label payload or filter behavior. +- Cold latency depends on cache state, idle window, replica pinning, storage architecture, and service warmup behavior. The idle and warmup rules must stay strict between products. +- Payload search rankings are workload-specific. IDs-only, scalar-label, vector-return, integer-filter, and label-filter runs can produce different winners. +- Cost Pareto results must be read together with recall, latency, payload profile, and readiness constraints rather than as a standalone ranking. diff --git a/tests/test_case_runner_reuse.py b/tests/test_case_runner_reuse.py new file mode 100644 index 000000000..55dfbda0b --- /dev/null +++ b/tests/test_case_runner_reuse.py @@ -0,0 +1,171 @@ +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig, MetricType +from vectordb_bench.backend.clients.doris.config import DorisCaseConfig, DorisConfig +from vectordb_bench.backend.clients.pinecone.config import PineconeConfig +from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferConfig, TurboPufferIndexConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus, TaskRunner +from vectordb_bench.interface import BenchMarkRunner +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseType, TaskConfig, TaskStage, TestResult + + +def make_runner( + *, + case_id: CaseType = CaseType.Performance1536D50K, + custom_case: dict | None = None, + db: DB = DB.TurboPuffer, + db_config=None, + db_case_config=None, + stages: list[TaskStage] | None = None, +) -> CaseRunner: + if db_config is None: + if db == DB.TurboPuffer: + db_config = TurboPufferConfig(api_key="key", region="aws-us-east-1") + elif db == DB.Pinecone: + db_config = PineconeConfig(api_key="key", index_name="idx") + elif db == DB.Doris: + db_config = DorisConfig(password=SecretStr("")) + else: + db_config = DB.Test.config_cls() + if db_case_config is None: + if db == DB.TurboPuffer: + db_case_config = TurboPufferIndexConfig(metric_type=MetricType.COSINE) + elif db == DB.Doris: + db_case_config = DorisCaseConfig(metric_type=MetricType.COSINE) + else: + db_case_config = EmptyDBCaseConfig() + + task = TaskConfig( + db=db, + db_config=db_config, + db_case_config=db_case_config, + case_config=CaseConfig(case_id=case_id, custom_case=custom_case or {}), + stages=stages or [TaskStage.DROP_OLD, TaskStage.LOAD, TaskStage.SEARCH_SERIAL], + ) + return CaseRunner( + run_id="run-id", + config=task, + ca=task.case_config.case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + +def assert_not_reusable(left: CaseRunner, right: CaseRunner) -> None: + assert left != right + assert right != left + assert hash(left) != hash(right) + + +def test_reuse_key_distinguishes_multitenant_routing_parameters(): + base_case = { + "dataset_with_size_type": DatasetWithSizeType.CohereSmall.value, + "tenant_count": 2, + "tenant_prefix": "tenant_", + "tenant_id_width": 4, + } + + assert_not_reusable( + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=base_case), + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case={**base_case, "tenant_count": 3}), + ) + assert_not_reusable( + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=base_case), + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case={**base_case, "tenant_prefix": "org_"}), + ) + assert_not_reusable( + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=base_case), + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case={**base_case, "tenant_id_width": 2}), + ) + + +def test_reuse_key_distinguishes_single_layout_from_multitenant_layout(): + dataset_case = {"dataset_with_size_type": DatasetWithSizeType.CohereSmall.value} + + assert_not_reusable( + make_runner(case_id=CaseType.CloudPayloadSearchCase, custom_case=dataset_case), + make_runner( + case_id=CaseType.CloudMultiTenantSearchCase, + custom_case={**dataset_case, "tenant_count": 2}, + ), + ) + + +def test_reuse_key_preserves_safe_payload_reuse(): + dataset_case = {"dataset_with_size_type": DatasetWithSizeType.CohereSmall.value} + + ids_only = make_runner( + case_id=CaseType.CloudPayloadSearchCase, + custom_case={**dataset_case, "payload_profile": "ids_only"}, + ) + vector = make_runner( + case_id=CaseType.CloudPayloadSearchCase, + custom_case={**dataset_case, "payload_profile": "vector"}, + ) + + assert ids_only == vector + assert hash(ids_only) == hash(vector) + + +def test_reuse_key_distinguishes_physical_db_targets(): + assert_not_reusable( + make_runner(db_config=TurboPufferConfig(api_key="key", region="aws-us-east-1", namespace="namespace_a")), + make_runner(db_config=TurboPufferConfig(api_key="key", region="aws-us-east-1", namespace="namespace_b")), + ) + assert_not_reusable( + make_runner(db=DB.Pinecone, db_config=PineconeConfig(api_key="key", index_name="index_a")), + make_runner(db=DB.Pinecone, db_config=PineconeConfig(api_key="key", index_name="index_b")), + ) + + +def test_reuse_key_distinguishes_doris_case_derived_table_names(): + assert_not_reusable( + make_runner(db=DB.Doris, case_id=CaseType.Performance768D1M), + make_runner( + db=DB.Doris, + case_id=CaseType.NewIntFilterPerformanceCase, + custom_case={ + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + "filter_rate": 0.01, + }, + ), + ) + + +def test_search_only_runner_does_not_suppress_later_full_load(monkeypatch): + calls: list[bool] = [] + search_only = make_runner( + stages=[TaskStage.SEARCH_SERIAL], + ) + full_load = make_runner( + stages=[TaskStage.DROP_OLD, TaskStage.LOAD, TaskStage.SEARCH_SERIAL], + ) + + def fake_run(self: CaseRunner, drop_old: bool = True) -> Metric: + calls.append(drop_old) + return Metric() + + class SendConn: + def __init__(self): + self.sent = [] + + def send(self, value): + self.sent.append(value) + + def close(self): + return None + + monkeypatch.setattr(CaseRunner, "run", fake_run) + monkeypatch.setattr(TestResult, "display", lambda self: None) + monkeypatch.setattr(TestResult, "flush", lambda self: None) + + BenchMarkRunner()._async_task_v2( + TaskRunner(run_id="run-id", task_label="task", case_runners=[search_only, full_load]), + SendConn(), + ) + + assert calls == [False, True] diff --git a/tests/test_cloud_cold_latency_case.py b/tests/test_cloud_cold_latency_case.py new file mode 100644 index 000000000..55f610b38 --- /dev/null +++ b/tests/test_cloud_cold_latency_case.py @@ -0,0 +1,470 @@ +import json +from contextlib import contextmanager +from pathlib import Path + +import numpy as np +import pytest + +from vectordb_bench.backend.assembler import Assembler +from vectordb_bench.backend.cases import CaseLabel, CaseType, CloudColdLatencyCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig +from vectordb_bench.backend.clients.pinecone.config import PineconeConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.result_collector import ResultCollector +from vectordb_bench.backend.runner.cold_warm_runner import ColdWarmSearchRunner +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus +from vectordb_bench.cli.cli import get_custom_case_config +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseResult, TaskConfig, TaskStage, TestResult + + +def test_cloud_cold_latency_case_defaults_to_laion_100m(): + case = CloudColdLatencyCase() + + assert case.case_id == CaseType.CloudColdLatencyCase + assert case.label == CaseLabel.CloudColdLatency + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.dataset.data.dim == 768 + assert case.payload_profile == PayloadProfile.IDS_ONLY + assert case.query_count == 1000 + assert case.filters.type == FilterOp.NonFilter + + +def test_cloud_cold_latency_case_accepts_payload_dataset_and_int_filter(): + case = CloudColdLatencyCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + payload_profile="vector", + filter_rate=0.9, + query_count=10, + ) + + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.name == "Cohere" + assert case.payload_profile == PayloadProfile.VECTOR + assert case.filter_rate == 0.9 + assert case.query_count == 10 + assert case.filters.type == FilterOp.NumGE + + +def test_cloud_cold_latency_case_accepts_label_filter(): + case = CloudColdLatencyCase(label_percentage=0.9) + + assert case.label_percentage == 0.9 + assert case.filters.type == FilterOp.StrEqual + + +def test_cloud_cold_latency_case_rejects_two_filter_types(): + with pytest.raises(ValueError, match="supports only one filter type"): + CloudColdLatencyCase(filter_rate=0.9, label_percentage=0.9) + + +def test_cloud_cold_latency_case_rejects_invalid_query_count(): + with pytest.raises(ValueError, match="query_count must be positive"): + CloudColdLatencyCase(query_count=0) + + +def test_case_config_builds_cloud_cold_latency_case_from_custom_case(): + case = CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={ + "payload_profile": "scalar_label", + "label_percentage": 0.9, + "query_count": 12, + }, + ).case + + assert isinstance(case, CloudColdLatencyCase) + assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.label_percentage == 0.9 + assert case.query_count == 12 + + +def test_cli_builds_cloud_cold_latency_custom_case_config(): + params = { + "case_type": "CloudColdLatencyCase", + "payload_profile": "vector", + "cloud_filter_rate": 0.9, + "cloud_label_percentage": None, + "cloud_cold_query_count": 1000, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + assert get_custom_case_config(params) == { + "payload_profile": "vector", + "filter_rate": 0.9, + "query_count": 1000, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + +def test_cli_keeps_cloud_cold_latency_default_dataset_as_laion(): + params = { + "case_type": "CloudColdLatencyCase", + "payload_profile": "ids_only", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + "cloud_cold_query_count": 1000, + "dataset_with_size_type": None, + } + + custom_case = get_custom_case_config(params) + case = CaseConfig(case_id=CaseType.CloudColdLatencyCase, custom_case=custom_case).case + + assert custom_case == { + "payload_profile": "ids_only", + "query_count": 1000, + } + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + + +def test_cloud_cold_latency_result_file_uses_cold_latency_metrics(tmp_path: Path): + cold_latency = { + "cold_stats": { + "first_query_latency": 0.2, + "p99_latency": 0.3, + "p95_latency": 0.25, + "avg_latency": 0.21, + }, + "warm_stats": { + "first_query_latency": 0.1, + "p99_latency": 0.15, + "p95_latency": 0.12, + "avg_latency": 0.11, + }, + "ratios": { + "first_query_latency": 2.0, + "p99_latency": 2.0, + "p95_latency": 2.0833, + "avg_latency": 1.9091, + }, + } + result = CaseResult( + task_config=TaskConfig( + db=DB.Pinecone, + db_config=PineconeConfig( + db_label="pinecone_cloud_cold_latency", + api_key="secret-key", + index_name="laion100m", + ), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"payload_profile": "vector", "query_count": 1000}, + ), + stages=[TaskStage.SEARCH_SERIAL], + load_concurrency=0, + ), + metrics=Metric( + insert_duration=0.0, + optimize_duration=0.0, + load_duration=0.0, + payload_profile="vector", + payload_estimated_bytes_per_query=309200, + additional_parameters={"cold_latency": cold_latency}, + ), + ) + test_result = TestResult(run_id="run-id", task_label="cloud_cold_latency_pinecone", results=[result]) + + test_result.write_db_file(tmp_path, test_result, "pinecone") + + result_file = next(tmp_path.glob("result_*_pinecone.json")) + raw_output = result_file.read_text() + assert raw_output.startswith('{\n "run_id"') + written = json.loads(raw_output) + assert written["results"][0]["metrics"] == { + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "payload_profile": "vector", + "payload_estimated_bytes_per_query": 309200, + "cold_latency": cold_latency, + } + assert written["results"][0]["task_config"]["db_config"]["api_key"] == "**********" + assert written["results"][0]["task_config"]["db_config"]["index_name"] == "laion100m" + assert written["results"][0]["task_config"]["case_config"] == { + "case_id": 700, + "custom_case": {"payload_profile": "vector", "query_count": 1000}, + } + + read_back = TestResult.read_file(result_file) + assert read_back.results[0].task_config.case_config.case_id == CaseType.CloudColdLatencyCase + assert read_back.results[0].task_config.case_config.custom_case == { + "payload_profile": "vector", + "query_count": 1000, + } + assert read_back.results[0].metrics.additional_parameters["cold_latency"] == cold_latency + + collected = ResultCollector.collect(tmp_path) + assert len(collected) == 1 + assert collected[0].results[0].metrics.additional_parameters["cold_latency"] == cold_latency + + +class FakeColdWarmDB: + name = "FakeColdWarmDB" + + def __init__(self, supported_payload_profiles: set[PayloadProfile] | None = None): + self.supported_payload_profiles = supported_payload_profiles or {PayloadProfile.IDS_ONLY} + self.calls = [] + self.prepare_filter_calls = [] + self.init_enter_count = 0 + + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in self.supported_payload_profiles + + def need_normalize_cosine(self) -> bool: + return False + + @contextmanager + def init(self): + self.init_enter_count += 1 + yield + + def prepare_filter(self, filters: Filter): + self.prepare_filter_calls.append(filters) + + def search_embedding(self, query: list[float], k: int = 100, **kwargs) -> list[int]: + self.calls.append((query, k, kwargs)) + return list(range(k)) + + +def test_cold_warm_runner_computes_stats_and_ratios(monkeypatch: pytest.MonkeyPatch): + db = FakeColdWarmDB() + # Cold latencies: 0.2, 0.2, 0.2. Warm latencies: 0.1, 0.1, 0.1. + perf_values = iter([0.0, 0.2, 0.2, 0.4, 0.4, 0.6, 0.6, 0.7, 0.7, 0.8, 0.8, 0.9]) + monkeypatch.setattr("vectordb_bench.backend.runner.cold_warm_runner.time.perf_counter", lambda: next(perf_values)) + + runner = ColdWarmSearchRunner( + db=db, + test_data=[[0.1], [0.2], [0.3]], + k=3, + query_count=3, + ) + + result = runner.run() + + assert result == { + "cold_stats": { + "first_query_latency": 0.2, + "p99_latency": 0.2, + "p95_latency": 0.2, + "avg_latency": 0.2, + }, + "warm_stats": { + "first_query_latency": 0.1, + "p99_latency": 0.1, + "p95_latency": 0.1, + "avg_latency": 0.1, + }, + "cold_warm_ratio": { + "first_query_latency_ratio": 2.0, + "p99_latency_ratio": 2.0, + "p95_latency_ratio": 2.0, + "avg_latency_ratio": 2.0, + }, + } + assert db.init_enter_count == 1 + assert len(db.prepare_filter_calls) == 1 + assert [call[0] for call in db.calls] == [[0.1], [0.2], [0.3], [0.1], [0.2], [0.3]] + + +def test_cold_warm_runner_passes_payload_profile_in_both_passes(monkeypatch: pytest.MonkeyPatch): + db = FakeColdWarmDB(supported_payload_profiles={PayloadProfile.IDS_ONLY, PayloadProfile.VECTOR}) + perf_values = iter([0.0, 0.1, 0.1, 0.2]) + monkeypatch.setattr("vectordb_bench.backend.runner.cold_warm_runner.time.perf_counter", lambda: next(perf_values)) + + runner = ColdWarmSearchRunner( + db=db, + test_data=[np.array([0.1])], + k=3, + payload_profile=PayloadProfile.VECTOR, + query_count=1, + ) + + runner.run() + + assert db.calls == [ + ([0.1], 3, {"payload_profile": PayloadProfile.VECTOR}), + ([0.1], 3, {"payload_profile": PayloadProfile.VECTOR}), + ] + + +def test_cold_warm_runner_omits_payload_profile_for_ids_only(monkeypatch: pytest.MonkeyPatch): + db = FakeColdWarmDB() + perf_values = iter([0.0, 0.1, 0.1, 0.2]) + monkeypatch.setattr("vectordb_bench.backend.runner.cold_warm_runner.time.perf_counter", lambda: next(perf_values)) + + runner = ColdWarmSearchRunner( + db=db, + test_data=[[0.1]], + k=3, + payload_profile=PayloadProfile.IDS_ONLY, + query_count=1, + ) + + runner.run() + + assert db.calls == [ + ([0.1], 3, {}), + ([0.1], 3, {}), + ] + + +def test_cold_warm_runner_fails_for_unsupported_payload_profile(): + db = FakeColdWarmDB() + + with pytest.raises(NotImplementedError, match="payload_profile=vector"): + ColdWarmSearchRunner( + db=db, + test_data=[[0.1]], + payload_profile=PayloadProfile.VECTOR, + query_count=1, + ) + + +def test_cold_warm_runner_rejects_too_few_queries(): + db = FakeColdWarmDB() + + with pytest.raises(ValueError, match="query_count=2 exceeds test_data size=1"): + ColdWarmSearchRunner(db=db, test_data=[[0.1]], query_count=2) + + +def test_assembler_schedules_cloud_cold_latency_case(): + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.SEARCH_SERIAL], + ) + + runner = Assembler.assemble_all("run-id", "task-label", [task], DatasetSource.S3) + + assert len(runner.case_runners) == 1 + assert runner.case_runners[0].ca.label == CaseLabel.CloudColdLatency + + +def test_case_runner_stores_cloud_cold_latency_metric(monkeypatch: pytest.MonkeyPatch): + case = CloudColdLatencyCase(query_count=1) + case.dataset.test_data = [[0.1]] + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.SEARCH_SERIAL], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + runner.db = FakeColdWarmDB() + + expected = { + "cold_stats": { + "first_query_latency": 0.2, + "p99_latency": 0.2, + "p95_latency": 0.2, + "avg_latency": 0.2, + }, + "warm_stats": { + "first_query_latency": 0.1, + "p99_latency": 0.1, + "p95_latency": 0.1, + "avg_latency": 0.1, + }, + "cold_warm_ratio": { + "first_query_latency_ratio": 2.0, + "p99_latency_ratio": 2.0, + "p95_latency_ratio": 2.0, + "avg_latency_ratio": 2.0, + }, + } + captured_kwargs = {} + + class FakeRunner: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + def run(self): + return expected + + monkeypatch.setattr("vectordb_bench.backend.task_runner.ColdWarmSearchRunner", FakeRunner) + + metric = runner._run_cloud_cold_latency_case(drop_old=False) + + assert metric.additional_parameters["cold_latency"] == expected + assert metric.payload_profile == "ids_only" + assert metric.payload_estimated_bytes_per_query == case.estimated_payload_bytes_per_query(task.case_config.k) + assert captured_kwargs == { + "db": runner.db, + "test_data": [[0.1]], + "filters": case.filters, + "k": task.case_config.k, + "payload_profile": case.payload_profile, + "query_count": case.query_count, + } + + +def test_cloud_cold_latency_case_rejects_drop_old(): + case = CloudColdLatencyCase(query_count=1) + case.dataset.test_data = [[0.1]] + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.SEARCH_SERIAL], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + with pytest.raises(ValueError, match="requires an existing cold collection"): + runner._run_cloud_cold_latency_case(drop_old=True) + + +def test_cloud_cold_latency_case_rejects_load_stage(): + case = CloudColdLatencyCase(query_count=1) + case.dataset.test_data = [[0.1]] + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.LOAD, TaskStage.SEARCH_SERIAL], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + with pytest.raises(ValueError, match="search-only"): + runner._run_cloud_cold_latency_case(drop_old=False) diff --git a/tests/test_cloud_insert_case.py b/tests/test_cloud_insert_case.py new file mode 100644 index 000000000..28e70f7bd --- /dev/null +++ b/tests/test_cloud_insert_case.py @@ -0,0 +1,1108 @@ +import json +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import MagicMock + +import numpy as np +import pandas as pd +import pytest + +from vectordb_bench import config +from vectordb_bench.backend.assembler import Assembler +from vectordb_bench.backend.cases import CaseLabel, CaseType, CloudInsertCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig, VectorDB +from vectordb_bench.backend.clients.milvus.milvus import Milvus +from vectordb_bench.backend.clients.pinecone.config import PineconeConfig +from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone +from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig +from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer +from vectordb_bench.backend.clients.zilliz_cloud.config import AutoIndexConfig, ZillizCloudConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.result_collector import ResultCollector +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner +from vectordb_bench.backend.runner.executor import TaskResult +from vectordb_bench.backend.task_runner import CaseRunner +from vectordb_bench.cli.cli import get_custom_case_config +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseResult, TaskConfig, TaskStage, TestResult + + +class _SerialTaskExecutor: + def __init__(self): + self.submitted = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return False + + def submit(self, fn): + self.submitted.append(fn) + + def wait_all(self): + results = [] + for fn in self.submitted: + try: + results.append(TaskResult(value=fn())) + except Exception as e: + results.append(TaskResult(error=e)) + return results + + +class _ConcurrentRunnerData: + train_id_field = "id" + train_vector_field = "vector" + + +class _ConcurrentRunnerDataset: + data = _ConcurrentRunnerData() + + def __init__(self, ids): + self.ids = ids + + def iter_batches(self, batch_size): + return iter([pd.DataFrame({"id": [row_id], "vector": [np.array([row_id / 10 + 0.1])]}) for row_id in self.ids]) + + +def test_cloud_insert_case_defaults_to_laion_100m(): + case = CloudInsertCase(batch_size=1000) + + assert case.case_id == CaseType.CloudInsertCase + assert case.label == CaseLabel.CloudInsert + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.batch_size == 1000 + assert case.duration is None + assert case.readiness_timeout is None + + +def test_case_config_builds_cloud_insert_case_from_custom_case(): + case = CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={ + "batch_size": 5000, + "duration": 1800, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + }, + ).case + + assert isinstance(case, CloudInsertCase) + assert case.batch_size == 5000 + assert case.duration == 1800 + assert case.dataset.data.name == "Cohere" + assert case.dataset.data.size == 1_000_000 + + +def test_case_config_builds_cloud_insert_case_from_laion_100m_dataset_option(): + case = CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={ + "batch_size": 10_000, + "dataset_with_size_type": "Large LAION (768dim, 100M)", + }, + ).case + + assert isinstance(case, CloudInsertCase) + assert case.batch_size == 10_000 + assert case.dataset_with_size_type == DatasetWithSizeType.LAIONLarge + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.dataset.data.dim == 768 + + +def test_laion_100m_dataset_option_uses_100m_timeouts(): + assert DatasetWithSizeType.LAIONLarge.get_load_timeout() == config.LOAD_TIMEOUT_768D_100M + assert DatasetWithSizeType.LAIONLarge.get_optimize_timeout() == config.OPTIMIZE_TIMEOUT_768D_100M + + +def test_cli_builds_cloud_insert_custom_case_config(): + params = { + "case_type": "CloudInsertCase", + "cloud_insert_batch_size": 10_000, + "cloud_insert_duration": 1800, + "cloud_insert_readiness_timeout": 7200, + "cloud_insert_readiness_poll_interval": 10, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + assert get_custom_case_config(params) == { + "batch_size": 10_000, + "duration": 1800, + "readiness_timeout": 7200, + "readiness_poll_interval": 10, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + +def test_cli_builds_cloud_insert_custom_case_config_with_laion_100m_dataset(): + cfg = get_custom_case_config( + { + "case_type": "CloudInsertCase", + "cloud_insert_batch_size": 10_000, + "cloud_insert_duration": None, + "cloud_insert_readiness_timeout": None, + "cloud_insert_readiness_poll_interval": None, + "dataset_with_size_type": DatasetWithSizeType.LAIONLarge.value, + } + ) + + assert cfg == { + "batch_size": 10_000, + "duration": None, + "dataset_with_size_type": DatasetWithSizeType.LAIONLarge.value, + } + + case = CaseConfig(case_id=CaseType.CloudInsertCase, custom_case=cfg).case + assert case.dataset_with_size_type == DatasetWithSizeType.LAIONLarge + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + + +def test_cli_builds_cloud_insert_custom_case_config_with_default_dataset(): + cfg = get_custom_case_config( + { + "case_type": "CloudInsertCase", + "cloud_insert_batch_size": 10_000, + "cloud_insert_duration": None, + "cloud_insert_readiness_timeout": None, + "cloud_insert_readiness_poll_interval": None, + "dataset_with_size_type": None, + } + ) + + assert cfg == { + "batch_size": 10_000, + "duration": None, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + case = CaseConfig(case_id=CaseType.CloudInsertCase, custom_case=cfg).case + assert case.dataset_with_size_type == DatasetWithSizeType.CohereMedium + assert case.dataset.data.size == 1_000_000 + + +def test_cli_builds_multitenant_custom_case_config(): + cfg = get_custom_case_config( + { + "case_type": "CloudMultiTenantSearchCase", + "dataset_with_size_type": "Small Cohere (768dim, 100K)", + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "vector", + "cloud_filter_rate": 0.01, + "cloud_label_percentage": None, + } + ) + + assert cfg == { + "dataset_with_size_type": "Small Cohere (768dim, 100K)", + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "vector", + "filter_rate": 0.01, + } + + +def test_cli_omits_multitenant_dataset_when_not_selected(): + cfg = get_custom_case_config( + { + "case_type": "CloudMultiTenantSearchCase", + "dataset_with_size_type": None, + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "ids_only", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + } + ) + + assert cfg == { + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "ids_only", + } + + case = CaseConfig(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=cfg).case + assert case.dataset_with_size_type == DatasetWithSizeType.CohereLarge + assert case.dataset.data.size == 10_000_000 + + +def test_assembler_schedules_cloud_insert_case(): + task = TaskConfig( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig(uri="https://example.com", user="db_admin"), + db_case_config=AutoIndexConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={ + "batch_size": 1000, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + }, + ), + stages=[TaskStage.DROP_OLD, TaskStage.LOAD], + ) + + runner = Assembler.assemble_all("run-id", "task-label", [task], DatasetSource.S3) + + assert len(runner.case_runners) == 1 + assert runner.case_runners[0].ca.label == CaseLabel.CloudInsert + + +def test_default_insert_readiness_is_immediately_ready(): + class FakeVectorDB(VectorDB): + def __init__(self, dim, db_config, db_case_config, collection_name="", drop_old=False): + pass + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None, **kwargs): + return len(embeddings), None + + def search_embedding(self, query, k=100, payload_profile=None): + return [] + + def optimize(self, data_size=None): + pass + + assert FakeVectorDB(1, {}, EmptyDBCaseConfig()).poll_insert_readiness(10) == { + "fully_searchable": True, + "fully_indexed": True, + "additional_parameters": {}, + } + + +def test_metric_contains_cloud_insert_output_fields(): + metric = Metric( + inserted_count=100, + insert_rows_per_second=83.33, + insert_completion_seconds=1.2, + searchable_after_insert_seconds=3.4, + indexed_after_searchable_seconds=5.6, + additional_parameters={"disable_backpressure": True}, + ) + + assert metric.inserted_count == 100 + assert metric.insert_rows_per_second == 83.33 + assert metric.insert_completion_seconds == 1.2 + assert metric.searchable_after_insert_seconds == 3.4 + assert metric.indexed_after_searchable_seconds == 5.6 + assert metric.additional_parameters == {"disable_backpressure": True} + + +def test_cloud_insert_result_file_uses_insert_only_metrics(tmp_path: Path): + result = CaseResult( + task_config=TaskConfig( + db=DB.Pinecone, + db_config=PineconeConfig( + db_label="pinecone_cloud_insert_laion100m_bs1k", + api_key="secret-key", + index_name="laion100m", + ), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={"batch_size": 1000, "duration": None}, + ), + stages=[TaskStage.DROP_OLD, TaskStage.LOAD], + load_concurrency=0, + ), + metrics=Metric( + inserted_count=100_000_000, + insert_rows_per_second=3919.9296, + insert_completion_seconds=255.1066, + searchable_after_insert_seconds=0.0, + indexed_after_searchable_seconds=28.5956, + additional_parameters={}, + ), + ) + test_result = TestResult(run_id="run-id", task_label="cloud_insert_pinecone_laion100m_bs1k", results=[result]) + + test_result.write_db_file(tmp_path, test_result, "pinecone") + + result_file = next(tmp_path.glob("result_*_pinecone.json")) + raw_output = result_file.read_text() + assert raw_output.startswith('{\n "run_id"') + written = json.loads(raw_output) + assert written["results"][0]["metrics"] == { + "inserted_count": 100_000_000, + "insert_rows_per_second": 3919.9296, + "insert_completion_seconds": 255.1066, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 28.5956, + "additional_parameters": {}, + } + assert written["results"][0]["task_config"]["db_config"]["api_key"] == "**********" + assert written["results"][0]["task_config"]["db_config"]["index_name"] == "laion100m" + assert written["results"][0]["task_config"]["case_config"] == { + "case_id": 600, + "custom_case": {"batch_size": 1000, "duration": None}, + } + + read_back = TestResult.read_file(result_file) + assert read_back.results[0].task_config.case_config.case_id == CaseType.CloudInsertCase + assert read_back.results[0].task_config.case_config.custom_case == {"batch_size": 1000, "duration": None} + + collected = ResultCollector.collect(tmp_path) + assert len(collected) == 1 + assert collected[0].results[0].metrics.inserted_count == 100_000_000 + + +def test_turbopuffer_insert_can_disable_backpressure(): + db = TurboPuffer.__new__(TurboPuffer) + db.with_scalar_labels = False + db._scalar_id_field = "id" + db._vector_field = "vector" + db.metric = "cosine_distance" + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=True) + + class Namespace: + kwargs = None + + def write(self, **kwargs): + self.kwargs = kwargs + + db.ns = Namespace() + + assert db.insert_embeddings([[0.1]], [1]) == (1, None) + assert db.ns.kwargs["disable_backpressure"] is True + + +def test_turbopuffer_insert_serializes_numpy_vectors(): + db = TurboPuffer.__new__(TurboPuffer) + db.with_scalar_labels = False + db._scalar_id_field = "id" + db._vector_field = "vector" + db.metric = "cosine_distance" + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=False) + + class Namespace: + kwargs = None + + def write(self, **kwargs): + self.kwargs = kwargs + + db.ns = Namespace() + + insert_count, error = db.insert_embeddings([np.array([0.1, 0.2])], [1]) + + assert error is None + assert insert_count == 1 + assert db.ns.kwargs["upsert_columns"]["vector"] == [[0.1, 0.2]] + + +def test_turbopuffer_insert_returns_write_error(): + db = TurboPuffer.__new__(TurboPuffer) + db.with_scalar_labels = False + db._scalar_id_field = "id" + db._vector_field = "vector" + db.metric = "cosine_distance" + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=False) + + class Namespace: + def write(self, **kwargs): + raise RuntimeError("write failed") + + db.ns = Namespace() + + insert_count, error = db.insert_embeddings([[0.1]], [1]) + + assert insert_count == 0 + assert isinstance(error, RuntimeError) + + +def test_milvus_insert_readiness_uses_entity_count_and_index_progress(): + db = Milvus.__new__(Milvus) + db.collection_name = "c" + db._vector_index_name = "vector_idx" + db.client = type( + "Client", + (), + { + "flush": lambda self, collection_name: None, + "get_collection_stats": lambda self, collection_name: {"row_count": "10"}, + "describe_index": lambda self, collection_name, index_name: {"pending_index_rows": 0}, + }, + )() + + assert db.poll_insert_readiness(10) == { + "fully_searchable": True, + "fully_indexed": True, + "additional_parameters": {}, + } + + +def test_pinecone_insert_readiness_uses_vector_count(): + db = Pinecone.__new__(Pinecone) + db.index = type("Index", (), {"describe_index_stats": lambda self: {"total_vector_count": 9}})() + + assert db.poll_insert_readiness(10)["fully_searchable"] is False + assert db.poll_insert_readiness(9)["fully_indexed"] is True + + +def test_pinecone_supports_cloud_payload_profiles(): + db = Pinecone.__new__(Pinecone) + + assert db.supports_payload_profile(PayloadProfile.IDS_ONLY) + assert db.supports_payload_profile(PayloadProfile.SCALAR_LABEL) + assert db.supports_payload_profile(PayloadProfile.VECTOR) + + +def test_pinecone_search_requests_metadata_for_scalar_label_payload(): + db = Pinecone.__new__(Pinecone) + db.expr = None + + class Index: + kwargs = None + + def query(self, **kwargs): + self.kwargs = kwargs + return {"matches": [{"id": "1"}]} + + db.index = Index() + + assert db.search_embedding([0.1], payload_profile=PayloadProfile.SCALAR_LABEL) == [1] + assert db.index.kwargs["include_metadata"] is True + assert db.index.kwargs["include_values"] is False + + +def test_pinecone_search_requests_values_for_vector_payload(): + db = Pinecone.__new__(Pinecone) + db.expr = {"meta": {"$gte": 10}} + + class Index: + kwargs = None + + def query(self, **kwargs): + self.kwargs = kwargs + return {"matches": [{"id": "2"}]} + + db.index = Index() + + assert db.search_embedding([0.1], payload_profile=PayloadProfile.VECTOR) == [2] + assert db.index.kwargs["include_metadata"] is False + assert db.index.kwargs["include_values"] is True + assert db.index.kwargs["filter"] == {"meta": {"$gte": 10}} + + +def test_pinecone_search_retries_rate_limited_queries(monkeypatch): + db = Pinecone.__new__(Pinecone) + db.expr = None + monkeypatch.setenv("PINECONE_QUERY_RETRY_SLEEP_SECONDS", "0.25") + sleeps = [] + monkeypatch.setattr("vectordb_bench.backend.clients.pinecone.pinecone.time.sleep", sleeps.append) + + class RateLimitError(Exception): + status = 429 + + class Index: + calls = 0 + + def query(self, **kwargs): + self.calls += 1 + if self.calls < 3: + raise RateLimitError("too many requests") + return {"matches": [{"id": "3"}]} + + db.index = Index() + + assert db.search_embedding([0.1]) == [3] + assert db.index.calls == 3 + assert sleeps == [0.25, 0.25] + + +def test_pinecone_search_stops_after_rate_limit_retry_budget(monkeypatch): + db = Pinecone.__new__(Pinecone) + db.expr = None + monkeypatch.setenv("PINECONE_QUERY_MAX_RETRIES", "1") + monkeypatch.setattr("vectordb_bench.backend.clients.pinecone.pinecone.time.sleep", lambda _: None) + + class RateLimitError(Exception): + status = 429 + + class Index: + def query(self, **kwargs): + raise RateLimitError("too many requests") + + db.index = Index() + + try: + db.search_embedding([0.1]) + except RateLimitError: + pass + else: + raise AssertionError("expected Pinecone rate limit error") + + +def test_pinecone_insert_tracks_last_write_lsn(): + db = Pinecone.__new__(Pinecone) + db.batch_size = 1000 + db.with_scalar_labels = False + db._scalar_id_field = "meta" + + class UpsertResponse: + _response_info = {"raw_headers": {"x-pinecone-request-lsn": "123"}} + + class Index: + def upsert(self, records): + return UpsertResponse() + + db.index = Index() + + insert_count, error = db.insert_embeddings([[0.1]], [1]) + + assert error is None + assert insert_count == 1 + assert db._last_write_lsn == 123 + + +def test_pinecone_insert_keeps_highest_write_lsn(): + db = Pinecone.__new__(Pinecone) + db.batch_size = 1 + db.with_scalar_labels = False + db._scalar_id_field = "meta" + responses = iter(["123", "122"]) + + class UpsertResponse: + def __init__(self, lsn): + self._response_info = {"raw_headers": {"x-pinecone-request-lsn": lsn}} + + class Index: + def upsert(self, records): + return UpsertResponse(next(responses)) + + db.index = Index() + + insert_count, error = db.insert_embeddings([[0.1], [0.2]], [1, 2]) + + assert error is None + assert insert_count == 2 + assert db._last_write_lsn == 123 + + +def test_pinecone_record_write_lsn_keeps_highest_value(): + db = Pinecone.__new__(Pinecone) + + db._record_write_lsn(123) + db._record_write_lsn(122) + + assert db._last_write_lsn == 123 + + +def test_pinecone_insert_readiness_uses_lsn_when_available(): + db = Pinecone.__new__(Pinecone) + db._last_write_lsn = 123 + db._readiness_probe_vector = [0.0] + + class QueryResponse(dict): + def __init__(self, indexed_lsn): + super().__init__({"matches": []}) + self._response_info = {"raw_headers": {"x-pinecone-max-indexed-lsn": str(indexed_lsn)}} + + class Index: + indexed_lsn = 122 + + def describe_index_stats(self): + return {"total_vector_count": 10} + + def query(self, **kwargs): + return QueryResponse(self.indexed_lsn) + + db.index = Index() + + assert db.poll_insert_readiness(10)["fully_searchable"] is False + db.index.indexed_lsn = 123 + assert db.poll_insert_readiness(10)["fully_indexed"] is True + + +def test_turbopuffer_insert_readiness_uses_unindexed_bytes(): + db = TurboPuffer.__new__(TurboPuffer) + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=True) + db.ns = type("Namespace", (), {"metadata": lambda self: {"unindexed_bytes": 1}})() + + status = db.poll_insert_readiness(10) + + assert status["fully_searchable"] is True + assert status["fully_indexed"] is False + assert status["additional_parameters"] == {"disable_backpressure": True} + + +def test_turbopuffer_insert_readiness_uses_nested_unindexed_bytes(): + db = TurboPuffer.__new__(TurboPuffer) + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=False) + db.ns = type( + "Namespace", + (), + {"metadata": lambda self: {"index": {"status": "updating", "unindexed_bytes": 1}}}, + )() + + status = db.poll_insert_readiness(10) + + assert status["fully_searchable"] is True + assert status["fully_indexed"] is False + assert status["additional_parameters"] == {"disable_backpressure": False} + + +def test_cloud_insert_runner_records_insert_and_readiness_metrics(monkeypatch): + class Data: + train_id_field = "id" + train_vector_field = "vector" + metric_type = "L2" + + class Dataset: + data = Data() + + def iter_batches(self, batch_size): + assert batch_size == 2 + return iter( + [ + pd.DataFrame({"id": [1, 2], "vector": [np.array([0.1]), np.array([0.2])]}), + pd.DataFrame({"id": [3], "vector": [np.array([0.3])]}), + ] + ) + + class DB: + thread_safe = True + name = "fake" + inserts = [] + readiness_calls = 0 + + @contextmanager + def init(self): + yield + + def need_normalize_cosine(self): + return False + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.inserts.append((embeddings, metadata)) + return len(metadata), None + + def poll_insert_readiness(self, expected_count): + self.readiness_calls += 1 + return { + "fully_searchable": self.readiness_calls >= 2, + "fully_indexed": self.readiness_calls >= 3, + "additional_parameters": {"example": "value"}, + } + + db = DB() + monkeypatch.setattr("vectordb_bench.backend.task_runner.time.sleep", lambda _: None) + case = CloudInsertCase(batch_size=2) + case.dataset = Dataset() + config = type("Config", (), {"load_concurrency": 1})() + runner = CaseRunner.construct(ca=case, db=db, config=config) + + metric = runner._run_cloud_insert_case() + + assert [metadata for _, metadata in db.inserts] == [[1, 2], [3]] + assert metric.inserted_count == 3 + assert metric.insert_rows_per_second > 0 + assert metric.insert_completion_seconds >= 0 + assert metric.searchable_after_insert_seconds >= 0 + assert metric.indexed_after_searchable_seconds >= 0 + assert metric.additional_parameters == {"example": "value"} + + +def test_cloud_insert_runner_times_out_when_readiness_never_completes(monkeypatch): + class Data: + train_id_field = "id" + train_vector_field = "vector" + metric_type = "L2" + + class Dataset: + data = Data() + + def iter_batches(self, batch_size): + return iter([pd.DataFrame({"id": [1], "vector": [np.array([0.1])]})]) + + class DB: + @contextmanager + def init(self): + yield + + def need_normalize_cosine(self): + return False + + def poll_insert_readiness(self, expected_count): + return { + "fully_searchable": False, + "fully_indexed": False, + "additional_parameters": {"reason": "stalled"}, + } + + class FakeConcurrentInsertRunner: + def __init__(self, *args, **kwargs): + pass + + def task(self): + return 1 + + def fail_on_sleep(_seconds): + raise AssertionError("readiness polling did not time out before sleeping") + + monkeypatch.setattr("vectordb_bench.backend.task_runner.ConcurrentInsertRunner", FakeConcurrentInsertRunner) + monkeypatch.setattr("vectordb_bench.backend.task_runner.time.sleep", fail_on_sleep) + case = CloudInsertCase(batch_size=1, readiness_timeout=0, readiness_poll_interval=0) + case.dataset = Dataset() + runner = CaseRunner.construct(ca=case, db=DB(), config=type("Config", (), {"load_concurrency": 1})()) + + with pytest.raises(TimeoutError, match="fully_searchable.*last_status.*stalled"): + runner._run_cloud_insert_case() + + +def test_cloud_insert_runner_uses_concurrent_insert_runner(monkeypatch): + created = {} + + class Data: + metric_type = "L2" + + class Dataset: + data = Data() + + class FakeConcurrentInsertRunner: + def __init__(self, db, dataset, normalize, filters, max_workers, batch_size, duration): + created.update( + { + "db": db, + "dataset": dataset, + "normalize": normalize, + "filters": filters, + "max_workers": max_workers, + "batch_size": batch_size, + "duration": duration, + } + ) + + def task(self): + return 3 + + class DB: + @contextmanager + def init(self): + yield + + def need_normalize_cosine(self): + return False + + def poll_insert_readiness(self, expected_count): + return {"fully_searchable": True, "fully_indexed": True, "additional_parameters": {}} + + monkeypatch.setattr("vectordb_bench.backend.task_runner.ConcurrentInsertRunner", FakeConcurrentInsertRunner) + case = CloudInsertCase(batch_size=1000, duration=60) + case.dataset = Dataset() + config = type("Config", (), {"load_concurrency": 7})() + runner = CaseRunner.construct(ca=case, db=DB(), config=config) + + metric = runner._run_cloud_insert_case() + + assert metric.inserted_count == 3 + assert created["batch_size"] == 1000 + assert created["duration"] == 60 + assert created["max_workers"] == 7 + assert created["dataset"] is case.dataset + + +def test_concurrent_insert_runner_does_not_retry_non_retryable_insert_errors(monkeypatch): + from vectordb_bench.backend.runner import concurrent_runner as concurrent_runner_module + + class NonRetryableInsertError(RuntimeError): + non_retryable = True + + class FakeDB: + def __init__(self): + self.calls = 0 + + def insert_embeddings(self, **kwargs): + self.calls += 1 + return 2, NonRetryableInsertError("partial tenant insert") + + monkeypatch.setattr(concurrent_runner_module.time, "sleep", lambda _seconds: None) + + runner = ConcurrentInsertRunner.__new__(ConcurrentInsertRunner) + db = FakeDB() + + with pytest.raises(RuntimeError, match="Non-retryable insert failure"): + runner._insert_batch_with_retry( + db, + embeddings=[[1.0], [2.0], [3.0]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + + assert db.calls == 1 + + +def test_concurrent_insert_runner_stops_handing_out_batches_after_non_retryable_error(): + class NonRetryableInsertError(RuntimeError): + non_retryable = True + + class DB: + thread_safe = True + name = "fake" + + def __init__(self): + self.calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.calls.append(metadata) + if len(self.calls) == 1: + return 0, NonRetryableInsertError("partial tenant insert") + return len(metadata), None + + db = DB() + runner = ConcurrentInsertRunner( + db, _ConcurrentRunnerDataset([0, 1, 2]), normalize=False, max_workers=2, batch_size=1 + ) + runner._create_executor = lambda: _SerialTaskExecutor() + + with pytest.raises(RuntimeError, match="Non-retryable insert failure"): + runner.task() + + assert db.calls == [[0]] + + +def test_concurrent_insert_runner_stops_handing_out_batches_after_retry_exhaustion(monkeypatch): + from vectordb_bench.backend.runner import concurrent_runner as concurrent_runner_module + + class DB: + thread_safe = True + name = "fake" + + def __init__(self): + self.calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.calls.append(metadata) + return 0, RuntimeError("insert failed") + + monkeypatch.setattr(concurrent_runner_module.config, "MAX_INSERT_RETRY", 0) + + db = DB() + runner = ConcurrentInsertRunner(db, _ConcurrentRunnerDataset([0, 1]), normalize=False, max_workers=2, batch_size=1) + runner._create_executor = lambda: _SerialTaskExecutor() + + with pytest.raises(RuntimeError, match="Insert failed and retried more than 0 times"): + runner.task() + + assert db.calls == [[0]] + + +def test_concurrent_insert_runner_uses_custom_batch_size_iterator(): + class Data: + train_id_field = "id" + train_vector_field = "vector" + + class Dataset: + data = Data() + requested_batch_size = None + + def iter_batches(self, batch_size): + self.requested_batch_size = batch_size + return iter( + [ + pd.DataFrame( + { + "id": [1, 2], + "vector": [np.array([0.1]), np.array([0.2])], + } + ) + ] + ) + + def __iter__(self): + raise AssertionError("ConcurrentInsertRunner should request an explicit batch size") + + class DB: + thread_safe = True + name = "fake" + + def __init__(self): + self.inserts = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.inserts.append((embeddings, metadata)) + return len(metadata), None + + dataset = Dataset() + db = DB() + runner = ConcurrentInsertRunner(db, dataset, normalize=False, max_workers=1, batch_size=1000) + + assert runner.task() == 2 + assert dataset.requested_batch_size == 1000 + assert db.inserts == [([[0.1], [0.2]], [1, 2])] + + +class TenantInsertProbeDB: + name = "TenantInsertProbeDB" + thread_safe = True + + def __init__(self): + self.calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None, tenant_labels_data=None): + self.calls.append( + { + "embeddings": embeddings, + "metadata": metadata, + "labels_data": labels_data, + "tenant_labels_data": tenant_labels_data, + } + ) + return len(embeddings), None + + +class TenantAwareCase: + is_multitenant = True + + def tenant_labels_for_ids(self, row_ids): + return [f"tenant_{int(row_id) % 3:04d}" for row_id in row_ids] + + +def test_concurrent_insert_runner_passes_tenant_labels(): + db = TenantInsertProbeDB() + dataset = MagicMock() + dataset.data.train_id_field = "id" + dataset.data.train_vector_field = "emb" + dataset.iter_batches.return_value = iter( + [ + pd.DataFrame( + { + "id": [0, 1, 5], + "emb": [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + } + ) + ] + ) + + runner = ConcurrentInsertRunner( + db=db, + dataset=dataset, + normalize=False, + max_workers=1, + batch_size=10, + tenant_case=TenantAwareCase(), + ) + + count = runner.task() + + assert count == 3 + assert db.calls[0]["metadata"] == [0, 1, 5] + assert db.calls[0]["tenant_labels_data"] == ["tenant_0000", "tenant_0001", "tenant_0002"] + + +def test_concurrent_insert_runner_passes_scalar_labels_for_scalar_payload_without_filter(): + db = TenantInsertProbeDB() + dataset = MagicMock() + dataset.data.train_id_field = "id" + dataset.data.train_vector_field = "emb" + dataset.data.scalar_labels_file_separated = False + dataset.iter_batches.return_value = iter( + [ + pd.DataFrame( + { + "id": [0, 1, 5], + "emb": [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + "labels": ["label_a", "label_b", "label_c"], + } + ) + ] + ) + + runner = ConcurrentInsertRunner( + db=db, + dataset=dataset, + normalize=False, + max_workers=1, + batch_size=10, + with_scalar_labels=True, + ) + + count = runner.task() + + assert count == 3 + assert db.calls[0]["metadata"] == [0, 1, 5] + assert db.calls[0]["labels_data"] == ["label_a", "label_b", "label_c"] + assert db.calls[0]["tenant_labels_data"] is None + + +def test_pre_run_prepares_separated_scalar_labels_for_scalar_payload(): + class Dataset: + def __init__(self): + self.prepare_kwargs = None + self.data = type("Data", (), {"dim": 2, "metric_type": "L2"})() + + def prepare(self, source, filters, with_train_files, with_scalar_labels=False): + self.prepare_kwargs = { + "source": source, + "filters": filters, + "with_train_files": with_train_files, + "with_scalar_labels": with_scalar_labels, + } + return True + + class Case: + is_multitenant = False + with_scalar_labels = True + filters = MagicMock() + dataset = Dataset() + + class DB: + init_args = None + + @classmethod + def init_cls(cls, **kwargs): + cls.init_args = kwargs + return cls() + + def need_normalize_cosine(self): + return False + + class DBConfig: + def to_dict(self): + return {} + + config = type( + "Config", + (), + { + "db": DB, + "db_config": DBConfig(), + "db_case_config": EmptyDBCaseConfig(), + "stages": [TaskStage.LOAD], + }, + )() + runner = CaseRunner.construct( + ca=Case(), + config=config, + dataset_source=DatasetSource.S3, + ) + + runner._pre_run() + + assert Case.dataset.prepare_kwargs["with_scalar_labels"] is True diff --git a/tests/test_cloud_payload_case.py b/tests/test_cloud_payload_case.py new file mode 100644 index 000000000..d227dd176 --- /dev/null +++ b/tests/test_cloud_payload_case.py @@ -0,0 +1,180 @@ +import pytest + +from vectordb_bench import config +from vectordb_bench.backend.cases import CaseType, CloudPayloadSearchCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.runner.mp_runner import MultiProcessingSearchRunner +from vectordb_bench.backend.runner.serial_runner import SerialSearchRunner +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus +from vectordb_bench.cli.cli import get_custom_case_config +from vectordb_bench.models import CaseConfig, TaskConfig + + +class FakeDB: + name = "FakeDB" + + def __init__(self, supported_payload_profiles=None): + self.supported_payload_profiles = supported_payload_profiles or {PayloadProfile.IDS_ONLY} + self.calls = [] + + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in self.supported_payload_profiles + + def search_embedding(self, query: list[float], k: int = 100, **kwargs) -> list[int]: + self.calls.append((query, k, kwargs)) + return list(range(k)) + + +def test_payload_profile_estimates_response_bytes(): + assert PayloadProfile.IDS_ONLY.estimated_bytes_per_query(k=10, dim=768) == 200 + assert PayloadProfile.VECTOR.estimated_bytes_per_query(k=10, dim=768) == 30_920 + + +def test_cloud_payload_case_defaults_to_laion_100m(): + case = CloudPayloadSearchCase(payload_profile="vector") + + assert case.case_id == CaseType.CloudPayloadSearchCase + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.dataset.data.dim == 768 + assert case.payload_profile == PayloadProfile.VECTOR + assert case.estimated_payload_bytes_per_query(config.K_DEFAULT) == 309_200 + + +def test_cloud_payload_case_accepts_sized_dataset(): + case = CloudPayloadSearchCase(dataset_with_size_type=DatasetWithSizeType.CohereSmall.value) + + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.name == "Cohere" + assert case.dataset.data.size == 100_000 + + +def test_case_config_builds_cloud_payload_case_from_custom_case(): + case = CaseConfig( + case_id=CaseType.CloudPayloadSearchCase, + custom_case={"payload_profile": "vector"}, + ).case + + assert isinstance(case, CloudPayloadSearchCase) + assert case.payload_profile == PayloadProfile.VECTOR + + +def test_case_runner_reuse_key_distinguishes_scalar_label_schema_requirement(): + ids_only_case = CloudPayloadSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + payload_profile=PayloadProfile.IDS_ONLY, + ) + scalar_label_case = CloudPayloadSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + payload_profile=PayloadProfile.SCALAR_LABEL, + ) + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig(case_id=CaseType.CloudPayloadSearchCase), + ) + + ids_only_runner = CaseRunner( + run_id="run-id", + config=task, + ca=ids_only_case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + scalar_label_runner = CaseRunner( + run_id="run-id", + config=task, + ca=scalar_label_case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + assert ids_only_case.with_scalar_labels is False + assert scalar_label_case.with_scalar_labels is True + assert ids_only_runner != scalar_label_runner + assert hash(ids_only_runner) != hash(scalar_label_runner) + + +def test_cli_propagates_cloud_payload_dataset_selection(): + custom_case = get_custom_case_config( + { + "case_type": "CloudPayloadSearchCase", + "dataset_with_size_type": DatasetWithSizeType.CohereSmall.value, + "payload_profile": "vector", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + } + ) + + assert custom_case["dataset_with_size_type"] == DatasetWithSizeType.CohereSmall.value + + case = CaseConfig(case_id=CaseType.CloudPayloadSearchCase, custom_case=custom_case).case + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.size == 100_000 + + +def test_cli_omits_cloud_payload_dataset_when_not_selected(): + custom_case = get_custom_case_config( + { + "case_type": "CloudPayloadSearchCase", + "dataset_with_size_type": None, + "payload_profile": "ids_only", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + } + ) + + assert "dataset_with_size_type" not in custom_case + + case = CaseConfig(case_id=CaseType.CloudPayloadSearchCase, custom_case=custom_case).case + assert case.dataset_with_size_type is None + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + + +def test_serial_runner_omits_payload_argument_for_ids_only(): + db = FakeDB() + runner = SerialSearchRunner(db=db, test_data=[[0.1]], ground_truth=[[0]], k=3) + + assert runner._get_db_search_res([0.1]) == [0, 1, 2] + assert db.calls == [([0.1], 3, {})] + + +def test_serial_runner_passes_payload_argument_for_vector_profile(): + db = FakeDB(supported_payload_profiles={PayloadProfile.IDS_ONLY, PayloadProfile.VECTOR}) + runner = SerialSearchRunner( + db=db, + test_data=[[0.1]], + ground_truth=[[0]], + k=3, + payload_profile=PayloadProfile.VECTOR, + ) + + assert runner._get_db_search_res([0.1]) == [0, 1, 2] + assert db.calls == [([0.1], 3, {"payload_profile": PayloadProfile.VECTOR})] + + +def test_search_runners_fail_fast_for_unsupported_payload_profile(): + db = FakeDB() + + with pytest.raises(NotImplementedError, match="payload_profile=vector"): + SerialSearchRunner( + db=db, + test_data=[[0.1]], + ground_truth=[[0]], + k=3, + payload_profile=PayloadProfile.VECTOR, + ) + + with pytest.raises(NotImplementedError, match="payload_profile=vector"): + MultiProcessingSearchRunner( + db=db, + test_data=[[0.1]], + k=3, + payload_profile=PayloadProfile.VECTOR, + ) diff --git a/tests/test_cloud_payload_search.py b/tests/test_cloud_payload_search.py new file mode 100644 index 000000000..cda8f63ba --- /dev/null +++ b/tests/test_cloud_payload_search.py @@ -0,0 +1,128 @@ +from contextlib import contextmanager + +import polars as pl + +from vectordb_bench.backend.cases import CloudPayloadSearchCase +from vectordb_bench.backend.clients.milvus.milvus import Milvus +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import Dataset +from vectordb_bench.backend.filter import FilterOp, non_filter +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.runner.serial_runner import SerialSearchRunner + + +class _Client: + def __init__(self): + self.search_kwargs = None + + def search(self, **kwargs): + self.search_kwargs = kwargs + return [[{"pk": 1}]] + + +def test_laion_100m_declares_scalar_label_assets(): + laion = Dataset.LAION.get(100_000_000) + + assert laion.with_scalar_labels is True + assert laion.scalar_labels_file == "scalar_labels.parquet" + assert 0.01 in laion.scalar_label_percentages + + +def test_scalar_label_payload_profile_estimates_small_string_payload(): + assert PayloadProfile.SCALAR_LABEL.estimated_bytes_per_query(k=100, dim=768) == 3600 + + +def test_scalar_label_payload_profile_requires_scalar_label_materialization_without_filter(): + case = CloudPayloadSearchCase(payload_profile="scalar_label") + + assert case.filters.type == FilterOp.NonFilter + assert case.with_scalar_labels is True + + +def test_dataset_prepare_loads_separated_scalar_labels_for_scalar_payload(monkeypatch): + dataset = Dataset.LAION.manager(100_000_000) + dataset.data.with_remote_resource = False + loaded_scalar_labels = object() + + def fake_read_file(file_name): + if file_name == dataset.data.scalar_labels_file: + return loaded_scalar_labels + return pl.DataFrame({dataset.data.test_vector_field: [], dataset.data.gt_neighbors_field: []}) + + monkeypatch.setattr(dataset, "_read_file", fake_read_file) + + dataset.prepare( + source=DatasetSource.S3, + filters=non_filter, + with_train_files=False, + with_scalar_labels=True, + ) + + assert dataset.scalar_labels is loaded_scalar_labels + + +def test_cloud_payload_case_can_combine_label_filter_with_scalar_label_payload(): + case = CloudPayloadSearchCase( + payload_profile="scalar_label", + label_percentage=0.01, + ) + + assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.filters.type == FilterOp.StrEqual + assert case.filters.label_value == "label_1p" + + +def test_milvus_scalar_label_payload_requests_label_output_field(): + db = Milvus.__new__(Milvus) + db.client = _Client() + db.case_config = type("CaseConfig", (), {"search_param": lambda self: {}})() + db.collection_name = "collection" + db.expr = "label == 'label_1p'" + db._primary_field = "pk" + db._vector_field = "vector" + db._scalar_label_field = "label" + + assert db.supports_payload_profile(PayloadProfile.SCALAR_LABEL) + assert db.search_embedding([0.1, 0.2], payload_profile=PayloadProfile.SCALAR_LABEL) == [1] + assert db.client.search_kwargs["output_fields"] == ["label"] + + +class TenantSearchProbeDB: + name = "TenantSearchProbeDB" + + def __init__(self): + self.tenants = [] + + def supports_payload_profile(self, payload_profile): + return True + + @contextmanager + def init(self): + yield + + def prepare_filter(self, filters): + return None + + def search_embedding(self, query, k=100, payload_profile=None, tenant=None): + self.tenants.append(tenant) + return [] + + +def test_serial_search_runner_passes_tenant_and_skips_recall(): + db = TenantSearchProbeDB() + runner = SerialSearchRunner( + db=db, + test_data=[[1.0, 0.0], [0.0, 1.0]], + ground_truth=None, + tenant_labels=["tenant_0000", "tenant_0001"], + measure_recall=False, + ) + + recall, ndcg, p99, p95 = runner.search((runner.test_data, runner.ground_truth)) + + assert recall == 0 + assert ndcg == 0 + assert p99 >= 0 + assert p95 >= 0 + assert set(db.tenants).issubset({"tenant_0000", "tenant_0001"}) + assert db.tenants diff --git a/tests/test_milvus.py b/tests/test_milvus.py index 1c5de7ce0..dfc88cad8 100644 --- a/tests/test_milvus.py +++ b/tests/test_milvus.py @@ -15,6 +15,7 @@ from vectordb_bench.backend.clients.api import IndexType from vectordb_bench.backend.clients.milvus.config import MilvusConfig from vectordb_bench.backend.clients.milvus.milvus import MILVUS_FORCE_MERGE_TARGET_SIZE_MB, Milvus +from vectordb_bench.backend.payload import PayloadProfile from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseConfig, TaskConfig @@ -84,3 +85,164 @@ def test_performance_1536d_50k(self): result = runner.get_results() log.info(f"test result: {result}") assert len(result) > 0 + + +def test_milvus_multitenant_search_uses_tenant_label_filter(): + captured = {} + + def search(**kwargs): + captured.update(kwargs) + return [[{"pk": 1}]] + + db = object.__new__(Milvus) + db.client = SimpleNamespace(search=search) + db.collection_name = "test_collection" + db._vector_field = "vector" + db._primary_field = "pk" + db._scalar_label_field = "label" + db.case_config = SimpleNamespace(search_param=lambda: {"metric_type": "COSINE"}) + db.expr = "" + + result = db.search_embedding([0.1, 0.2], k=3, payload_profile=PayloadProfile.IDS_ONLY, tenant="tenant_0003") + + assert result == [1] + assert captured["filter"] == "label == 'tenant_0003'" + + +def test_milvus_validate_multitenant_schema_accepts_partition_key_label( + monkeypatch: pytest.MonkeyPatch, +) -> None: + closed = [] + + class FakeMilvusClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def describe_collection(self, _collection_name: str) -> dict: + return { + "fields": [ + {"name": "pk", "is_primary": True}, + {"name": "label", "is_partition_key": True}, + ] + } + + def close(self) -> None: + closed.append(True) + + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.MilvusClient", FakeMilvusClient) + + db = object.__new__(Milvus) + db.name = "Milvus" + db.db_config = {"uri": "http://example.invalid", "user": None, "password": None, "token": ""} + db.collection_name = "existing" + db._scalar_label_field = "label" + + db.validate_multitenant_schema() + + assert closed == [True] + + +def test_milvus_validate_multitenant_schema_rejects_non_partition_key_label( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeMilvusClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def describe_collection(self, _collection_name: str) -> dict: + return {"fields": [{"name": "label", "is_partition_key": False}]} + + def close(self) -> None: + pass + + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.MilvusClient", FakeMilvusClient) + + db = object.__new__(Milvus) + db.name = "Milvus" + db.db_config = {"uri": "http://example.invalid", "user": None, "password": None, "token": ""} + db.collection_name = "existing" + db._scalar_label_field = "label" + + with pytest.raises(ValueError, match="label field is not a partition key"): + db.validate_multitenant_schema() + + +def test_milvus_validate_multitenant_schema_uses_existing_labels_partition_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured = {} + + class FakeMilvusClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def describe_collection(self, _collection_name: str) -> dict: + return { + "fields": [ + {"name": "pk", "is_primary": True}, + {"name": "labels", "is_partition_key": True}, + {"name": "scalar_label", "nullable": True}, + ] + } + + def close(self) -> None: + pass + + def search(**kwargs): + captured.update(kwargs) + return [[{"pk": 1}]] + + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.MilvusClient", FakeMilvusClient) + + db = object.__new__(Milvus) + db.name = "Milvus" + db.db_config = {"uri": "http://example.invalid", "user": None, "password": None, "token": ""} + db.collection_name = "existing" + db._vector_field = "vector" + db._primary_field = "pk" + db._scalar_label_field = "label" + db.case_config = SimpleNamespace(search_param=lambda: {"metric_type": "COSINE"}) + db.expr = "" + + db.validate_multitenant_schema() + db.client = SimpleNamespace(search=search) + + db.search_embedding([0.1, 0.2], payload_profile=PayloadProfile.SCALAR_LABEL, tenant="tenant_0003") + + assert captured["filter"] == "labels == 'tenant_0003'" + assert captured["output_fields"] == ["scalar_label"] + + +def test_milvus_multitenant_insert_writes_tenant_and_scalar_payload_labels() -> None: + inserted = {} + + def insert(collection_name, batch_data): + inserted["collection_name"] = collection_name + inserted["batch_data"] = batch_data + return {"insert_count": len(batch_data)} + + db = object.__new__(Milvus) + db.client = SimpleNamespace(insert=insert) + db.collection_name = "test_collection" + db.batch_size = 100 + db._primary_field = "pk" + db._scalar_id_field = "id" + db._vector_field = "vector" + db._scalar_label_field = "label" + db._scalar_payload_label_field = "scalar_label" + db._multitenant_partition_key_field = "labels" + db.with_scalar_labels = True + + count, err = db.insert_embeddings( + embeddings=[[0.1, 0.2], [0.3, 0.4]], + metadata=[1, 2], + labels_data=["label_a", "label_b"], + tenant_labels_data=["tenant_0001", "tenant_0002"], + ) + + assert count == 2 + assert err is None + assert inserted["batch_data"] == [ + {"pk": 1, "id": 1, "vector": [0.1, 0.2], "labels": "tenant_0001", "scalar_label": "label_a"}, + {"pk": 2, "id": 2, "vector": [0.3, 0.4], "labels": "tenant_0002", "scalar_label": "label_b"}, + ] diff --git a/tests/test_milvus_zilliz_cli.py b/tests/test_milvus_zilliz_cli.py new file mode 100644 index 000000000..a028c535b --- /dev/null +++ b/tests/test_milvus_zilliz_cli.py @@ -0,0 +1,57 @@ +from click.testing import CliRunner +from pytest import MonkeyPatch + +from vectordb_bench.backend.clients.milvus import cli as milvus_cli +from vectordb_bench.backend.clients.zilliz_cloud import cli as zilliz_cli + + +def test_milvus_autoindex_cli_enables_partition_key_for_multitenant_case( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(milvus_cli, "run", fake_run) + + result = CliRunner().invoke( + milvus_cli.MilvusAutoIndex, + [ + "--case-type", + "CloudMultiTenantSearchCase", + "--uri", + "http://localhost:19530", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].use_partition_key is True + + +def test_zilliz_autoindex_cli_enables_partition_key_for_multitenant_case( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(zilliz_cli, "run", fake_run) + + result = CliRunner().invoke( + zilliz_cli.ZillizAutoIndex, + [ + "--case-type", + "CloudMultiTenantSearchCase", + "--uri", + "https://example.api.gcp-us-west1.zillizcloud.com", + "--token", + "secret", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].use_partition_key is True diff --git a/tests/test_multitenant_case.py b/tests/test_multitenant_case.py new file mode 100644 index 000000000..45939c32a --- /dev/null +++ b/tests/test_multitenant_case.py @@ -0,0 +1,400 @@ +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest + +from vectordb_bench.backend.cases import CaseType, CloudMultiTenantSearchCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig, VectorDB +from vectordb_bench.backend.clients.zilliz_cloud.config import AutoIndexConfig, ZillizCloudConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetManager, DatasetWithSizeType +from vectordb_bench.backend.filter import FilterOp +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus +from vectordb_bench.models import CaseConfig, TaskConfig, TaskStage + + +def test_multitenant_case_defaults_to_cohere_large_1000_tenants(): + case = CloudMultiTenantSearchCase() + + assert case.case_id == CaseType.CloudMultiTenantSearchCase + assert case.dataset_with_size_type == DatasetWithSizeType.CohereLarge + assert case.dataset.data.size == 10_000_000 + assert case.tenant_count == 1000 + assert case.tenant_prefix == "tenant_" + assert case.tenant_id_width == 4 + assert case.measure_recall is False + assert case.is_multitenant is True + assert case.filters.type == FilterOp.NonFilter + + +def test_multitenant_case_accepts_dataset_and_tenant_count(): + case = CloudMultiTenantSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + tenant_count=7, + tenant_prefix="acct_", + tenant_id_width=2, + payload_profile=PayloadProfile.VECTOR.value, + filter_rate=0.01, + ) + + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.size == 100_000 + assert case.payload_profile == PayloadProfile.VECTOR + assert case.estimated_payload_bytes_per_query(k=10) == PayloadProfile.VECTOR.estimated_bytes_per_query( + k=10, + dim=case.dataset.data.dim, + ) + assert case.filters.type == FilterOp.NumGE + assert case.tenant_count == 7 + assert case.tenant_for_id(0) == "acct_00" + assert case.tenant_for_id(8) == "acct_01" + assert case.tenant_labels_for_ids([0, 1, 8, 13]) == ["acct_00", "acct_01", "acct_01", "acct_06"] + + +def test_case_config_constructs_multitenant_case(): + case = CaseType.CloudMultiTenantSearchCase.case_cls( + { + "dataset_with_size_type": DatasetWithSizeType.CohereSmall.value, + "tenant_count": 5, + "payload_profile": PayloadProfile.SCALAR_LABEL.value, + "label_percentage": 0.01, + } + ) + + assert isinstance(case, CloudMultiTenantSearchCase) + assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.filters.type == FilterOp.StrEqual + assert case.tenant_labels() == ["tenant_0000", "tenant_0001", "tenant_0002", "tenant_0003", "tenant_0004"] + + +class TenantApiProbeDB(VectorDB): + name = "TenantApiProbeDB" + + def __init__(self, dim=2, db_config=None, db_case_config=None, collection_name="test", drop_old=False, **kwargs): + self.insert_calls = [] + self.search_calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None, tenant_labels_data=None, **kwargs): + self.insert_calls.append((embeddings, metadata, labels_data, tenant_labels_data)) + return len(embeddings), None + + def search_embedding(self, query, k=100, payload_profile=None, tenant=None): + self.search_calls.append((query, k, payload_profile, tenant)) + return [] + + def optimize(self, data_size=None): + return None + + +def test_vector_db_accepts_optional_tenant_context(): + db = TenantApiProbeDB(db_case_config=EmptyDBCaseConfig()) + + count, err = db.insert_embeddings([[0.1, 0.2]], [42], tenant_labels_data=["tenant_0002"]) + result = db.search_embedding([0.1, 0.2], tenant="tenant_0002") + + assert count == 1 + assert err is None + assert result == [] + assert db.insert_calls[0][3] == ["tenant_0002"] + assert db.search_calls[0][3] == "tenant_0002" + + +def test_search_only_zilliz_multitenant_validates_existing_partition_key_schema( + monkeypatch: pytest.MonkeyPatch, +) -> None: + case = CloudMultiTenantSearchCase() + task = TaskConfig( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig(uri="http://example.invalid", collection_name="existing"), + db_case_config=AutoIndexConfig(use_partition_key=False), + case_config=CaseConfig(case_id=CaseType.CloudMultiTenantSearchCase), + stages=[TaskStage.SEARCH_CONCURRENT], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + calls: list[tuple[str, object]] = [] + + class ExistingCollectionDB: + def supports_multitenant(self) -> bool: + return True + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + calls.append(("set_context", tenant_labels)) + + def validate_multitenant_schema(self) -> None: + calls.append(("validate_schema", None)) + + def fake_init_db(self: CaseRunner, _drop_old: bool = True) -> None: + self.db = ExistingCollectionDB() + + def fake_prepare(*_args: object, **_kwargs: object) -> None: + calls.append(("prepare", None)) + + monkeypatch.setattr(CaseRunner, "init_db", fake_init_db) + monkeypatch.setattr(DatasetManager, "prepare", fake_prepare) + + runner._pre_run(drop_old=False) + + assert ("validate_schema", None) in calls + assert calls[0][0] == "set_context" + + +def test_zilliz_multitenant_create_still_requires_partition_key() -> None: + case = CloudMultiTenantSearchCase() + task = TaskConfig( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig(uri="http://example.invalid", collection_name="new_collection"), + db_case_config=AutoIndexConfig(use_partition_key=False), + case_config=CaseConfig(case_id=CaseType.CloudMultiTenantSearchCase), + stages=[TaskStage.DROP_OLD, TaskStage.LOAD], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + with pytest.raises(ValueError, match="requires use_partition_key=True"): + runner._pre_run(drop_old=True) + + +class FakeTurboNamespace: + def __init__(self): + self.write_calls = [] + self.query_calls = [] + + def write(self, **kwargs): + self.write_calls.append(kwargs) + + def query(self, **kwargs): + self.query_calls.append(kwargs) + return SimpleNamespace(rows=[SimpleNamespace(id="10")]) + + def metadata(self): + return {"index": {"unindexed_bytes": 0}} + + +class FakeTurboClient: + def __init__(self): + self.namespaces = {} + + def namespace(self, name): + self.namespaces.setdefault(name, FakeTurboNamespace()) + return self.namespaces[name] + + +def test_turbopuffer_groups_multitenant_insert_and_search(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": None, + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + ) + db.set_multitenant_context(["tenant_0000", "tenant_0001"]) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + result = db.search_embedding([1.0, 0.0], k=1, tenant="tenant_0001") + + assert count == 3 + assert err is None + assert result == [10] + assert fake_client.namespaces["mt_tenant_0000"].write_calls[0]["upsert_columns"]["id"] == [0, 2] + assert fake_client.namespaces["mt_tenant_0001"].write_calls[0]["upsert_columns"]["id"] == [1] + assert fake_client.namespaces["mt_tenant_0001"].query_calls + + +def test_turbopuffer_multitenant_insert_preserves_scalar_payload_labels(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": None, + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + "scalar_payload_label_field": "scalar_label", + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + with_scalar_labels=True, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0]], + metadata=[0, 1], + labels_data=["label_a", "label_b"], + tenant_labels_data=["tenant_0000", "tenant_0001"], + ) + + assert count == 2 + assert err is None + assert fake_client.namespaces["mt_tenant_0000"].write_calls[0]["upsert_columns"]["scalar_label"] == ["label_a"] + assert fake_client.namespaces["mt_tenant_0001"].write_calls[0]["upsert_columns"]["scalar_label"] == ["label_b"] + + +def test_turbopuffer_multitenant_partial_insert_failure_is_explicit(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + class FailingNamespace(FakeTurboNamespace): + def __init__(self, fail_write: bool = False): + super().__init__() + self.fail_write = fail_write + + def write(self, **kwargs): + self.write_calls.append(kwargs) + if self.fail_write: + raise RuntimeError("tenant write failed") + + class FailingTurboClient: + def __init__(self): + self.namespaces = {} + + def namespace(self, name): + self.namespaces.setdefault(name, FailingNamespace(fail_write=name == "mt_tenant_0001")) + return self.namespaces[name] + + fake_client = FailingTurboClient() + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": None, + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + + assert count == 2 + assert getattr(err, "non_retryable", False) is True + assert getattr(err, "inserted_count") == 2 + assert getattr(err, "successful_tenants") == {"tenant_0000": 2} + assert getattr(err, "failed_tenant") == "tenant_0001" + assert "tenant_0001" in str(err) + assert fake_client.namespaces["mt_tenant_0000"].write_calls + assert fake_client.namespaces["mt_tenant_0001"].write_calls + + +def test_turbopuffer_pins_multitenant_namespaces_on_init(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer import turbopuffer as turbopuffer_module + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + calls = [] + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, namespace, payload, api_base_url)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", namespace, replicas, timeout)) + return {"pinning": {"replicas": replicas, "status": {"ready_replicas": replicas}}} + + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + monkeypatch.setattr(turbopuffer_module, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_module, "wait_for_namespace_pinning", fake_wait_for_pinning) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": "https://tpuf.example", + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + "pin_namespace": True, + "pin_replicas": 2, + "pin_timeout": 30, + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + ) + db.set_multitenant_context(["tenant_0000", "tenant_0001"]) + + with db.init(): + pass + + assert calls == [ + ("PATCH", "mt_tenant_0000", {"pinning": {"replicas": 2}}, "https://tpuf.example"), + ("WAIT", "mt_tenant_0000", 2, 30), + ("PATCH", "mt_tenant_0001", {"pinning": {"replicas": 2}}, "https://tpuf.example"), + ("WAIT", "mt_tenant_0001", 2, 30), + ] + + +def test_turbopuffer_supports_scalar_label_payload_for_multitenant_search() -> None: + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + db = TurboPuffer.__new__(TurboPuffer) + db.client = fake_client + db.namespace = "single" + db.multitenant_namespace_prefix = "mt_" + db._ns_cache = {} + db._vector_field = "vector" + db._scalar_label_field = "label" + db._scalar_payload_label_field = "scalar_label" + db.expr = None + + assert db.supports_payload_profile(PayloadProfile.SCALAR_LABEL) + assert db.search_embedding( + [1.0, 0.0], + k=50, + payload_profile=PayloadProfile.SCALAR_LABEL, + tenant="tenant_0001", + ) == [10] + assert fake_client.namespaces["mt_tenant_0001"].query_calls[0]["include_attributes"] == ["scalar_label"] diff --git a/tests/test_pinecone_multitenant.py b/tests/test_pinecone_multitenant.py new file mode 100644 index 000000000..1a81d55ff --- /dev/null +++ b/tests/test_pinecone_multitenant.py @@ -0,0 +1,214 @@ +import threading +from types import SimpleNamespace + + +class FakePineconeIndex: + def __init__(self): + self.upserts = [] + self.queries = [] + self.deletes = [] + + def describe_index_stats(self): + return {"dimension": 2, "total_vector_count": 3, "namespaces": {"mt_tenant_0000": {}, "mt_tenant_0001": {}}} + + def upsert(self, vectors, namespace=None): + self.upserts.append((vectors, namespace)) + return SimpleNamespace(_response_info={"raw_headers": {"x-pinecone-request-lsn": "7"}}) + + def query(self, **kwargs): + self.queries.append(kwargs) + return SimpleNamespace( + matches=[{"id": "11"}], + _response_info={"raw_headers": {"x-pinecone-max-indexed-lsn": "7"}}, + ) + + def delete(self, delete_all=False, namespace=None): + self.deletes.append((delete_all, namespace)) + + +class FailingPineconeIndex(FakePineconeIndex): + def upsert(self, vectors, namespace=None): + self.upserts.append((vectors, namespace)) + if namespace == "mt_tenant_0001": + raise RuntimeError("tenant upsert failed") + return SimpleNamespace(_response_info={"raw_headers": {"x-pinecone-request-lsn": "7"}}) + + +class FakePineconeClient: + def __init__(self, index): + self.index = index + + def Index(self, name): + return self.index + + +def test_pinecone_groups_multitenant_upsert_and_query(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + fake_index = FakePineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + ) + db.set_multitenant_context(["tenant_0000", "tenant_0001"]) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + result = db.search_embedding([1.0, 0.0], k=1, tenant="tenant_0001") + + assert count == 3 + assert err is None + assert result == [11] + assert fake_index.upserts[0][1] == "mt_tenant_0000" + assert fake_index.upserts[1][1] == "mt_tenant_0001" + assert fake_index.queries[-1]["namespace"] == "mt_tenant_0001" + + +def test_pinecone_multitenant_upsert_preserves_scalar_payload_labels(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + fake_index = FakePineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + with_scalar_labels=True, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0]], + metadata=[0, 1], + labels_data=["label_a", "label_b"], + tenant_labels_data=["tenant_0000", "tenant_0001"], + ) + + assert count == 2 + assert err is None + assert fake_index.upserts[0][0][0][2]["label"] == "label_a" + assert fake_index.upserts[1][0][0][2]["label"] == "label_b" + + +def test_pinecone_multitenant_partial_insert_failure_is_explicit(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + fake_index = FailingPineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + + assert count == 2 + assert getattr(err, "non_retryable", False) is True + assert getattr(err, "inserted_count") == 2 + assert getattr(err, "successful_tenants") == {"tenant_0000": 2} + assert getattr(err, "failed_tenant") == "tenant_0001" + assert getattr(err, "failed_tenant_count") == 1 + assert db._multitenant_insert_counts == {"tenant_0000": 2} + + +def test_pinecone_multitenant_insert_counts_are_thread_safe(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + class RacingCounts(dict): + def __init__(self, lock_getter): + super().__init__() + self.barrier = threading.Barrier(2) + self.lock_getter = lock_getter + + def get(self, key, default=None): + value = super().get(key, default) + lock = self.lock_getter() + if key == "tenant_0000" and (lock is None or not lock.locked()): + self.barrier.wait(timeout=5) + return value + + fake_index = FakePineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + ) + db._multitenant_insert_counts = RacingCounts(lambda: getattr(db, "_multitenant_insert_counts_lock", None)) + results = [] + + def insert_one(row_id): + results.append( + db.insert_embeddings( + embeddings=[[float(row_id), 0.0]], + metadata=[row_id], + tenant_labels_data=["tenant_0000"], + ) + ) + + with db.init(): + threads = [threading.Thread(target=insert_one, args=(row_id,)) for row_id in (1, 2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(fake_index.upserts) == 2 + assert sorted(results) == [(1, None), (1, None)] + assert dict(db._multitenant_insert_counts) == {"tenant_0000": 2} diff --git a/tests/test_turbopuffer_cli.py b/tests/test_turbopuffer_cli.py new file mode 100644 index 000000000..b167a319e --- /dev/null +++ b/tests/test_turbopuffer_cli.py @@ -0,0 +1,293 @@ +from types import SimpleNamespace + +from click.testing import CliRunner +from pytest import MonkeyPatch + +from vectordb_bench.backend.clients.api import MetricType +from vectordb_bench.backend.clients.turbopuffer import cli as turbopuffer_cli +from vectordb_bench.backend.clients.turbopuffer import turbopuffer as turbopuffer_client +from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + +def test_turbopuffer_cli_accepts_multitenant_namespace_prefix_and_metric_type( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--skip-drop-old", + "--skip-load", + "--skip-search-serial", + "--search-concurrent", + "--case-type", + "CloudMultiTenantSearchCase", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "cohere10m_multitenant", + "--multitenant-namespace-prefix", + "cohere10m_", + "--scalar-payload-label-field", + "scalar_label", + "--metric-type", + "COSINE", + "--dry-run", + ], + ) + + assert result.exit_code == 0 + assert captured["db_config"].multitenant_namespace_prefix == "cohere10m_" + assert captured["db_config"].scalar_payload_label_field == "scalar_label" + assert captured["db_case_config"].metric_type == MetricType.COSINE + assert captured["db_case_config"].multitenant_warmup_policy == "none" + + +def test_turbopuffer_cli_skips_pin_namespace_during_dry_run(monkeypatch: MonkeyPatch) -> None: + calls = [] + captured = {} + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, payload, api_key, region, namespace, api_base_url)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", replicas, api_key, region, namespace, api_base_url, timeout)) + return {"pinning": {"replicas": replicas, "status": {"ready_replicas": replicas}}} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_client, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_client, "wait_for_namespace_pinning", fake_wait_for_pinning) + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--skip-drop-old", + "--skip-load", + "--skip-search-serial", + "--search-concurrent", + "--case-type", + "CloudPayloadSearchCase", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "laion100m", + "--pin-namespace", + "--pin-replicas", + "2", + "--pin-timeout", + "7200", + "--metric-type", + "COSINE", + "--disable-backpressure", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [] + assert captured["db_config"].pin_namespace is False + assert captured["db_config"].pin_namespace_requested is True + assert captured["db_config"].pin_replicas == 2 + assert captured["db_config"].pin_timeout == 7200 + assert captured["db_config"].pin_target_namespace_count == 1 + assert captured["db_case_config"].metric_type == MetricType.COSINE + assert captured["db_case_config"].disable_backpressure is True + + +def test_turbopuffer_cli_accepts_multitenant_warmup_policy( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--case-type", + "CloudMultiTenantSearchCase", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--multitenant-warmup-policy", + "all", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].multitenant_warmup_policy == "all" + + +def test_turbopuffer_multitenant_optimize_skips_base_namespace_by_default( + monkeypatch: MonkeyPatch, +) -> None: + warmed = [] + + class FakeNamespace: + def __init__(self, name: str): + self.name = name + + def hint_cache_warm(self): + warmed.append(self.name) + + class FakeClient: + def namespace(self, name: str): + return FakeNamespace(name) + + monkeypatch.setattr(turbopuffer_client.time, "sleep", lambda _seconds: None) + db = object.__new__(TurboPuffer) + db.client = FakeClient() + db.ns = FakeNamespace("base") + db.namespace = "base" + db.multitenant_namespace_prefix = "mt_" + db.multitenant_tenant_labels = ["tenant_0000", "tenant_0001"] + db._ns_cache = {} + db.db_case_config = SimpleNamespace(time_wait_warmup=1, multitenant_warmup_policy="none") + + db.optimize() + + assert warmed == [] + + +def test_turbopuffer_multitenant_optimize_can_warm_all_tenant_namespaces( + monkeypatch: MonkeyPatch, +) -> None: + warmed = [] + + class FakeNamespace: + def __init__(self, name: str): + self.name = name + + def hint_cache_warm(self): + warmed.append(self.name) + + class FakeClient: + def namespace(self, name: str): + return FakeNamespace(name) + + monkeypatch.setattr(turbopuffer_client.time, "sleep", lambda _seconds: None) + db = object.__new__(TurboPuffer) + db.client = FakeClient() + db.ns = FakeNamespace("base") + db.namespace = "base" + db.multitenant_namespace_prefix = "mt_" + db.multitenant_tenant_labels = ["tenant_0000", "tenant_0001"] + db._ns_cache = {} + db.db_case_config = SimpleNamespace(time_wait_warmup=1, multitenant_warmup_policy="all") + + db.optimize() + + assert warmed == ["mt_tenant_0000", "mt_tenant_0001"] + + +def test_turbopuffer_cli_skips_multitenant_pin_namespaces_during_dry_run( + monkeypatch: MonkeyPatch, +) -> None: + calls = [] + captured = {} + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, namespace, payload)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", namespace, replicas)) + return {"pinning": {"replicas": replicas, "status": {"ready_replicas": replicas}}} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_client, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_client, "wait_for_namespace_pinning", fake_wait_for_pinning) + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--skip-drop-old", + "--skip-load", + "--skip-search-serial", + "--search-concurrent", + "--case-type", + "CloudMultiTenantSearchCase", + "--tenant-count", + "2", + "--tenant-prefix", + "tenant_", + "--tenant-id-width", + "4", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "unused_single_namespace", + "--multitenant-namespace-prefix", + "cohere10m_", + "--pin-namespace", + "--pin-replicas", + "1", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [] + assert captured["db_config"].pin_namespace is False + assert captured["db_config"].pin_namespace_requested is True + assert captured["db_config"].pin_target_namespace_count == 2 + assert captured["db_config"].multitenant_namespace_prefix == "cohere10m_" + + +def test_turbopuffer_unpin_namespace_uses_pin_timeout(monkeypatch: MonkeyPatch) -> None: + calls = [] + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, payload, api_key, region, namespace, api_base_url)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", replicas, api_key, region, namespace, api_base_url, timeout)) + return {"pinning": None} + + monkeypatch.setattr(turbopuffer_client, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_client, "wait_for_namespace_pinning", fake_wait_for_pinning) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPufferUnpin, + [ + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "laion100m", + "--pin-timeout", + "7200", + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [ + ("PATCH", {"pinning": None}, "secret", "aws-us-west-2", "laion100m", None), + ("WAIT", None, "secret", "aws-us-west-2", "laion100m", None, 7200), + ] diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index fc1813b38..9b3cbdff8 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -35,6 +35,8 @@ class config: CONCURRENCY_DURATION = 30 CONCURRENCY_TIMEOUT = 3600 + CLOUD_INSERT_READINESS_TIMEOUT = env.float("CLOUD_INSERT_READINESS_TIMEOUT", None) + CLOUD_INSERT_READINESS_POLL_INTERVAL = env.float("CLOUD_INSERT_READINESS_POLL_INTERVAL", 5.0) RESULTS_LOCAL_DIR = env.path( "RESULTS_LOCAL_DIR", diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index 3268cde35..b1177f7f4 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -48,10 +48,14 @@ def assemble_all( load_runners = [r for r in runners if r.ca.label == CaseLabel.Load] perf_runners = [r for r in runners if r.ca.label == CaseLabel.Performance] streaming_runners = [r for r in runners if r.ca.label == CaseLabel.Streaming] + cloud_insert_runners = [r for r in runners if r.ca.label == CaseLabel.CloudInsert] + cloud_cold_latency_runners = [r for r in runners if r.ca.label == CaseLabel.CloudColdLatency] + + search_filter_runners = [*perf_runners, *cloud_cold_latency_runners] # group by db db2runner: dict[DB, list[CaseRunner]] = {} - for r in perf_runners: + for r in search_filter_runners: db = r.config.db if db not in db2runner: db2runner[db] = [] @@ -71,6 +75,7 @@ def assemble_all( all_runners = [] all_runners.extend(load_runners) all_runners.extend(streaming_runners) + all_runners.extend(cloud_insert_runners) for v in db2runner.values(): all_runners.extend(v) diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index edfcdea19..b93bd04c3 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -5,6 +5,7 @@ from vectordb_bench import config from vectordb_bench.backend.clients.api import MetricType from vectordb_bench.backend.filter import Filter, FilterOp, IntFilter, LabelFilter, NewIntFilter, NonFilter, non_filter +from vectordb_bench.backend.payload import PayloadProfile from vectordb_bench.base import BaseModel from vectordb_bench.frontend.components.custom.getCustomConfig import CustomDatasetConfig @@ -56,6 +57,10 @@ class CaseType(Enum): LabelFilterPerformanceCase = 300 NewIntFilterPerformanceCase = 400 + CloudPayloadSearchCase = 500 + CloudInsertCase = 600 + CloudColdLatencyCase = 700 + CloudMultiTenantSearchCase = 800 def case_cls(self, custom_configs: dict | None = None) -> type["Case"]: if custom_configs is None: @@ -79,6 +84,8 @@ class CaseLabel(Enum): Load = auto() Performance = auto() Streaming = auto() + CloudInsert = auto() + CloudColdLatency = auto() class Case(BaseModel): @@ -102,14 +109,24 @@ class Case(BaseModel): optimize_timeout: float | int | None = None filter_rate: float | None = None + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY @property def filters(self) -> Filter: return non_filter + def estimated_payload_bytes_per_query(self, k: int | None) -> int: + if k is None: + k = config.K_DEFAULT + return self.payload_profile.estimated_bytes_per_query(k=k, dim=self.dataset.data.dim) + + @property + def is_multitenant(self) -> bool: + return False + @property def with_scalar_labels(self) -> bool: - return self.filters.type == FilterOp.StrEqual + return self.filters.type == FilterOp.StrEqual or self.payload_profile == PayloadProfile.SCALAR_LABEL def check_scalar_labels(self) -> None: if self.with_scalar_labels and not self.dataset.data.with_scalar_labels: @@ -594,6 +611,259 @@ def filters(self) -> Filter: return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) +class CloudPayloadSearchCase(PerformanceCase): + case_id: CaseType = CaseType.CloudPayloadSearchCase + dataset_with_size_type: DatasetWithSizeType | None = None + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY + filter_rate: float | None = None + label_percentage: float | None = None + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str | None = None, + payload_profile: PayloadProfile | str = PayloadProfile.IDS_ONLY, + filter_rate: float | None = None, + label_percentage: float | None = None, + **kwargs, + ): + if filter_rate is not None and label_percentage is not None: + msg = "CloudPayloadSearchCase supports only one filter type per run" + raise ValueError(msg) + if dataset_with_size_type is not None and not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + if not isinstance(payload_profile, PayloadProfile): + payload_profile = PayloadProfile(payload_profile) + + if dataset_with_size_type is None: + dataset = Dataset.LAION.manager(100_000_000) + load_timeout = config.LOAD_TIMEOUT_768D_100M + optimize_timeout = config.OPTIMIZE_TIMEOUT_768D_100M + dataset_name = "LAION 100M (768dim)" + else: + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + dataset_name = dataset_with_size_type.value + + name = f"Cloud Payload Search - {payload_profile.value} - {dataset_name}" + description = ( + "Cloud leaderboard search envelope case with explicit response payload profile. " + f"Payload profile: {payload_profile.value}; dataset: {dataset_name}." + ) + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + dataset_with_size_type=dataset_with_size_type, + payload_profile=payload_profile, + filter_rate=filter_rate, + label_percentage=label_percentage, + **kwargs, + ) + + @property + def filters(self) -> Filter: + if self.label_percentage is not None: + return LabelFilter(label_percentage=self.label_percentage) + if self.filter_rate is None: + return non_filter + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + +class CloudColdLatencyCase(Case): + case_id: CaseType = CaseType.CloudColdLatencyCase + label: CaseLabel = CaseLabel.CloudColdLatency + dataset_with_size_type: DatasetWithSizeType | None = None + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY + filter_rate: float | None = None + label_percentage: float | None = None + query_count: int = 1000 + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str | None = None, + payload_profile: PayloadProfile | str = PayloadProfile.IDS_ONLY, + filter_rate: float | None = None, + label_percentage: float | None = None, + query_count: int = 1000, + **kwargs, + ): + if filter_rate is not None and label_percentage is not None: + msg = "CloudColdLatencyCase supports only one filter type per run" + raise ValueError(msg) + if query_count <= 0: + msg = "query_count must be positive" + raise ValueError(msg) + if dataset_with_size_type is not None and not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + if not isinstance(payload_profile, PayloadProfile): + payload_profile = PayloadProfile(payload_profile) + + if dataset_with_size_type is None: + dataset = Dataset.LAION.manager(100_000_000) + load_timeout = config.LOAD_TIMEOUT_768D_100M + optimize_timeout = config.OPTIMIZE_TIMEOUT_768D_100M + dataset_name = "LAION 100M (768dim)" + else: + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + dataset_name = dataset_with_size_type.value + + name = f"Cloud Cold Latency - {payload_profile.value} - {dataset_name}" + description = ( + "Cloud leaderboard cold/warm serial latency case with explicit response payload profile. " + f"Payload profile: {payload_profile.value}; dataset: {dataset_name}; query count: {query_count}." + ) + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + dataset_with_size_type=dataset_with_size_type, + payload_profile=payload_profile, + filter_rate=filter_rate, + label_percentage=label_percentage, + query_count=query_count, + **kwargs, + ) + + @property + def filters(self) -> Filter: + if self.label_percentage is not None: + return LabelFilter(label_percentage=self.label_percentage) + if self.filter_rate is None: + return non_filter + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + +class CloudInsertCase(Case): + case_id: CaseType = CaseType.CloudInsertCase + label: CaseLabel = CaseLabel.CloudInsert + batch_size: int + duration: float | None = None + readiness_timeout: float | None = config.CLOUD_INSERT_READINESS_TIMEOUT + readiness_poll_interval: float = config.CLOUD_INSERT_READINESS_POLL_INTERVAL + dataset_with_size_type: DatasetWithSizeType | None = None + + def __init__( + self, + batch_size: int, + duration: float | None = None, + readiness_timeout: float | None = config.CLOUD_INSERT_READINESS_TIMEOUT, + readiness_poll_interval: float = config.CLOUD_INSERT_READINESS_POLL_INTERVAL, + dataset_with_size_type: DatasetWithSizeType | str | None = None, + **kwargs, + ): + if dataset_with_size_type is not None and not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + dataset = ( + Dataset.LAION.manager(100_000_000) + if dataset_with_size_type is None + else dataset_with_size_type.get_manager() + ) + super().__init__( + name=f"Cloud Insert - batch {batch_size}", + description="Cloud leaderboard insert-only case with readiness polling.", + dataset=dataset, + batch_size=batch_size, + duration=duration, + readiness_timeout=readiness_timeout, + readiness_poll_interval=readiness_poll_interval, + dataset_with_size_type=dataset_with_size_type, + **kwargs, + ) + + +class CloudMultiTenantSearchCase(PerformanceCase): + case_id: CaseType = CaseType.CloudMultiTenantSearchCase + dataset_with_size_type: DatasetWithSizeType = DatasetWithSizeType.CohereLarge + tenant_count: int = 1000 + tenant_prefix: str = "tenant_" + tenant_id_width: int = 4 + tenant_distribution: str = "uniform_by_id_mod" + measure_recall: bool = False + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY + filter_rate: float | None = None + label_percentage: float | None = None + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str = DatasetWithSizeType.CohereLarge, + tenant_count: int = 1000, + tenant_prefix: str = "tenant_", + tenant_id_width: int = 4, + payload_profile: PayloadProfile | str = PayloadProfile.IDS_ONLY, + filter_rate: float | None = None, + label_percentage: float | None = None, + **kwargs, + ): + if filter_rate is not None and label_percentage is not None: + msg = "CloudMultiTenantSearchCase supports only one filter type per run" + raise ValueError(msg) + if not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + if not isinstance(payload_profile, PayloadProfile): + payload_profile = PayloadProfile(payload_profile) + if tenant_count <= 0: + msg = "tenant_count must be greater than 0" + raise ValueError(msg) + if tenant_id_width <= 0: + msg = "tenant_id_width must be greater than 0" + raise ValueError(msg) + + dataset = dataset_with_size_type.get_manager() + super().__init__( + name=f"Cloud Multi-Tenant Search - {dataset_with_size_type.value}, {tenant_count} tenants", + description=( + "Multi-tenant QPS/latency benchmark with deterministic tenant routing " + f"({dataset_with_size_type.value}, {tenant_count} tenants)." + ), + dataset=dataset, + load_timeout=dataset_with_size_type.get_load_timeout(), + optimize_timeout=dataset_with_size_type.get_optimize_timeout(), + dataset_with_size_type=dataset_with_size_type, + tenant_count=tenant_count, + tenant_prefix=tenant_prefix, + tenant_id_width=tenant_id_width, + payload_profile=payload_profile, + filter_rate=filter_rate, + label_percentage=label_percentage, + **kwargs, + ) + + @property + def is_multitenant(self) -> bool: + return True + + def tenant_for_id(self, row_id: int) -> str: + tenant_id = int(row_id) % self.tenant_count + return f"{self.tenant_prefix}{tenant_id:0{self.tenant_id_width}d}" + + def tenant_labels_for_ids(self, row_ids: list[int]) -> list[str]: + return [self.tenant_for_id(row_id) for row_id in row_ids] + + def tenant_labels(self) -> list[str]: + return [f"{self.tenant_prefix}{tenant_id:0{self.tenant_id_width}d}" for tenant_id in range(self.tenant_count)] + + @property + def filters(self) -> Filter: + if self.label_percentage is not None: + return LabelFilter(label_percentage=self.label_percentage) + if self.filter_rate is None: + return non_filter + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + class LabelFilterPerformanceCase(PerformanceCase): case_id: CaseType = CaseType.LabelFilterPerformanceCase dataset_with_size_type: DatasetWithSizeType @@ -655,4 +925,8 @@ def filters(self) -> Filter: CaseType.StreamingCustomDataset: StreamingCustomDataset, CaseType.NewIntFilterPerformanceCase: NewIntFilterPerformanceCase, CaseType.LabelFilterPerformanceCase: LabelFilterPerformanceCase, + CaseType.CloudPayloadSearchCase: CloudPayloadSearchCase, + CaseType.CloudInsertCase: CloudInsertCase, + CaseType.CloudColdLatencyCase: CloudColdLatencyCase, + CaseType.CloudMultiTenantSearchCase: CloudMultiTenantSearchCase, } diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 118c505ff..37a5c71dc 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, model_validator from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile class MetricType(StrEnum): @@ -63,6 +64,29 @@ class SQType(StrEnum): FP32 = "FP32" +class NonRetryableInsertError(RuntimeError): + non_retryable = True + + +class PartialInsertError(NonRetryableInsertError): + def __init__( + self, + message: str, + *, + inserted_count: int, + successful_tenants: dict[str, int] | None = None, + failed_tenant: str | None = None, + failed_tenant_count: int | None = None, + cause: Exception | None = None, + ): + super().__init__(message) + self.inserted_count = inserted_count + self.successful_tenants = successful_tenants or {} + self.failed_tenant = failed_tenant + self.failed_tenant_count = failed_tenant_count + self.__cause__ = cause + + class DBConfig(ABC, BaseModel): """DBConfig contains the connection info of vector database @@ -216,12 +240,28 @@ def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" return False + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile == PayloadProfile.IDS_ONLY + + def poll_insert_readiness(self, expected_count: int) -> dict: + return {"fully_searchable": True, "fully_indexed": True, "additional_parameters": {}} + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + self.multitenant_tenant_labels = tenant_labels + + def supports_multitenant(self) -> bool: + return False + + def validate_multitenant_schema(self) -> None: + return None + @abstractmethod def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert the embeddings to the vector database. The default number of embeddings for @@ -242,6 +282,8 @@ def search_embedding( self, query: list[float], k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: """Get k most similar embeddings to query vector. diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index ae7269801..146d23aed 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -1,7 +1,7 @@ from typing import Annotated, TypedDict, Unpack import click -from pydantic import SecretStr +from pydantic import BaseModel, SecretStr from vectordb_bench.backend.clients import DB from vectordb_bench.cli.cli import ( @@ -16,6 +16,17 @@ DBTYPE = DB.Milvus +def _use_partition_key(parameters: dict) -> bool: + explicit = parameters.get("use_partition_key") + if explicit is not None: + return explicit + return parameters.get("case_type") == "CloudMultiTenantSearchCase" + + +def _with_partition_key(db_case_config: BaseModel, parameters: dict) -> BaseModel: + return db_case_config.model_copy(update={"use_partition_key": _use_partition_key(parameters)}) + + class MilvusTypedDict(TypedDict): uri: Annotated[ str, @@ -51,6 +62,17 @@ class MilvusTypedDict(TypedDict): show_default=True, ), ] + use_partition_key: Annotated[ + bool | None, + click.option( + "--use-partition-key/--no-use-partition-key", + default=None, + help=( + "Use the Milvus partition key on the label field. " + "Defaults to enabled for CloudMultiTenantSearchCase and disabled otherwise." + ), + ), + ] class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ... @@ -71,7 +93,7 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=AutoIndexConfig(), + db_case_config=_with_partition_key(AutoIndexConfig(), parameters), **parameters, ) @@ -91,7 +113,7 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=FLATConfig(), + db_case_config=_with_partition_key(FLATConfig(), parameters), **parameters, ) @@ -114,10 +136,13 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], + db_case_config=_with_partition_key( + HNSWConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + ), + parameters, ), **parameters, ) @@ -179,14 +204,17 @@ def MilvusHNSWPQ(**parameters: Unpack[MilvusHNSWPQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWPQConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], - nbits=parameters["nbits"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], + db_case_config=_with_partition_key( + HNSWPQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + nbits=parameters["nbits"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + parameters, ), **parameters, ) @@ -223,15 +251,18 @@ def MilvusHNSWPRQ(**parameters: Unpack[MilvusHNSWPRQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWPRQConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], - nbits=parameters["nbits"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], - nrq=parameters["nrq"], + db_case_config=_with_partition_key( + HNSWPRQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + nbits=parameters["nbits"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + nrq=parameters["nrq"], + ), + parameters, ), **parameters, ) @@ -264,14 +295,17 @@ def MilvusHNSWSQ(**parameters: Unpack[MilvusHNSWSQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWSQConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], - sq_type=parameters["sq_type"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], + db_case_config=_with_partition_key( + HNSWSQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + sq_type=parameters["sq_type"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + parameters, ), **parameters, ) @@ -295,9 +329,12 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=IVFFlatConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], + db_case_config=_with_partition_key( + IVFFlatConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + ), + parameters, ), **parameters, ) @@ -318,9 +355,12 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=IVFSQ8Config( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], + db_case_config=_with_partition_key( + IVFSQ8Config( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + ), + parameters, ), **parameters, ) @@ -380,13 +420,16 @@ def MilvusIVFRabitQ(**parameters: Unpack[MilvusIVFRABITQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=IVFRABITQConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], - rbq_bits_query=parameters["rbq_bits_query"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], + db_case_config=_with_partition_key( + IVFRABITQConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + rbq_bits_query=parameters["rbq_bits_query"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + parameters, ), **parameters, ) @@ -411,8 +454,11 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=DISKANNConfig( - search_list=parameters["search_list"], + db_case_config=_with_partition_key( + DISKANNConfig( + search_list=parameters["search_list"], + ), + parameters, ), **parameters, ) @@ -441,11 +487,14 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUIVFFlatConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], - cache_dataset_on_device=parameters["cache_dataset_on_device"], - refine_ratio=parameters.get("refine_ratio"), + db_case_config=_with_partition_key( + GPUIVFFlatConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters.get("refine_ratio"), + ), + parameters, ), **parameters, ) @@ -477,9 +526,12 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUBruteForceConfig( - metric_type=parameters["metric_type"], - limit=parameters["limit"], # top-k for search + db_case_config=_with_partition_key( + GPUBruteForceConfig( + metric_type=parameters["metric_type"], + limit=parameters["limit"], # top-k for search + ), + parameters, ), **parameters, ) @@ -567,13 +619,16 @@ def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=SVSVamanaConfig( - svs_graph_max_degree=parameters["svs_graph_max_degree"], - svs_construction_window_size=parameters["svs_construction_window_size"], - svs_alpha=parameters["svs_alpha"], - svs_storage_kind=parameters["svs_storage_kind"], - svs_search_window_size=parameters["svs_search_window_size"], - svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + db_case_config=_with_partition_key( + SVSVamanaConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + parameters, ), **parameters, ) @@ -594,13 +649,16 @@ def MilvusSVSVamanaLVQ(**parameters: Unpack[MilvusSVSVamanaTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=SVSVamanaLVQConfig( - svs_graph_max_degree=parameters["svs_graph_max_degree"], - svs_construction_window_size=parameters["svs_construction_window_size"], - svs_alpha=parameters["svs_alpha"], - svs_storage_kind=parameters["svs_storage_kind"], - svs_search_window_size=parameters["svs_search_window_size"], - svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + db_case_config=_with_partition_key( + SVSVamanaLVQConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + parameters, ), **parameters, ) @@ -635,14 +693,17 @@ def MilvusSVSVamanaLeanVec(**parameters: Unpack[MilvusSVSVamanaLeanVecTypedDict] num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=SVSVamanaLeanVecConfig( - svs_graph_max_degree=parameters["svs_graph_max_degree"], - svs_construction_window_size=parameters["svs_construction_window_size"], - svs_alpha=parameters["svs_alpha"], - svs_storage_kind=parameters["svs_storage_kind"], - svs_search_window_size=parameters["svs_search_window_size"], - svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], - svs_leanvec_dim=parameters["svs_leanvec_dim"], + db_case_config=_with_partition_key( + SVSVamanaLeanVecConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + svs_leanvec_dim=parameters["svs_leanvec_dim"], + ), + parameters, ), **parameters, ) @@ -673,13 +734,16 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUIVFPQConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], - m=parameters["m"], - nbits=parameters["nbits"], - cache_dataset_on_device=parameters["cache_dataset_on_device"], - refine_ratio=parameters["refine_ratio"], + db_case_config=_with_partition_key( + GPUIVFPQConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + m=parameters["m"], + nbits=parameters["nbits"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters["refine_ratio"], + ), + parameters, ), **parameters, ) @@ -714,17 +778,20 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUCAGRAConfig( - intermediate_graph_degree=parameters["intermediate_graph_degree"], - graph_degree=parameters["graph_degree"], - itopk_size=parameters["itopk_size"], - team_size=parameters["team_size"], - search_width=parameters["search_width"], - min_iterations=parameters["min_iterations"], - max_iterations=parameters["max_iterations"], - build_algo=parameters["build_algo"], - cache_dataset_on_device=parameters["cache_dataset_on_device"], - refine_ratio=parameters["refine_ratio"], + db_case_config=_with_partition_key( + GPUCAGRAConfig( + intermediate_graph_degree=parameters["intermediate_graph_degree"], + graph_degree=parameters["graph_degree"], + itopk_size=parameters["itopk_size"], + team_size=parameters["team_size"], + search_width=parameters["search_width"], + min_iterations=parameters["min_iterations"], + max_iterations=parameters["max_iterations"], + build_algo=parameters["build_algo"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters["refine_ratio"], + ), + parameters, ), **parameters, ) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 740c89509..063faf3bd 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -4,10 +4,12 @@ import time from collections.abc import Iterable from contextlib import contextmanager +from typing import Any from pymilvus import DataType, MilvusClient, MilvusException from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile from ..api import VectorDB from .config import MilvusIndexConfig @@ -47,6 +49,13 @@ def __init__( self._primary_field = "pk" self._scalar_id_field = "id" self._scalar_label_field = "label" + self._scalar_payload_label_field = self._scalar_label_field + self._multitenant_partition_key_field = self._scalar_label_field + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) + if self.multitenant_tenant_labels: + self._multitenant_partition_key_field = "labels" + if self.with_scalar_labels: + self._scalar_payload_label_field = "scalar_label" self._vector_field = "vector" self._vector_index_name = "vector_idx" self._scalar_id_index_name = "id_sort_idx" @@ -56,6 +65,7 @@ def __init__( uri=self.db_config.get("uri"), user=self.db_config.get("user"), password=self.db_config.get("password"), + token=self.db_config.get("token", ""), timeout=30, ) @@ -69,16 +79,27 @@ def __init__( schema.add_field(self._scalar_id_field, DataType.INT64) schema.add_field(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) - if self.with_scalar_labels: - is_partition_key = db_case_config.use_partition_key - log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") + if self.multitenant_tenant_labels: schema.add_field( - self._scalar_label_field, + self._multitenant_partition_key_field, DataType.VARCHAR, max_length=256, - is_partition_key=is_partition_key, + is_partition_key=True, ) + if self.with_scalar_labels: + is_partition_key = db_case_config.use_partition_key + log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") + if not self.multitenant_tenant_labels or ( + self._scalar_payload_label_field != self._multitenant_partition_key_field + ): + schema.add_field( + self._scalar_payload_label_field, + DataType.VARCHAR, + max_length=256, + is_partition_key=is_partition_key and not self.multitenant_tenant_labels, + ) + log.info(f"{self.name} create collection: {self.collection_name}") index_params = self._build_index_params() @@ -113,12 +134,59 @@ def _build_index_params(self): ) if self.with_scalar_labels: index_params.add_index( - field_name=self._scalar_label_field, + field_name=self._scalar_payload_label_field, index_name=self._scalar_labels_index_name, index_type="BITMAP", ) return index_params + def supports_multitenant(self) -> bool: + return True + + def validate_multitenant_schema(self) -> None: + client = MilvusClient( + uri=self.db_config.get("uri"), + user=self.db_config.get("user"), + password=self.db_config.get("password"), + token=self.db_config.get("token", ""), + timeout=30, + ) + try: + desc = client.describe_collection(self.collection_name) + fields = desc.get("fields", []) if isinstance(desc, dict) else [] + fields_by_name = {self._field_property(field, "name"): field for field in fields} + partition_key_field = self._find_multitenant_partition_key_field(fields_by_name) + if partition_key_field is None: + label_field = fields_by_name.get(self._scalar_label_field) + if label_field is None: + msg = f"{self.name} multitenant collection {self.collection_name} is missing tenant label field" + raise ValueError(msg) + msg = f"{self.name} multitenant collection {self.collection_name} label field is not a partition key" + raise ValueError(msg) + self._multitenant_partition_key_field = partition_key_field + if "scalar_label" in fields_by_name: + self._scalar_payload_label_field = "scalar_label" + finally: + client.close() + + def _find_multitenant_partition_key_field(self, fields_by_name: dict[str, dict | object]) -> str | None: + for field_name in [self._scalar_label_field, "labels"]: + field = fields_by_name.get(field_name) + if field is not None and self._field_property(field, "is_partition_key", False): + return field_name + return None + + @staticmethod + def _field_property(field: dict | object, name: str, default: Any = None): + if isinstance(field, dict): + if name in field: + return field[name] + params = field.get("params") + if isinstance(params, dict) and name in params: + return params[name] + return default + return getattr(field, name, default) + @contextmanager def init(self): """ @@ -132,6 +200,7 @@ def init(self): uri=self.db_config.get("uri"), user=self.db_config.get("user"), password=self.db_config.get("password"), + token=self.db_config.get("token", ""), timeout=60, ) yield @@ -211,6 +280,7 @@ def insert_embeddings( embeddings: Iterable[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" @@ -227,8 +297,10 @@ def insert_embeddings( self._scalar_id_field: metadata[i], self._vector_field: embeddings[i], } + if tenant_labels_data is not None: + row[self._multitenant_partition_key_field] = tenant_labels_data[i] if self.with_scalar_labels: - row[self._scalar_label_field] = labels_data[i] + row[self._scalar_payload_label_field] = labels_data[i] batch_data.append(row) res = self.client.insert(self.collection_name, batch_data) insert_count += res["insert_count"] @@ -243,27 +315,62 @@ def prepare_filter(self, filters: Filter): elif filters.type == FilterOp.NumGE: self.expr = f"{self._scalar_id_field} >= {filters.int_value}" elif filters.type == FilterOp.StrEqual: - self.expr = f"{self._scalar_label_field} == '{filters.label_value}'" + self.expr = f"{self._scalar_payload_label_field} == '{filters.label_value}'" else: msg = f"Not support Filter for Milvus - {filters}" raise ValueError(msg) + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in { + PayloadProfile.IDS_ONLY, + PayloadProfile.VECTOR, + PayloadProfile.SCALAR_LABEL, + } + + def poll_insert_readiness(self, expected_count: int) -> dict: + assert self.client is not None + self.client.flush(self.collection_name) + stats = self.client.get_collection_stats(self.collection_name) + count = int(stats.get("row_count", stats.get("num_entities", 0))) + progress = self.client.describe_index(self.collection_name, self._vector_index_name) + return { + "fully_searchable": count >= expected_count, + "fully_indexed": progress.get("pending_index_rows", -1) == 0, + "additional_parameters": {}, + } + def search_embedding( self, query: list[float], k: int = 100, timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: """Perform a search on a query embedding and return results.""" assert self.client is not None - res = self.client.search( - collection_name=self.collection_name, - data=[query], - anns_field=self._vector_field, - search_params=self.case_config.search_param(), - limit=k, - filter=self.expr, - ) + output_fields = None + if payload_profile == PayloadProfile.VECTOR: + output_fields = [self._vector_field] + elif payload_profile == PayloadProfile.SCALAR_LABEL: + output_fields = [getattr(self, "_scalar_payload_label_field", self._scalar_label_field)] + + expr = self.expr + if tenant is not None: + tenant_field = getattr(self, "_multitenant_partition_key_field", self._scalar_label_field) + tenant_expr = f"{tenant_field} == '{tenant}'" + expr = tenant_expr if not expr else f"({expr}) and ({tenant_expr})" + + search_kwargs = { + "collection_name": self.collection_name, + "data": [query], + "anns_field": self._vector_field, + "search_params": self.case_config.search_param(), + "limit": k, + "filter": expr, + "output_fields": output_fields, + } + res = self.client.search(**search_kwargs) return [result[self._primary_field] for result in res[0]] diff --git a/vectordb_bench/backend/clients/pinecone/config.py b/vectordb_bench/backend/clients/pinecone/config.py index fe1a039ed..c42c876ff 100644 --- a/vectordb_bench/backend/clients/pinecone/config.py +++ b/vectordb_bench/backend/clients/pinecone/config.py @@ -6,9 +6,11 @@ class PineconeConfig(DBConfig): api_key: SecretStr index_name: str + multitenant_namespace_prefix: str = "vdbbench_mt_" def to_dict(self) -> dict: return { "api_key": self.api_key.get_secret_value(), "index_name": self.index_name, + "multitenant_namespace_prefix": self.multitenant_namespace_prefix, } diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index 9c2b38888..f123ae231 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -1,18 +1,27 @@ """Wrapper around the Pinecone vector database over VectorDB""" import logging +import os +import threading +import time from contextlib import contextmanager +from typing import Any import pinecone from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile -from ..api import DBCaseConfig, VectorDB +from ..api import DBCaseConfig, PartialInsertError, VectorDB log = logging.getLogger(__name__) PINECONE_MAX_NUM_PER_BATCH = 1000 PINECONE_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB +PINECONE_QUERY_MAX_RETRIES_ENV = "PINECONE_QUERY_MAX_RETRIES" +PINECONE_QUERY_RETRY_SLEEP_ENV = "PINECONE_QUERY_RETRY_SLEEP_SECONDS" +PINECONE_QUERY_DEFAULT_MAX_RETRIES = 10 +PINECONE_QUERY_DEFAULT_RETRY_SLEEP_SECONDS = 0.5 class Pinecone(VectorDB): @@ -34,15 +43,28 @@ def __init__( """Initialize wrapper around the milvus vector database.""" self.index_name = db_config.get("index_name", "") self.api_key = db_config.get("api_key", "") + self.multitenant_namespace_prefix = db_config.get("multitenant_namespace_prefix", "vdbbench_mt_") + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) + self._multitenant_insert_counts: dict[str, int] = {} + self._multitenant_insert_counts_lock = threading.Lock() self.batch_size = int( min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH), ) + self._last_write_lsn: int | None = None + self._last_write_lsn_lock = threading.Lock() + self._readiness_probe_vector = [0.0] * dim pc = pinecone.Pinecone(api_key=self.api_key) index = pc.Index(self.index_name) self.with_scalar_labels = with_scalar_labels - if drop_old: + self.expr = None + if drop_old and self.multitenant_tenant_labels: + for tenant in self.multitenant_tenant_labels: + namespace = self._namespace_for_tenant(tenant) + log.info(f"Pinecone index delete multitenant namespace: {namespace}") + index.delete(delete_all=True, namespace=namespace) + elif drop_old: index_stats = index.describe_index_stats() index_dim = index_stats["dimension"] if index_dim != dim: @@ -61,18 +83,154 @@ def init(self): self.index = pc.Index(self.index_name) yield + def __getstate__(self): + state = self.__dict__.copy() + state.pop("_last_write_lsn_lock", None) + state.pop("_multitenant_insert_counts_lock", None) + return state + def optimize(self, data_size: int | None = None): pass + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in { + PayloadProfile.IDS_ONLY, + PayloadProfile.SCALAR_LABEL, + PayloadProfile.VECTOR, + } + + def supports_multitenant(self) -> bool: + return True + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + self.multitenant_tenant_labels = tenant_labels + + def _namespace_for_tenant(self, tenant: str | None) -> str | None: + if tenant is None: + return None + return f"{self.multitenant_namespace_prefix}{tenant}" + + def poll_insert_readiness(self, expected_count: int) -> dict: + stats = self.index.describe_index_stats() + if getattr(self, "multitenant_tenant_labels", []): + namespaces = stats.get("namespaces", {}) + expected_by_tenant = self._expected_multitenant_counts(expected_count) + count_ready = True + for tenant, expected_tenant_count in expected_by_tenant.items(): + namespace = self._namespace_for_tenant(tenant) + namespace_stats = namespaces.get(namespace, {}) + namespace_count = namespace_stats.get("vector_count", 0) + count_ready = count_ready and namespace_count >= expected_tenant_count + fresh = self._multitenant_lsn_ready(expected_by_tenant) + return { + "fully_searchable": count_ready and fresh, + "fully_indexed": count_ready and fresh, + "additional_parameters": {}, + } + + count = stats.get("total_vector_count", 0) + count_ready = count >= expected_count + last_write_lsn = getattr(self, "_last_write_lsn", None) + if last_write_lsn is None: + return { + "fully_searchable": count_ready, + "fully_indexed": count_ready, + "additional_parameters": {}, + } + query_res = self.index.query(vector=self._readiness_probe_vector, top_k=1) + indexed_lsn = self._extract_lsn(query_res, "x-pinecone-max-indexed-lsn") + if indexed_lsn is None: + return { + "fully_searchable": count_ready, + "fully_indexed": count_ready, + "additional_parameters": {}, + } + fresh = indexed_lsn >= last_write_lsn + return { + "fully_searchable": count_ready and fresh, + "fully_indexed": count_ready and fresh, + "additional_parameters": {}, + } + + def _expected_multitenant_counts(self, expected_count: int) -> dict[str, int]: + insert_counts = self._multitenant_insert_count_snapshot() + if insert_counts: + return insert_counts + tenant_labels = self.multitenant_tenant_labels + tenant_count = len(tenant_labels) + base_count = expected_count // tenant_count if tenant_count else 0 + remainder = expected_count % tenant_count if tenant_count else 0 + return {tenant: base_count + (1 if idx < remainder else 0) for idx, tenant in enumerate(tenant_labels)} + + def _multitenant_insert_count_snapshot(self) -> dict[str, int]: + if not hasattr(self, "_multitenant_insert_counts_lock"): + self._multitenant_insert_counts_lock = threading.Lock() + with self._multitenant_insert_counts_lock: + return dict(getattr(self, "_multitenant_insert_counts", {})) + + def _record_multitenant_insert_count(self, tenant: str, count: int) -> None: + if not hasattr(self, "_multitenant_insert_counts_lock"): + self._multitenant_insert_counts_lock = threading.Lock() + with self._multitenant_insert_counts_lock: + # Pinecone readiness is count-based per tenant namespace. Unlike + # providers that only track touched tenants, this counter must keep + # every successful write from concurrent insert workers. + self._multitenant_insert_counts[tenant] = self._multitenant_insert_counts.get(tenant, 0) + count + + def _multitenant_lsn_ready(self, expected_by_tenant: dict[str, int]) -> bool: + last_write_lsn = getattr(self, "_last_write_lsn", None) + if last_write_lsn is None: + return True + for tenant, expected_tenant_count in expected_by_tenant.items(): + if expected_tenant_count <= 0: + continue + query_res = self.index.query( + vector=self._readiness_probe_vector, + top_k=1, + namespace=self._namespace_for_tenant(tenant), + ) + indexed_lsn = self._extract_lsn(query_res, "x-pinecone-max-indexed-lsn") + if indexed_lsn is not None and indexed_lsn < last_write_lsn: + return False + return True + + @staticmethod + def _extract_lsn(response: Any, header_name: str) -> int | None: + response_info = getattr(response, "_response_info", None) + if not response_info: + return None + raw_headers = response_info.get("raw_headers", {}) + value = raw_headers.get(header_name.lower()) or raw_headers.get(header_name) + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + def _record_write_lsn(self, write_lsn: int) -> None: + if not hasattr(self, "_last_write_lsn_lock"): + self._last_write_lsn_lock = threading.Lock() + with self._last_write_lsn_lock: + self._last_write_lsn = max(getattr(self, "_last_write_lsn", 0) or 0, write_lsn) + + @staticmethod + def _matches(response: Any) -> list: + if isinstance(response, dict): + return response["matches"] + return response.matches + def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: assert len(embeddings) == len(metadata) insert_count = 0 + successful_tenants: dict[str, int] = {} try: for batch_start_offset in range(0, len(embeddings), self.batch_size): batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) @@ -87,7 +245,46 @@ def insert_embeddings( metadata_dict, ) insert_datas.append(insert_data) - self.index.upsert(insert_datas) + if tenant_labels_data is None: + upsert_res = self.index.upsert(insert_datas) + write_lsn = self._extract_lsn(upsert_res, "x-pinecone-request-lsn") + if write_lsn is not None: + self._record_write_lsn(write_lsn) + else: + batch_tenant_labels = tenant_labels_data[batch_start_offset:batch_end_offset] + for tenant in sorted(set(batch_tenant_labels)): + tenant_insert_datas = [ + insert_data + for insert_data, tenant_label in zip(insert_datas, batch_tenant_labels, strict=True) + if tenant_label == tenant + ] + try: + upsert_res = self.index.upsert( + tenant_insert_datas, + namespace=self._namespace_for_tenant(tenant), + ) + except Exception as e: + msg = ( + "Pinecone multitenant insert failed for " + f"tenant={tenant} after writing {insert_count} rows; " + f"successful_tenants={successful_tenants}; " + f"failed_tenant_count={len(tenant_insert_datas)}" + ) + return insert_count, PartialInsertError( + msg, + inserted_count=insert_count, + successful_tenants=successful_tenants, + failed_tenant=tenant, + failed_tenant_count=len(tenant_insert_datas), + cause=e, + ) + write_lsn = self._extract_lsn(upsert_res, "x-pinecone-request-lsn") + if write_lsn is not None: + self._record_write_lsn(write_lsn) + insert_count += len(tenant_insert_datas) + successful_tenants[tenant] = successful_tenants.get(tenant, 0) + len(tenant_insert_datas) + self._record_multitenant_insert_count(tenant, len(tenant_insert_datas)) + continue insert_count += batch_end_offset - batch_start_offset except Exception as e: return insert_count, e @@ -98,15 +295,38 @@ def search_embedding( query: list[float], k: int = 100, timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: pinecone_filters = self.expr - res = self.index.query( - top_k=k, - vector=query, - filter=pinecone_filters, - )["matches"] + include_metadata = payload_profile == PayloadProfile.SCALAR_LABEL + include_values = payload_profile == PayloadProfile.VECTOR + max_retries = int(os.getenv(PINECONE_QUERY_MAX_RETRIES_ENV, PINECONE_QUERY_DEFAULT_MAX_RETRIES)) + retry_sleep = float( + os.getenv(PINECONE_QUERY_RETRY_SLEEP_ENV, PINECONE_QUERY_DEFAULT_RETRY_SLEEP_SECONDS), + ) + for retry_idx in range(max_retries + 1): + try: + query_res = self.index.query( + top_k=k, + vector=query, + filter=pinecone_filters, + include_metadata=include_metadata, + include_values=include_values, + namespace=self._namespace_for_tenant(tenant), + ) + res = self._matches(query_res) + break + except Exception as exc: + if not self._is_rate_limited(exc) or retry_idx >= max_retries: + raise + time.sleep(retry_sleep) return [int(one_res["id"]) for one_res in res] + @staticmethod + def _is_rate_limited(exc: Exception) -> bool: + return getattr(exc, "status", None) == 429 + def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.expr = None diff --git a/vectordb_bench/backend/clients/turbopuffer/cli.py b/vectordb_bench/backend/clients/turbopuffer/cli.py index d510889a0..def442d69 100644 --- a/vectordb_bench/backend/clients/turbopuffer/cli.py +++ b/vectordb_bench/backend/clients/turbopuffer/cli.py @@ -10,42 +10,134 @@ run, ) from .. import DB +from ..api import MetricType +from .config import TurboPufferMultitenantWarmupPolicy + +DEFAULT_PIN_TIMEOUT = 45 * 60 + +ApiKeyOption = Annotated[ + str, + click.option("--api-key", type=str, help="TurboPuffer API key", required=True), +] +RegionOption = Annotated[ + str, + click.option( + "--region", + type=str, + help="TurboPuffer region (e.g. aws-us-east-1, gcp-us-central1)", + required=True, + ), +] +ApiBaseUrlOption = Annotated[ + str, + click.option( + "--api-base-url", + type=str, + help="Override the region-based API URL", + required=False, + default="", + show_default=False, + ), +] +NamespaceOption = Annotated[ + str, + click.option( + "--namespace", + type=str, + help="TurboPuffer namespace", + required=False, + default="vdbbench_test", + show_default=True, + ), +] +PinTimeoutOption = Annotated[ + int, + click.option( + "--pin-timeout", + type=click.IntRange(min=1), + default=DEFAULT_PIN_TIMEOUT, + show_default=True, + help="Seconds to wait for TurboPuffer namespace pinning or unpinning to complete", + ), +] class TurboPufferTypedDict(TypedDict): - api_key: Annotated[ - str, - click.option("--api-key", type=str, help="TurboPuffer API key", required=True), - ] - region: Annotated[ + api_key: ApiKeyOption + region: RegionOption + api_base_url: ApiBaseUrlOption + namespace: NamespaceOption + multitenant_namespace_prefix: Annotated[ str, click.option( - "--region", + "--multitenant-namespace-prefix", type=str, - help="TurboPuffer region (e.g. aws-us-east-1, gcp-us-central1)", - required=True, + help="Namespace prefix for CloudMultiTenantSearchCase tenant namespaces", + required=False, + default="vdbbench_mt_", + show_default=True, ), ] - api_base_url: Annotated[ + scalar_payload_label_field: Annotated[ str, click.option( - "--api-base-url", + "--scalar-payload-label-field", type=str, - help="Override the region-based API URL", + help="TurboPuffer attribute used for scalar_label payload and label filtering", required=False, - default="", - show_default=False, + default="label", + show_default=True, ), ] - namespace: Annotated[ + metric_type: Annotated[ str, click.option( - "--namespace", - type=str, - help="TurboPuffer namespace", + "--metric-type", + type=click.Choice([MetricType.COSINE.value, MetricType.L2.value]), + help="TurboPuffer distance metric type", required=False, - default="vdbbench_test", + default=MetricType.COSINE.value, + show_default=True, + ), + ] + disable_backpressure: Annotated[ + bool, + click.option( + "--disable-backpressure/--enable-backpressure", + type=bool, + default=False, + show_default=True, + help="Disable Turbopuffer write backpressure", + ), + ] + pin_namespace: Annotated[ + bool, + click.option( + "--pin-namespace/--no-pin-namespace", + default=False, + show_default=True, + help="Pin TurboPuffer namespace(s) before benchmark workers run", + ), + ] + pin_replicas: Annotated[ + int, + click.option( + "--pin-replicas", + type=click.IntRange(min=1), + default=1, + show_default=True, + help="Number of TurboPuffer pinning replicas to request", + ), + ] + pin_timeout: PinTimeoutOption + multitenant_warmup_policy: Annotated[ + str, + click.option( + "--multitenant-warmup-policy", + type=click.Choice([policy.value for policy in TurboPufferMultitenantWarmupPolicy]), + default=TurboPufferMultitenantWarmupPolicy.NONE.value, show_default=True, + help="TurboPuffer cache warmup policy for CloudMultiTenantSearchCase tenant namespaces", ), ] @@ -53,11 +145,67 @@ class TurboPufferTypedDict(TypedDict): class TurboPufferIndexTypedDict(CommonTypedDict, TurboPufferTypedDict): ... +class TurboPufferUnpinTypedDict(TypedDict): + """Options for explicit TurboPuffer namespace pinning cleanup. + + Namespace pinning is persistent service state: enabling it before a benchmark + reserves replicas for the namespace until it is cleared. Unpinning is kept + as a separate command so failed or interrupted benchmark runs can be cleaned + up later, and so a billing-affecting teardown action is not hidden behind + ordinary benchmark flags. + """ + + api_key: ApiKeyOption + region: RegionOption + api_base_url: ApiBaseUrlOption + namespace: NamespaceOption + pin_timeout: PinTimeoutOption + + +def target_namespaces_for_pinning(parameters: TurboPufferIndexTypedDict) -> list[str]: + if parameters.get("case_type") != "CloudMultiTenantSearchCase": + return [parameters["namespace"]] + + namespace_prefix = parameters["multitenant_namespace_prefix"] + tenant_prefix = parameters["tenant_prefix"] + tenant_id_width = parameters["tenant_id_width"] + return [ + f"{namespace_prefix}{tenant_prefix}{tenant_id:0{tenant_id_width}d}" + for tenant_id in range(parameters["tenant_count"]) + ] + + +def pin_namespaces_once(parameters: TurboPufferIndexTypedDict) -> None: + from .turbopuffer import namespace_metadata_request, wait_for_namespace_pinning + + for namespace in target_namespaces_for_pinning(parameters): + namespace_metadata_request( + parameters["api_key"], + parameters["region"], + namespace, + "PATCH", + {"pinning": {"replicas": parameters["pin_replicas"]}}, + parameters["api_base_url"] or None, + ) + wait_for_namespace_pinning( + parameters["api_key"], + parameters["region"], + namespace, + parameters["pin_replicas"], + parameters["api_base_url"] or None, + parameters["pin_timeout"], + ) + + @cli.command() @click_parameter_decorators_from_typed_dict(TurboPufferIndexTypedDict) def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): from .config import TurboPufferConfig, TurboPufferIndexConfig + pin_target_namespace_count = len(target_namespaces_for_pinning(parameters)) if parameters["pin_namespace"] else 0 + if parameters["pin_namespace"] and not parameters["dry_run"]: + pin_namespaces_once(parameters) + run( db=DB.TurboPuffer, db_config=TurboPufferConfig( @@ -66,7 +214,42 @@ def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): region=parameters["region"], api_base_url=parameters["api_base_url"] or None, namespace=parameters["namespace"], + multitenant_namespace_prefix=parameters["multitenant_namespace_prefix"], + scalar_payload_label_field=parameters["scalar_payload_label_field"], + pin_namespace=False, + pin_namespace_requested=parameters["pin_namespace"], + pin_replicas=parameters["pin_replicas"], + pin_timeout=parameters["pin_timeout"], + pin_target_namespace_count=pin_target_namespace_count, + ), + db_case_config=TurboPufferIndexConfig( + metric_type=MetricType(parameters["metric_type"]), + disable_backpressure=parameters["disable_backpressure"], + multitenant_warmup_policy=TurboPufferMultitenantWarmupPolicy(parameters["multitenant_warmup_policy"]), ), - db_case_config=TurboPufferIndexConfig(), **parameters, ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(TurboPufferUnpinTypedDict) +def TurboPufferUnpin(**parameters: Unpack[TurboPufferUnpinTypedDict]): + from .turbopuffer import namespace_metadata_request, wait_for_namespace_pinning + + namespace_metadata_request( + parameters["api_key"], + parameters["region"], + parameters["namespace"], + "PATCH", + {"pinning": None}, + parameters["api_base_url"] or None, + ) + meta = wait_for_namespace_pinning( + parameters["api_key"], + parameters["region"], + parameters["namespace"], + None, + parameters["api_base_url"] or None, + parameters["pin_timeout"], + ) + click.echo(f"TurboPuffer namespace unpinned: {parameters['namespace']} pinning={meta.get('pinning')}") diff --git a/vectordb_bench/backend/clients/turbopuffer/config.py b/vectordb_bench/backend/clients/turbopuffer/config.py index 88e797351..ed28675f8 100644 --- a/vectordb_bench/backend/clients/turbopuffer/config.py +++ b/vectordb_bench/backend/clients/turbopuffer/config.py @@ -1,13 +1,27 @@ +from enum import StrEnum + from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType +class TurboPufferMultitenantWarmupPolicy(StrEnum): + NONE = "none" + ALL = "all" + + class TurboPufferConfig(DBConfig): api_key: SecretStr region: str api_base_url: str | None = None namespace: str = "vdbbench_test" + multitenant_namespace_prefix: str = "vdbbench_mt_" + scalar_payload_label_field: str = "label" + pin_namespace: bool = False + pin_namespace_requested: bool = False + pin_replicas: int = 1 + pin_timeout: int = 45 * 60 + pin_target_namespace_count: int = 0 def to_dict(self) -> dict: return { @@ -15,6 +29,13 @@ def to_dict(self) -> dict: "region": self.region, "api_base_url": self.api_base_url, "namespace": self.namespace, + "multitenant_namespace_prefix": self.multitenant_namespace_prefix, + "scalar_payload_label_field": self.scalar_payload_label_field, + "pin_namespace": self.pin_namespace, + "pin_namespace_requested": self.pin_namespace_requested, + "pin_replicas": self.pin_replicas, + "pin_timeout": self.pin_timeout, + "pin_target_namespace_count": self.pin_target_namespace_count, } @@ -22,6 +43,8 @@ class TurboPufferIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None use_multi_ns_for_filter: bool = False time_wait_warmup: int = 60 * 1 # 1min + disable_backpressure: bool = False + multitenant_warmup_policy: TurboPufferMultitenantWarmupPolicy = TurboPufferMultitenantWarmupPolicy.NONE def parse_metric(self) -> str: if self.metric_type == MetricType.COSINE: diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 6de0df21d..e28e38ae3 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -3,15 +3,81 @@ import logging import time from contextlib import contextmanager +from json import dumps, loads +from typing import Any +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import Request, urlopen import turbopuffer as tpuf -from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig +from vectordb_bench.backend.clients.turbopuffer.config import ( + TurboPufferIndexConfig, + TurboPufferMultitenantWarmupPolicy, +) from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile -from ..api import VectorDB +from ..api import PartialInsertError, VectorDB log = logging.getLogger(__name__) +TURBOPUFFER_SEARCHABLE_UNINDEXED_BYTES = 2 * 1024 * 1024 * 1024 +PINNING_POLL_INTERVAL = 10 +PINNING_TIMEOUT = 45 * 60 + + +def namespace_metadata_request( + api_key: str, + region: str, + namespace: str, + method: str, + payload: dict[str, Any] | None = None, + api_base_url: str | None = None, +) -> dict: + base_url = api_base_url or f"https://{region}.turbopuffer.com" + url = f"{base_url.rstrip('/')}/v1/namespaces/{quote(namespace, safe='')}/metadata" + req = Request( # noqa: S310 + url, + data=dumps(payload).encode() if payload is not None else None, + method=method, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + ) + try: + with urlopen(req, timeout=60) as resp: # noqa: S310 + return loads(resp.read().decode() or "{}") + except HTTPError as e: + detail = e.read().decode(errors="replace") + msg = f"Failed to update TurboPuffer namespace metadata: {e.code} {detail}" + raise RuntimeError(msg) from e + + +def wait_for_namespace_pinning( + api_key: str, + region: str, + namespace: str, + replicas: int | None, + api_base_url: str | None = None, + timeout: int = PINNING_TIMEOUT, +) -> dict: + deadline = time.monotonic() + timeout + while True: + meta = namespace_metadata_request(api_key, region, namespace, "GET", api_base_url=api_base_url) + pinning = meta.get("pinning") + if replicas is None: + if pinning is None: + return meta + else: + status = pinning.get("status", {}) if isinstance(pinning, dict) else {} + if pinning and pinning.get("replicas") == replicas and status.get("ready_replicas") == replicas: + return meta + if time.monotonic() >= deadline: + msg = f"Timed out waiting for TurboPuffer pinning state on namespace {namespace}" + raise TimeoutError(msg) + log.info("Waiting for TurboPuffer pinning state on %s: %s", namespace, pinning) + time.sleep(PINNING_POLL_INTERVAL) class TurboPuffer(VectorDB): @@ -34,23 +100,40 @@ def __init__( self.region = db_config.get("region", "") self.api_base_url = db_config.get("api_base_url") self.namespace = db_config.get("namespace", "") + self.multitenant_namespace_prefix = db_config.get("multitenant_namespace_prefix", "vdbbench_mt_") + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) + self._multitenant_touched_tenants: set[str] = set() + self._ns_cache = {} + self.pin_namespace = db_config.get("pin_namespace", False) + self.pin_replicas = db_config.get("pin_replicas", 1) + self.pin_timeout = db_config.get("pin_timeout", PINNING_TIMEOUT) + self._pinning_applied = False self.db_case_config = db_case_config self.metric = db_case_config.parse_metric() self._vector_field = "vector" self._scalar_id_field = "id" self._scalar_label_field = "label" + self._scalar_payload_label_field = db_config.get("scalar_payload_label_field", self._scalar_label_field) self.with_scalar_labels = with_scalar_labels + self.expr = None if drop_old: - log.info(f"Drop old. delete the namespace: {self.namespace}") tmp_client = self._create_client() - ns = tmp_client.namespace(self.namespace) - try: - ns.delete_all() - except Exception as e: - log.warning(f"Failed to delete all. Error: {e}") + if self.multitenant_tenant_labels: + for tenant in self.multitenant_tenant_labels: + try: + tmp_client.namespace(self._namespace_name_for_tenant(tenant)).delete_all() + except Exception as e: + log.warning(f"Failed to delete multitenant namespace {tenant}. Error: {e}") + else: + log.info(f"Drop old. delete the namespace: {self.namespace}") + ns = tmp_client.namespace(self.namespace) + try: + ns.delete_all() + except Exception as e: + log.warning(f"Failed to delete all. Error: {e}") tmp_client = None def _create_client(self) -> tpuf.Turbopuffer: @@ -59,61 +142,227 @@ def _create_client(self) -> tpuf.Turbopuffer: client_kwargs["base_url"] = self.api_base_url return tpuf.Turbopuffer(**client_kwargs) + def _apply_namespace_pinning(self): + if not self.pin_namespace or self._pinning_applied: + return + for namespace in self._target_namespaces_for_pinning(): + namespace_metadata_request( + self.api_key, + self.region, + namespace, + "PATCH", + {"pinning": {"replicas": self.pin_replicas}}, + self.api_base_url, + ) + meta = wait_for_namespace_pinning( + self.api_key, + self.region, + namespace, + self.pin_replicas, + self.api_base_url, + self.pin_timeout, + ) + pinning = meta.get("pinning", {}) + status = pinning.get("status", {}) if isinstance(pinning, dict) else {} + log.info( + "TurboPuffer pinning requested for %s: replicas=%s ready_replicas=%s", + namespace, + pinning.get("replicas", self.pin_replicas) if isinstance(pinning, dict) else self.pin_replicas, + status.get("ready_replicas"), + ) + self._pinning_applied = True + + def _target_namespaces_for_pinning(self) -> list[str]: + if self.multitenant_tenant_labels: + return [self._namespace_name_for_tenant(tenant) for tenant in self.multitenant_tenant_labels] + return [self.namespace] + @contextmanager def init(self): self.client = self._create_client() + self._ns_cache = {} self.ns = self.client.namespace(self.namespace) + self._apply_namespace_pinning() yield + def supports_multitenant(self) -> bool: + return True + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + self.multitenant_tenant_labels = tenant_labels + + def _namespace_name_for_tenant(self, tenant: str | None) -> str: + if tenant is None: + return self.namespace + return f"{self.multitenant_namespace_prefix}{tenant}" + + def _namespace_for_tenant(self, tenant: str | None): + name = self._namespace_name_for_tenant(tenant) + ns = self._ns_cache.get(name) + if ns is None: + ns = self.client.namespace(name) + self._ns_cache[name] = ns + return ns + def optimize(self, data_size: int | None = None): # turbopuffer responds to the request # once the cache warming operation has been started. # It does not wait for the operation to complete, # which can take multiple minutes for large namespaces. - self.ns.hint_cache_warm() + warmed_namespaces = self._warmup_target_namespaces() + for namespace in warmed_namespaces: + self._namespace_for_tenant(namespace).hint_cache_warm() + if not warmed_namespaces: + log.info("TurboPuffer cache warmup skipped") + return log.info(f"warming up but no api waiting for complete. just sleep {self.db_case_config.time_wait_warmup}s") time.sleep(self.db_case_config.time_wait_warmup) + def _warmup_target_namespaces(self) -> list[str | None]: + if not self.multitenant_tenant_labels: + return [None] + policy = getattr( + self.db_case_config, + "multitenant_warmup_policy", + TurboPufferMultitenantWarmupPolicy.NONE, + ) + if policy == TurboPufferMultitenantWarmupPolicy.ALL: + return self.multitenant_tenant_labels + return [] + def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: + vectors = [embedding.tolist() if hasattr(embedding, "tolist") else embedding for embedding in embeddings] + if tenant_labels_data is not None: + inserted = 0 + successful_tenants: dict[str, int] = {} + for tenant in sorted(set(tenant_labels_data)): + self._multitenant_touched_tenants.add(tenant) + idxs = [i for i, label in enumerate(tenant_labels_data) if label == tenant] + try: + upsert_columns = { + self._scalar_id_field: [metadata[i] for i in idxs], + self._vector_field: [vectors[i] for i in idxs], + } + if self.with_scalar_labels: + upsert_columns[self._scalar_payload_label_field] = [labels_data[i] for i in idxs] + self._namespace_for_tenant(tenant).write( + upsert_columns=upsert_columns, + distance_metric=self.metric, + disable_backpressure=self.db_case_config.disable_backpressure, + ) + except Exception as e: + msg = ( + "TurboPuffer multitenant insert failed for " + f"tenant={tenant} after writing {inserted} rows; " + f"successful_tenants={successful_tenants}; " + f"failed_tenant_count={len(idxs)}" + ) + err = PartialInsertError( + msg, + inserted_count=inserted, + successful_tenants=successful_tenants, + failed_tenant=tenant, + failed_tenant_count=len(idxs), + cause=e, + ) + log.warning(f"Failed to insert. Error: {err}") + return inserted, err + inserted += len(idxs) + successful_tenants[tenant] = len(idxs) + return inserted, None try: if self.with_scalar_labels: self.ns.write( upsert_columns={ self._scalar_id_field: metadata, - self._vector_field: embeddings, - self._scalar_label_field: labels_data, + self._vector_field: vectors, + self._scalar_payload_label_field: labels_data, }, distance_metric=self.metric, + disable_backpressure=self.db_case_config.disable_backpressure, ) else: self.ns.write( upsert_columns={ self._scalar_id_field: metadata, - self._vector_field: embeddings, + self._vector_field: vectors, }, distance_metric=self.metric, + disable_backpressure=self.db_case_config.disable_backpressure, ) except Exception as e: log.warning(f"Failed to insert. Error: {e}") + return 0, e return len(embeddings), None + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in { + PayloadProfile.IDS_ONLY, + PayloadProfile.SCALAR_LABEL, + PayloadProfile.VECTOR, + } + + def poll_insert_readiness(self, expected_count: int) -> dict: + if getattr(self, "multitenant_tenant_labels", []): + unindexed_by_tenant = {} + tenant_labels = ( + sorted(getattr(self, "_multitenant_touched_tenants", set())) or self.multitenant_tenant_labels + ) + for tenant in tenant_labels: + metadata = self._namespace_for_tenant(tenant).metadata() + if not isinstance(metadata, dict): + metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else vars(metadata) + index = metadata.get("index", {}) + if not isinstance(index, dict): + index = index.model_dump() if hasattr(index, "model_dump") else vars(index) + unindexed_by_tenant[tenant] = metadata.get("unindexed_bytes", index.get("unindexed_bytes", 0)) + max_unindexed_bytes = max(unindexed_by_tenant.values(), default=0) + return { + "fully_searchable": max_unindexed_bytes <= TURBOPUFFER_SEARCHABLE_UNINDEXED_BYTES, + "fully_indexed": max_unindexed_bytes == 0, + "additional_parameters": { + "disable_backpressure": self.db_case_config.disable_backpressure, + "max_unindexed_bytes": max_unindexed_bytes, + }, + } + metadata = self.ns.metadata() + if not isinstance(metadata, dict): + metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else vars(metadata) + index = metadata.get("index", {}) + if not isinstance(index, dict): + index = index.model_dump() if hasattr(index, "model_dump") else vars(index) + unindexed_bytes = metadata.get("unindexed_bytes", index.get("unindexed_bytes", 0)) + return { + "fully_searchable": unindexed_bytes <= TURBOPUFFER_SEARCHABLE_UNINDEXED_BYTES, + "fully_indexed": unindexed_bytes == 0, + "additional_parameters": {"disable_backpressure": self.db_case_config.disable_backpressure}, + } + def search_embedding( self, query: list[float], k: int = 100, timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: - res = self.ns.query( - rank_by=("vector", "ANN", query), - top_k=k, - filters=self.expr, - ) + query_kwargs = { + "rank_by": ("vector", "ANN", query), + "top_k": k, + "filters": self.expr, + } + if payload_profile == PayloadProfile.VECTOR: + query_kwargs["include_attributes"] = [self._vector_field] + elif payload_profile == PayloadProfile.SCALAR_LABEL: + query_kwargs["include_attributes"] = [self._scalar_payload_label_field] + res = self._namespace_for_tenant(tenant).query(**query_kwargs) return [int(row.id) for row in res.rows] if res.rows is not None else [] def prepare_filter(self, filters: Filter): @@ -122,7 +371,7 @@ def prepare_filter(self, filters: Filter): elif filters.type == FilterOp.NumGE: self.expr = (self._scalar_id_field, "Gte", filters.int_value) elif filters.type == FilterOp.StrEqual: - self.expr = (self._scalar_label_field, "Eq", filters.label_value) + self.expr = (self._scalar_payload_label_field, "Eq", filters.label_value) else: msg = f"Not support Filter for TurboPuffer - {filters}" raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/cli.py b/vectordb_bench/backend/clients/zilliz_cloud/cli.py index a8d177ee5..030a00661 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/cli.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/cli.py @@ -13,6 +13,13 @@ ) +def _use_partition_key(parameters: dict) -> bool: + explicit = parameters.get("use_partition_key") + if explicit is not None: + return explicit + return parameters.get("case_type") == "CloudMultiTenantSearchCase" + + class ZillizTypedDict(CommonTypedDict): uri: Annotated[ str, @@ -20,7 +27,7 @@ class ZillizTypedDict(CommonTypedDict): ] user_name: Annotated[ str, - click.option("--user-name", type=str, help="Db username", required=True), + click.option("--user-name", type=str, help="Db username", default=""), ] password: Annotated[ str, @@ -32,6 +39,16 @@ class ZillizTypedDict(CommonTypedDict): show_default="$ZILLIZ_PASSWORD", ), ] + token: Annotated[ + str, + click.option( + "--token", + type=str, + help="Zilliz API token", + default=lambda: os.environ.get("ZILLIZ_TOKEN", ""), + show_default="$ZILLIZ_TOKEN", + ), + ] level: Annotated[ str, click.option("--level", type=str, help="Zilliz index level", required=False), @@ -58,6 +75,17 @@ class ZillizTypedDict(CommonTypedDict): show_default=True, ), ] + use_partition_key: Annotated[ + bool | None, + click.option( + "--use-partition-key/--no-use-partition-key", + default=None, + help=( + "Use the Zilliz Cloud partition key on the label field. " + "Defaults to enabled for CloudMultiTenantSearchCase and disabled otherwise." + ), + ), + ] @cli.command() @@ -72,12 +100,14 @@ def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + token=SecretStr(parameters["token"]), num_shards=parameters["num_shards"], collection_name=parameters["collection_name"], ), db_case_config=AutoIndexConfig( level=int(parameters["level"]) if parameters["level"] else 1, num_shards=parameters["num_shards"], + use_partition_key=_use_partition_key(parameters), ), **parameters, ) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/config.py b/vectordb_bench/backend/clients/zilliz_cloud/config.py index f0b3fd000..8ab45caa2 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/config.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/config.py @@ -6,16 +6,22 @@ class ZillizCloudConfig(DBConfig): uri: SecretStr - user: str - password: SecretStr + user: str = "" + password: SecretStr = SecretStr("") + token: SecretStr = SecretStr("") num_shards: int = 1 collection_name: str = "ZillizCloudVDBBench" + @staticmethod + def common_long_configs() -> list[str]: + return [*DBConfig.common_long_configs(), "user", "password", "token"] + def to_dict(self) -> dict: return { "uri": self.uri.get_secret_value(), "user": self.user, "password": self.password.get_secret_value(), + "token": self.token.get_secret_value(), "num_shards": self.num_shards, "collection_name": self.collection_name, } diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 94216532f..c249f91c9 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -138,6 +138,8 @@ class LAION(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False with_gt: bool = True + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] _size_label: ClassVar[dict] = { 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -338,11 +340,16 @@ def data_dir(self) -> pathlib.Path: def __iter__(self): return DataSetIterator(self) + def iter_batches(self, batch_size: int): + return DataSetIterator(self, batch_size=batch_size) + # TODO passing use_shuffle from outside def prepare( self, source: DatasetSource = DatasetSource.S3, filters: Filter = non_filter, + with_train_files: bool = True, + with_scalar_labels: bool = False, ) -> bool: """Download the dataset from DatasetSource url = f"{source}/{self.data.dir_name}" @@ -356,7 +363,7 @@ def prepare( bool: whether the dataset is successfully prepared """ - self.train_files = self.data.train_files + self.train_files = self.data.train_files if with_train_files else [] gt_file, test_file = None, None if self.data.with_gt: gt_file, test_file = filters.groundtruth_file, self.data.test_file @@ -373,12 +380,10 @@ def prepare( local_ds_root=self.data_dir, ) + needs_scalar_labels = filters.type == FilterOp.StrEqual or with_scalar_labels + # read scalar_labels_file if separated - if ( - filters.type == FilterOp.StrEqual - and self.data.with_scalar_labels - and self.data.scalar_labels_file_separated - ): + if needs_scalar_labels and self.data.with_scalar_labels and self.data.scalar_labels_file_separated: self.scalar_labels = self._read_file(self.data.scalar_labels_file) if gt_file is not None and test_file is not None: @@ -401,8 +406,9 @@ def _read_file(self, file_name: str) -> pl.DataFrame: class DataSetIterator: - def __init__(self, dataset: DatasetManager): + def __init__(self, dataset: DatasetManager, batch_size: int = config.NUM_PER_BATCH): self._ds = dataset + self._batch_size = batch_size self._idx = 0 # file number self._cur = None self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file @@ -428,7 +434,7 @@ def _get_iter(self, file_name: str): msg = f"No such file: {p}" log.warning(msg) raise IndexError(msg) - return ParquetFile(p, memory_map=True, pre_buffer=True).iter_batches(config.NUM_PER_BATCH) + return ParquetFile(p, memory_map=True, pre_buffer=True).iter_batches(self._batch_size) def __next__(self) -> pd.DataFrame: """return the data in the next file of the training list""" @@ -478,6 +484,7 @@ class DatasetWithSizeType(Enum): CohereSmall = "Small Cohere (768dim, 100K)" CohereMedium = "Medium Cohere (768dim, 1M)" CohereLarge = "Large Cohere (768dim, 10M)" + LAIONLarge = "Large LAION (768dim, 100M)" BioasqMedium = "Medium Bioasq (1024dim, 1M)" BioasqLarge = "Large Bioasq (1024dim, 10M)" OpenAISmall = "Small OpenAI (1536dim, 50K)" @@ -491,6 +498,8 @@ def get_manager(self) -> DatasetManager: return DatasetWithSizeMap.get(self) def get_load_timeout(self) -> float: + if self is DatasetWithSizeType.LAIONLarge: + return config.LOAD_TIMEOUT_768D_100M if "small" in self.value.lower(): return config.LOAD_TIMEOUT_768D_100K if "medium" in self.value.lower(): @@ -501,6 +510,8 @@ def get_load_timeout(self) -> float: raise KeyError(msg) def get_optimize_timeout(self) -> float: + if self is DatasetWithSizeType.LAIONLarge: + return config.OPTIMIZE_TIMEOUT_768D_100M if "small" in self.value.lower(): return config.OPTIMIZE_TIMEOUT_768D_100K if "medium" in self.value.lower(): @@ -514,6 +525,7 @@ def get_optimize_timeout(self) -> float: DatasetWithSizeType.CohereSmall: Dataset.COHERE.manager(100_000), DatasetWithSizeType.CohereMedium: Dataset.COHERE.manager(1_000_000), DatasetWithSizeType.CohereLarge: Dataset.COHERE.manager(10_000_000), + DatasetWithSizeType.LAIONLarge: Dataset.LAION.manager(100_000_000), DatasetWithSizeType.BioasqMedium: Dataset.BIOASQ.manager(1_000_000), DatasetWithSizeType.BioasqLarge: Dataset.BIOASQ.manager(10_000_000), DatasetWithSizeType.OpenAISmall: Dataset.OPENAI.manager(50_000), diff --git a/vectordb_bench/backend/payload.py b/vectordb_bench/backend/payload.py new file mode 100644 index 000000000..49050c85f --- /dev/null +++ b/vectordb_bench/backend/payload.py @@ -0,0 +1,21 @@ +from enum import StrEnum + + +class PayloadProfile(StrEnum): + IDS_ONLY = "ids_only" + VECTOR = "vector" + SCALAR_LABEL = "scalar_label" + + def estimated_bytes_per_query(self, *, k: int, dim: int) -> int: + # Approximate payload size used for cloud leaderboard cost expansion. + # ID + distance is about 20 bytes per hit; vector is float32. + id_distance_bytes = 20 + scalar_label_bytes = 16 + if self == PayloadProfile.IDS_ONLY: + return k * id_distance_bytes + if self == PayloadProfile.VECTOR: + return k * (id_distance_bytes + dim * 4) + if self == PayloadProfile.SCALAR_LABEL: + return k * (id_distance_bytes + scalar_label_bytes) + msg = f"Unsupported payload profile: {self}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/runner/__init__.py b/vectordb_bench/backend/runner/__init__.py index d56fe0ff8..ddee99554 100644 --- a/vectordb_bench/backend/runner/__init__.py +++ b/vectordb_bench/backend/runner/__init__.py @@ -1,9 +1,11 @@ +from .cold_warm_runner import ColdWarmSearchRunner from .concurrent_runner import ConcurrentInsertRunner from .mp_runner import MultiProcessingSearchRunner from .read_write_runner import ReadWriteRunner from .serial_runner import SerialInsertRunner, SerialSearchRunner __all__ = [ + "ColdWarmSearchRunner", "ConcurrentInsertRunner", "MultiProcessingSearchRunner", "ReadWriteRunner", diff --git a/vectordb_bench/backend/runner/cold_warm_runner.py b/vectordb_bench/backend/runner/cold_warm_runner.py new file mode 100644 index 000000000..51bde6ff8 --- /dev/null +++ b/vectordb_bench/backend/runner/cold_warm_runner.py @@ -0,0 +1,120 @@ +import logging +import time + +import numpy as np + +from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.payload import PayloadProfile + +from ... import config +from ..clients import api + +log = logging.getLogger(__name__) + + +class ColdWarmSearchRunner: + def __init__( + self, + db: api.VectorDB, + test_data: list[list[float]], + k: int = 100, + filters: Filter = non_filter, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + query_count: int = 1000, + ): + if query_count <= 0: + msg = "query_count must be positive" + raise ValueError(msg) + if len(test_data) < query_count: + msg = f"query_count={query_count} exceeds test_data size={len(test_data)}" + raise ValueError(msg) + + self.db = db + self.k = k + self.filters = filters + self.payload_profile = payload_profile + self.query_count = query_count + if not self.db.supports_payload_profile(self.payload_profile): + msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) + + self.test_data = [ + query.tolist() if isinstance(query, np.ndarray) else query for query in test_data[:query_count] + ] + + def _search_embedding(self, emb: list[float]) -> list[int]: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile) + + def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + try: + results = self._search_embedding(emb) + except Exception as e: + log.warning(f"Cold/warm search failed, retry_idx={retry_idx}, Exception: {e}") + if retry_idx < config.MAX_SEARCH_RETRY: + return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + + msg = f"Cold/warm search failed and retried more than {config.MAX_SEARCH_RETRY} times" + raise RuntimeError(msg) from e + + return results + + @staticmethod + def _latency_stats(latencies: list[float]) -> dict[str, float]: + return { + "first_query_latency": round(float(latencies[0]), 4), + "p99_latency": round(float(np.percentile(latencies, 99)), 4), + "p95_latency": round(float(np.percentile(latencies, 95)), 4), + "avg_latency": round(float(np.mean(latencies)), 4), + } + + @staticmethod + def _safe_ratio(numerator: float, denominator: float) -> float: + if denominator == 0: + return 0.0 + return round(float(numerator / denominator), 4) + + def _ratio_stats(self, cold_stats: dict[str, float], warm_stats: dict[str, float]) -> dict[str, float]: + return { + "first_query_latency_ratio": self._safe_ratio( + cold_stats["first_query_latency"], + warm_stats["first_query_latency"], + ), + "p99_latency_ratio": self._safe_ratio(cold_stats["p99_latency"], warm_stats["p99_latency"]), + "p95_latency_ratio": self._safe_ratio(cold_stats["p95_latency"], warm_stats["p95_latency"]), + "avg_latency_ratio": self._safe_ratio(cold_stats["avg_latency"], warm_stats["avg_latency"]), + } + + def _run_pass(self, pass_name: str) -> dict[str, float]: + latencies = [] + for emb in self.test_data: + start = time.perf_counter() + self._get_db_search_res(emb) + latencies.append(time.perf_counter() - start) + + if len(latencies) % 100 == 0: + log.debug(f"{pass_name} search_count={len(latencies):3}, latest_latency={latencies[-1]}") + + stats = self._latency_stats(latencies) + log.info( + f"{pass_name} search pass: " + f"queries={len(latencies)}, " + f"first_query_latency={stats['first_query_latency']}, " + f"avg_latency={stats['avg_latency']}, " + f"p99={stats['p99_latency']}, " + f"p95={stats['p95_latency']}" + ) + return stats + + def run(self) -> dict[str, dict[str, float]]: + with self.db.init(): + self.db.prepare_filter(self.filters) + cold_stats = self._run_pass("cold") + warm_stats = self._run_pass("warm") + + return { + "cold_stats": cold_stats, + "warm_stats": warm_stats, + "cold_warm_ratio": self._ratio_stats(cold_stats, warm_stats), + } diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 7c8aeb24f..650795535 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -64,6 +64,10 @@ def __init__( timeout: float | None = None, max_workers: int | None = None, backend: ExecutorBackend = ExecutorBackend.THREADING, + batch_size: int = config.NUM_PER_BATCH, + duration: float | None = None, + with_scalar_labels: bool = False, + tenant_case=None, # noqa: ANN001 ): self.timeout = timeout if isinstance(timeout, int | float) else None self.dataset: DatasetManager = dataset @@ -71,6 +75,10 @@ def __init__( self.normalize = normalize self.filters = filters self.backend = backend + self.batch_size = batch_size + self.duration = duration if isinstance(duration, int | float) else None + self.with_scalar_labels = with_scalar_labels + self.tenant_case = tenant_case effective_workers = max_workers or min(mp.cpu_count(), 4) if not db.thread_safe: @@ -87,6 +95,7 @@ def __getstate__(self): state = self.__dict__.copy() state.pop("_iter_lock", None) state.pop("_dataset_iter", None) + state.pop("_stop_event", None) return state def _create_executor(self) -> TaskExecutor: @@ -109,20 +118,34 @@ def _insert_batch_with_retry( embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, retry_idx: int = 0, ) -> int: """Insert a single batch with retry logic. Returns inserted count.""" - insert_count, error = db.insert_embeddings( - embeddings=embeddings, - metadata=metadata, - labels_data=labels_data, - ) + insert_kwargs = { + "embeddings": embeddings, + "metadata": metadata, + "labels_data": labels_data, + } + if tenant_labels_data is not None: + insert_kwargs["tenant_labels_data"] = tenant_labels_data + insert_count, error = db.insert_embeddings(**insert_kwargs) if error is not None: log.warning(f"Insert failed, try_idx={retry_idx}, Exception: {error}") + if getattr(error, "non_retryable", False): + msg = f"Non-retryable insert failure after {insert_count} inserted rows: {error}" + raise RuntimeError(msg) from error retry_idx += 1 if retry_idx <= config.MAX_INSERT_RETRY: time.sleep(retry_idx) - return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, retry_idx) + return self._insert_batch_with_retry( + db, + embeddings, + metadata, + labels_data, + tenant_labels_data, + retry_idx, + ) msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" raise RuntimeError(msg) return insert_count @@ -132,18 +155,27 @@ def _worker_insert( embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, ) -> int: """Worker function: insert a batch with retry.""" db = self._get_thread_db() - return self._insert_batch_with_retry(db, embeddings, metadata, labels_data) + return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, tenant_labels_data) - def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None] | None: + def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None, list[str] | None] | None: """Pull the next batch from the shared dataset iterator. Thread-safe: only one thread reads from the iterator at a time. Returns None when the iterator is exhausted. """ + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None and stop_event.is_set(): + return None + if self._deadline is not None and time.perf_counter() >= self._deadline: + return None with self._iter_lock: + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None and stop_event.is_set(): + return None try: data_df = next(self._dataset_iter) except StopIteration: @@ -158,35 +190,48 @@ def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None] | del emb_np labels_data = None - if self.filters.type == FilterOp.StrEqual: + if self.filters.type == FilterOp.StrEqual or self.with_scalar_labels: + label_field = self.filters.label_field if self.filters.type == FilterOp.StrEqual else "labels" if self.dataset.data.scalar_labels_file_separated: - labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() + labels_data = self.dataset.scalar_labels[label_field][all_metadata].to_list() else: - labels_data = data_df[self.filters.label_field].tolist() + labels_data = data_df[label_field].tolist() + + tenant_labels_data = None + if self.tenant_case is not None and getattr(self.tenant_case, "is_multitenant", False): + tenant_labels_data = self.tenant_case.tenant_labels_for_ids(all_metadata) - return all_embeddings, all_metadata, labels_data + return all_embeddings, all_metadata, labels_data, tenant_labels_data def _worker_loop(self) -> int: """Worker loop: pull batches from the shared iterator and insert them.""" total = 0 - while True: - batch = self._next_batch() - if batch is None: - break - embeddings, metadata, labels_data = batch - total += self._worker_insert(embeddings, metadata, labels_data) + try: + while True: + batch = self._next_batch() + if batch is None: + break + embeddings, metadata, labels_data, tenant_labels_data = batch + total += self._worker_insert(embeddings, metadata, labels_data, tenant_labels_data) + except Exception: + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None: + stop_event.set() + raise return total def task(self) -> int: """Insert entire dataset using concurrent executor. Runs in subprocess.""" count = 0 self._iter_lock = threading.Lock() - self._dataset_iter = iter(self.dataset) + self._stop_event = threading.Event() + self._deadline = None if self.duration is None else time.perf_counter() + self.duration + self._dataset_iter = self.dataset.iter_batches(self.batch_size) with self.db.init(): log.info( f"({mp.current_process().name:16}) Start concurrent insert, " - f"batch_size={config.NUM_PER_BATCH}, max_workers={self.max_workers}" + f"batch_size={self.batch_size}, max_workers={self.max_workers}" ) start = time.perf_counter() diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index b7823af37..bb867b3f1 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -12,6 +12,7 @@ from hdrh.histogram import HdrHistogram from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.payload import PayloadProfile from ... import config from ...models import ConcurrencySlotTimeoutError @@ -45,10 +46,17 @@ def __init__( concurrencies: Iterable[int] = config.NUM_CONCURRENCY, duration: int = config.CONCURRENCY_DURATION, concurrency_timeout: int = config.CONCURRENCY_TIMEOUT, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant_labels: list[str] | None = None, ): self.db = db self.k = k self.filters = filters + self.payload_profile = payload_profile + self.tenant_labels = tenant_labels or [] + if not self.db.supports_payload_profile(self.payload_profile): + msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) self.concurrencies = concurrencies self.duration = duration self.concurrency_timeout = concurrency_timeout @@ -56,6 +64,15 @@ def __init__( self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") + def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list[int]: + if tenant is None: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile) + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k, tenant=tenant) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile, tenant=tenant) + def search( self, test_data: list[list[float]], @@ -75,6 +92,7 @@ def search( with self.db.init(): self.db.prepare_filter(self.filters) num, idx = len(test_data), random.randint(0, len(test_data) - 1) + tenant_rng = random.Random(mp.current_process().pid or 0) start_time = time.perf_counter() count = 0 @@ -82,7 +100,12 @@ def search( while time.perf_counter() < start_time + self.duration: s = time.perf_counter() try: - self.db.search_embedding(test_data[idx], self.k) + tenant = ( + self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] + if self.tenant_labels + else None + ) + self._search_embedding(test_data[idx], tenant=tenant) count += 1 latencies.append(time.perf_counter() - s) except Exception as e: @@ -341,7 +364,7 @@ def search_by_dur( while time.perf_counter() < start_time + dur: s = time.perf_counter() try: - self.db.search_embedding(test_data[idx], self.k) + self._search_embedding(test_data[idx]) success_count += 1 latency_us = int((time.perf_counter() - s) * US_TO_SECONDS) histogram.record_value(min(latency_us, HDR_HISTOGRAM_MAX_US)) diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index be0c6322d..3fc37e0ce 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -2,6 +2,7 @@ import logging import math import multiprocessing as mp +import random import time import traceback @@ -9,6 +10,7 @@ from vectordb_bench.backend.dataset import DatasetManager from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.payload import PayloadProfile from ... import config from ...metric import calc_ndcg, calc_recall, get_ideal_dcg @@ -130,10 +132,19 @@ def __init__( ground_truth: list[list[int]], k: int = 100, filters: Filter = non_filter, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant_labels: list[str] | None = None, + measure_recall: bool = True, ): self.db = db self.k = k self.filters = filters + self.payload_profile = payload_profile + self.tenant_labels = tenant_labels or [] + self.measure_recall = measure_recall + if not self.db.supports_payload_profile(self.payload_profile): + msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] @@ -141,13 +152,22 @@ def __init__( self.test_data = test_data self.ground_truth = ground_truth - def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list[int]: + if tenant is None: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile) + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k, tenant=tenant) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile, tenant=tenant) + + def _get_db_search_res(self, emb: list[float], tenant: str | None = None, retry_idx: int = 0) -> list[int]: try: - results = self.db.search_embedding(emb, self.k) + results = self._search_embedding(emb, tenant=tenant) except Exception as e: log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") if retry_idx < config.MAX_SEARCH_RETRY: - return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + return self._get_db_search_res(emb=emb, tenant=tenant, retry_idx=retry_idx + 1) msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" raise RuntimeError(msg) from e @@ -162,20 +182,24 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa ideal_dcg = get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {len(ground_truth)}") + log.debug(f"ground truth size: {len(ground_truth) if ground_truth is not None else 0}") latencies, recalls, ndcgs = [], [], [] + tenant_rng = random.Random(0) for idx, emb in enumerate(test_data): + tenant = ( + self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] if self.tenant_labels else None + ) s = time.perf_counter() try: - results = self._get_db_search_res(emb) + results = self._get_db_search_res(emb, tenant=tenant) except Exception as e: log.warning(f"VectorDB search_embedding error: {e}") raise e from None latencies.append(time.perf_counter() - s) - if ground_truth is not None: + if self.measure_recall and ground_truth is not None: gt = ground_truth[idx] recalls.append(calc_recall(self.k, gt[: self.k], results)) ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 6b51d1277..3a65c4d04 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -2,6 +2,7 @@ import hashlib import logging import re +import time import traceback from enum import Enum, auto @@ -15,6 +16,7 @@ from .clients import DB, MetricType, api from .data_source import DatasetSource from .runner import ( + ColdWarmSearchRunner, ConcurrentInsertRunner, MultiProcessingSearchRunner, ReadWriteRunner, @@ -56,28 +58,86 @@ class CaseRunner(BaseModel): search_runner: MultiProcessingSearchRunner | None = None final_search_runner: MultiProcessingSearchRunner | None = None read_write_runner: ReadWriteRunner | None = None + cold_warm_search_runner: ColdWarmSearchRunner | None = None def __eq__(self, obj: any): if isinstance(obj, CaseRunner): - return ( - self.ca.label == CaseLabel.Performance - and self.config.db == obj.config.db - and self.config.db_case_config == obj.config.db_case_config - and self.ca.dataset == obj.ca.dataset - ) + key = self.load_reuse_key() + return key is not None and key == obj.load_reuse_key() return False def __hash__(self) -> int: """Hash method to maintain consistency with __eq__ method.""" - return hash( - ( - self.ca.label, - self.config.db, - self.config.db_case_config, - self.ca.dataset, - ) + return hash(self.load_reuse_key()) + + def load_reuse_key(self) -> tuple | None: + if self.ca.label != CaseLabel.Performance: + return None + return ( + self.config.db.value, + self._db_config_hash_key(), + self._db_case_config_hash_key(), + self._collection_name_hash_key(), + self._dataset_hash_key(), + self.ca.with_scalar_labels, + self.ca.is_multitenant, + self._multitenant_routing_hash_key(), ) + @classmethod + def _hashable_value(cls, value: object) -> object: + if isinstance(value, dict): + hashable = tuple(sorted((str(k), cls._hashable_value(v)) for k, v in value.items())) + elif isinstance(value, (list, tuple)): + hashable = tuple(cls._hashable_value(v) for v in value) + elif isinstance(value, (set, frozenset)): + hashable = tuple(sorted((cls._hashable_value(v) for v in value), key=repr)) + elif isinstance(value, Enum): + hashable = value.value + elif hasattr(value, "model_dump"): + hashable = cls._hashable_value(value.model_dump(mode="json")) + elif hasattr(value, "get_secret_value"): + hashable = value.get_secret_value() + else: + hashable = value + return hashable + + def _db_config_hash_key(self) -> object: + db_config = self.config.db_config + if hasattr(db_config, "to_dict"): + return self._hashable_value(db_config.to_dict()) + return self._hashable_value(db_config) + + def _db_case_config_hash_key(self) -> object: + return self._hashable_value(self.config.db_case_config) + + def _collection_name_hash_key(self) -> str | None: + return self._doris_collection_name() + + def _dataset_hash_key(self) -> object: + return self._hashable_value(self.ca.dataset.data) + + def _multitenant_routing_hash_key(self) -> tuple | None: + if not self.ca.is_multitenant: + return None + return ( + getattr(self.ca, "tenant_count", None), + getattr(self.ca, "tenant_prefix", None), + getattr(self.ca, "tenant_id_width", None), + getattr(self.ca, "tenant_distribution", None), + ) + + def _doris_collection_name(self) -> str | None: + if self.config.db != DB.Doris: + return None + case_type_name = self.config.case_config.case_id.name + base = f"{case_type_name.lower()}" + base = re.sub(r"[^a-z0-9_]+", "_", base).strip("_") + if len(base) > 63: + h = hashlib.md5(base.encode(), usedforsecurity=False).hexdigest()[:6] + base = f"{base[:(63-7)]}_{h}" + return base + def display(self) -> dict: c_dict = self.ca.dict( include={ @@ -108,17 +168,7 @@ def init_db(self, drop_old: bool = True) -> None: # Compose a compact, case-unique collection/table name for Doris to avoid cross-case interference collection_name = None try: - if self.config.db == DB.Doris: - # Primary identifier = case-type enum name from CLI (e.g., Performance768D10M) - case_type_name = self.config.case_config.case_id.name - base = f"{case_type_name.lower()}" - # Sanitize to [a-z0-9_] - base = re.sub(r"[^a-z0-9_]+", "_", base).strip("_") - # Cap to 63 chars; add short hash if truncated - if len(base) > 63: - h = hashlib.md5(base.encode(), usedforsecurity=False).hexdigest()[:6] - base = f"{base[:(63-7)]}_{h}" - collection_name = base + collection_name = self._doris_collection_name() except Exception: # If anything goes wrong, fall back silently; Doris will use its default name logic collection_name = None @@ -128,23 +178,66 @@ def init_db(self, drop_old: bool = True) -> None: if "collection_name" in db_config_dict and not collection_name: collection_name = db_config_dict.pop("collection_name") + extra_db_kwargs = {} + if collection_name: + extra_db_kwargs["collection_name"] = collection_name + if self.ca.is_multitenant: + extra_db_kwargs["multitenant_tenant_labels"] = self.ca.tenant_labels() + self.db = db_cls( dim=self.ca.dataset.data.dim, db_config=db_config_dict, db_case_config=self.config.db_case_config, drop_old=drop_old, with_scalar_labels=self.ca.with_scalar_labels, - **({"collection_name": collection_name} if collection_name else {}), + **extra_db_kwargs, ) def _pre_run(self, drop_old: bool = True): try: + self._validate_cloud_cold_latency_config(drop_old) + creates_multitenant_collection = ( + TaskStage.DROP_OLD in self.config.stages or TaskStage.LOAD in self.config.stages + ) + if ( + self.ca.is_multitenant + and self.config.db in {DB.Milvus, DB.ZillizCloud} + and creates_multitenant_collection + and not getattr(self.config.db_case_config, "use_partition_key", False) + ): + msg = "CloudMultiTenantSearchCase requires use_partition_key=True for Milvus/ZillizCloud" + raise ValueError(msg) self.init_db(drop_old) - self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filters) + if self.ca.is_multitenant and self.db is not None: + if not self.db.supports_multitenant(): + msg = f"{self.config.db_name} does not support CloudMultiTenantSearchCase" + raise NotImplementedError(msg) + self.db.set_multitenant_context(self.ca.tenant_labels()) + if self.config.db in {DB.Milvus, DB.ZillizCloud} and not creates_multitenant_collection: + self.db.validate_multitenant_schema() + self.ca.dataset.prepare( + self.dataset_source, + filters=self.ca.filters, + with_train_files=TaskStage.LOAD in self.config.stages, + with_scalar_labels=self.ca.with_scalar_labels, + ) except ModuleNotFoundError as e: log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") raise e from None + def _validate_cloud_cold_latency_config(self, drop_old: bool) -> None: + if getattr(self.ca, "label", None) != CaseLabel.CloudColdLatency: + return + if drop_old: + msg = ( + "CloudColdLatencyCase requires an existing cold collection. " + "Run with --skip-drop-old and --skip-load." + ) + raise ValueError(msg) + if TaskStage.LOAD in self.config.stages: + msg = "CloudColdLatencyCase is search-only. Run with --skip-load." + raise ValueError(msg) + def run(self, drop_old: bool = True) -> Metric: log.info("Starting run") @@ -156,6 +249,10 @@ def run(self, drop_old: bool = True) -> Metric: return self._run_perf_case(drop_old) if self.ca.label == CaseLabel.Streaming: return self._run_streaming_case() + if self.ca.label == CaseLabel.CloudInsert: + return self._run_cloud_insert_case() + if self.ca.label == CaseLabel.CloudColdLatency: + return self._run_cloud_cold_latency_case(drop_old) msg = f"unknown case type: {self.ca.label}" log.warning(msg) raise ValueError(msg) @@ -223,6 +320,8 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() m.recall, m.ndcg, m.serial_latency_p99, m.serial_latency_p95 = search_results + m.payload_profile = self.ca.payload_profile.value + m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query(self.config.case_config.k) except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") @@ -245,10 +344,111 @@ def _run_streaming_case(self) -> Metric: log.info(f"Streaming case got result: {m}") return m + def _run_cloud_insert_case(self) -> Metric: + assert self.db is not None + started = time.perf_counter() + runner_kwargs = {} + if self.ca.is_multitenant: + runner_kwargs["tenant_case"] = self.ca + runner = ConcurrentInsertRunner( + self.db, + self.ca.dataset, + self.normalize, + self.ca.filters, + max_workers=self.config.load_concurrency or None, + batch_size=self.ca.batch_size, + duration=self.ca.duration, + **runner_kwargs, + ) + count = runner.task() + insert_done = time.perf_counter() + readiness_timeout = self.ca.readiness_timeout + readiness_poll_interval = self.ca.readiness_poll_interval + readiness_deadline = None if readiness_timeout is None else time.perf_counter() + readiness_timeout + with self.db.init(): + status = self.db.poll_insert_readiness(count) + searchable_started = time.perf_counter() + while not status["fully_searchable"]: + if readiness_deadline is not None and time.perf_counter() >= readiness_deadline: + msg = ( + "Cloud insert readiness timed out waiting for fully_searchable " + f"after {readiness_timeout}s; last_status={status}" + ) + raise TimeoutError(msg) + time.sleep(readiness_poll_interval) + status = self.db.poll_insert_readiness(count) + indexed_started = time.perf_counter() + while not status["fully_indexed"]: + if readiness_deadline is not None and time.perf_counter() >= readiness_deadline: + msg = ( + "Cloud insert readiness timed out waiting for fully_indexed " + f"after {readiness_timeout}s; last_status={status}" + ) + raise TimeoutError(msg) + time.sleep(readiness_poll_interval) + status = self.db.poll_insert_readiness(count) + return Metric( + inserted_count=count, + insert_rows_per_second=round(count / max(insert_done - started, 0.001), 4), + insert_completion_seconds=round(insert_done - started, 4), + searchable_after_insert_seconds=round(indexed_started - searchable_started, 4), + indexed_after_searchable_seconds=round(time.perf_counter() - indexed_started, 4), + additional_parameters=status.get("additional_parameters", {}), + ) + + def _init_cold_warm_search_runner(self) -> None: + if self.normalize: + test_emb = np.stack(self.ca.dataset.test_data) + test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] + self.test_emb = test_emb.tolist() + else: + self.test_emb = self.ca.dataset.test_data + + self.cold_warm_search_runner = ColdWarmSearchRunner( + db=self.db, + test_data=self.test_emb, + filters=self.ca.filters, + k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + query_count=self.ca.query_count, + ) + + def _run_cloud_cold_latency_case(self, drop_old: bool = True) -> Metric: + log.info("Start cloud cold latency case") + try: + self._validate_cloud_cold_latency_config(drop_old) + m = Metric() + if drop_old: + if TaskStage.LOAD in self.config.stages: + _, load_dur = self._load_train_data() + build_dur = self._optimize() + m.insert_duration = round(load_dur, 4) + m.optimize_duration = round(build_dur, 4) + m.load_duration = round(load_dur + build_dur, 4) + else: + log.info("Data loading skipped") + + self._init_cold_warm_search_runner() + m.additional_parameters = { + "cold_latency": self.cold_warm_search_runner.run(), + } + m.payload_profile = self.ca.payload_profile.value + m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query(self.config.case_config.k) + except Exception as e: + log.warning(f"Failed to run cloud cold latency case, reason = {e}") + traceback.print_exc() + raise e from None + else: + log.info(f"Cloud cold latency case got result: {m}") + return m + @utils.time_it def _load_train_data(self): """Insert train data concurrently and get the insert_duration""" try: + runner_kwargs = {} + if self.ca.is_multitenant: + runner_kwargs["tenant_case"] = self.ca runner = ConcurrentInsertRunner( self.db, self.ca.dataset, @@ -256,6 +456,8 @@ def _load_train_data(self): self.ca.filters, self.ca.load_timeout, max_workers=self.config.load_concurrency or None, + with_scalar_labels=self.ca.with_scalar_labels, + **runner_kwargs, ) runner.run() except Exception as e: @@ -320,7 +522,9 @@ def _init_search_runner(self): else: self.test_emb = self.ca.dataset.test_data - gt_df = self.ca.dataset.gt_data + tenant_labels = self.ca.tenant_labels() if self.ca.is_multitenant else None + measure_recall = getattr(self.ca, "measure_recall", True) + gt_df = self.ca.dataset.gt_data if measure_recall else None if TaskStage.SEARCH_SERIAL in self.config.stages: self.serial_search_runner = SerialSearchRunner( @@ -329,6 +533,9 @@ def _init_search_runner(self): ground_truth=gt_df, filters=self.ca.filters, k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + tenant_labels=tenant_labels, + measure_recall=measure_recall, ) if TaskStage.SEARCH_CONCURRENT in self.config.stages: self.search_runner = MultiProcessingSearchRunner( @@ -339,6 +546,8 @@ def _init_search_runner(self): duration=self.config.case_config.concurrency_search_config.concurrency_duration, concurrency_timeout=self.config.case_config.concurrency_search_config.concurrency_timeout, k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + tenant_labels=tenant_labels, ) def _init_read_write_runner(self): diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 94b13762a..cf5c8fecd 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -19,6 +19,7 @@ from .. import config from ..backend.clients import DB from ..backend.clients.api import MetricType +from ..backend.dataset import DatasetWithSizeType from ..interface import benchmark_runner from ..models import ( CaseConfig, @@ -35,6 +36,20 @@ except ImportError: from yaml import Loader +DEFAULT_DATASET_WITH_SIZE_TYPE = DatasetWithSizeType.CohereMedium.value +SUPPORTED_DATASET_WITH_SIZE_TYPES = "|".join(dataset.value for dataset in DatasetWithSizeType) + + +def copy_if_not_none( + custom_case_config: dict[str, Any], + parameters: dict[str, Any], + key: str, + target_key: str | None = None, +) -> None: + value = parameters[key] + if value is not None: + custom_case_config[target_key or key] = value + def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 if value: @@ -165,6 +180,7 @@ def check_custom_case_parameters(ctx: any, param: any, value: any): # noqa: ARG def get_custom_case_config(parameters: dict) -> dict: custom_case_config = {} + dataset_with_size_type = parameters["dataset_with_size_type"] or DEFAULT_DATASET_WITH_SIZE_TYPE if parameters["case_type"] == "PerformanceCustomDataset": custom_case_config = { "name": parameters["custom_case_name"], @@ -184,14 +200,56 @@ def get_custom_case_config(parameters: dict) -> dict: } elif parameters["case_type"] == "NewIntFilterPerformanceCase": custom_case_config = { - "dataset_with_size_type": parameters["dataset_with_size_type"], + "dataset_with_size_type": dataset_with_size_type, "filter_rate": parameters["filter_rate"], } elif parameters["case_type"] == "LabelFilterPerformanceCase": custom_case_config = { - "dataset_with_size_type": parameters["dataset_with_size_type"], + "dataset_with_size_type": dataset_with_size_type, "label_percentage": parameters["label_percentage"], } + elif parameters["case_type"] == "CloudPayloadSearchCase": + custom_case_config = { + "payload_profile": parameters["payload_profile"], + } + copy_if_not_none(custom_case_config, parameters, "dataset_with_size_type") + if parameters["cloud_filter_rate"] is not None: + custom_case_config["filter_rate"] = parameters["cloud_filter_rate"] + if parameters["cloud_label_percentage"] is not None: + custom_case_config["label_percentage"] = parameters["cloud_label_percentage"] + elif parameters["case_type"] == "CloudColdLatencyCase": + custom_case_config = { + "payload_profile": parameters["payload_profile"], + "query_count": parameters["cloud_cold_query_count"], + } + copy_if_not_none(custom_case_config, parameters, "dataset_with_size_type") + copy_if_not_none(custom_case_config, parameters, "cloud_filter_rate", "filter_rate") + copy_if_not_none(custom_case_config, parameters, "cloud_label_percentage", "label_percentage") + elif parameters["case_type"] == "CloudInsertCase": + custom_case_config = { + "batch_size": parameters["cloud_insert_batch_size"], + "duration": parameters["cloud_insert_duration"], + "dataset_with_size_type": dataset_with_size_type, + } + copy_if_not_none(custom_case_config, parameters, "cloud_insert_readiness_timeout", "readiness_timeout") + copy_if_not_none( + custom_case_config, + parameters, + "cloud_insert_readiness_poll_interval", + "readiness_poll_interval", + ) + elif parameters["case_type"] == "CloudMultiTenantSearchCase": + custom_case_config = { + "tenant_count": parameters["tenant_count"], + "tenant_prefix": parameters["tenant_prefix"], + "tenant_id_width": parameters["tenant_id_width"], + "payload_profile": parameters["payload_profile"], + } + copy_if_not_none(custom_case_config, parameters, "dataset_with_size_type") + if parameters["cloud_filter_rate"] is not None: + custom_case_config["filter_rate"] = parameters["cloud_filter_rate"] + if parameters["cloud_label_percentage"] is not None: + custom_case_config["label_percentage"] = parameters["cloud_label_percentage"] return custom_case_config @@ -436,14 +494,13 @@ class CommonTypedDict(TypedDict): ] task_label: Annotated[str, click.option("--task-label", help="Task label")] dataset_with_size_type: Annotated[ - str, + str | None, click.option( "--dataset-with-size-type", - help="Dataset with size type for NewIntFilterPerformanceCase/LabelFilterPerformanceCase, you can use " - "Medium Cohere (768dim, 1M)|Large Cohere (768dim, 10M)|Medium Bioasq (1024dim, 1M)|" - "Large Bioasq (1024dim, 10M)|Large OpenAI (1536dim, 5M)|Medium OpenAI (1536dim, 500K)", - default="Medium Cohere (768dim, 1M)", - show_default=True, + help="Dataset with size type. When omitted, filter/insert cases use Medium Cohere (768dim, 1M), " + "CloudPayloadSearchCase and CloudColdLatencyCase use LAION 100M, and CloudMultiTenantSearchCase " + f"uses Large Cohere (768dim, 10M). Supported values include {SUPPORTED_DATASET_WITH_SIZE_TYPES}", + default=None, ), ] filter_rate: Annotated[ @@ -464,6 +521,111 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + payload_profile: Annotated[ + str, + click.option( + "--payload-profile", + type=click.Choice(["ids_only", "vector", "scalar_label"]), + help="Response payload profile for CloudPayloadSearchCase and CloudColdLatencyCase", + default="ids_only", + show_default=True, + ), + ] + cloud_filter_rate: Annotated[ + float | None, + click.option( + "--cloud-filter-rate", + type=float, + default=None, + help="Optional int filter rate for CloudPayloadSearchCase and CloudColdLatencyCase", + ), + ] + cloud_label_percentage: Annotated[ + float | None, + click.option( + "--cloud-label-percentage", + type=float, + default=None, + help="Optional label percentage for CloudPayloadSearchCase and CloudColdLatencyCase", + ), + ] + cloud_cold_query_count: Annotated[ + int, + click.option( + "--cloud-cold-query-count", + type=int, + default=1000, + show_default=True, + help="Number of serial queries per cold/warm pass for CloudColdLatencyCase", + ), + ] + cloud_insert_batch_size: Annotated[ + int, + click.option( + "--cloud-insert-batch-size", + type=int, + default=5000, + show_default=True, + help="Insert batch size for CloudInsertCase", + ), + ] + cloud_insert_duration: Annotated[ + float | None, + click.option( + "--cloud-insert-duration", + type=float, + default=None, + help="Optional insert duration in seconds for CloudInsertCase", + ), + ] + cloud_insert_readiness_timeout: Annotated[ + float | None, + click.option( + "--cloud-insert-readiness-timeout", + type=float, + default=None, + help="Optional readiness polling timeout in seconds for CloudInsertCase", + ), + ] + cloud_insert_readiness_poll_interval: Annotated[ + float | None, + click.option( + "--cloud-insert-readiness-poll-interval", + type=float, + default=None, + help="Optional readiness polling interval in seconds for CloudInsertCase", + ), + ] + tenant_count: Annotated[ + int, + click.option( + "--tenant-count", + type=int, + default=1000, + show_default=True, + help="Tenant count for CloudMultiTenantSearchCase", + ), + ] + tenant_prefix: Annotated[ + str, + click.option( + "--tenant-prefix", + type=str, + default="tenant_", + show_default=True, + help="Tenant label prefix for CloudMultiTenantSearchCase", + ), + ] + tenant_id_width: Annotated[ + int, + click.option( + "--tenant-id-width", + type=int, + default=4, + show_default=True, + help="Zero-padding width for CloudMultiTenantSearchCase tenant IDs", + ), + ] class HNSWBaseTypedDict(TypedDict): diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index eca3dbc52..13c9687c7 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -39,7 +39,7 @@ from ..backend.clients.tencent_elasticsearch.cli import TencentElasticsearch from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB -from ..backend.clients.turbopuffer.cli import TurboPuffer +from ..backend.clients.turbopuffer.cli import TurboPuffer, TurboPufferUnpin from ..backend.clients.vectorchord.cli import VectorChordGraph, VectorChordRQ from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate @@ -83,6 +83,7 @@ cli.add_command(AliSQLHNSW) cli.add_command(Doris) cli.add_command(TurboPuffer) +cli.add_command(TurboPufferUnpin) cli.add_command(Chroma) cli.add_command(Zvec) cli.add_command(Endee) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 0d4119e93..8b603be24 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -163,7 +163,7 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non return c_results = [] - latest_runner, cached_load_duration = None, None + latest_loaded_reuse_key, cached_load_duration = None, None for idx, runner in enumerate(running_task.case_runners): case_res = CaseResult( metrics=Metric(), @@ -171,7 +171,10 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non ) drop_old = TaskStage.DROP_OLD in runner.config.stages - if (latest_runner and runner == latest_runner) or not self.drop_old: + reuse_key = runner.load_reuse_key() + if reuse_key is not None and reuse_key == latest_loaded_reuse_key: + drop_old = False + if not self.drop_old: drop_old = False num_cases = running_task.num_cases() try: @@ -182,14 +185,12 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non f"result={case_res.metrics}, label={case_res.label}" ) - # cache the latest succeeded runner - latest_runner = runner - - # cache the latest drop_old=True load_duration of the latest succeeded runner - cached_load_duration = case_res.metrics.load_duration if drop_old else cached_load_duration + if drop_old and TaskStage.LOAD in runner.config.stages and reuse_key is not None: + latest_loaded_reuse_key = reuse_key + cached_load_duration = case_res.metrics.load_duration # use the cached load duration if this case didn't drop the existing collection - if not drop_old: + if not drop_old and reuse_key is not None and reuse_key == latest_loaded_reuse_key: case_res.metrics.load_duration = cached_load_duration if cached_load_duration else 0.0 except (LoadTimeoutError, PerformanceTimeoutError) as e: log.warning(f"[{idx+1}/{num_cases}] case {runner.display()} failed to run, reason={e}") diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index 3634b2114..5a7c14e82 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -29,6 +29,15 @@ class Metric: conc_latency_p99_list: list[float] = field(default_factory=list) conc_latency_p95_list: list[float] = field(default_factory=list) conc_latency_avg_list: list[float] = field(default_factory=list) + payload_profile: str = "ids_only" + payload_estimated_bytes_per_query: int = 0 + + inserted_count: int = 0 + insert_rows_per_second: float = 0.0 + insert_completion_seconds: float = 0.0 + searchable_after_insert_seconds: float = 0.0 + indexed_after_searchable_seconds: float = 0.0 + additional_parameters: dict = field(default_factory=dict) # for streaming cases st_ideal_insert_duration: int = 0 diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index a7e7c09f1..dc1709cc0 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -1,8 +1,9 @@ import logging import pathlib +from dataclasses import asdict from datetime import date, datetime from enum import Enum, StrEnum -from typing import Self +from typing import Any, ClassVar, Self import ujson @@ -149,6 +150,7 @@ class CaseConfigParamType(Enum): dataset_with_size_type = "dataset_with_size_type" filter_rate = "filter_rate" + payload_profile = "payload_profile" insert_rate = "insert_rate" search_stages = "search_stages" concurrencies = "concurrencies" @@ -276,6 +278,68 @@ class TestResult(BaseModel): file_fmt: str = "result_{}_{}_{}.json" # result_20230718_statndard_milvus.json timestamp: float = 0.0 + sensitive_output_fields: ClassVar[set[str]] = {"api_key", "password", "token"} + + @classmethod + def _redact_sensitive_fields(cls, value: Any) -> Any: + if isinstance(value, dict): + return { + key: ( + "**********" + if key.lower() in cls.sensitive_output_fields and item + else cls._redact_sensitive_fields(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [cls._redact_sensitive_fields(item) for item in value] + return value + + @staticmethod + def _output_metrics_for_case(case_result: CaseResult) -> dict: + metrics = asdict(case_result.metrics) + case_id = case_result.task_config.case_config.case_id + + if case_id == CaseType.CloudInsertCase: + return { + "inserted_count": metrics["inserted_count"], + "insert_rows_per_second": metrics["insert_rows_per_second"], + "insert_completion_seconds": metrics["insert_completion_seconds"], + "searchable_after_insert_seconds": metrics["searchable_after_insert_seconds"], + "indexed_after_searchable_seconds": metrics["indexed_after_searchable_seconds"], + "additional_parameters": metrics["additional_parameters"], + } + + if case_id == CaseType.CloudColdLatencyCase: + return { + "insert_duration": metrics["insert_duration"], + "optimize_duration": metrics["optimize_duration"], + "load_duration": metrics["load_duration"], + "payload_profile": metrics["payload_profile"], + "payload_estimated_bytes_per_query": metrics["payload_estimated_bytes_per_query"], + "cold_latency": metrics["additional_parameters"].get("cold_latency", {}), + } + + return metrics + + @staticmethod + def _output_case_config_for_case(case_result: CaseResult) -> dict: + case_config = case_result.task_config.case_config + + if case_config.case_id in {CaseType.CloudInsertCase, CaseType.CloudColdLatencyCase}: + return { + "case_id": case_config.case_id.value, + "custom_case": case_config.custom_case, + } + + return case_config.model_dump(mode="json") + + def model_dump_for_output(self) -> dict: + output = self.model_dump(mode="json", serialize_as_any=True) + for idx, case_result in enumerate(self.results): + output["results"][idx]["metrics"] = self._output_metrics_for_case(case_result) + output["results"][idx]["task_config"]["case_config"] = self._output_case_config_for_case(case_result) + return self._redact_sensitive_fields(output) def flush(self): db2case = self.get_db_results() @@ -314,8 +378,8 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"write results to disk {result_file}") with pathlib.Path(result_file).open("w") as f: - b = partial.model_dump_json(exclude={"db_config": {"password", "api_key"}}) - f.write(b) + f.write(ujson.dumps(partial.model_dump_for_output(), indent=2)) + f.write("\n") def get_case_config(case_config: CaseConfig) -> dict[CaseConfig]: if case_config["case_id"] in {6, 7, 8, 9, 12, 13, 14, 15}: @@ -360,27 +424,31 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: task_config["case_config"] = cls.get_case_config(case_config=case_config) case_result["task_config"] = task_config - - if trans_unit: - cur_max_count = case_result["metrics"]["max_load_count"] - case_result["metrics"]["max_load_count"] = ( - cur_max_count / 1000 if int(cur_max_count) > 0 else cur_max_count - ) - - cur_latency = case_result["metrics"]["serial_latency_p99"] - case_result["metrics"]["serial_latency_p99"] = ( - cur_latency * 1000 if cur_latency > 0 else cur_latency - ) - - # Handle P95 latency for backward compatibility with existing result files - if "serial_latency_p95" in case_result["metrics"]: - cur_latency_p95 = case_result["metrics"]["serial_latency_p95"] - case_result["metrics"]["serial_latency_p95"] = ( + metrics = case_result.get("metrics") + if ( + metrics + and CaseType(case_config.get("case_id")) == CaseType.CloudColdLatencyCase + and "cold_latency" in metrics + ): + metrics.setdefault("additional_parameters", {})["cold_latency"] = metrics.pop("cold_latency") + + if trans_unit and metrics: + if "max_load_count" in metrics: + cur_max_count = metrics["max_load_count"] + metrics["max_load_count"] = cur_max_count / 1000 if int(cur_max_count) > 0 else cur_max_count + + if "serial_latency_p99" in metrics: + cur_latency = metrics["serial_latency_p99"] + metrics["serial_latency_p99"] = cur_latency * 1000 if cur_latency > 0 else cur_latency + + # Handle P95 latency for backward compatibility with existing result files. + if "serial_latency_p95" in metrics: + cur_latency_p95 = metrics["serial_latency_p95"] + metrics["serial_latency_p95"] = ( cur_latency_p95 * 1000 if cur_latency_p95 > 0 else cur_latency_p95 ) - else: - # Default to 0 for older result files that don't have P95 data - case_result["metrics"]["serial_latency_p95"] = 0.0 + elif "serial_latency_p99" in metrics: + metrics["serial_latency_p95"] = 0.0 return TestResult.model_validate(test_result) def display(self, dbs: list[DB] | None = None): From 625a64ab06cd5f5aeb3535ee60cb97b3320b1cc6 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Tue, 7 Jul 2026 22:14:03 -0700 Subject: [PATCH 33/38] Make Antfly VDBBench queries ids-only --- vectordb_bench/backend/clients/antfly/antfly.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index e29d109e8..bf4ff5063 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -11,6 +11,7 @@ import httpx from ..api import DBCaseConfig, MetricType, VectorDB +from ...payload import PayloadProfile log = logging.getLogger(__name__) @@ -442,6 +443,7 @@ def _metadata_query_body(self, query: list[float], k: int) -> dict[str, Any]: return { "embeddings": {"vec": self._serialize_query_vector(query)}, "limit": k, + "fields": [], **self.case_config.search_param(), } @@ -548,10 +550,15 @@ def search_embedding( self, query: list[float], k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, filters: dict | None = None, timeout: int | None = None, **kwargs: Any, ) -> list[int]: + if payload_profile != PayloadProfile.IDS_ONLY: + raise NotImplementedError( + f"Antfly VDBBench adapter only supports payload_profile={PayloadProfile.IDS_ONLY.value}" + ) if self._uses_cosine_distance(): query = self._normalize_vector(query) From 4bd5621f9455f53bb96da7fb93ecf354f6f02dae Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Wed, 8 Jul 2026 00:54:02 -0700 Subject: [PATCH 34/38] Use case metric for Antfly VDBBench runs --- .../backend/clients/antfly/antfly.py | 12 ++++++------ vectordb_bench/backend/clients/antfly/cli.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index bf4ff5063..27826eb40 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -111,14 +111,14 @@ def __init__( ) log.info(f"Create table response: {r.status_code}") r.raise_for_status() + self._wait_for_shard_ready(client) + # Wait for the write path before touching indexes: legacy (Go) + # binaries can permanently orphan a shard if an index-add lands + # while the shard is still initializing. + self._wait_for_write_ready(client) else: log.info("Reusing existing table: %s", self.collection_name) - - self._wait_for_shard_ready(client) - # Wait for the write path before touching indexes: legacy (Go) - # binaries can permanently orphan a shard if an index-add lands - # while the shard is still initializing. - self._wait_for_write_ready(client) + self._wait_for_shard_ready(client) if self._get_index_status(client) is None: index_def = { diff --git a/vectordb_bench/backend/clients/antfly/cli.py b/vectordb_bench/backend/clients/antfly/cli.py index 2c8566521..0bb40db99 100644 --- a/vectordb_bench/backend/clients/antfly/cli.py +++ b/vectordb_bench/backend/clients/antfly/cli.py @@ -9,6 +9,8 @@ click_parameter_decorators_from_typed_dict, run, ) +from ...cases import CaseType +from ..api import MetricType from .. import DB @@ -56,6 +58,16 @@ class AntflyTypedDict(TypedDict): show_default=True, ), ] + metric_type: Annotated[ + str | None, + click.option( + "--metric-type", + type=click.Choice([metric.value for metric in MetricType]), + help="Distance metric override. Defaults to the selected VDBBench case dataset metric.", + default=None, + show_default=True, + ), + ] class AntflyAKNNTypedDict(CommonTypedDict, AntflyTypedDict): ... @@ -66,6 +78,12 @@ class AntflyAKNNTypedDict(CommonTypedDict, AntflyTypedDict): ... def AntflyAKNN(**parameters: Unpack[AntflyAKNNTypedDict]): from .config import AntflyConfig, AntflyIndexConfig + metric_type = ( + MetricType(parameters["metric_type"]) + if parameters["metric_type"] + else CaseType[parameters["case_type"]].case_cls(parameters.get("custom_case")).dataset.data.metric_type + ) + run( db=DB.Antfly, db_config=AntflyConfig( @@ -81,6 +99,7 @@ def AntflyAKNN(**parameters: Unpack[AntflyAKNNTypedDict]): pack_query_vectors=parameters["pack_query_vectors"], ), db_case_config=AntflyIndexConfig( + metric_type=metric_type, num_shards=parameters["num_shards"], search_effort=parameters["search_effort"], ), From 8a8e2a8244b74930cfcf8e7cd759c005b12216a6 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Wed, 8 Jul 2026 17:13:14 -0700 Subject: [PATCH 35/38] Tune Antfly VDBBench sync behavior --- vectordb_bench/backend/clients/antfly/antfly.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index 27826eb40..3a1d01694 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -36,6 +36,12 @@ def _pack_dense_f32(values: list[float]) -> str: def _make_client(base_url: str, timeout: float) -> httpx.Client: + raw_timeout = os.environ.get("ANTFLY_VDBBENCH_HTTP_TIMEOUT") + if raw_timeout: + try: + timeout = float(raw_timeout) + except ValueError: + log.warning("Ignoring invalid ANTFLY_VDBBENCH_HTTP_TIMEOUT=%r", raw_timeout) return httpx.Client( base_url=base_url, timeout=timeout, @@ -89,6 +95,10 @@ def __init__( self._store_port = db_config.get("store_port") self._use_direct_store_search = bool(db_config.get("use_direct_store_search")) self._pack_query_vectors = bool(db_config.get("pack_query_vectors")) + self._write_sync_level = os.environ.get( + "ANTFLY_VDBBENCH_SYNC_LEVEL", + "write" if self._legacy_api else "full_index", + ) self._direct_shard_id: str | None = None self._bench_status_last_log = 0.0 num_shards = db_config.get("num_shards", 1) @@ -535,7 +545,7 @@ def insert_embeddings( SOURCE_FIELD: str(metadata[i]), "_embeddings": {"vec": serialized_embedding}, } - payload = {"inserts": inserts, "sync_level": "write"} + payload = {"inserts": inserts, "sync_level": self._write_sync_level} r = self.client.post( f"/tables/{self.collection_name}/batch", json=payload ) From 4af3ec43dee350c082390868e4efce5da4223f5a Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Thu, 9 Jul 2026 15:13:14 -0700 Subject: [PATCH 36/38] Keep Antfly write probe alive and pace async-sync loads Two fixes for invalid 1M benchmark rows: - The write-readiness probe no longer deletes its doc. The tombstone permanently disabled the all-docs-visible fast path, so every dense query materialized the full live-doc set as a positive filter (~40ms/query at 1M docs, linear in table size). - TEMPORARY: pace inserts on the server's catch-up backlog (catch_up_target_sequence - applied) when using async sync levels. Unthrottled write-sync loads outrun dense catch-up + LSM compaction and wedge the write path mid-load. Env-tunable via ANTFLY_VDBBENCH_MAX_LAG_SEQ / RESUME_LAG_SEQ / PACE_EVERY; remove once the server applies its own ingest backpressure. --- .../backend/clients/antfly/antfly.py | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index 3a1d01694..21c148882 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -97,8 +97,20 @@ def __init__( self._pack_query_vectors = bool(db_config.get("pack_query_vectors")) self._write_sync_level = os.environ.get( "ANTFLY_VDBBENCH_SYNC_LEVEL", - "write" if self._legacy_api else "full_index", + "write", ) + # TEMPORARY client-side backpressure for async sync levels, until the + # server applies its own under sustained ingest. With sync_level + # "write" nothing throttles inserts, and a sustained 1M load outruns + # dense catch-up + LSM compaction until the write path stalls + # (observed as insert timeouts around 200-600k docs). Pause between + # batches whenever the server-reported catch-up backlog exceeds + # max_lag sequences (~100 docs per sequence), resuming at resume_lag. + # Remove once the server keeps catch-up healthy on its own. + self._pace_max_lag = int(os.environ.get("ANTFLY_VDBBENCH_MAX_LAG_SEQ", "200")) + self._pace_resume_lag = int(os.environ.get("ANTFLY_VDBBENCH_RESUME_LAG_SEQ", "50")) + self._pace_check_every = int(os.environ.get("ANTFLY_VDBBENCH_PACE_EVERY", "5")) + self._pace_batches_since_check = 0 self._direct_shard_id: str | None = None self._bench_status_last_log = 0.0 num_shards = db_config.get("num_shards", 1) @@ -195,7 +207,11 @@ def _wait_for_write_ready(self, client: httpx.Client): # legacy (Go) binaries return 500 "shard is still initializing" for a # few seconds after table creation. Probe with a throwaway document # (no embedding, so it never lands in the vector index) until a batch - # write succeeds, then delete it. + # write succeeds. The probe doc is intentionally left in place: deleting + # it would leave a tombstone, and a table that has ever deleted a doc + # loses the "all docs visible" fast path — every dense query then + # materializes the full live-doc set as a positive filter, which costs + # O(table size) per query (~40ms/query at 1M docs). probe_key = "key:__circus_write_probe__" deadline = time.monotonic() + TABLE_READY_TIMEOUT last_error = None @@ -206,10 +222,6 @@ def _wait_for_write_ready(self, client: httpx.Client): json={"inserts": {probe_key: {"id": -1}}, "sync_level": "write"}, ) if r.is_success: - client.post( - f"/tables/{self.collection_name}/batch", - json={"deletes": [probe_key], "sync_level": "write"}, - ) log.info("Write path is ready") return last_error = f"{r.status_code}: {r.text[:120]}" @@ -521,6 +533,33 @@ def optimize(self, data_size: int | None = None): self._wait_for_index_ready(client, expected_total=data_size) self._maybe_log_bench_status(client, "optimize_end", force=True) + def _catch_up_lag_sequences(self) -> int | None: + try: + payload = self._get_index_status(self.client) + status = (payload or {}).get("status") or {} + applied = status.get("catch_up_applied_sequence") + target = status.get("catch_up_target_sequence") + if applied is None or target is None: + return None + return max(0, int(target) - int(applied)) + except Exception: + return None + + def _pace_async_indexing(self) -> None: + if self._pace_max_lag <= 0 or self._write_sync_level == "full_index": + return + self._pace_batches_since_check += 1 + if self._pace_batches_since_check < self._pace_check_every: + return + self._pace_batches_since_check = 0 + lag = self._catch_up_lag_sequences() + if lag is None or lag <= self._pace_max_lag: + return + log.info("Antfly pacing: catch-up lag %d sequences, waiting", lag) + while lag is not None and lag > self._pace_resume_lag: + time.sleep(1) + lag = self._catch_up_lag_sequences() + def insert_embeddings( self, embeddings: list[list[float]], @@ -551,6 +590,7 @@ def insert_embeddings( ) r.raise_for_status() self._maybe_log_bench_status(self.client, "insert") + self._pace_async_indexing() except Exception as e: log.warning(f"Antfly insert error: {e}") return 0, e From 01d38075644dd42f22584e1f5f51d512bfb99800 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Thu, 9 Jul 2026 23:45:15 -0700 Subject: [PATCH 37/38] Fix benchmark client compatibility --- pyproject.toml | 8 ++++---- vectordb_bench/backend/clients/chroma/chroma.py | 6 +++--- vectordb_bench/backend/clients/pgvector/pgvector.py | 5 ++++- .../backend/clients/weaviate_cloud/weaviate_cloud.py | 3 +++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b8f11e37..a8ba51413 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,8 +56,8 @@ all = [ "grpcio-tools==1.53.0", # for qdrant-client and pymilvus "qdrant-client", "pinecone", - "weaviate-client", - "elasticsearch", + "weaviate-client>=3.26.7,<4.0.0", + "elasticsearch==8.16.0", "sqlalchemy", "redis", "chromadb", @@ -83,8 +83,8 @@ all = [ ] qdrant = [ "qdrant-client" ] pinecone = [ "pinecone" ] -weaviate = [ "weaviate-client" ] -elastic = [ "elasticsearch" ] +weaviate = [ "weaviate-client>=3.26.7,<4.0.0" ] +elastic = [ "elasticsearch==8.16.0" ] # For elastic and aliyun_elasticsearch pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index 6942f4fc9..f2065ab8b 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -35,10 +35,10 @@ def __init__( if drop_old: try: - client.reset() - except Exception: - drop_old = False log.info(f"Chroma client drop_old collection: {self.collection_name}") + client.delete_collection(self.collection_name) + except Exception as e: + log.info(f"Chroma client collection was not dropped: {self.collection_name}, error: {e!s}") self.client = None self.collection = None diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 41060af27..002987fad 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -90,8 +90,11 @@ def __init__( @staticmethod def _create_connection(**kwargs) -> tuple[Connection, Cursor]: conn = psycopg.connect(**kwargs) - register_vector(conn) conn.autocommit = False + with conn.cursor() as cursor: + cursor.execute("CREATE EXTENSION IF NOT EXISTS vector") + conn.commit() + register_vector(conn) cursor = conn.cursor() assert conn is not None, "Connection is not initialized" diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index d6111c8da..0bcfc56d6 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -13,6 +13,8 @@ class WeaviateCloud(VectorDB): + thread_safe: bool = False + def __init__( self, dim: int, @@ -23,6 +25,7 @@ def __init__( **kwargs, ): """Initialize wrapper around the weaviate vector database.""" + self.name = "WeaviateCloud" db_config.update( { "auth_client_secret": weaviate.AuthApiKey( From d385cea2a21a97b2578a68bf292bb02d2406e0ad Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Sun, 12 Jul 2026 21:50:31 -0700 Subject: [PATCH 38/38] Add filtered ANN adapter support --- .../backend/clients/antfly/antfly.py | 40 ++++++++++++++++++- vectordb_bench/backend/clients/antfly/cli.py | 5 ++- .../backend/clients/chroma/chroma.py | 20 +++++++++- vectordb_bench/backend/clients/chroma/cli.py | 1 + .../clients/elastic_cloud/elastic_cloud.py | 4 +- .../backend/clients/qdrant_local/cli.py | 5 ++- .../clients/qdrant_local/qdrant_local.py | 37 +++++++++++------ .../clients/weaviate_cloud/weaviate_cloud.py | 28 +++++++++---- vectordb_bench/backend/dataset.py | 7 ++++ 9 files changed, 120 insertions(+), 27 deletions(-) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index 21c148882..f3e4cbfac 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -11,6 +11,7 @@ import httpx from ..api import DBCaseConfig, MetricType, VectorDB +from ...filter import Filter, FilterOp from ...payload import PayloadProfile log = logging.getLogger(__name__) @@ -65,6 +66,12 @@ def _detect_api_root(host: str, port: int) -> str: class Antfly(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, @@ -72,12 +79,15 @@ def __init__( db_case_config: DBCaseConfig, collection_name: str = "vdbbench", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name self.dim = dim + self.with_scalar_labels = with_scalar_labels + self._filter_query: dict[str, Any] | None = None # Antfly v0.1 used /api/v1; current antfly-zig serves the public DB API # at /db/v1. Auto-detect unless ANTFLY_API_ROOT pins it explicitly. @@ -462,12 +472,15 @@ def _serialize_insert_vector(self, vector: list[float]) -> str | list[float]: return self._pack_vector(vector) def _metadata_query_body(self, query: list[float], k: int) -> dict[str, Any]: - return { + body = { "embeddings": {"vec": self._serialize_query_vector(query)}, "limit": k, "fields": [], **self.case_config.search_param(), } + if getattr(self, "_filter_query", None) is not None: + body["filter_query"] = self._filter_query + return body def _store_query_body(self, query: list[float], k: int) -> dict[str, Any]: search_params = self.case_config.search_param() @@ -533,6 +546,24 @@ def optimize(self, data_size: int | None = None): self._wait_for_index_ready(client, expected_total=data_size) self._maybe_log_bench_status(client, "optimize_end", force=True) + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self._filter_query = None + elif filters.type == FilterOp.NumGE: + self._filter_query = { + "numeric_range": { + "field": filters.int_field, + "min": filters.int_value, + "inclusive_min": True, + } + } + elif filters.type == FilterOp.StrEqual: + self._filter_query = { + "term": {filters.label_field: filters.label_value} + } + else: + raise ValueError(f"Unsupported Antfly filter: {filters}") + def _catch_up_lag_sequences(self) -> int | None: try: payload = self._get_index_status(self.client) @@ -564,6 +595,7 @@ def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs: Any, ) -> tuple[int, Exception]: total = len(embeddings) @@ -584,6 +616,10 @@ def insert_embeddings( SOURCE_FIELD: str(metadata[i]), "_embeddings": {"vec": serialized_embedding}, } + if self.with_scalar_labels: + if labels_data is None: + raise ValueError("Antfly label-filter load requires labels_data") + inserts[key]["labels"] = labels_data[i] payload = {"inserts": inserts, "sync_level": self._write_sync_level} r = self.client.post( f"/tables/{self.collection_name}/batch", json=payload @@ -613,6 +649,8 @@ def search_embedding( query = self._normalize_vector(query) if self._use_direct_store_search: + if self._filter_query is not None: + raise ValueError("Antfly filtered ANN requires the public metadata query API") if self._direct_shard_id is None: self._refresh_direct_search_routing(self.client) r = self.store_client.post( diff --git a/vectordb_bench/backend/clients/antfly/cli.py b/vectordb_bench/backend/clients/antfly/cli.py index 0bb40db99..0f5389631 100644 --- a/vectordb_bench/backend/clients/antfly/cli.py +++ b/vectordb_bench/backend/clients/antfly/cli.py @@ -7,6 +7,7 @@ CommonTypedDict, cli, click_parameter_decorators_from_typed_dict, + get_custom_case_config, run, ) from ...cases import CaseType @@ -81,7 +82,9 @@ def AntflyAKNN(**parameters: Unpack[AntflyAKNNTypedDict]): metric_type = ( MetricType(parameters["metric_type"]) if parameters["metric_type"] - else CaseType[parameters["case_type"]].case_cls(parameters.get("custom_case")).dataset.data.metric_type + else CaseType[parameters["case_type"]] + .case_cls(get_custom_case_config(parameters)) + .dataset.data.metric_type ) run( diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index f2065ab8b..1b09d4cb7 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -3,6 +3,8 @@ import chromadb +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import ChromaIndexConfig @@ -17,6 +19,11 @@ class ChromaClient(VectorDB): To change to running in process, modify the HttpClient() in __init__() and init(). """ + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + def __init__( self, dim: int, @@ -42,6 +49,7 @@ def __init__( self.client = None self.collection = None + self._where_filter: dict | None = None @contextmanager def init(self): @@ -85,10 +93,18 @@ def search_embedding( self, query: list[float], k: int = 100, filters: dict | None = None, timeout: int | None = None ) -> list[int]: assert self.client is not None, "Please call self.init() before" - if filters: + if self._where_filter is not None: results = self.collection.query( - query_embeddings=[query], n_results=k, where={"id": {"$gt": filters.get("id")}} + query_embeddings=[query], n_results=k, where=self._where_filter ) else: results = self.collection.query(query_embeddings=[query], n_results=k) return [int(idx) for idx in results["ids"][0]] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self._where_filter = None + elif filters.type == FilterOp.NumGE: + self._where_filter = {"index": {"$gte": filters.int_value}} + else: + raise ValueError(f"Unsupported Chroma filter: {filters}") diff --git a/vectordb_bench/backend/clients/chroma/cli.py b/vectordb_bench/backend/clients/chroma/cli.py index 64a2f972f..3445660a8 100644 --- a/vectordb_bench/backend/clients/chroma/cli.py +++ b/vectordb_bench/backend/clients/chroma/cli.py @@ -50,6 +50,7 @@ def Chroma(**parameters: Unpack[ChromaTypeDict]): run( db=DBTYPE, db_config=ChromaConfig( + db_label=parameters["db_label"], user=parameters["user"], password=SecretStr(parameters["password"]) if parameters["password"] else None, host=SecretStr(parameters["host"]), diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index ae3350ddf..43a72faea 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -50,7 +50,7 @@ def __init__( from elasticsearch import Elasticsearch - client = Elasticsearch(**self.db_config) + client = Elasticsearch(**self.db_config, request_timeout=180) if drop_old: log.info(f"Elasticsearch client drop_old indices: {self.indice}") @@ -154,7 +154,7 @@ def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.filter = [] elif filters.type == FilterOp.NumGE: - self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + self.filter = {"range": {self.id_col_name: {"gte": filters.int_value}}} elif filters.type == FilterOp.StrEqual: self.filter = {"term": {self.label_col_name: filters.label_value}} if self.case_config.use_routing: diff --git a/vectordb_bench/backend/clients/qdrant_local/cli.py b/vectordb_bench/backend/clients/qdrant_local/cli.py index 7995b99b3..ee3309dc2 100644 --- a/vectordb_bench/backend/clients/qdrant_local/cli.py +++ b/vectordb_bench/backend/clients/qdrant_local/cli.py @@ -49,7 +49,10 @@ def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]): run( db=DBTYPE, - db_config=QdrantLocalConfig(url=SecretStr(parameters["url"])), + db_config=QdrantLocalConfig( + db_label=parameters["db_label"], + url=SecretStr(parameters["url"]), + ), db_case_config=QdrantLocalIndexConfig( on_disk=parameters["on_disk"], m=parameters["m"], diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py index 15c790c61..585389597 100644 --- a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -19,6 +19,8 @@ VectorParams, ) +from vectordb_bench.backend.filter import Filter as BenchFilter, FilterOp + from ..api import VectorDB from .config import QdrantLocalIndexConfig @@ -40,6 +42,11 @@ def qdrant_collection_exists(client: QdrantClient, collection_name: str) -> bool class QdrantLocal(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + def __init__( self, dim: int, @@ -60,6 +67,7 @@ def __init__( self._primary_field = "pk" self._vector_field = "vector" + self._query_filter: Filter | None = None client = QdrantClient(**self.db_config) @@ -209,25 +217,28 @@ def search_embedding( """ assert self.client is not None - f = None - if filters: - f = Filter( - must=[ - FieldCondition( - key=self._primary_field, - range=Range( - gt=filters.get("id"), - ), - ), - ], - ) res = self.client.query_points( collection_name=self.collection_name, query=query, limit=k, - query_filter=f, + query_filter=self._query_filter, search_params=SearchParams(**self.search_parameter), timeout=timeout, ).points return [result.id for result in res] + + def prepare_filter(self, filters: BenchFilter): + if filters.type == FilterOp.NonFilter: + self._query_filter = None + elif filters.type == FilterOp.NumGE: + self._query_filter = Filter( + must=[ + FieldCondition( + key=self._primary_field, + range=Range(gte=filters.int_value), + ) + ] + ) + else: + raise ValueError(f"Unsupported QdrantLocal filter: {filters}") diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index 0bcfc56d6..899c0143d 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -7,6 +7,8 @@ import weaviate from weaviate.exceptions import WeaviateBaseError +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) @@ -14,6 +16,10 @@ class WeaviateCloud(VectorDB): thread_safe: bool = False + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] def __init__( self, @@ -40,6 +46,7 @@ def __init__( self._scalar_field = "key" self._vector_field = "vector" self._index_name = "vector_idx" + self._where_filter: dict | None = None # If local setup is used, we if db_config["no_auth"]: @@ -148,16 +155,23 @@ def search_embedding( .with_near_vector({"vector": query}) .with_limit(k) ) - if filters: - where_filter = { - "path": "key", - "operator": "GreaterThanEqual", - "valueInt": filters.get("id"), - } - query_obj = query_obj.with_where(where_filter) + if self._where_filter is not None: + query_obj = query_obj.with_where(self._where_filter) # Perform the search. res = query_obj.do() # Organize results. return [result[self._scalar_field] for result in res["data"]["Get"][self.collection_name]] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self._where_filter = None + elif filters.type == FilterOp.NumGE: + self._where_filter = { + "path": [self._scalar_field], + "operator": "GreaterThanEqual", + "valueInt": filters.int_value, + } + else: + raise ValueError(f"Unsupported Weaviate filter: {filters}") diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index c249f91c9..fc3c9381f 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -5,6 +5,7 @@ """ import logging +import os import pathlib from enum import Enum from typing import Any, ClassVar, NamedTuple @@ -374,6 +375,12 @@ def prepare( if self.data.with_scalar_labels and self.data.scalar_labels_file_separated: download_files.append(self.data.scalar_labels_file) download_files = [file for file in download_files if file is not None] + if ( + os.environ.get("VDBB_USE_LOCAL_FILTER_GT") == "1" + and gt_file is not None + and self.data_dir.joinpath(gt_file).exists() + ): + download_files = [file for file in download_files if file != gt_file] source.reader().read( dataset=self.data.dir_name.lower(), files=download_files,