From 42ebf08d963400f020db8d1aa49ac52fe7105d29 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Thu, 19 Mar 2026 11:08:39 -0700 Subject: [PATCH 1/6] feat: add Antfly vector database integration Add VectorDBBench integration for Antfly using httpx REST client directly (no SDK dependency). Uses two-step table creation with Termite embedder and direct pre-computed embeddings for queries. New files: - vectordb_bench/backend/clients/antfly/ (antfly.py, config.py, cli.py) Modified: - DB enum, config_cls, case_config_cls registrations - CLI command registration (AntflyAKNN) - pyproject.toml optional dependency (httpx) --- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 16 ++ .../backend/clients/antfly/__init__.py | 0 .../backend/clients/antfly/antfly.py | 168 ++++++++++++++++++ vectordb_bench/backend/clients/antfly/cli.py | 47 +++++ .../backend/clients/antfly/config.py | 30 ++++ vectordb_bench/cli/vectordbbench.py | 2 + 7 files changed, 264 insertions(+) create mode 100644 vectordb_bench/backend/clients/antfly/__init__.py create mode 100644 vectordb_bench/backend/clients/antfly/antfly.py create mode 100644 vectordb_bench/backend/clients/antfly/cli.py create mode 100644 vectordb_bench/backend/clients/antfly/config.py diff --git a/pyproject.toml b/pyproject.toml index c5e30aa6b..f19759c7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] +antfly = [ "httpx" ] [project.urls] Repository = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index b5f3a4d6c..ebb3b3b13 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -59,6 +59,7 @@ class DB(Enum): Zvec = "Zvec" Endee = "Endee" Lindorm = "Lindorm" + Antfly = "Antfly" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -246,6 +247,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormVector + if self == DB.Antfly: + from .antfly.antfly import Antfly + + return Antfly + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -435,6 +441,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormConfig + if self == DB.Antfly: + from .antfly.config import AntflyConfig + + return AntflyConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -601,6 +612,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _lindorm_vector_case_config.get(index_type) + if self == DB.Antfly: + from .antfly.config import AntflyIndexConfig + + return AntflyIndexConfig + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/antfly/__init__.py b/vectordb_bench/backend/clients/antfly/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py new file mode 100644 index 000000000..8e00adc03 --- /dev/null +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -0,0 +1,168 @@ +import logging +import time +from contextlib import contextmanager +from typing import Any + +import httpx + +from ..api import DBCaseConfig, VectorDB + +log = logging.getLogger(__name__) + +BATCH_CHUNK_SIZE = 500 +TABLE_READY_TIMEOUT = 30 +TABLE_READY_POLL_INTERVAL = 2 + + +class Antfly(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "vdbbench", + drop_old: bool = False, + **kwargs, + ): + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + self.dim = dim + + base_url = f"http://{db_config['host']}:{db_config['port']}/api/v1" + self._base_url = base_url + num_shards = db_config.get("num_shards", 1) + + client = httpx.Client(base_url=base_url, timeout=60) + try: + if drop_old: + r = client.delete(f"/tables/{self.collection_name}") + log.info(f"Drop table response: {r.status_code}") + + # Two-step table creation to avoid Pebble lock issue: + # 1. Create table without embeddings index + r = client.post(f"/tables/{self.collection_name}", json={"num_shards": num_shards}) + log.info(f"Create table response: {r.status_code}") + + # Wait for shard to initialize + self._wait_for_shard_ready(client) + + # 2. Add embeddings index with Termite embedder + index_def = { + "type": "embeddings", + "dimension": dim, + "field": "vec_data", + "template": "{{vec_data}}", + "embedder": {"provider": "termite", "model": "BAAI/bge-small-en-v1.5"}, + } + r = client.post(f"/tables/{self.collection_name}/indexes/vec", json=index_def) + if r.status_code >= 400: + # Fallback: try without embedder (field-only index) + log.warning(f"Index creation with embedder failed ({r.status_code}), trying field-only") + index_def = {"type": "embeddings", "dimension": dim, "field": "vec_data"} + r = client.post(f"/tables/{self.collection_name}/indexes/vec", json=index_def) + log.info(f"Add embeddings index response: {r.status_code}") + finally: + client.close() + + def _wait_for_shard_ready(self, client: httpx.Client): + """Wait for shard to be initialized and accepting writes.""" + deadline = time.monotonic() + TABLE_READY_TIMEOUT + while time.monotonic() < deadline: + try: + r = client.post( + f"/tables/{self.collection_name}/batch", + json={"inserts": {"_healthcheck": {"_probe": True}}}, + ) + if r.status_code < 500: + # Delete the probe doc + client.post( + f"/tables/{self.collection_name}/batch", + json={"deletes": ["_healthcheck"]}, + ) + log.info("Shard is ready (accepts writes)") + return + except Exception: + pass + time.sleep(TABLE_READY_POLL_INTERVAL) + log.warning(f"Shard readiness timeout after {TABLE_READY_TIMEOUT}s, proceeding anyway") + + @contextmanager + def init(self): + self.client = httpx.Client(base_url=self._base_url, timeout=120) + try: + yield + finally: + self.client.close() + self.client = None + + def ready_to_search(self) -> bool: + pass + + def optimize(self, data_size: int | None = None): + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception]: + try: + total = len(embeddings) + for start in range(0, total, BATCH_CHUNK_SIZE): + end = min(start + BATCH_CHUNK_SIZE, total) + inserts = {} + for i in range(start, end): + key = f"key:{metadata[i]}" + inserts[key] = { + "id": metadata[i], + "metadata": metadata[i], + "_embeddings": {"vec": embeddings[i]}, + } + r = self.client.post( + f"/tables/{self.collection_name}/batch", + json={"inserts": inserts}, + ) + r.raise_for_status() + return total, None + except Exception as e: + log.warning(f"Antfly insert error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + # Workaround: omit semantic_search and indexes, send embeddings directly. + # This bypasses generateQueryEmbeddings() and uses the direct vector path. + body = { + "embeddings": {"vec": query}, + "limit": k, + "fields": ["id"], + } + r = self.client.post( + f"/tables/{self.collection_name}/query", + json=body, + ) + r.raise_for_status() + data = r.json() + # Response format: {"responses": [{"hits": {"hits": [{"_id": "key:42", ...}, ...]}}]} + resp = data.get("responses", [{}])[0] + hits_obj = resp.get("hits") or {} + hits = hits_obj.get("hits") or [] + results = [] + for hit in hits: + if "id" in hit: + results.append(int(hit["id"])) + else: + doc_key = hit.get("_id", "") + try: + results.append(int(doc_key.split(":", 1)[1])) + except (IndexError, ValueError): + log.warning(f"Could not parse id from _id: {doc_key}") + return results diff --git a/vectordb_bench/backend/clients/antfly/cli.py b/vectordb_bench/backend/clients/antfly/cli.py new file mode 100644 index 000000000..4410d638b --- /dev/null +++ b/vectordb_bench/backend/clients/antfly/cli.py @@ -0,0 +1,47 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class AntflyTypedDict(TypedDict): + host: Annotated[str, click.option("--host", type=str, help="Antfly host", default="localhost", show_default=True)] + port: Annotated[int, click.option("--port", type=int, help="Antfly port", default=8080, show_default=True)] + username: Annotated[str, click.option("--username", type=str, help="Antfly username", default=None)] + password: Annotated[str, click.option("--password", type=str, help="Antfly password", default=None)] + num_shards: Annotated[ + int, click.option("--num-shards", type=int, help="Number of shards", default=1, show_default=True) + ] + + +class AntflyAKNNTypedDict(CommonTypedDict, AntflyTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(AntflyAKNNTypedDict) +def AntflyAKNN(**parameters: Unpack[AntflyAKNNTypedDict]): + from .config import AntflyConfig, AntflyIndexConfig + + run( + db=DB.Antfly, + db_config=AntflyConfig( + db_label=parameters["db_label"], + host=parameters["host"], + port=parameters["port"], + username=SecretStr(parameters["username"]) if parameters["username"] else None, + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=parameters["num_shards"], + ), + db_case_config=AntflyIndexConfig( + num_shards=parameters["num_shards"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/antfly/config.py b/vectordb_bench/backend/clients/antfly/config.py new file mode 100644 index 000000000..e87103bf6 --- /dev/null +++ b/vectordb_bench/backend/clients/antfly/config.py @@ -0,0 +1,30 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig + + +class AntflyConfig(DBConfig): + host: str = "localhost" + port: int = 8080 + username: SecretStr | None = None + password: SecretStr | None = None + num_shards: int = 1 + + def to_dict(self) -> dict: + return { + "host": self.host, + "port": self.port, + "username": self.username.get_secret_value() if self.username else None, + "password": self.password.get_secret_value() if self.password else None, + "num_shards": self.num_shards, + } + + +class AntflyIndexConfig(BaseModel, DBCaseConfig): + num_shards: int = 1 + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index dd704ae91..8c89d76c2 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,4 +1,5 @@ from ..backend.clients.alisql.cli import AliSQLHNSW +from ..backend.clients.antfly.cli import AntflyAKNN from ..backend.clients.alloydb.cli import AlloyDBScaNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch from ..backend.clients.chroma.cli import Chroma @@ -82,6 +83,7 @@ cli.add_command(LindormHNSW) cli.add_command(LindormIVFBQ) cli.add_command(Pinecone) +cli.add_command(AntflyAKNN) if __name__ == "__main__": From bc8a0ff0e6fcccbe786aa23366b2121fef5835e4 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Thu, 19 Mar 2026 11:09:48 -0700 Subject: [PATCH 2/6] fix: use field-only embeddings index, no Termite dependency Field-only indexes work with the direct embeddings query workaround (omit semantic_search and indexes fields). This removes the Termite embedder requirement and supports any vector dimension. --- vectordb_bench/backend/clients/antfly/antfly.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index 8e00adc03..e630b51a1 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -47,20 +47,9 @@ def __init__( # Wait for shard to initialize self._wait_for_shard_ready(client) - # 2. Add embeddings index with Termite embedder - index_def = { - "type": "embeddings", - "dimension": dim, - "field": "vec_data", - "template": "{{vec_data}}", - "embedder": {"provider": "termite", "model": "BAAI/bge-small-en-v1.5"}, - } + # 2. Add field-only embeddings index (pre-computed vectors, no embedder needed) + index_def = {"type": "embeddings", "dimension": dim, "field": "vec_data"} r = client.post(f"/tables/{self.collection_name}/indexes/vec", json=index_def) - if r.status_code >= 400: - # Fallback: try without embedder (field-only index) - log.warning(f"Index creation with embedder failed ({r.status_code}), trying field-only") - index_def = {"type": "embeddings", "dimension": dim, "field": "vec_data"} - r = client.post(f"/tables/{self.collection_name}/indexes/vec", json=index_def) log.info(f"Add embeddings index response: {r.status_code}") finally: client.close() From 9da3de48944b9a8470cc30c918870f63efa03555 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Thu, 19 Mar 2026 11:10:24 -0700 Subject: [PATCH 3/6] fix: add metric_type field to AntflyIndexConfig --- vectordb_bench/backend/clients/antfly/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/antfly/config.py b/vectordb_bench/backend/clients/antfly/config.py index e87103bf6..d630b0dc2 100644 --- a/vectordb_bench/backend/clients/antfly/config.py +++ b/vectordb_bench/backend/clients/antfly/config.py @@ -1,6 +1,6 @@ from pydantic import BaseModel, SecretStr -from ..api import DBCaseConfig, DBConfig +from ..api import DBCaseConfig, DBConfig, MetricType class AntflyConfig(DBConfig): @@ -21,6 +21,7 @@ def to_dict(self) -> dict: class AntflyIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None num_shards: int = 1 def index_param(self) -> dict: From 2e0cefab317049ae81439f771d7d09edc8f25c01 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Thu, 19 Mar 2026 13:14:51 -0700 Subject: [PATCH 4/6] feat: add Makefile with benchmark targets for Antfly, Qdrant, Milvus Includes 50K and 1M dataset targets, Docker infrastructure commands, and dev targets (unittest, format, lint). --- Makefile | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/Makefile b/Makefile index ef8207c55..8a899c7b1 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,77 @@ +PYTHON ?= /home/rowan/.local/share/uv/tools/pip/bin/python +BENCH = $(PYTHON) -m vectordb_bench.cli.vectordbbench +COMMON = --drop-old --skip-search-concurrent + +help: + @echo "VectorDBBench — Antfly Integration" + @echo "" + @echo "Benchmarks (50K vectors, 1536 dim):" + @echo " make bench-antfly-50k Antfly SPANN" + @echo " make bench-qdrant-50k Qdrant HNSW" + @echo " make bench-milvus-50k Milvus AutoIndex" + @echo " make bench-all-50k All three (50K)" + @echo "" + @echo "Benchmarks (1M vectors, 768 dim):" + @echo " make bench-antfly-1m Antfly SPANN" + @echo " make bench-qdrant-1m Qdrant HNSW" + @echo " make bench-milvus-1m Milvus AutoIndex" + @echo " make bench-all-1m All three (1M)" + @echo "" + @echo "Infrastructure:" + @echo " make start-qdrant Start Qdrant (Docker)" + @echo " make start-milvus Start Milvus (Docker)" + @echo " make stop-all Stop Docker containers" + @echo "" + @echo "Dev:" + @echo " make unittest Run unit tests" + @echo " make format / lint Code formatting" + +# --- 50K benchmarks --- +bench-antfly-50k: + $(BENCH) antflyaknn --host localhost --port 8080 --num-shards 1 \ + --case-type Performance1536D50K $(COMMON) --db-label antfly-local + +bench-qdrant-50k: + $(BENCH) qdrantlocal --url http://localhost:6333 \ + --case-type Performance1536D50K $(COMMON) --db-label qdrant-local \ + --m 16 --ef-construct 128 --hnsw-ef 64 + +bench-milvus-50k: + $(BENCH) milvusautoindex --uri http://localhost:19530 \ + --case-type Performance1536D50K $(COMMON) --db-label milvus-local + +bench-all-50k: bench-antfly-50k bench-qdrant-50k bench-milvus-50k + +# --- 1M benchmarks --- +bench-antfly-1m: + $(BENCH) antflyaknn --host localhost --port 8080 --num-shards 1 \ + --case-type Performance768D1M $(COMMON) --db-label antfly-local + +bench-qdrant-1m: + $(BENCH) qdrantlocal --url http://localhost:6333 \ + --case-type Performance768D1M $(COMMON) --db-label qdrant-local \ + --m 16 --ef-construct 128 --hnsw-ef 64 + +bench-milvus-1m: + $(BENCH) milvusautoindex --uri http://localhost:19530 \ + --case-type Performance768D1M $(COMMON) --db-label milvus-local + +bench-all-1m: bench-antfly-1m bench-qdrant-1m bench-milvus-1m + +# --- Infrastructure --- +start-qdrant: + docker run -d --name qdrant_bench -p 6333:6333 -p 6334:6334 qdrant/qdrant:latest + +start-milvus: + docker run -d --name milvus_bench -p 19530:19530 -p 9091:9091 \ + -e ETCD_USE_EMBED=true -e COMMON_STORAGETYPE=local \ + milvusdb/milvus:latest milvus run standalone + +stop-all: + -docker stop qdrant_bench milvus_bench 2>/dev/null + -docker rm qdrant_bench milvus_bench 2>/dev/null + +# --- Dev --- unittest: PYTHONPATH=`pwd` python3 -m pytest tests/test_dataset.py::TestDataSet::test_download_small -svv @@ -8,3 +82,7 @@ format: lint: PYTHONPATH=`pwd` python3 -m black vectordb_bench --check PYTHONPATH=`pwd` python3 -m ruff check vectordb_bench + +.PHONY: help bench-antfly-50k bench-qdrant-50k bench-milvus-50k bench-all-50k \ + bench-antfly-1m bench-qdrant-1m bench-milvus-1m bench-all-1m \ + start-qdrant start-milvus stop-all unittest format lint From 2144918cf6d3966be87b1affb8800aac24df7fbb Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Fri, 20 Mar 2026 15:46:59 -0700 Subject: [PATCH 5/6] Improve Antfly cosine handling and document upstream failure --- antfly-recall-antfly-only.md | 176 ++++++++++++++++++ .../backend/clients/antfly/antfly.py | 147 +++++++++++++-- .../backend/clients/antfly/config.py | 9 +- 3 files changed, 318 insertions(+), 14 deletions(-) create mode 100644 antfly-recall-antfly-only.md diff --git a/antfly-recall-antfly-only.md b/antfly-recall-antfly-only.md new file mode 100644 index 000000000..4c249f91b --- /dev/null +++ b/antfly-recall-antfly-only.md @@ -0,0 +1,176 @@ +# Antfly Recall Assessment: Antfly-Only Findings + +## Scope + +This note is intentionally limited to **Antfly-side** explanations for the low recall seen in `#antfly > vector-db-bench`. + +I am **not** treating the original `VectorDBBench` client wiring issues as part of this diagnosis. Those confounders were removed before the rerun summarized below. The question here is narrower: after controlling for the obvious external issues, does the remaining low recall look like an Antfly problem? + +## Bottom line + +Yes, at this point the low recall looks much more like an **Antfly-side issue** than a benchmark wiring mistake. + +After rerunning with the benchmark-side confounders removed, recall stayed essentially unchanged: + +- `recall@100 = 0.5008` +- `ndcg = 0.6166` + +That means the remaining plausible causes are now mostly inside Antfly itself: + +1. a concrete cosine/HBC correctness bug visible in the source-built upstream server +2. hardcoded HBC search defaults that may be too aggressive for this workload +3. dynamic pruning behavior in HBC +4. quantized leaf search +5. possible drift between the running local Antfly binary and the checked-out source/API expectations + +## Findings + +### 1. The remaining problem is no longer well explained by client wiring + +I reran the 50K case after removing the main external confounders and the result remained about the same as before. That materially weakens the hypothesis that recall was low simply because the benchmark client was wired incorrectly. + +So the updated diagnosis is: + +- the original setup did have avoidable confounders +- but those confounders were **not** the main reason recall ended up near 50% + +That pushes the investigation onto Antfly itself. + +### 2. The source-built upstream Antfly still fails after receiving unit-normalized cosine vectors + +I then reran against a fresh source-built upstream checkout at `~/Documents/antfly/antfly-upstream` after hardening the client further: + +- correct cosine `distance_metric` +- explicit cosine normalization on insert/query in the Antfly client +- `need_normalize_cosine()` enabled so `VectorDBBench` normalizes cosine datasets before calling the client + +For the first insert batch, the benchmark logged: + +- `metric=COSINE` +- `use_cosine=True` +- `first_sqnorm=1.000000` + +So at that point the benchmark was definitely sending a unit-normalized cosine vector. + +Even with that in place, the upstream source build still panicked inside Antfly during HBC split/reclustering with: + +- `vector is not a unit vector` +- stack rooted in `BalancedBinaryKmeans.validateVectors()` via `HBCIndex.splitVectorSet()` + +That is the strongest current evidence in this whole investigation. It means there is at least one remaining Antfly-side cosine/HBC correctness problem that is **not** explained by the benchmark simply forgetting to normalize vectors. + +### 3. Antfly's dense-vector path is using hardcoded HBC defaults that could plausibly cap recall + +The embeddings index builds an HBC index with a fixed configuration in source: + +- quantization enabled +- `Episilon2 = 7` +- `BranchingFactor = 168` +- `LeafSize = 168` +- `SearchWidth = 1008` + +Relevant source: + +- [`src/store/db/indexes/embeddings_index.go:538`](/home/rowan/Documents/antfly/antfly/src/store/db/indexes/embeddings_index.go#L538) + +Those are not obviously conservative "maximize recall" settings. They look like fixed performance-oriented defaults. If recall is poor on this dataset, this config is a very credible place to look first. + +### 4. HBC is doing aggressive dynamic pruning during search + +The HBC search code prunes internal nodes and leaves based on the current best distance threshold and `Episilon2`: + +- internal-node pruning: + [`lib/vectorindex/hbc.go:1885`](/home/rowan/Documents/antfly/antfly/lib/vectorindex/hbc.go#L1885) +- default distance metric fallback: + [`lib/vectorindex/hbc.go:288`](/home/rowan/Documents/antfly/antfly/lib/vectorindex/hbc.go#L288) + +The specific pruning logic here is aggressive enough that it could absolutely explain a recall ceiling if the heuristic is not well tuned for this workload. + +I would treat this as one of the strongest Antfly-side suspects. + +### 5. Antfly is searching quantized vectors by default + +The embeddings index turns quantization on by default in the HBC config: + +- [`src/store/db/indexes/embeddings_index.go:544`](/home/rowan/Documents/antfly/antfly/src/store/db/indexes/embeddings_index.go#L544) + +And leaf search uses quantized vectors when available: + +- [`lib/vectorindex/hbc.go:1902`](/home/rowan/Documents/antfly/antfly/lib/vectorindex/hbc.go#L1902) + +That does not prove quantization is the reason recall is low, but it is another real Antfly-side approximation step in the retrieval path. Between quantization and dynamic pruning, there is enough approximation here that a ~50% recall result is not inherently surprising. + +### 6. The running local Antfly binary appears inconsistent with the checked-out source/API expectations + +This matters because it raises the possibility that the benchmark is not actually exercising the exact implementation implied by the local source tree. + +Observed on the running local Antfly used for the rerun: + +- creating the vector index with type `"embeddings"` failed with HTTP 400 +- creating it with `"aknn_v0"` succeeded +- inserts using `SyncLevelEmbeddings`/`aknn` semantics for precomputed `_embeddings` failed with a server-side `index not found: vec` +- explicitly including `"name": "vec"` in the index config was necessary for the field-only precomputed-embedding path to work reliably + +Those runtime behaviors are notable because the checked-out source clearly presents an embeddings index abstraction: + +- the API type includes `EmbeddingsIndexConfig` and documents `distance_metric`: + [`src/store/db/indexes/openapi.gen.go:170`](/home/rowan/Documents/antfly/antfly/src/store/db/indexes/openapi.gen.go#L170) +- the CLI expects an explicit `name` field in an index definition: + [`cmd/antfly/cmd/cli/table.go:133`](/home/rowan/Documents/antfly/antfly/cmd/antfly/cmd/cli/table.go#L133) +- the DB tests explicitly expect vector writes to support embeddings-level sync before search: + [`src/store/db/embeddings_test.go:792`](/home/rowan/Documents/antfly/antfly/src/store/db/embeddings_test.go#L792) +- `SyncLevelEmbeddings` is a first-class sync level: + [`src/store/db/ops.proto:41`](/home/rowan/Documents/antfly/antfly/src/store/db/ops.proto#L41) + +So there is at least some evidence of **runtime/build drift or compatibility mismatch**: + +- either the running binary is older or behaviorally different from the checked-out source +- or the field-only precomputed-embedding path is not as solid as the source/tests suggest + +Either way, that is an Antfly-side problem, not a benchmark problem. + +### 7. The direct query shape itself does not look like the main failure point + +The Antfly query path for direct embeddings appears coherent in source: + +- request embeddings are copied into the remote index query: + [`src/metadata/api_query.go:45`](/home/rowan/Documents/antfly/antfly/src/metadata/api_query.go#L45) +- if `indexes` is omitted, Antfly skips query-time embedding generation: + [`src/metadata/api_query.go:496`](/home/rowan/Documents/antfly/antfly/src/metadata/api_query.go#L496) +- remote search accepts direct embedding searches: + [`src/store/db/indexes/remoteindex.go:1430`](/home/rowan/Documents/antfly/antfly/src/store/db/indexes/remoteindex.go#L1430) + +So my current read is not "the query API is fundamentally broken." The stronger Antfly-side story is that the query reaches the right subsystem, but the subsystem's current implementation and/or runtime configuration is not producing good enough nearest-neighbor quality. + +## Most likely Antfly-side explanations, in order + +1. There is a cosine/HBC bug in Antfly's indexing path, visible even on the source-built upstream server after the benchmark sends unit-normalized vectors. +2. HBC pruning and search-width defaults are too aggressive for this benchmark. +3. Quantization is costing too much recall on this workload. +4. The running local Antfly binary does not match the behavior implied by the checked-out source, especially around embeddings index creation and sync semantics. +5. There is a field-only precomputed-embedding bug or edge case in the current implementation. + +## What I would test next in Antfly + +1. Reproduce the source-build cosine panic in a focused Antfly test around `HBCIndex.splitVectorSet()` / `BalancedBinaryKmeans.validateVectors()` with precomputed unit vectors. +2. Verify why Antfly later observes a non-unit vector after the benchmark has already sent `sqnorm=1.0`. +3. Verify the exact binary/version/config that was running for the local ~50% recall rerun. +4. Run the same 50K case with a less approximate Antfly configuration: + - higher `SearchWidth` + - lower or disabled pruning + - quantization disabled +5. Check whether the runtime binary is actually the same code as `~/Documents/antfly/antfly`. +6. Reproduce the `SyncLevelEmbeddings` failure in a focused Antfly test using precomputed `_embeddings`. + +## Verdict + +The general concern in the thread now looks directionally correct: the remaining low recall appears to be an **Antfly problem**. + +The specific root cause is not yet proven to be only "HBC/SPANN pruning," but the evidence now points much more strongly to: + +- an Antfly cosine/HBC bug on the source-built path +- Antfly search heuristics/defaults +- Antfly approximation behavior +- and possibly Antfly runtime/build inconsistency + +not to simple benchmark miswiring. diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index e630b51a1..827ec96ad 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -1,17 +1,23 @@ import logging +import math import time from contextlib import contextmanager from typing import Any import httpx -from ..api import DBCaseConfig, VectorDB +from ..api import DBCaseConfig, MetricType, VectorDB log = logging.getLogger(__name__) BATCH_CHUNK_SIZE = 500 TABLE_READY_TIMEOUT = 30 TABLE_READY_POLL_INTERVAL = 2 +INDEX_READY_TIMEOUT = 1800 +INDEX_READY_POLL_INTERVAL = 2 +INDEX_NAME = "vec" +INDEX_TYPES = ("embeddings", "aknn_v0") +SOURCE_FIELD = "vec_data" class Antfly(VectorDB): @@ -48,9 +54,26 @@ def __init__( self._wait_for_shard_ready(client) # 2. Add field-only embeddings index (pre-computed vectors, no embedder needed) - index_def = {"type": "embeddings", "dimension": dim, "field": "vec_data"} - r = client.post(f"/tables/{self.collection_name}/indexes/vec", json=index_def) - log.info(f"Add embeddings index response: {r.status_code}") + index_def = { + "name": INDEX_NAME, + "dimension": dim, + "field": SOURCE_FIELD, + **self.case_config.index_param(), + } + index_error = None + for index_type in INDEX_TYPES: + r = client.post( + f"/tables/{self.collection_name}/indexes/{INDEX_NAME}", + json={"type": index_type, **index_def}, + ) + log.info(f"Add embeddings index response ({index_type}): {r.status_code}") + if r.is_success: + index_error = None + break + index_error = r + if index_error is not None: + index_error.raise_for_status() + self._wait_for_index_ready(client, expected_total=0) finally: client.close() @@ -61,13 +84,13 @@ def _wait_for_shard_ready(self, client: httpx.Client): try: r = client.post( f"/tables/{self.collection_name}/batch", - json={"inserts": {"_healthcheck": {"_probe": True}}}, + json={"inserts": {"_healthcheck": {"_probe": True}}, "sync_level": "write"}, ) if r.status_code < 500: # Delete the probe doc client.post( f"/tables/{self.collection_name}/batch", - json={"deletes": ["_healthcheck"]}, + json={"deletes": ["_healthcheck"], "sync_level": "write"}, ) log.info("Shard is ready (accepts writes)") return @@ -76,6 +99,58 @@ def _wait_for_shard_ready(self, client: httpx.Client): time.sleep(TABLE_READY_POLL_INTERVAL) log.warning(f"Shard readiness timeout after {TABLE_READY_TIMEOUT}s, proceeding anyway") + def _get_index_status(self, client: httpx.Client) -> dict | None: + r = client.get(f"/tables/{self.collection_name}/indexes/{INDEX_NAME}") + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + + def _index_status_is_ready( + self, + payload: dict | None, + status: dict | None, + expected_total: int | None = None, + ) -> bool: + if payload is None: + return False + if status is None: + return expected_total == 0 + + rebuilding = bool(status.get("rebuilding")) + wal_backlog = int(status.get("wal_backlog", 0) or 0) + total_indexed = int(status.get("total_indexed", 0) or 0) + has_error = bool(status.get("error")) + + if has_error or rebuilding or wal_backlog > 0: + return False + if expected_total is not None and total_indexed < expected_total: + return False + return True + + def _wait_for_index_ready(self, client: httpx.Client, expected_total: int | None = None): + deadline = time.monotonic() + INDEX_READY_TIMEOUT + last_status = None + + while time.monotonic() < deadline: + try: + payload = self._get_index_status(client) + status = payload.get("status") if payload else None + last_status = status + if self._index_status_is_ready(payload, status, expected_total): + log.info(f"Embeddings index is ready: {status}") + return + except Exception as e: + last_status = {"error": str(e)} + time.sleep(INDEX_READY_POLL_INTERVAL) + + log.warning( + "Embeddings index readiness timeout after %ss, expected_total=%s, last_status=%s", + INDEX_READY_TIMEOUT, + expected_total, + last_status, + ) + @contextmanager def init(self): self.client = httpx.Client(base_url=self._base_url, timeout=120) @@ -85,11 +160,39 @@ def init(self): self.client.close() self.client = None + def need_normalize_cosine(self) -> bool: + # TaskRunner already gates normalization on the dataset metric being COSINE. + # Returning True here avoids relying on mutable case_config state to decide + # whether Antfly should receive unit-normalized vectors. + return True + + def _uses_cosine_distance(self) -> bool: + try: + return self.case_config.index_param().get("distance_metric") == "cosine" + except Exception: + return getattr(self.case_config, "metric_type", None) == MetricType.COSINE + + @staticmethod + def _normalize_vector(vector: list[float]) -> list[float]: + norm = math.sqrt(sum(value * value for value in vector)) + if norm == 0: + return vector + return [value / norm for value in vector] + def ready_to_search(self) -> bool: - pass + if getattr(self, "client", None) is not None: + payload = self._get_index_status(self.client) + return self._index_status_is_ready(payload, payload.get("status") if payload else None) + with httpx.Client(base_url=self._base_url, timeout=120) as client: + payload = self._get_index_status(client) + return self._index_status_is_ready(payload, payload.get("status") if payload else None) def optimize(self, data_size: int | None = None): - pass + if getattr(self, "client", None) is not None: + self._wait_for_index_ready(self.client, expected_total=data_size) + return + with httpx.Client(base_url=self._base_url, timeout=120) as client: + self._wait_for_index_ready(client, expected_total=data_size) def insert_embeddings( self, @@ -99,20 +202,36 @@ def insert_embeddings( ) -> tuple[int, Exception]: try: total = len(embeddings) + use_cosine = self._uses_cosine_distance() for start in range(0, total, BATCH_CHUNK_SIZE): end = min(start + BATCH_CHUNK_SIZE, total) inserts = {} for i in range(start, end): key = f"key:{metadata[i]}" + embedding = embeddings[i] + if use_cosine: + embedding = self._normalize_vector(embedding) inserts[key] = { "id": metadata[i], "metadata": metadata[i], - "_embeddings": {"vec": embeddings[i]}, + # Antfly derives a hashID for precomputed embeddings from the + # configured source field. Give it a stable string value instead + # of forcing the "missing field" path for every document. + SOURCE_FIELD: str(metadata[i]), + "_embeddings": {"vec": embedding}, } - r = self.client.post( - f"/tables/{self.collection_name}/batch", - json={"inserts": inserts}, - ) + payload = {"inserts": inserts, "sync_level": "aknn"} + r = self.client.post(f"/tables/{self.collection_name}/batch", json=payload) + if not r.is_success: + log.warning( + "Antfly aknn batch write failed (%s), falling back to sync_level=write: %s", + r.status_code, + r.text, + ) + r = self.client.post( + f"/tables/{self.collection_name}/batch", + json={"inserts": inserts, "sync_level": "write"}, + ) r.raise_for_status() return total, None except Exception as e: @@ -129,6 +248,8 @@ def search_embedding( ) -> list[int]: # Workaround: omit semantic_search and indexes, send embeddings directly. # This bypasses generateQueryEmbeddings() and uses the direct vector path. + if self._uses_cosine_distance(): + query = self._normalize_vector(query) body = { "embeddings": {"vec": query}, "limit": k, diff --git a/vectordb_bench/backend/clients/antfly/config.py b/vectordb_bench/backend/clients/antfly/config.py index d630b0dc2..db3c6c809 100644 --- a/vectordb_bench/backend/clients/antfly/config.py +++ b/vectordb_bench/backend/clients/antfly/config.py @@ -24,8 +24,15 @@ class AntflyIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None num_shards: int = 1 + def parse_metric(self) -> str: + if self.metric_type == MetricType.COSINE: + return "cosine" + if self.metric_type in (MetricType.IP, MetricType.DP): + return "inner_product" + return "l2_squared" + def index_param(self) -> dict: - return {} + return {"distance_metric": self.parse_metric()} def search_param(self) -> dict: return {} From 30b918f36bfa291839b0699800530f6a811db756 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Fri, 20 Mar 2026 16:29:57 -0700 Subject: [PATCH 6/6] Fix CI lint failures for Antfly integration --- vectordb_bench/backend/clients/antfly/antfly.py | 12 +++++------- vectordb_bench/cli/vectordbbench.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py index 827ec96ad..057c6397a 100644 --- a/vectordb_bench/backend/clients/antfly/antfly.py +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -94,8 +94,8 @@ def _wait_for_shard_ready(self, client: httpx.Client): ) log.info("Shard is ready (accepts writes)") return - except Exception: - pass + except Exception as exc: + log.debug("Shard readiness probe failed", exc_info=exc) time.sleep(TABLE_READY_POLL_INTERVAL) log.warning(f"Shard readiness timeout after {TABLE_READY_TIMEOUT}s, proceeding anyway") @@ -124,9 +124,7 @@ def _index_status_is_ready( if has_error or rebuilding or wal_backlog > 0: return False - if expected_total is not None and total_indexed < expected_total: - return False - return True + return expected_total is None or total_indexed >= expected_total def _wait_for_index_ready(self, client: httpx.Client, expected_total: int | None = None): deadline = time.monotonic() + INDEX_READY_TIMEOUT @@ -200,8 +198,8 @@ def insert_embeddings( metadata: list[int], **kwargs: Any, ) -> tuple[int, Exception]: + total = len(embeddings) try: - total = len(embeddings) use_cosine = self._uses_cosine_distance() for start in range(0, total, BATCH_CHUNK_SIZE): end = min(start + BATCH_CHUNK_SIZE, total) @@ -233,10 +231,10 @@ def insert_embeddings( json={"inserts": inserts, "sync_level": "write"}, ) r.raise_for_status() - return total, None except Exception as e: log.warning(f"Antfly insert error: {e}") return 0, e + return total, None def search_embedding( self, diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 8c89d76c2..751eaed66 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,6 +1,6 @@ from ..backend.clients.alisql.cli import AliSQLHNSW -from ..backend.clients.antfly.cli import AntflyAKNN from ..backend.clients.alloydb.cli import AlloyDBScaNN +from ..backend.clients.antfly.cli import AntflyAKNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch from ..backend.clients.chroma.cli import Chroma from ..backend.clients.clickhouse.cli import Clickhouse