Skip to content

Fix cuvs-bench opensearch recall and save results #2369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jrbourbeau wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Fix cuvs-bench opensearch recall and save results #2369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
756b691
Initial commit
jrbourbeau Jul 28, 2026
34656e4
Update
jrbourbeau Jul 28, 2026
b7b4c1a
Merge branch 'main' of https://github.com/rapidsai/cuvs into cuvs-ben…
jrbourbeau Jul 28, 2026
6ed6909
Merge branch 'main' of https://github.com/rapidsai/cuvs into cuvs-ben…
jrbourbeau Jul 29, 2026
e1baea2
Add ef_search fix
jrbourbeau Jul 31, 2026
e9ae31f
Merge branch 'main' of https://github.com/rapidsai/cuvs into cuvs-ben…
jrbourbeau Jul 31, 2026
ef38e5a
Merge branch 'main' into cuvs-bench-fixup
cjnolet Aug 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions python/cuvs_bench/cuvs_bench/backends/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""
Expand Down Expand Up @@ -274,8 +274,8 @@ class SearchResult:
algorithm : str
Algorithm name
search_params : List[Dict[str, Any]]
List of search parameter combinations used (e.g., [{"nprobe": 1}, {"nprobe": 5}])
All are batched into one C++ command (matches runners.py behavior)
Search parameter combinations represented by this result. Backends
normally return one result per combination.
latency_percentiles : Optional[Dict[str, float]]
Latency percentiles in milliseconds (p50, p95, p99)
gpu_time_seconds : Optional[float]
Expand Down Expand Up @@ -421,7 +421,7 @@ def search(
force: bool = False,
search_threads: Optional[int] = None,
dry_run: bool = False,
) -> SearchResult:
) -> List[SearchResult]:
"""
Search for nearest neighbors using the built indexes.

Expand Down Expand Up @@ -452,8 +452,11 @@ def search(

Returns
-------
SearchResult
Search timing, results, and recall metrics
List[SearchResult]
One or more search result objects. Backends normally return one
result per independently measurable search point, but may return
an aggregate result when their native output remains authoritative
(for example, the C++ Google Benchmark backend).

Raises
------
Expand Down
187 changes: 100 additions & 87 deletions python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""
Expand Down Expand Up @@ -309,7 +309,7 @@ def search(
force: bool = False,
search_threads: Optional[int] = None,
dry_run: bool = False,
) -> SearchResult:
) -> List[SearchResult]:
"""
Search using C++ Google Benchmark executable.

Expand Down Expand Up @@ -339,40 +339,45 @@ def search(

Returns
-------
SearchResult
Search timing, recall, and QPS (aggregated across all indexes)
List[SearchResult]
A single aggregate result containing search timing, recall, and
QPS across all indexes.
"""
if not indexes:
return SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=0.0,
queries_per_second=0.0,
recall=0.0,
algorithm="",
search_params=[],
metadata={"skipped": True, "reason": "no_indexes"},
success=True,
)
return [
SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=0.0,
queries_per_second=0.0,
recall=0.0,
algorithm="",
search_params=[],
metadata={"skipped": True, "reason": "no_indexes"},
success=True,
)
]

first_index = indexes[0]

# Pre-flight check (GPU, network, etc.)
skip_reason = self._pre_flight_check()
if skip_reason:
return SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=0.0,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params
if first_index.search_params
else [],
metadata={"skipped": True, "reason": skip_reason},
success=True,
)
return [
SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=0.0,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params
if first_index.search_params
else [],
metadata={"skipped": True, "reason": skip_reason},
success=True,
)
]

# Note: runners.py doesn't validate and lets C++ fail. We validate here for
# better Python-side error messages.
Expand Down Expand Up @@ -470,21 +475,23 @@ def search(
f"Benchmark command for {self.output_filename[1]}:\n{' '.join(cmd)}\n"
)
Path(temp_config_path).unlink(missing_ok=True)
return SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=0.0,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params,
metadata={
"dry_run": True,
"num_indexes": len(indexes),
"total_search_configs": total_search_configs,
},
success=True,
)
return [
SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=0.0,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params,
metadata={
"dry_run": True,
"num_indexes": len(indexes),
"total_search_configs": total_search_configs,
},
success=True,
)
]

# Execute subprocess
start_time = time.perf_counter()
Expand Down Expand Up @@ -544,54 +551,60 @@ def search(

# Note: C++ Google Benchmark doesn't return actual neighbors/distances
# This is a limitation of the current system
return SearchResult(
neighbors=np.array([]), # Not available from C++ benchmark
distances=np.array([]), # Not available from C++ benchmark
search_time_ms=total_search_time,
queries_per_second=avg_qps,
recall=avg_recall,
algorithm=first_index.algo,
search_params=first_index.search_params,
metadata={
"num_indexes": len(indexes),
"num_benchmarks": len(benchmarks),
"elapsed_time": elapsed_time,
"latency_us": benchmarks[0].get("Latency")
if benchmarks
else None,
"end_to_end": benchmarks[0].get("end_to_end")
if benchmarks
else None,
"context": gbench_results.get("context", {}),
},
success=True,
)
return [
SearchResult(
neighbors=np.array([]), # Not available from C++ benchmark
distances=np.array([]), # Not available from C++ benchmark
search_time_ms=total_search_time,
queries_per_second=avg_qps,
recall=avg_recall,
algorithm=first_index.algo,
search_params=first_index.search_params,
metadata={
"num_indexes": len(indexes),
"num_benchmarks": len(benchmarks),
"elapsed_time": elapsed_time,
"latency_us": benchmarks[0].get("Latency")
if benchmarks
else None,
"end_to_end": benchmarks[0].get("end_to_end")
if benchmarks
else None,
"context": gbench_results.get("context", {}),
},
success=True,
)
]

except subprocess.CalledProcessError as e:
return SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=time.perf_counter() - start_time,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params,
success=False,
error_message=f"Search failed: {e.stderr}",
)
return [
SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=time.perf_counter() - start_time,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params,
success=False,
error_message=f"Search failed: {e.stderr}",
)
]

except Exception as e:
return SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=time.perf_counter() - start_time,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params,
success=False,
error_message=f"Search error: {str(e)}",
)
return [
SearchResult(
neighbors=np.array([]),
distances=np.array([]),
search_time_ms=time.perf_counter() - start_time,
queries_per_second=0.0,
recall=0.0,
algorithm=first_index.algo,
search_params=first_index.search_params,
success=False,
error_message=f"Search error: {str(e)}",
)
]

finally:
# Cleanup temporary config
Expand Down
Loading
Loading