From 9161eb325d4c09f39368b6716a3253a6c5133730 Mon Sep 17 00:00:00 2001 From: erkrali Date: Thu, 19 Nov 2026 17:51:52 +0000 Subject: [PATCH] bench: ask the community row's plan probes about the cards the run will use --community rightly refuses --gpu, since a row has to be comparable, and then picks the cards the run uses the way the server does, from CUDA_VISIBLE_DEVICES. The plan probes that read the row's model name, its cache format and its fixtures still asked about "--gpu 0". On a box whose first visible card is not the main card's architecture - a Blackwell card with an Ada card built in by -DPEER_ADA=ON, where enumeration may name the older one first - that probe plans an engine that could never start: the answer has no plan in it, the model name comes back empty, and the run is refused with a line pointing at the pack's metadata. The boxes -DPEER_ADA=ON exists for are then the ones that cannot file the rows that would validate it. Move the card selection above the probes and hand the probes the value the run is given. A probe is not a measurement, so no figure in a row changes. Two small things ride along, both about reaching the real cause: a refused community run now quotes the plan log's own error line, and an engine that fails to start leaves behind the temp directory its message names instead of deleting it. docs/community.md says what such a box should pass in CUDA_VISIBLE_DEVICES, which is the question this reached as. --- docs/community.md | 2 +- src/program/bench_main.cpp | 74 ++++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/docs/community.md b/docs/community.md index 5a830e6aa..6743412ac 100644 --- a/docs/community.md +++ b/docs/community.md @@ -4,7 +4,7 @@ ## Add your run -1. Stop any running Basalt server (the GPUs must be free), then run `basalt-bench --community --model PACK` (a Flash-Next, a 27B or a DeepSeek pack). It starts its own engine (three times, once per measurement), shows the load progress on stderr while each one starts, adds its row to `docs/community/benchmarks.json` and prints it. It uses every visible Blackwell GPU: a second card is the peer, and `CUDA_VISIBLE_DEVICES=0` gives a single-card row. +1. Stop any running Basalt server (the GPUs must be free), then run `basalt-bench --community --model PACK` (a Flash-Next, a 27B or a DeepSeek pack). It starts its own engine (three times, once per measurement), shows the load progress on stderr while each one starts, adds its row to `docs/community/benchmarks.json` and prints it. It uses every visible Blackwell GPU: a second card is the peer, and `CUDA_VISIBLE_DEVICES=0` gives a single-card row. Where the driver enumerates an older card first - a Blackwell card with an `-DPEER_ADA=ON` peer - put the Blackwell card first anyway: `CUDA_VISIBLE_DEVICES=1` for a single-card row, `CUDA_VISIBLE_DEVICES=1,0` with the peer, because the harness samples power and clocks from the first visible card while it measures. 2. `--submitter NAME` is required: the name you want shown (one name with no spaces, up to 40 characters; it is not verified). `notes` (up to 200 characters) is optional and added by hand. 3. Review the row in `docs/community/benchmarks.json`, commit it and open a pull request. If the file already has a row of your setup (see below), the bench replaces that row in place, keeps its `notes` and says so; check the notes still apply. If the bench could not write the file, it prints the row and says where to paste it. diff --git a/src/program/bench_main.cpp b/src/program/bench_main.cpp index 6bc49d54c..7ff0bd68c 100644 --- a/src/program/bench_main.cpp +++ b/src/program/bench_main.cpp @@ -679,6 +679,19 @@ std::string hardware_text(const Hardware& h, const std::vector used; + // a community run uses every visible card; CUDA_VISIBLE_DEVICES picks them + const std::string want = o.community ? "auto" : has_arg(args, "--gpu") ? arg_value(args, "--gpu") : o.gpu; + if (!basalt::program::select_gpus(want, "basalt-bench", gpu_value, used, gerr)) { + basalt::log::error("basalt-bench", "%s", gerr.c_str()); + return 2; + } + set_arg(args, "--gpu", gpu_value); + } + // the plan probes ask about these cards and not about GPU 0: on a box whose first visible card is not the main + // card's architecture - a Blackwell card with an older expert-tier peer, where enumeration may name the older + // one first - a probe at --gpu 0 plans an engine that could never start, and an answer with no plan in it has + // no model name in it either. A probe is not a measurement, so a row made this way is the same row. + const std::string probe_gpu = gpu_value.empty() ? std::string("0") : gpu_value; if (o.community && (!o.passthrough.empty() || !o.sets.empty())) { // any engine flag is refused; the engine's plan for them says which are lossy, so the refusal can name those std::vector extra = o.passthrough; @@ -743,7 +778,7 @@ int main(int argc, char** argv) { extra.push_back(kv.first); extra.push_back(kv.second); } - std::vector plan_args = {"--model", fs::absolute(o.model).string(), "--gpu", "0"}; + std::vector plan_args = {"--model", fs::absolute(o.model).string(), "--gpu", probe_gpu}; merge_args(plan_args, extra); std::string answer; const std::string plan_log = basalt::platform::temp_dir() + "/basalt-bench-plan-" + std::to_string((long long) basalt::platform::process_id()); @@ -760,20 +795,21 @@ int main(int argc, char** argv) { return 2; } } - std::string community_model, community_arch; + std::string community_model, community_arch, community_plan_err; std::string community_kv; // the cache format a community run passes; empty when the model takes neither if (o.fixtures.empty() || o.community) { // the model's own fixtures, when it has them; the community row's model name std::string answer; const std::string plan_log = basalt::platform::temp_dir() + "/basalt-bench-plan-" + std::to_string((long long) basalt::platform::process_id()); const std::string abs_model = fs::absolute(o.model).string(); - const int asked = o.model.empty() ? 0 : engine_plan(engine_bin, {"--model", abs_model, "--gpu", "0"}, plan_log, answer); + const int asked = o.model.empty() ? 0 : engine_plan(engine_bin, {"--model", abs_model, "--gpu", probe_gpu}, plan_log, answer); if (o.community) { community_model = asked == 0 ? basalt::program::plan_field(answer, "model") : std::string(); community_arch = basalt::program::plan_field(answer, "arch"); + if (asked != 0) community_plan_err = plan_complaint(plan_log); // read it before the probes below reuse the log // the cache format users run, when the engine accepts one of the flags for this model std::string unused; - if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", "0", "--kv", "int8"}, plan_log, unused) == 0) community_kv = "int8"; - else if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", "0", "--kv", "fp8"}, plan_log, unused) == 0) community_kv = "fp8"; + if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", probe_gpu, "--kv", "int8"}, plan_log, unused) == 0) community_kv = "int8"; + else if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", probe_gpu, "--kv", "fp8"}, plan_log, unused) == 0) community_kv = "fp8"; } std::error_code rm_ec; fs::remove(plan_log, rm_ec); @@ -783,7 +819,7 @@ int main(int argc, char** argv) { if (do_quality && !o.model.empty()) { // what the gate starts this model with, and the window it scores std::string answer; const std::string plan_log = basalt::platform::temp_dir() + "/basalt-bench-plan-" + std::to_string((long long) basalt::platform::process_id()) + ".log"; - const int asked = engine_plan(engine_bin, {"--model", fs::absolute(o.model).string(), "--gpu", "0", "--ppl-base", o.kld_base}, plan_log, answer); + const int asked = engine_plan(engine_bin, {"--model", fs::absolute(o.model).string(), "--gpu", probe_gpu, "--ppl-base", o.kld_base}, plan_log, answer); std::error_code rm_ec; fs::remove(plan_log, rm_ec); if (asked != 0) { @@ -794,23 +830,6 @@ int main(int argc, char** argv) { const int window = std::atoi(basalt::program::plan_field(answer, "quality_window").c_str()); if (window > 0) o.quality_t = std::min(o.quality_t, window); } - for (const auto& kv : o.sets) set_arg(args, kv.first, kv.second); - if (!o.spec.empty()) set_arg(args, "--spec", o.spec); - if (o.model.empty()) for (const auto& a : o.passthrough) args.push_back(a); - else merge_args(args, o.passthrough); - // the GPUs, chosen here once as the server chooses them: the engine sees only those cards and says nothing again - if (!o.model.empty()) { - std::string gpu_value, gerr; - std::vector used; - // a community run uses every visible card; CUDA_VISIBLE_DEVICES picks them - const std::string want = o.community ? "auto" : has_arg(args, "--gpu") ? arg_value(args, "--gpu") : o.gpu; - if (!basalt::program::select_gpus(want, "basalt-bench", gpu_value, used, gerr)) { - basalt::log::error("basalt-bench", "%s", gerr.c_str()); - return 2; - } - set_arg(args, "--gpu", gpu_value); - } - // fixtures, read before anything is started std::vector> decode_ids(3); std::vector> prefill_parts; @@ -852,8 +871,10 @@ int main(int argc, char** argv) { return 0; } if (o.community && community_model.empty()) { - basalt::log::error("basalt-bench", "--community: the engine reports no community model name for %s (architecture '%s'); " - "no row was written", o.model.c_str(), community_arch.c_str()); + const std::string why = fmt("--community: the engine reports no community model name for %s (architecture '%s'); " + "no row was written", o.model.c_str(), community_arch.c_str()) + + (community_plan_err.empty() ? std::string() : "\n the engine's plan said: " + community_plan_err); + basalt::log::error("basalt-bench", "%s", why.c_str()); return 1; } @@ -941,7 +962,8 @@ int main(int argc, char** argv) { Engine eng; basalt::log::info("basalt-bench", "starting %s (log: %s)", engine_bin.c_str(), log_path.c_str()); if (!eng.start(engine_bin, perf_args, log_path, err)) { - basalt::log::error("basalt-bench", "%s", err.c_str()); + cleanup.on = false; // the line above names this log: keep the directory it sits in + basalt::log::error("basalt-bench", "%s (engine log: %s)", err.c_str(), log_path.c_str()); return 1; } { // fixtures of another tokenizer would fail at the first request with ids past the vocabulary