diff --git a/docs/community.md b/docs/community.md index 5a830e6aa..6743412ac 100644 --- a/docs/community.md +++ b/docs/community.md @@ -4,7 +4,7 @@ ## Add your run -1. Stop any running Basalt server (the GPUs must be free), then run `basalt-bench --community --model PACK` (a Flash-Next, a 27B or a DeepSeek pack). It starts its own engine (three times, once per measurement), shows the load progress on stderr while each one starts, adds its row to `docs/community/benchmarks.json` and prints it. It uses every visible Blackwell GPU: a second card is the peer, and `CUDA_VISIBLE_DEVICES=0` gives a single-card row. +1. Stop any running Basalt server (the GPUs must be free), then run `basalt-bench --community --model PACK` (a Flash-Next, a 27B or a DeepSeek pack). It starts its own engine (three times, once per measurement), shows the load progress on stderr while each one starts, adds its row to `docs/community/benchmarks.json` and prints it. It uses every visible Blackwell GPU: a second card is the peer, and `CUDA_VISIBLE_DEVICES=0` gives a single-card row. Where the driver enumerates an older card first - a Blackwell card with an `-DPEER_ADA=ON` peer - put the Blackwell card first anyway: `CUDA_VISIBLE_DEVICES=1` for a single-card row, `CUDA_VISIBLE_DEVICES=1,0` with the peer, because the harness samples power and clocks from the first visible card while it measures. 2. `--submitter NAME` is required: the name you want shown (one name with no spaces, up to 40 characters; it is not verified). `notes` (up to 200 characters) is optional and added by hand. 3. Review the row in `docs/community/benchmarks.json`, commit it and open a pull request. If the file already has a row of your setup (see below), the bench replaces that row in place, keeps its `notes` and says so; check the notes still apply. If the bench could not write the file, it prints the row and says where to paste it. diff --git a/src/program/bench_main.cpp b/src/program/bench_main.cpp index 6bc49d54c..7ff0bd68c 100644 --- a/src/program/bench_main.cpp +++ b/src/program/bench_main.cpp @@ -679,6 +679,19 @@ std::string hardware_text(const Hardware& h, const std::vector used; + // a community run uses every visible card; CUDA_VISIBLE_DEVICES picks them + const std::string want = o.community ? "auto" : has_arg(args, "--gpu") ? arg_value(args, "--gpu") : o.gpu; + if (!basalt::program::select_gpus(want, "basalt-bench", gpu_value, used, gerr)) { + basalt::log::error("basalt-bench", "%s", gerr.c_str()); + return 2; + } + set_arg(args, "--gpu", gpu_value); + } + // the plan probes ask about these cards and not about GPU 0: on a box whose first visible card is not the main + // card's architecture - a Blackwell card with an older expert-tier peer, where enumeration may name the older + // one first - a probe at --gpu 0 plans an engine that could never start, and an answer with no plan in it has + // no model name in it either. A probe is not a measurement, so a row made this way is the same row. + const std::string probe_gpu = gpu_value.empty() ? std::string("0") : gpu_value; if (o.community && (!o.passthrough.empty() || !o.sets.empty())) { // any engine flag is refused; the engine's plan for them says which are lossy, so the refusal can name those std::vector extra = o.passthrough; @@ -743,7 +778,7 @@ int main(int argc, char** argv) { extra.push_back(kv.first); extra.push_back(kv.second); } - std::vector plan_args = {"--model", fs::absolute(o.model).string(), "--gpu", "0"}; + std::vector plan_args = {"--model", fs::absolute(o.model).string(), "--gpu", probe_gpu}; merge_args(plan_args, extra); std::string answer; const std::string plan_log = basalt::platform::temp_dir() + "/basalt-bench-plan-" + std::to_string((long long) basalt::platform::process_id()); @@ -760,20 +795,21 @@ int main(int argc, char** argv) { return 2; } } - std::string community_model, community_arch; + std::string community_model, community_arch, community_plan_err; std::string community_kv; // the cache format a community run passes; empty when the model takes neither if (o.fixtures.empty() || o.community) { // the model's own fixtures, when it has them; the community row's model name std::string answer; const std::string plan_log = basalt::platform::temp_dir() + "/basalt-bench-plan-" + std::to_string((long long) basalt::platform::process_id()); const std::string abs_model = fs::absolute(o.model).string(); - const int asked = o.model.empty() ? 0 : engine_plan(engine_bin, {"--model", abs_model, "--gpu", "0"}, plan_log, answer); + const int asked = o.model.empty() ? 0 : engine_plan(engine_bin, {"--model", abs_model, "--gpu", probe_gpu}, plan_log, answer); if (o.community) { community_model = asked == 0 ? basalt::program::plan_field(answer, "model") : std::string(); community_arch = basalt::program::plan_field(answer, "arch"); + if (asked != 0) community_plan_err = plan_complaint(plan_log); // read it before the probes below reuse the log // the cache format users run, when the engine accepts one of the flags for this model std::string unused; - if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", "0", "--kv", "int8"}, plan_log, unused) == 0) community_kv = "int8"; - else if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", "0", "--kv", "fp8"}, plan_log, unused) == 0) community_kv = "fp8"; + if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", probe_gpu, "--kv", "int8"}, plan_log, unused) == 0) community_kv = "int8"; + else if (engine_plan(engine_bin, {"--model", abs_model, "--gpu", probe_gpu, "--kv", "fp8"}, plan_log, unused) == 0) community_kv = "fp8"; } std::error_code rm_ec; fs::remove(plan_log, rm_ec); @@ -783,7 +819,7 @@ int main(int argc, char** argv) { if (do_quality && !o.model.empty()) { // what the gate starts this model with, and the window it scores std::string answer; const std::string plan_log = basalt::platform::temp_dir() + "/basalt-bench-plan-" + std::to_string((long long) basalt::platform::process_id()) + ".log"; - const int asked = engine_plan(engine_bin, {"--model", fs::absolute(o.model).string(), "--gpu", "0", "--ppl-base", o.kld_base}, plan_log, answer); + const int asked = engine_plan(engine_bin, {"--model", fs::absolute(o.model).string(), "--gpu", probe_gpu, "--ppl-base", o.kld_base}, plan_log, answer); std::error_code rm_ec; fs::remove(plan_log, rm_ec); if (asked != 0) { @@ -794,23 +830,6 @@ int main(int argc, char** argv) { const int window = std::atoi(basalt::program::plan_field(answer, "quality_window").c_str()); if (window > 0) o.quality_t = std::min(o.quality_t, window); } - for (const auto& kv : o.sets) set_arg(args, kv.first, kv.second); - if (!o.spec.empty()) set_arg(args, "--spec", o.spec); - if (o.model.empty()) for (const auto& a : o.passthrough) args.push_back(a); - else merge_args(args, o.passthrough); - // the GPUs, chosen here once as the server chooses them: the engine sees only those cards and says nothing again - if (!o.model.empty()) { - std::string gpu_value, gerr; - std::vector used; - // a community run uses every visible card; CUDA_VISIBLE_DEVICES picks them - const std::string want = o.community ? "auto" : has_arg(args, "--gpu") ? arg_value(args, "--gpu") : o.gpu; - if (!basalt::program::select_gpus(want, "basalt-bench", gpu_value, used, gerr)) { - basalt::log::error("basalt-bench", "%s", gerr.c_str()); - return 2; - } - set_arg(args, "--gpu", gpu_value); - } - // fixtures, read before anything is started std::vector> decode_ids(3); std::vector> prefill_parts; @@ -852,8 +871,10 @@ int main(int argc, char** argv) { return 0; } if (o.community && community_model.empty()) { - basalt::log::error("basalt-bench", "--community: the engine reports no community model name for %s (architecture '%s'); " - "no row was written", o.model.c_str(), community_arch.c_str()); + const std::string why = fmt("--community: the engine reports no community model name for %s (architecture '%s'); " + "no row was written", o.model.c_str(), community_arch.c_str()) + + (community_plan_err.empty() ? std::string() : "\n the engine's plan said: " + community_plan_err); + basalt::log::error("basalt-bench", "%s", why.c_str()); return 1; } @@ -941,7 +962,8 @@ int main(int argc, char** argv) { Engine eng; basalt::log::info("basalt-bench", "starting %s (log: %s)", engine_bin.c_str(), log_path.c_str()); if (!eng.start(engine_bin, perf_args, log_path, err)) { - basalt::log::error("basalt-bench", "%s", err.c_str()); + cleanup.on = false; // the line above names this log: keep the directory it sits in + basalt::log::error("basalt-bench", "%s (engine log: %s)", err.c_str(), log_path.c_str()); return 1; } { // fixtures of another tokenizer would fail at the first request with ids past the vocabulary