From 480ebd2a6f07e07455d280aa8a3d8c062a06ca85 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 22:19:07 -0700 Subject: [PATCH 01/20] Sort SASO nonzeros within each major-axis vector during sampling The SASO sampling path emits each major-axis vector's vec_nnz nonzeros in Fisher-Yates draw order, so a wide SASO's COO is column-ordered but NOT row-ordered within a column -- coo_arrays_determine_sort labels it None, and apply_coo_via_csc then pays a deepcopy + re-sort to CSC on every apply (only the vec_nnz==1 CountSketch path, which short-circuits to the i.i.d. sampler, escaped this). Sort each contiguous vec_nnz block by its major coordinate in the SASO branch of fill_sparse_unpacked, before Phase-2 compaction. A wide SASO now emits CSC-sorted COO (tall SASO -> CSR, which the right_spmm transpose turns into CSC under the row/col swap), so the per-apply re-sort is skipped. The operator is mathematically identical -- only the within-vector storage order changes -- so thread-independence and the RNG stream are preserved. vec_nnz is small, so a no-alloc insertion sort is used. Scoped to operator construction; the public repeated_fisher_yates index utility is untouched. All 435 ctest pass (the structural and KS tests are order-independent). Co-Authored-By: Claude Opus 4.8 (1M context) --- RandBLAS/sparse_skops.hh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index 9284d25d..e140ed8e 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -645,6 +645,27 @@ state_t fill_sparse_unpacked( work_state, vec_nnz, dim_major, num_major_sub, idxs_major, idxs_minor, vals ); total = vec_nnz * num_major_sub; + // Emit each major-axis vector in ascending major-coordinate order. Fisher-Yates + // draws the vec_nnz nonzeros in shuffle order; sorting each contiguous block by + // its major coordinate (rows for a wide SASO, cols for a tall one) makes the COO + // natively CSC- (wide) / CSR- (tall) sorted, so apply_coo_via_csc skips its + // per-apply deepcopy + re-sort. idxs_minor is constant within a block, so only + // (idxs_major, vals) move. vec_nnz is small, so a no-alloc insertion sort is best. + for (int64_t b = 0; b < num_major_sub; ++b) { + sint_t* blk_major = idxs_major + b * vec_nnz; + T* blk_vals = vals + b * vec_nnz; + for (int64_t a = 1; a < vec_nnz; ++a) { + sint_t key = blk_major[a]; + T v = blk_vals[a]; + int64_t c = a - 1; + for (; c >= 0 && blk_major[c] > key; --c) { + blk_major[c+1] = blk_major[c]; + blk_vals[c+1] = blk_vals[c]; + } + blk_major[c+1] = key; + blk_vals[c+1] = v; + } + } } else { // LASO: each major-axis vector is sampled with replacement and merged in place, // advancing through the output buffers exactly as the full operator does. From c7834f2646aca5e54b66f61fce9efc8b5080d292 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 22:19:22 -0700 Subject: [PATCH 02/20] Add sketch_general performance benchmark with work-normalized metrics Benchmarks the full sketch_general() call with real SparseSkOps (wide SASO, CountSketch, wide LASO; left and right sketch; both layouts) -- the structured operators spmm_performance.cc never produces, so it exercises the regular-CSC path and the COO sort/re-sort behavior that sketching actually hits. Reports work-normalized metrics so results are comparable across vec_nnz and SASO/LASO: ns/elt = time/(nnz*work_mult), and model GB/s as a fraction of a measured STREAM-triad ceiling (%STR). The byte model bounds A-read traffic by min(nnz, contract_dim) so sparse LASO operators don't report impossible bandwidth. --no-stream skips calibration; --scaling sweeps thread counts and reports speedup / parallel efficiency / %STR per count. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/CMakeLists.txt | 10 + .../sketch_general_performance.cc | 555 ++++++++++++++++++ 2 files changed, 565 insertions(+) create mode 100644 examples/simple-kernel-benchmarks/sketch_general_performance.cc diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0d373186..a4d0a14c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -102,5 +102,15 @@ target_link_libraries( spmm_performance PUBLIC RandBLAS blaspp lapackpp ) +add_executable( + sketch_general_performance simple-kernel-benchmarks/sketch_general_performance.cc +) +target_include_directories( + sketch_general_performance PUBLIC ${Random123_DIR} +) +target_link_libraries( + sketch_general_performance PUBLIC RandBLAS blaspp lapackpp +) + diff --git a/examples/simple-kernel-benchmarks/sketch_general_performance.cc b/examples/simple-kernel-benchmarks/sketch_general_performance.cc new file mode 100644 index 00000000..24e06d53 --- /dev/null +++ b/examples/simple-kernel-benchmarks/sketch_general_performance.cc @@ -0,0 +1,555 @@ +// Copyright, 2026. See LICENSE for copyright holder information. +// +// ============================================================================ +// SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse sketching operators) +// ============================================================================ +// +// Analogous to spmm_performance.cc, but it benchmarks the full sketching call +// sketch_general() with a real SparseSkOp -- not the low-level left_spmm on a +// generic uniform-random matrix. This exercises the *structured* operators that +// spmm_performance never produces: +// +// * Wide SASO SparseDist(d, m, k, Short) -- fixed k nnz per column +// * CountSketch SparseDist(d, m, 1, Short) -- exactly one nnz per column +// * Wide LASO SparseDist(d, m, k, Long) -- <= k nnz per row, empty cols +// +// and (via the right-sketch / tall operators) the transpose reduction that maps +// a tall SASO right-sketch onto the same column-regular kernel as a wide SASO. +// +// WHY WORK-NORMALIZED METRICS: raw wall-time conflates "how much work" with "how +// efficiently it's done". A vec_nnz=8 operator has 4x the nonzeros of vec_nnz=2, +// and a wide SASO has ~m/d x the nonzeros of a wide LASO at the same vec_nnz, so +// raw times are not comparable across those axes. We therefore report: +// +// * ns/elt = time / (nnz * work_mult): nanoseconds per nonzero-output-element. +// work_mult = n for a left-sketch (each S-nonzero touches a length-n output +// row), m for a right-sketch. This IS comparable across vec_nnz and SASO/LASO; +// for an appropriately optimized kernel it stays roughly flat as vec_nnz grows. +// * GB/s (%STR) = a MODEL byte count / time, as a fraction of measured STREAM +// bandwidth. These kernels are memory-bound (arithmetic intensity ~ vec_nnz/4 +// flop/byte), so % of STREAM is the roofline ceiling: ~80% means "done", ~20% +// means headroom. The byte count is an idealized-reuse model, not a hardware +// counter (perf counters are SIP-gated on macOS) -- read it as a bound. Two +// caveats: (a) A-read traffic is bounded by the DISTINCT rows/cols of A actually +// touched (<= min(nnz, contract_dim)), so a sparse LASO reads far less than all +// of A; (b) STREAM is a DRAM ceiling, so a small cache-resident problem (e.g. a +// tiny LASO) can legitimately exceed 100% %STR -- that flags "fits in cache", +// not a measurement error. +// +// Parallel efficiency needs the SAME kernel run at multiple thread counts, so it +// is NOT free per run; it lives behind --scaling (see USAGE). +// +// The cost of a sparse sketch has three phases; we time them separately: +// 1. SAMPLE fill_sparse(S): populate (rows, cols, vals). +// 2. CONVERT the COO->CSC sort inside apply_coo_via_csc (deepcopy + re-sort, +// incurred every apply when the operator's COO is not CSC-sorted). +// 3. KERNEL apply_csc_jki_p11 (ColMajor) / apply_csc_kib_1p1_rowmajor (RowMajor). +// WARM apply (S pre-sampled; phases 2+3) is the primary number; COLD (1+2+3) and +// CONVERT-only appear in the notes of the left/ColMajor table. +// +// NOTATION (left-sketch): B(d x n) = alpha * S(d x m) * A(m x n) +// (right-sketch): B(m x d) = alpha * A(m x n) * S(n x d) +// +// USAGE: +// ./sketch_general_performance [flags] # default sweep +// ./sketch_general_performance [flags] d m n vec_nnz major [trials] # single cfg +// major: 0 = Short (SASO), 1 = Long (LASO) +// flags: +// --no-stream skip the startup STREAM calibration (%STR prints '-') +// --scaling thread-scaling mode: speedup/efficiency/%STR per thread +// --threads=1,2,4,8 thread counts for --scaling (default 1,2,4,8) +// +// EXAMPLES: +// env OMP_NUM_THREADS=4 ./sketch_general_performance +// env OMP_NUM_THREADS=4 ./sketch_general_performance 200 2000 2000 4 0 +// ./sketch_general_performance --scaling --threads=1,2,4,8 200 2000 2000 4 0 +// +// ============================================================================ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "RandBLAS/config.h" +#include "RandBLAS/sparse_data/spmm_dispatch.hh" +#if defined(RandBLAS_HAS_OpenMP) +#include +#endif + +using namespace std::chrono; +using blas::Layout; +using blas::Op; +using RandBLAS::Axis; +using RandBLAS::SparseDist; +using RandBLAS::SparseSkOp; + +// Machine bandwidth ceiling (GB/s) used as the %STR denominator; <0 disables %STR. +static double g_stream_gbps = -1.0; + +// ---- OpenMP shims (no-ops without OpenMP) ---------------------------------- +static int current_threads() { +#if defined(RandBLAS_HAS_OpenMP) + return omp_get_max_threads(); +#else + return 1; +#endif +} +static void set_threads(int t) { +#if defined(RandBLAS_HAS_OpenMP) + omp_set_num_threads(t); +#else + (void)t; +#endif +} + +// Run num_trials repetitions, return {min, median} times in microseconds. +template +std::pair run_trials(Func&& func, int num_trials) { + std::vector times; + times.reserve(num_trials); + for (int t = 0; t < num_trials; ++t) { + auto start = steady_clock::now(); + func(); + auto end = steady_clock::now(); + times.push_back(duration_cast(end - start).count()); + } + std::sort(times.begin(), times.end()); + return {times[0], times[num_trials / 2]}; +} + +// STREAM triad (a = b + scalar*c) at the current OpenMP thread count. Returns +// achievable bandwidth in GB/s (min time over reps; 3*N*8 bytes per pass). +static double measure_stream_gbps(int64_t N = (int64_t)1 << 24, int reps = 10) { + std::vector a(N), b(N), c(N); + #pragma omp parallel for schedule(static) + for (int64_t i = 0; i < N; ++i) { a[i] = 0.0; b[i] = 1.0; c[i] = 2.0; } + const double scalar = 3.0; + // warmup + #pragma omp parallel for schedule(static) + for (int64_t i = 0; i < N; ++i) a[i] = b[i] + scalar * c[i]; + auto best = std::numeric_limits::max(); + for (int r = 0; r < reps; ++r) { + auto s = steady_clock::now(); + #pragma omp parallel for schedule(static) + for (int64_t i = 0; i < N; ++i) a[i] = b[i] + scalar * c[i]; + auto e = steady_clock::now(); + best = std::min(best, duration_cast(e - s).count()); + } + volatile double sink = a[N - 1]; (void)sink; // defeat dead-store elision + double bytes = 3.0 * (double)N * sizeof(double); + return best > 0 ? bytes / (double)best / 1000.0 : -1.0; // bytes/us/1000 == GB/s +} + +// ---- per-row metrics ------------------------------------------------------- +struct Row { + std::string label; + long min_us = 0, med_us = 0; + double ns_per_elt = -1; // time / (nnz * work_mult) + double gbps = -1; // model bytes / time + double pct_stream = -1; // gbps / g_stream_gbps + std::string notes; +}; + +// work_mult = output cols touched per nonzero (n for left-sketch, m for right). +// read_dense = elements of A read; out_elems = elements of the dense output. +static void fill_metrics(Row& r, long min_us, int64_t nnz, int64_t work_mult, + int64_t read_dense, int64_t out_elems) { + int64_t work = nnz * work_mult; + r.ns_per_elt = (work > 0) ? (double)min_us * 1000.0 / (double)work : -1; + double bytes = (double)(read_dense + 2 * out_elems) * sizeof(double) + + (double)nnz * (sizeof(double) + sizeof(int64_t)); + r.gbps = (min_us > 0) ? bytes / (double)min_us / 1000.0 : -1; + r.pct_stream = (g_stream_gbps > 0 && r.gbps > 0) ? r.gbps / g_stream_gbps * 100.0 : -1; +} + +static std::string fcell(double v, int prec) { + if (v < 0) return "-"; + std::ostringstream o; o << std::fixed << std::setprecision(prec) << v; return o.str(); +} + +static void print_metric_header(const std::string& title) { + std::cout << " " << title << "\n"; + std::cout << " " << std::left << std::setw(26) << "Operator" << std::right + << std::setw(9) << "Min(us)" << std::setw(9) << "Med(us)" + << std::setw(9) << "ns/elt" << std::setw(8) << "GB/s" + << std::setw(7) << "%STR" << " notes\n"; + std::cout << " " << std::string(76, '-') << "\n"; +} + +static void print_metric_row(const Row& r) { + std::cout << " " << std::left << std::setw(26) << r.label << std::right + << std::setw(9) << r.min_us << std::setw(9) << r.med_us + << std::setw(9) << fcell(r.ns_per_elt, 3) + << std::setw(8) << fcell(r.gbps, 1) + << std::setw(7) << fcell(r.pct_stream, 0) + << " " << r.notes << "\n"; +} + +static const char* sort_name(RandBLAS::sparse_data::NonzeroSort s) { + using NS = RandBLAS::sparse_data::NonzeroSort; + switch (s) { + case NS::CSC: return "CSC"; + case NS::CSR: return "CSR"; + default: return "None"; + } +} + +// One distribution config, benchmarked across both layouts on one side. +struct OpSpec { + std::string label; + int64_t vec_nnz; + Axis axis; +}; + +// --------------------------------------------------------------------------- +// LEFT-SKETCH: B(d x n) = S(d x m) * A(m x n), S ~ SparseDist(d, m, k, axis). +// work_mult = n; read_dense(A) = m*n; out_elems(B) = d*n. +// --------------------------------------------------------------------------- +void run_left(int64_t d, int64_t m, int64_t n, + const std::vector& specs, int num_trials) { + using T = double; + namespace rb = RandBLAS; + uint64_t seed = 12345; + + std::cout << "=== LEFT-SKETCH B(" << d << "x" << n << ") = S(" << d << "x" << m + << ") * A(" << m << "x" << n << "), trials=" << num_trials << " ===\n\n"; + + std::vector A_cm(m * n), A_rm(m * n); + rb::DenseDist DA(m, n); + auto st = rb::fill_dense(DA, A_cm.data(), rb::RNGState<>(seed)); + for (int64_t i = 0; i < m; ++i) + for (int64_t j = 0; j < n; ++j) + A_rm[i * n + j] = A_cm[i + j * m]; + + std::vector B_cm(d * n), B_rm(d * n), B_ref(d * n), result(d * n); + std::vector S_dense(d * m); + + std::vector rows_cm, rows_rm; + Row oracle; oracle.label = "densify(S)+GEMM"; oracle.notes = "dense oracle"; + + for (const auto& spec : specs) { + SparseDist dist(d, m, spec.vec_nnz, spec.axis); + SparseSkOp S(dist, st); + rb::fill_sparse(S); + auto view = rb::sparse::coo_view_of_skop(S); + int64_t nnz = S.nnz; + std::string base_note = std::string(sort_name(view.sort)) + " nnz=" + std::to_string(nnz); + + // Oracle: densify S then GEMM (ColMajor); also the correctness oracle. + rb::sparse_data::coo::coo_to_dense(view, Layout::ColMajor, S_dense.data()); + auto [t_gemm, m_gemm] = run_trials([&]() { + std::fill(B_ref.begin(), B_ref.end(), 0.0); + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S_dense.data(), d, A_cm.data(), m, 0.0, B_ref.data(), d); + }, num_trials); + oracle.min_us = t_gemm; oracle.med_us = m_gemm; + + auto [wcm, wcm_med] = run_trials([&]() { + std::fill(B_cm.begin(), B_cm.end(), 0.0); + rb::sketch_general(Layout::ColMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S, 0, 0, A_cm.data(), m, 0.0, B_cm.data(), d); + }, num_trials); + auto [wrm, wrm_med] = run_trials([&]() { + std::fill(B_rm.begin(), B_rm.end(), 0.0); + rb::sketch_general(Layout::RowMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S, 0, 0, A_rm.data(), n, 0.0, B_rm.data(), n); + }, num_trials); + + // COLD ColMajor: fresh unsampled operator each trial (sample+convert+kernel). + auto [ccm, ccm_med] = run_trials([&]() { + SparseSkOp Sc(dist, st); + std::fill(B_cm.begin(), B_cm.end(), 0.0); + rb::sketch_general(Layout::ColMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, Sc, 0, 0, A_cm.data(), m, 0.0, B_cm.data(), d); + }, num_trials); + (void)ccm_med; + // CONVERT-only: worst-case COO->CSC re-sort (deepcopy + sort). + auto [cvt, cvt_med] = run_trials([&]() { + auto cp = view.deepcopy(); + cp.sort_arrays(rb::sparse_data::NonzeroSort::CSC); + }, num_trials); + (void)cvt_med; + + // Correctness vs the ColMajor oracle (both layouts). + std::fill(result.begin(), result.end(), 0.0); + rb::sketch_general(Layout::ColMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S, 0, 0, A_cm.data(), m, 0.0, result.data(), d); + double maxdiff = 0; + for (int64_t i = 0; i < d * n; ++i) + maxdiff = std::max(maxdiff, std::abs(result[i] - B_ref[i])); + if (maxdiff > 1e-9) + std::cout << " FAIL " << spec.label << " ColMajor max|diff|=" + << std::scientific << maxdiff << std::fixed << "\n"; + std::fill(result.begin(), result.end(), 0.0); + rb::sketch_general(Layout::RowMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S, 0, 0, A_rm.data(), n, 0.0, result.data(), n); + maxdiff = 0; + for (int64_t i = 0; i < d; ++i) + for (int64_t j = 0; j < n; ++j) + maxdiff = std::max(maxdiff, std::abs(result[i * n + j] - B_ref[i + j * d])); + if (maxdiff > 1e-9) + std::cout << " FAIL " << spec.label << " RowMajor max|diff|=" + << std::scientific << maxdiff << std::fixed << "\n"; + + // A-read traffic is bounded by the DISTINCT rows of A touched (<= min(nnz,m)), + // not all m rows: a wide LASO has mostly-empty columns. Using m here would + // overcount sparse operators (and push %STR past 100%). + int64_t a_read = std::min(nnz, m) * n; + Row rc; rc.label = spec.label; rc.min_us = wcm; rc.med_us = wcm_med; + fill_metrics(rc, wcm, nnz, n, a_read, d * n); + rc.notes = base_note + " cold=" + std::to_string(ccm) + " cvt=" + std::to_string(cvt); + rows_cm.push_back(rc); + + Row rr; rr.label = spec.label; rr.min_us = wrm; rr.med_us = wrm_med; + fill_metrics(rr, wrm, nnz, n, a_read, d * n); + rr.notes = base_note; + rows_rm.push_back(rr); + } + + print_metric_header("warm apply, ColMajor dense (jik/jki kernels)"); + for (auto& r : rows_cm) print_metric_row(r); + print_metric_row(oracle); + std::cout << "\n"; + print_metric_header("warm apply, RowMajor dense (ikb/kib kernels)"); + for (auto& r : rows_rm) print_metric_row(r); + std::cout << "\n"; +} + +// --------------------------------------------------------------------------- +// RIGHT-SKETCH: B(m x d) = A(m x n) * S(n x d), S ~ SparseDist(n, d, k, axis). +// work_mult = m; read_dense(A) = m*n; out_elems(B) = m*d. +// --------------------------------------------------------------------------- +void run_right(int64_t d, int64_t m, int64_t n, + const std::vector& specs, int num_trials) { + using T = double; + namespace rb = RandBLAS; + uint64_t seed = 67890; + + std::cout << "=== RIGHT-SKETCH B(" << m << "x" << d << ") = A(" << m << "x" << n + << ") * S(" << n << "x" << d << "), trials=" << num_trials << " ===\n\n"; + + std::vector A_cm(m * n), A_rm(m * n); + rb::DenseDist DA(m, n); + auto st = rb::fill_dense(DA, A_cm.data(), rb::RNGState<>(seed)); + for (int64_t i = 0; i < m; ++i) + for (int64_t j = 0; j < n; ++j) + A_rm[i * n + j] = A_cm[i + j * m]; + + std::vector B_cm(m * d), B_rm(m * d), B_ref(m * d), result(m * d); + std::vector S_dense(n * d); + + std::vector rows_cm, rows_rm; + + for (const auto& spec : specs) { + SparseDist dist(n, d, spec.vec_nnz, spec.axis); + SparseSkOp S(dist, st); + rb::fill_sparse(S); + auto view = rb::sparse::coo_view_of_skop(S); + int64_t nnz = S.nnz; + std::string base_note = std::string(sort_name(view.sort)) + " nnz=" + std::to_string(nnz); + + rb::sparse_data::coo::coo_to_dense(view, Layout::ColMajor, S_dense.data()); + std::fill(B_ref.begin(), B_ref.end(), 0.0); + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, m, d, n, + 1.0, A_cm.data(), m, S_dense.data(), n, 0.0, B_ref.data(), m); + + auto [wcm, wcm_med] = run_trials([&]() { + std::fill(B_cm.begin(), B_cm.end(), 0.0); + rb::sketch_general(Layout::ColMajor, Op::NoTrans, Op::NoTrans, m, d, n, + 1.0, A_cm.data(), m, S, 0.0, B_cm.data(), m); + }, num_trials); + auto [wrm, wrm_med] = run_trials([&]() { + std::fill(B_rm.begin(), B_rm.end(), 0.0); + rb::sketch_general(Layout::RowMajor, Op::NoTrans, Op::NoTrans, m, d, n, + 1.0, A_rm.data(), n, S, 0.0, B_rm.data(), d); + }, num_trials); + + std::fill(result.begin(), result.end(), 0.0); + rb::sketch_general(Layout::ColMajor, Op::NoTrans, Op::NoTrans, m, d, n, + 1.0, A_cm.data(), m, S, 0.0, result.data(), m); + double maxdiff = 0; + for (int64_t i = 0; i < m * d; ++i) + maxdiff = std::max(maxdiff, std::abs(result[i] - B_ref[i])); + if (maxdiff > 1e-9) + std::cout << " FAIL " << spec.label << " ColMajor max|diff|=" + << std::scientific << maxdiff << std::fixed << "\n"; + + // A-read traffic bounded by distinct columns of A touched (<= min(nnz,n)). + int64_t a_read = std::min(nnz, n) * m; + Row rc; rc.label = spec.label; rc.min_us = wcm; rc.med_us = wcm_med; + fill_metrics(rc, wcm, nnz, m, a_read, m * d); rc.notes = base_note; + rows_cm.push_back(rc); + Row rr; rr.label = spec.label; rr.min_us = wrm; rr.med_us = wrm_med; + fill_metrics(rr, wrm, nnz, m, a_read, m * d); rr.notes = base_note; + rows_rm.push_back(rr); + } + + print_metric_header("warm apply, ColMajor dense"); + for (auto& r : rows_cm) print_metric_row(r); + std::cout << "\n"; + print_metric_header("warm apply, RowMajor dense"); + for (auto& r : rows_rm) print_metric_row(r); + std::cout << "\n"; +} + +// --------------------------------------------------------------------------- +// SCALING: re-run the left-sketch ColMajor warm apply at each thread count and +// report speedup, parallel efficiency, and %STREAM. STREAM is recalibrated per +// thread count so %STR is apples-to-apples. Note: a memory-bound kernel that has +// already saturated bandwidth SHOULD show efficiency < 1 -- read efficiency next +// to %STR, not in isolation. +// --------------------------------------------------------------------------- +void run_scaling(int64_t d, int64_t m, int64_t n, const std::vector& specs, + int num_trials, const std::vector& threads, bool do_stream) { + using T = double; + namespace rb = RandBLAS; + uint64_t seed = 12345; + + std::cout << "=== SCALING (left-sketch, ColMajor warm) d=" << d << " m=" << m + << " n=" << n << ", trials=" << num_trials << " ===\n"; +#if !defined(RandBLAS_HAS_OpenMP) + std::cout << " (built without OpenMP -- thread sweep is a no-op)\n"; +#endif + std::cout << "\n"; + + std::vector A_cm(m * n); + rb::DenseDist DA(m, n); + auto st = rb::fill_dense(DA, A_cm.data(), rb::RNGState<>(seed)); + std::vector B_cm(d * n); + + for (const auto& spec : specs) { + SparseDist dist(d, m, spec.vec_nnz, spec.axis); + SparseSkOp S(dist, st); + rb::fill_sparse(S); + int64_t nnz = S.nnz; + + std::cout << " " << spec.label << " (nnz=" << nnz << ")\n"; + std::cout << " " << std::right << std::setw(6) << "Thr" + << std::setw(10) << "Min(us)" << std::setw(10) << "Speedup" + << std::setw(8) << "Eff" << std::setw(9) << "GB/s" + << std::setw(7) << "%STR" << "\n"; + std::cout << " " << std::string(48, '-') << "\n"; + + long base_us = 0; + int base_t = threads.empty() ? 1 : threads.front(); + for (int t : threads) { + set_threads(t); + double stream = do_stream ? measure_stream_gbps() : -1.0; + auto [mn, md] = run_trials([&]() { + std::fill(B_cm.begin(), B_cm.end(), 0.0); + rb::sketch_general(Layout::ColMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S, 0, 0, A_cm.data(), m, 0.0, B_cm.data(), d); + }, num_trials); + (void)md; + if (t == base_t) base_us = mn; + double speedup = (mn > 0) ? (double)base_us / (double)mn : 0.0; + double eff = speedup / ((double)t / (double)base_t); + double bytes = (double)(std::min(nnz, m) * n + 2 * d * n) * sizeof(double) + + (double)nnz * (sizeof(double) + sizeof(int64_t)); + double gbps = (mn > 0) ? bytes / (double)mn / 1000.0 : -1; + double pct = (stream > 0 && gbps > 0) ? gbps / stream * 100.0 : -1; + std::cout << " " << std::right << std::setw(6) << t + << std::setw(10) << mn + << std::setw(10) << fcell(speedup, 2) + << std::setw(8) << fcell(eff, 2) + << std::setw(9) << fcell(gbps, 1) + << std::setw(7) << fcell(pct, 0) << "\n"; + } + std::cout << "\n"; + } +} + +std::vector sweep_specs() { + return { + {"Wide SASO k=2", 2, Axis::Short}, + {"Wide SASO k=4", 4, Axis::Short}, + {"Wide SASO k=8", 8, Axis::Short}, + {"CountSketch k=1", 1, Axis::Short}, + {"Wide LASO k=2", 2, Axis::Long}, + {"Wide LASO k=4", 4, Axis::Long}, + {"Wide LASO k=8", 8, Axis::Long}, + }; +} + +static std::vector parse_threads(const std::string& csv) { + std::vector out; + std::stringstream ss(csv); + std::string tok; + while (std::getline(ss, tok, ',')) { + if (!tok.empty()) out.push_back(std::atoi(tok.c_str())); + } + if (out.empty()) out = {1, 2, 4, 8}; + return out; +} + +int main(int argc, char** argv) { + bool no_stream = false, scaling = false; + std::vector threads = {1, 2, 4, 8}; + std::vector pos; + for (int i = 1; i < argc; ++i) { + std::string s = argv[i]; + if (s == "--no-stream") no_stream = true; + else if (s == "--scaling") scaling = true; + else if (s.rfind("--threads=", 0) == 0) threads = parse_threads(s.substr(10)); + else pos.push_back(s); + } + + std::cout << "\n============================================================\n"; + std::cout << "SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators)\n"; + std::cout << "============================================================\n"; + std::cout << "Threads (OMP_NUM_THREADS/max): " << current_threads() << "\n"; + + bool have_cfg = pos.size() >= 5; + int64_t d = have_cfg ? std::atoll(pos[0].c_str()) : 0; + int64_t m = have_cfg ? std::atoll(pos[1].c_str()) : 0; + int64_t n = have_cfg ? std::atoll(pos[2].c_str()) : 0; + int64_t k = have_cfg ? std::atoll(pos[3].c_str()) : 0; + Axis axis = (have_cfg && std::atoi(pos[4].c_str()) != 0) ? Axis::Long : Axis::Short; + int trials = (pos.size() > 5) ? std::atoi(pos[5].c_str()) : (have_cfg ? 20 : 10); + + if (scaling) { + // STREAM is calibrated per thread count inside run_scaling. + std::vector specs = have_cfg + ? std::vector{{"single", k, axis}} : sweep_specs(); + int64_t sd = have_cfg ? d : 200, sm = have_cfg ? m : 2000, sn = have_cfg ? n : 2000; + std::cout << "\n"; + run_scaling(sd, sm, sn, specs, trials, threads, !no_stream); + return 0; + } + + if (!no_stream) { + std::cout << "Calibrating STREAM triad..." << std::flush; + g_stream_gbps = measure_stream_gbps(); + std::cout << " " << std::fixed << std::setprecision(1) << g_stream_gbps + << " GB/s (= 100% on the %STR column)\n"; + } else { + std::cout << "STREAM calibration skipped (--no-stream); %STR shows '-'\n"; + } + std::cout << "\n"; + + if (have_cfg) { + std::vector specs = {{"single", k, axis}}; + run_left(d, m, n, specs, trials); + run_right(d, m, n, specs, trials); + return 0; + } + + auto specs = sweep_specs(); + std::cout << "########## SECTION 1: vary sketch size d (m=n=2000) ##########\n\n"; + for (int64_t dd : {40, 200, 500}) { + run_left(dd, 2000, 2000, specs, trials); + run_right(dd, 2000, 2000, specs, trials); + } + std::cout << "########## SECTION 2: narrow n (SpMV-ish) ##########\n\n"; + run_left(200, 2000, 1, specs, trials); + run_left(200, 2000, 10, specs, trials); + return 0; +} From e273fabe83a1ac1bb9af412b78159d9a316cdec0 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 22:19:31 -0700 Subject: [PATCH 03/20] Add sparse-sketching optimization proposal, bench plan, and results Design docs and measured baselines for the sparse-sketching SPMM work: - sparsesketching_opt.md: proposal framed on row- vs column-structured operators (transposition is free), with candidate kernels (CountSketch, bounded-CSR for wide LASO, implicit +/-1 values) and a benchmark-first recommendation. - sparsesketching_bench_plan.md: plan for the sketch_general benchmark, plus the measured finding that motivated the sampling fix. - sketch_general_results.txt: pre-fix baseline (raw timings). - sketch_general_results_postfix.txt: post-fix run with the work-normalized metric columns. These are working/analysis artifacts; drop if they shouldn't live in-tree. Co-Authored-By: Claude Opus 4.8 (1M context) --- sketch_general_results.txt | 209 +++++++++++++++++++++++++++ sketch_general_results_postfix.txt | 219 +++++++++++++++++++++++++++++ sparsesketching_bench_plan.md | 184 ++++++++++++++++++++++++ sparsesketching_opt.md | 185 ++++++++++++++++++++++++ 4 files changed, 797 insertions(+) create mode 100644 sketch_general_results.txt create mode 100644 sketch_general_results_postfix.txt create mode 100644 sparsesketching_bench_plan.md create mode 100644 sparsesketching_opt.md diff --git a/sketch_general_results.txt b/sketch_general_results.txt new file mode 100644 index 00000000..fe371e78 --- /dev/null +++ b/sketch_general_results.txt @@ -0,0 +1,209 @@ +# Environment: Apple M3, OMP_NUM_THREADS=4, Release build (MKL off) +# RandBLAS commit: 705f09e | Date: 2026-05-31 | Binary: build-randblas-examples/sketch_general_performance +# Correctness: all configs PASS (0 FAIL lines) + +============================================================ +SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) +============================================================ + +########## SECTION 1: vary sketch size d (m=n=2000) ########## + +=== LEFT-SKETCH B(40x2000) = S(40x2000) * A(2000x2000), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 1453 1469 3.29x nnz=4000 coo-sort=None cold=1483 convert=28 + Wide SASO k=4 2279 2337 5.16x nnz=8000 coo-sort=None cold=2357 convert=77 + Wide SASO k=8 4418 4528 10.00x nnz=16000 coo-sort=None cold=4363 convert=180 + CountSketch k=1 832 838 1.88x nnz=2000 coo-sort=CSC cold=859 convert=3 + Wide LASO k=2 553 757 1.25x nnz=78 coo-sort=None cold=483 convert=0 + Wide LASO k=4 442 450 1.00x nnz=158 coo-sort=None cold=460 convert=1 + Wide LASO k=8 477 488 1.08x nnz=318 coo-sort=None cold=506 convert=3 + densify(S)+GEMM 1566 1566 3.54x dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 767 779 27.39x nnz=4000 coo-sort=None + Wide SASO k=4 1173 1185 41.89x nnz=8000 coo-sort=None + Wide SASO k=8 2003 2063 71.54x nnz=16000 coo-sort=None + CountSketch k=1 515 518 18.39x nnz=2000 coo-sort=CSC + Wide LASO k=2 28 31 1.00x nnz=78 coo-sort=None + Wide LASO k=4 35 39 1.25x nnz=158 coo-sort=None + Wide LASO k=8 53 57 1.89x nnz=318 coo-sort=None + +=== RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === + + warm apply, ColMajor dense + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 733 758 23.65x nnz=4000 coo-sort=None + Wide SASO k=4 1174 1203 37.87x nnz=8000 coo-sort=None + Wide SASO k=8 2107 2217 67.97x nnz=16000 coo-sort=None + CountSketch k=1 472 550 15.23x nnz=2000 coo-sort=CSR + Wide LASO k=2 31 34 1.00x nnz=80 coo-sort=None + Wide LASO k=4 37 39 1.19x nnz=160 coo-sort=None + Wide LASO k=8 57 66 1.84x nnz=320 coo-sort=None + + warm apply, RowMajor dense + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 1443 1452 3.06x nnz=4000 coo-sort=None + Wide SASO k=4 2277 2317 4.83x nnz=8000 coo-sort=None + Wide SASO k=8 4158 4459 8.83x nnz=16000 coo-sort=None + CountSketch k=1 852 1287 1.81x nnz=2000 coo-sort=CSR + Wide LASO k=2 471 517 1.00x nnz=80 coo-sort=None + Wide LASO k=4 471 495 1.00x nnz=160 coo-sort=None + Wide LASO k=8 495 500 1.05x nnz=320 coo-sort=None + +=== LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 1491 1552 2.62x nnz=4000 coo-sort=None cold=1524 convert=28 + Wide SASO k=4 2183 2211 3.84x nnz=8000 coo-sort=None cold=2259 convert=76 + Wide SASO k=8 4189 4240 7.36x nnz=16000 coo-sort=None cold=4202 convert=178 + CountSketch k=1 875 886 1.54x nnz=2000 coo-sort=CSC cold=897 convert=3 + Wide LASO k=2 569 590 1.00x nnz=398 coo-sort=None cold=606 convert=4 + Wide LASO k=4 798 826 1.40x nnz=797 coo-sort=None cold=859 convert=10 + Wide LASO k=8 1247 1364 2.19x nnz=1594 coo-sort=None cold=1400 convert=23 + densify(S)+GEMM 4670 4670 8.21x dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 772 827 7.64x nnz=4000 coo-sort=None + Wide SASO k=4 1149 1245 11.38x nnz=8000 coo-sort=None + Wide SASO k=8 1780 2121 17.62x nnz=16000 coo-sort=None + CountSketch k=1 514 559 5.09x nnz=2000 coo-sort=CSC + Wide LASO k=2 101 106 1.00x nnz=398 coo-sort=None + Wide LASO k=4 167 180 1.65x nnz=797 coo-sort=None + Wide LASO k=8 400 407 3.96x nnz=1594 coo-sort=None + +=== RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === + + warm apply, ColMajor dense + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 786 814 7.63x nnz=4000 coo-sort=None + Wide SASO k=4 1241 1300 12.05x nnz=8000 coo-sort=None + Wide SASO k=8 2087 2309 20.26x nnz=16000 coo-sort=None + CountSketch k=1 558 560 5.42x nnz=2000 coo-sort=CSR + Wide LASO k=2 103 114 1.00x nnz=400 coo-sort=None + Wide LASO k=4 153 163 1.49x nnz=800 coo-sort=None + Wide LASO k=8 397 448 3.85x nnz=1599 coo-sort=None + + warm apply, RowMajor dense + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 1496 1539 2.62x nnz=4000 coo-sort=None + Wide SASO k=4 2186 2228 3.83x nnz=8000 coo-sort=None + Wide SASO k=8 4064 4135 7.12x nnz=16000 coo-sort=None + CountSketch k=1 876 883 1.53x nnz=2000 coo-sort=CSR + Wide LASO k=2 571 601 1.00x nnz=400 coo-sort=None + Wide LASO k=4 779 802 1.36x nnz=800 coo-sort=None + Wide LASO k=8 1178 1319 2.06x nnz=1599 coo-sort=None + +=== LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 1592 1615 1.63x nnz=4000 coo-sort=None cold=1632 convert=29 + Wide SASO k=4 2323 2434 2.38x nnz=8000 coo-sort=None cold=2329 convert=76 + Wide SASO k=8 4163 4279 4.26x nnz=16000 coo-sort=None cold=4201 convert=184 + CountSketch k=1 978 985 1.00x nnz=2000 coo-sort=CSC cold=999 convert=3 + Wide LASO k=2 1035 1084 1.06x nnz=998 coo-sort=None cold=1083 convert=14 + Wide LASO k=4 1652 1735 1.69x nnz=1996 coo-sort=None cold=1759 convert=29 + Wide LASO k=8 2117 2150 2.16x nnz=3990 coo-sort=None cold=2326 convert=66 + densify(S)+GEMM 11075 11075 11.32x dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 923 985 2.19x nnz=4000 coo-sort=None + Wide SASO k=4 1343 1374 3.19x nnz=8000 coo-sort=None + Wide SASO k=8 2141 2220 5.09x nnz=16000 coo-sort=None + CountSketch k=1 776 786 1.84x nnz=2000 coo-sort=CSC + Wide LASO k=2 421 456 1.00x nnz=998 coo-sort=None + Wide LASO k=4 616 657 1.46x nnz=1996 coo-sort=None + Wide LASO k=8 990 1097 2.35x nnz=3990 coo-sort=None + +=== RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === + + warm apply, ColMajor dense + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 927 957 2.26x nnz=4000 coo-sort=None + Wide SASO k=4 1352 1398 3.30x nnz=8000 coo-sort=None + Wide SASO k=8 2190 2356 5.34x nnz=16000 coo-sort=None + CountSketch k=1 713 917 1.74x nnz=2000 coo-sort=CSR + Wide LASO k=2 410 452 1.00x nnz=1000 coo-sort=None + Wide LASO k=4 626 683 1.53x nnz=2000 coo-sort=None + Wide LASO k=8 944 1044 2.30x nnz=3993 coo-sort=None + + warm apply, RowMajor dense + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 1600 1705 1.58x nnz=4000 coo-sort=None + Wide SASO k=4 2258 2333 2.23x nnz=8000 coo-sort=None + Wide SASO k=8 4047 4176 4.00x nnz=16000 coo-sort=None + CountSketch k=1 1012 1446 1.00x nnz=2000 coo-sort=CSR + Wide LASO k=2 1014 1028 1.00x nnz=1000 coo-sort=None + Wide LASO k=4 1557 1635 1.54x nnz=2000 coo-sort=None + Wide LASO k=8 2096 2229 2.07x nnz=3993 coo-sort=None + +########## SECTION 2: narrow n (SpMV-ish) ########## + +=== LEFT-SKETCH B(200x1) = S(200x2000) * A(2000x1), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 46 50 3.29x nnz=4000 coo-sort=None cold=83 convert=29 + Wide SASO k=4 100 115 7.14x nnz=8000 coo-sort=None cold=167 convert=78 + Wide SASO k=8 216 266 15.43x nnz=16000 coo-sort=None cold=339 convert=176 + CountSketch k=1 14 15 1.00x nnz=2000 coo-sort=CSC cold=37 convert=3 + Wide LASO k=2 16 22 1.14x nnz=400 coo-sort=None cold=40 convert=4 + Wide LASO k=4 27 37 1.93x nnz=799 coo-sort=None cold=75 convert=10 + Wide LASO k=8 44 60 3.14x nnz=1595 coo-sort=None cold=135 convert=22 + densify(S)+GEMM 13 13 0.93x dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 85 86 3.86x nnz=4000 coo-sort=None + Wide SASO k=4 174 176 7.91x nnz=8000 coo-sort=None + Wide SASO k=8 355 437 16.14x nnz=16000 coo-sort=None + CountSketch k=1 35 35 1.59x nnz=2000 coo-sort=CSC + Wide LASO k=2 22 24 1.00x nnz=400 coo-sort=None + Wide LASO k=4 34 36 1.55x nnz=799 coo-sort=None + Wide LASO k=8 62 73 2.82x nnz=1595 coo-sort=None + +=== LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 55 59 2.75x nnz=4000 coo-sort=None cold=89 convert=29 + Wide SASO k=4 125 161 6.25x nnz=8000 coo-sort=None cold=182 convert=76 + Wide SASO k=8 261 357 13.05x nnz=16000 coo-sort=None cold=364 convert=182 + CountSketch k=1 20 21 1.00x nnz=2000 coo-sort=CSC cold=42 convert=3 + Wide LASO k=2 21 23 1.05x nnz=400 coo-sort=None cold=47 convert=4 + Wide LASO k=4 30 33 1.50x nnz=800 coo-sort=None cold=77 convert=10 + Wide LASO k=8 45 54 2.25x nnz=1599 coo-sort=None cold=139 convert=22 + densify(S)+GEMM 35 35 1.75x dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min (us) Med (us) vs best notes + -------------------------------------------------------------------- + Wide SASO k=2 79 84 3.59x nnz=4000 coo-sort=None + Wide SASO k=4 170 228 7.73x nnz=8000 coo-sort=None + Wide SASO k=8 334 345 15.18x nnz=16000 coo-sort=None + CountSketch k=1 33 35 1.50x nnz=2000 coo-sort=CSC + Wide LASO k=2 22 24 1.00x nnz=400 coo-sort=None + Wide LASO k=4 35 36 1.59x nnz=800 coo-sort=None + Wide LASO k=8 59 60 2.68x nnz=1599 coo-sort=None + diff --git a/sketch_general_results_postfix.txt b/sketch_general_results_postfix.txt new file mode 100644 index 00000000..5ddeb277 --- /dev/null +++ b/sketch_general_results_postfix.txt @@ -0,0 +1,219 @@ +# POST-FIX run with work-normalized metrics (ns/elt, GB/s, %STR). +# Sampling fix: within-vector sort in fill_sparse_unpacked SASO branch +# (RandBLAS/sparse_skops.hh) -> wide SASO vec_nnz>=2 emits CSC-sorted COO +# (tall SASO -> CSR), so apply_coo_via_csc skips its per-apply re-sort. +# ns/elt = us*1000/(nnz*work_mult); work_mult = n (left) or m (right). +# GB/s = model bytes / time; %STR = GB/s as a fraction of the STREAM triad +# below. Byte model bounds A traffic by min(nnz, contract_dim); STREAM is a +# DRAM ceiling, so tiny cache-resident LASO cases can exceed 100% %STR. +# Pre-fix baseline (no metric columns): sketch_general_results.txt +# Environment: Apple M3, OMP_NUM_THREADS=4, Release build (MKL off) +# RandBLAS commit: 705f09e + uncommitted sampling fix | Correctness: 0 FAIL + +============================================================ +SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) +============================================================ +Threads (OMP_NUM_THREADS/max): 4 +Calibrating STREAM triad... 85.4 GB/s (= 100% on the %STR column) + +########## SECTION 1: vary sketch size d (m=n=2000) ########## + +=== LEFT-SKETCH B(40x2000) = S(40x2000) * A(2000x2000), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1097 1111 0.137 30.4 36 CSC nnz=4000 cold=1153 cvt=7 + Wide SASO k=4 2223 2332 0.139 15.0 18 CSC nnz=8000 cold=2308 cvt=14 + Wide SASO k=8 3934 3988 0.123 8.5 10 CSC nnz=16000 cold=4423 cvt=26 + CountSketch k=1 834 844 0.208 39.9 47 CSC nnz=2000 cold=852 cvt=3 + Wide LASO k=2 420 597 2.625 6.1 7 None nnz=80 cold=689 cvt=0 + Wide LASO k=4 401 414 1.253 9.6 11 None nnz=160 cold=464 cvt=1 + Wide LASO k=8 485 501 0.763 13.1 15 None nnz=318 cold=517 cvt=3 + densify(S)+GEMM 1574 1599 - - - dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 740 754 0.092 45.1 53 CSC nnz=4000 + Wide SASO k=4 1073 1112 0.067 31.1 36 CSC nnz=8000 + Wide SASO k=8 1539 1839 0.048 21.8 26 CSC nnz=16000 + CountSketch k=1 497 523 0.124 67.0 78 CSC nnz=2000 + Wide LASO k=2 30 32 0.188 85.4 100 None nnz=80 + Wide LASO k=4 36 38 0.113 106.7 125 None nnz=160 + Wide LASO k=8 57 63 0.090 111.8 131 None nnz=318 + +=== RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === + + warm apply, ColMajor dense + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 649 719 0.081 51.4 60 CSR nnz=4000 + Wide SASO k=4 1103 1138 0.069 30.3 35 CSR nnz=8000 + Wide SASO k=8 1505 1849 0.047 22.3 26 CSR nnz=16000 + CountSketch k=1 474 509 0.118 70.3 82 CSR nnz=2000 + Wide LASO k=2 28 31 0.175 91.5 107 None nnz=80 + Wide LASO k=4 37 39 0.116 103.9 122 None nnz=160 + Wide LASO k=8 53 55 0.083 120.9 141 None nnz=320 + + warm apply, RowMajor dense + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1095 1104 0.137 30.5 36 CSR nnz=4000 + Wide SASO k=4 2197 2206 0.137 15.2 18 CSR nnz=8000 + Wide SASO k=8 3938 4008 0.123 8.5 10 CSR nnz=16000 + CountSketch k=1 833 843 0.208 40.0 47 CSR nnz=2000 + Wide LASO k=2 528 857 3.300 4.9 6 None nnz=80 + Wide LASO k=4 458 463 1.431 8.4 10 None nnz=160 + Wide LASO k=8 496 505 0.775 12.9 15 None nnz=320 + +=== LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1134 1193 0.142 33.9 40 CSC nnz=4000 cold=1180 cvt=6 + Wide SASO k=4 2134 2349 0.133 18.1 21 CSC nnz=8000 cold=3055 cvt=15 + Wide SASO k=8 3752 3956 0.117 10.3 12 CSC nnz=16000 cold=4091 cvt=26 + CountSketch k=1 875 884 0.219 43.9 51 CSC nnz=2000 cold=893 cvt=3 + Wide LASO k=2 571 607 0.714 22.4 26 None nnz=400 cold=606 cvt=4 + Wide LASO k=4 880 1047 0.550 21.8 26 None nnz=800 cold=847 cvt=10 + Wide LASO k=8 1356 1436 0.425 23.6 28 None nnz=1597 cold=1465 cvt=22 + densify(S)+GEMM 4726 4855 - - - dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 717 859 0.090 53.6 63 CSC nnz=4000 + Wide SASO k=4 1074 1230 0.067 35.9 42 CSC nnz=8000 + Wide SASO k=8 1785 1908 0.056 21.7 25 CSC nnz=16000 + CountSketch k=1 560 573 0.140 68.6 80 CSC nnz=2000 + Wide LASO k=2 101 107 0.126 126.8 148 None nnz=400 + Wide LASO k=4 174 211 0.109 110.4 129 None nnz=800 + Wide LASO k=8 406 414 0.127 78.8 92 None nnz=1597 + +=== RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === + + warm apply, ColMajor dense + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 769 782 0.096 50.0 59 CSR nnz=4000 + Wide SASO k=4 1133 1174 0.071 34.0 40 CSR nnz=8000 + Wide SASO k=8 1657 1903 0.052 23.3 27 CSR nnz=16000 + CountSketch k=1 563 656 0.141 68.3 80 CSR nnz=2000 + Wide LASO k=2 105 114 0.131 122.0 143 None nnz=400 + Wide LASO k=4 166 171 0.104 115.5 135 None nnz=798 + Wide LASO k=8 417 458 0.131 76.7 90 None nnz=1597 + + warm apply, RowMajor dense + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1134 1153 0.142 33.9 40 CSR nnz=4000 + Wide SASO k=4 2111 2175 0.132 18.3 21 CSR nnz=8000 + Wide SASO k=8 3727 4065 0.116 10.4 12 CSR nnz=16000 + CountSketch k=1 881 913 0.220 43.6 51 CSR nnz=2000 + Wide LASO k=2 584 600 0.730 21.9 26 None nnz=400 + Wide LASO k=4 801 836 0.502 23.9 28 None nnz=798 + Wide LASO k=8 1259 1326 0.394 25.4 30 None nnz=1597 + +=== LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1242 1281 0.155 38.7 45 CSC nnz=4000 cold=1281 cvt=7 + Wide SASO k=4 2193 2391 0.137 21.9 26 CSC nnz=8000 cold=2314 cvt=14 + Wide SASO k=8 3854 4161 0.120 12.5 15 CSC nnz=16000 cold=4149 cvt=26 + CountSketch k=1 997 1400 0.249 48.2 56 CSC nnz=2000 cold=997 cvt=3 + Wide LASO k=2 1032 1104 0.516 31.0 36 None nnz=1000 cold=1089 cvt=13 + Wide LASO k=4 1580 1767 0.395 30.4 36 None nnz=1999 cold=2217 cvt=30 + Wide LASO k=8 2173 2294 0.272 22.1 26 None nnz=3994 cold=2408 cvt=65 + densify(S)+GEMM 11199 11712 - - - dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 945 979 0.118 50.9 60 CSC nnz=4000 + Wide SASO k=4 1307 1493 0.082 36.8 43 CSC nnz=8000 + Wide SASO k=8 1995 2040 0.062 24.2 28 CSC nnz=16000 + CountSketch k=1 708 807 0.177 67.8 79 CSC nnz=2000 + Wide LASO k=2 411 418 0.205 77.9 91 None nnz=1000 + Wide LASO k=4 650 742 0.163 73.9 86 None nnz=1999 + Wide LASO k=8 977 1113 0.122 49.2 58 None nnz=3994 + +=== RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === + + warm apply, ColMajor dense + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 926 1070 0.116 51.9 61 CSR nnz=4000 + Wide SASO k=4 1272 1357 0.080 37.8 44 CSR nnz=8000 + Wide SASO k=8 1973 2116 0.062 24.5 29 CSR nnz=16000 + CountSketch k=1 773 787 0.193 62.1 73 CSR nnz=2000 + Wide LASO k=2 434 453 0.217 73.7 86 None nnz=998 + Wide LASO k=4 668 699 0.167 71.9 84 None nnz=1998 + Wide LASO k=8 979 1088 0.123 49.1 57 None nnz=3993 + + warm apply, RowMajor dense + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1241 1333 0.155 38.7 45 CSR nnz=4000 + Wide SASO k=4 2179 2255 0.136 22.1 26 CSR nnz=8000 + Wide SASO k=8 3595 3936 0.112 13.4 16 CSR nnz=16000 + CountSketch k=1 1003 1010 0.251 47.9 56 CSR nnz=2000 + Wide LASO k=2 1047 1168 0.525 30.5 36 None nnz=998 + Wide LASO k=4 1575 1607 0.394 30.5 36 None nnz=1998 + Wide LASO k=8 2112 2255 0.264 22.8 27 None nnz=3993 + +########## SECTION 2: narrow n (SpMV-ish) ########## + +=== LEFT-SKETCH B(200x1) = S(200x2000) * A(2000x1), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 16 30 4.000 5.2 6 CSC nnz=4000 cold=75 cvt=7 + Wide SASO k=4 23 36 2.875 6.4 7 CSC nnz=8000 cold=119 cvt=14 + Wide SASO k=8 39 40 2.438 7.1 8 CSC nnz=16000 cold=245 cvt=26 + CountSketch k=1 13 15 6.500 3.9 5 CSC nnz=2000 cold=34 cvt=3 + Wide LASO k=2 17 21 42.500 0.8 1 None nnz=400 cold=42 cvt=4 + Wide LASO k=4 25 33 31.250 0.9 1 None nnz=800 cold=75 cvt=10 + Wide LASO k=8 37 47 23.139 1.1 1 None nnz=1599 cold=139 cvt=22 + densify(S)+GEMM 13 14 - - - dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 77 80 19.250 1.1 1 CSC nnz=4000 + Wide SASO k=4 89 91 11.125 1.7 2 CSC nnz=8000 + Wide SASO k=8 172 246 10.750 1.6 2 CSC nnz=16000 + CountSketch k=1 35 36 17.500 1.5 2 CSC nnz=2000 + Wide LASO k=2 22 23 55.000 0.6 1 None nnz=400 + Wide LASO k=4 38 47 47.500 0.6 1 None nnz=800 + Wide LASO k=8 65 81 40.650 0.6 1 None nnz=1599 + +=== LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === + + warm apply, ColMajor dense (jik/jki kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 26 29 0.650 9.8 12 CSC nnz=4000 cold=74 cvt=7 + Wide SASO k=4 41 43 0.512 7.8 9 CSC nnz=8000 cold=138 cvt=14 + Wide SASO k=8 64 65 0.400 7.0 8 CSC nnz=16000 cold=269 cvt=25 + CountSketch k=1 21 22 1.050 10.7 12 CSC nnz=2000 cold=38 cvt=3 + Wide LASO k=2 21 24 5.250 3.4 4 None nnz=400 cold=47 cvt=4 + Wide LASO k=4 29 32 3.625 3.8 4 None nnz=800 cold=78 cvt=10 + Wide LASO k=8 45 55 2.820 4.1 5 None nnz=1596 cold=143 cvt=22 + densify(S)+GEMM 35 35 - - - dense oracle + + warm apply, RowMajor dense (ikb/kib kernels) + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 54 56 1.350 4.7 6 CSC nnz=4000 + Wide SASO k=4 94 100 1.175 3.4 4 CSC nnz=8000 + Wide SASO k=8 169 174 1.056 2.7 3 CSC nnz=16000 + CountSketch k=1 44 44 2.200 5.1 6 CSC nnz=2000 + Wide LASO k=2 21 23 5.250 3.4 4 None nnz=400 + Wide LASO k=4 36 43 4.500 3.0 4 None nnz=800 + Wide LASO k=8 58 64 3.634 3.2 4 None nnz=1596 + diff --git a/sparsesketching_bench_plan.md b/sparsesketching_bench_plan.md new file mode 100644 index 00000000..4c001c8d --- /dev/null +++ b/sparsesketching_bench_plan.md @@ -0,0 +1,184 @@ +# Plan: a `sketch_general` benchmark, analogous to `spmm_performance.cc` + +## Goal + +`spmm_performance.cc` benchmarks the low-level `left_spmm` kernels on **generic +uniform-random** sparse matrices. It therefore never exercises the *structured* +sketching operators (fixed nnz per column/row, ±1 values, mostly-empty columns), +never hits the regular-CSC fast path, and never goes through `sketch_general`. +This benchmark closes that gap: it measures the **full sketching call** +(`sketch_general` with a real `SparseSkOp`) across the operator shapes identified +in `sparsesketching_opt.md`, so we get a baseline for the proposed kernels +(CountSketch, bounded-CSR for wide LASO, regular-path layout coverage, ±1 +implicit-value) before any of them is written. + +New executable: `examples/simple-kernel-benchmarks/sketch_general_performance.cc`. + +## What `sketch_general` does, and the cost lifecycle we must measure + +Left-sketch form (the headline case): + +``` +sketch_general(layout, opS, opA, d, n, m, alpha, S, ro_s, co_s, A, lda, beta, B, ldb) + // B(d x n) = alpha * op(S)(d x m) * op(A)(m x n) + beta * B +``` + +with `S` a `SparseSkOp` of distribution `SparseDist(d, m, vec_nnz, major_axis)`. +For a sparse `S` this routes `lskges → coo_view_of_skop → left_spmm`. The cost has +**three distinct phases** that the benchmark must be able to separate, because the +opt-doc proposals target different ones: + +1. **Sample / fill** — `fill_sparse(S)` populates `(rows, cols, vals)`. Happens + once in real build-once/apply-many use; if `S.nnz < 0` at call time, + `sketch_general` materializes a *temporary* representation **every call** and + frees it. We must pre-sample (`fill_sparse(S)`) to measure apply-only cost, and + *separately* measure the cold path. +2. **View + convert/sort** — `apply_coo_via_csc` determines the COO sort order and, + if not already CSC-sorted, **deepcopies + re-sorts to CSC on every call**. This + is the per-apply re-sort flagged for wide-LASO left-sketch; it must be timed in + isolation. +3. **Kernel** — `apply_csc_jki_p11` (ColMajor) / `apply_csc_kib_1p1_rowmajor` + (RowMajor), with the regular-CSC specialization firing only in the `jki` path + when nnz-per-column is exactly fixed. + +> Design choice: report **warm apply** (phases 2+3, `S` pre-sampled) as the primary +> number, plus a **cold** number (phases 1+2+3) and a **convert-only** number, so +> the re-sort cost is visible. This mirrors `spmm_performance`'s +> `run_split_trials` (densify vs compute) but with a sketching-specific split. + +## Relationship to `spmm_performance.cc` (reuse vs. change) + +**Reuse verbatim:** +- `run_trials` / `run_split_trials` (min + median over N trials), `print_row`, + `print_table_header`, the densify+GEMM correctness oracle pattern. +- Both dense layouts (`ColMajor`, `RowMajor`) and both `op` flags — the dispatch + selects the kernel purely from `(layout_opB, layout_C)`, exactly as documented in + the spmm benchmark header. +- The "compute the identical logical product in both layouts and check against one + ColMajor oracle" structure. + +**Change / add:** +- Operand is a `SparseSkOp` built from a `SparseDist`, **not** `random_coo`. So the + regular-CSC path and CountSketch are actually represented. +- New sweep axes: `major_axis ∈ {Short, Long}` and `vec_nnz ∈ {1,2,4,8}`. +- Both **sides** (left-sketch and right-sketch). Right-sketch uses the tall + operator and, per the transpose reduction, should match the corresponding + left-sketch RowMajor/ColMajor kernel — the benchmark lets us confirm this + empirically (a useful cross-check, not just a measurement). +- Phase-split timing (sample / convert / kernel) described above. +- Oracle is `densify(S) + GEMM`, where `densify(S)` comes from the COO view of the + sampled operator (reuse `coo` → dense, or build dense directly from + `rows/cols/vals`). + +## Operator shapes to sweep (the structured taxonomy) + +Map each `SparseDist` onto the opt-doc taxonomy. For left-sketch, `S` is `d×m` +with `d < m` (wide); for right-sketch, `S` is `n×d` with `n > d` (tall). + +| Config | dist | side | structured axis | what it isolates | +|---|---|---|---|---| +| Wide SASO | `SparseDist(d,m,k,Short)` | left | fixed `k`/col (CSC-regular) | regular-CSC fast path | +| CountSketch | `SparseDist(d,m,1,Short)` | left | 1/col | the `vec_nnz=1` special case | +| Wide LASO | `SparseDist(d,m,k,Long)` | left | ≤`k`/row, empty cols | the uncovered CSR-gather case + re-sort cost | +| Tall SASO | `SparseDist(n,d,k,Short)` | right | →CSC-regular via transpose | confirms transpose reduction == wide SASO | +| Tall LASO | `SparseDist(n,d,k,Long)` | right | ≤`k`/col after transpose | general CSC scatter (no regular) | + +For each: `× {ColMajor, RowMajor} × {opS, opA NoTrans/Trans as applicable}`. + +## Parameters and sweep + +CLI mirrors `spmm_performance`: + +``` +./sketch_general_performance # default sweep +./sketch_general_performance d m n vec_nnz major_axis [trials] # single config + major_axis: 0 = Short (SASO), 1 = Long (LASO) +``` + +**Default sweep** (10 trials each), two sections: + +1. **Square-ish data, varying sketch size** — fix `m=n` large, sweep + `d ∈ {m/50, m/10, m/4}`; e.g. `m=n=2000`, `d ∈ {40,200,500}`. Covers the + common "tall data, modest embedding" regime. +2. **`vec_nnz` / `major_axis` cross** — fix one geometry (e.g. `d=200, m=n=2000`) + and sweep `vec_nnz ∈ {1,2,4,8} × major_axis ∈ {Short, Long}`, both sides. This + is the section that exercises CountSketch, wide LASO, and the regular path. + +Also expose, via env (as in the existing benchmark): `OMP_NUM_THREADS`. Per the +perf notes, sweep `{1,4,8}` on the M3 and additionally on an Arm SVE box. + +Include the **`n=1` / narrow-`n`** SpMV regime explicitly (a small extra config), +since that is the design fork for the CountSketch bucketed kernel. + +## Measurement methodology + +For each (shape × layout × side): + +- **Pre-sample** `S` once with `fill_sparse(S)`; then time the warm + `sketch_general` call (re-zeroing `B` each trial), reporting `{min, median}`. +- **Cold** variant: reconstruct an unsampled `SparseSkOp` each trial and time the + full call (captures sample + convert + kernel) — report `min` only. +- **Convert-only**: time `coo_view_of_skop` + the COO→CSC sort path in isolation + (or infer as cold − warm − sample) to expose the per-apply re-sort. +- **References**: `densify(S)+GEMM` (oracle, split densify vs GEMM like spmm), and + optionally a `DenseSkOp` (Gaussian) sketch via `lskge3` at the same `d,m,n` for a + "dense operator" cost anchor. +- **Correctness**: one extra call per (shape×layout×side) compared against the + ColMajor densify+GEMM oracle, `max|diff| < 1e-10`, printed as a PASS/FAIL line. + +Report a SUMMARY per config: best warm (ColMajor vs RowMajor), regular-vs-general +gap where both are reachable, and convert/re-sort as a fraction of warm apply. + +## Output tables (per config) + +Mirror the spmm layout — one table per (side × layout): + +``` +SKETCH_GENERAL left, ColMajor d x n = A applied + Operator Min(us) Med(us) vs best notes + Wide SASO k=4 ... (regular-CSC) + CountSketch k=1 ... (regular, 1/col) + Wide LASO k=4 ... (general CSC + resort) + densify(S)+GEMM ... (densify .. + GEMM ..) +``` + +Plus the RowMajor table, the right-sketch (tall) tables, and a convert-cost line. + +## File / build wiring + +- Add `sketch_general_performance.cc` under + `examples/simple-kernel-benchmarks/`. +- In `examples/CMakeLists.txt`, add an `add_executable` / + `target_include_directories(... ${Random123_DIR})` / + `target_link_libraries(... RandBLAS blaspp lapackpp)` block copied from the + `spmm_performance` block (lines ~95–103). +- Headers: `` is enough for `sketch_general` / `SparseSkOp` / + `SparseDist` / `fill_sparse`; pull `RandBLAS/sparse_data/conversions.hh` only if + densifying via the conversion helpers. + +## Things to confirm while implementing + +1. Whether `sketch_general` with a pre-sampled `S` still re-sorts COO→CSC on each + call (expected yes, via `apply_coo_via_csc`) — the convert-only timing settles + this. +2. Which kernel each (side × layout) actually lands on, to confirm the regular + fast path is/ isn't invoked (e.g. ColMajor right-sketch → RowMajor axpy kernel, + no regular specialization). +3. That a sampled wide SASO's COO is in CSC sort order (so no re-sort) while a + wide LASO's is not — the structural prediction the benchmark is meant to verify. + +## Measured findings (first run, M3, OMP=4) + +The benchmark is implemented as `sketch_general_performance.cc` and runs clean +(all correctness checks pass). First results already correct a prediction: + +- **Only CountSketch (`k=1`) is `coo-sort=CSC`** and skips the re-sort + (convert ≈ 3µs). **Wide SASO `k≥2` is `coo-sort=None`** and re-sorts on *every* + apply (convert ≈ 30/77/171µs for k=2/4/8 at d=40) — i.e. the per-apply re-sort + is **not** limited to wide LASO; SASO pays it too because within-column row + indices aren't sorted. This strengthens the case for structure-aware dispatch. +- **RowMajor is ~2× faster than ColMajor** for these operators (e.g. wide SASO + k=4, d=40: 1144 vs 2276µs), consistent with the prior SpMM findings. +- Wide LASO has very low nnz (≤ `k·d` minus merge collisions) and is the cheapest + case by a wide margin. +``` diff --git a/sparsesketching_opt.md b/sparsesketching_opt.md new file mode 100644 index 00000000..c3248e83 --- /dev/null +++ b/sparsesketching_opt.md @@ -0,0 +1,185 @@ +# Optimizing SPMM kernels for sparse sketching operators + +A proposal for new structure-aware SPMM kernels in RandBLAS, plus a recommendation +to gather targeted performance data before hand-tuning. + +> **Revision note.** An earlier draft listed *tall SASO (right-sketching)* as a +> motivating case for a new regular-CSR kernel. That was wrong: free operator +> transposition plus the `right_spmm`→`left_spmm` reduction already turn that case +> into the existing wide-SASO / CSC-regular kernel. This draft reframes the +> taxonomy around the operator's *structured axis* (row vs column), which is the +> distinction that survives transposition. + +## How sketching operators reach the kernels today + +Every `SparseSkOp` is stored as **COO** (`rows, cols, vals`). Sketching +(`lskges`/`rskges`) takes a `coo_view_of_skop` and calls `left_spmm`/`right_spmm`. + +Two reductions happen *before* any kernel runs, and they are central to this +analysis: + +- **`right_spmm` reduces to `left_spmm`** by transposing the operation and + **flipping the dense layout** (`trans_layout`: ColMajor↔RowMajor), so the sparse + operand always lands as the left operand of `left_spmm`. +- **`left_spmm` resolves `opA=Trans` for free**: it takes a lightweight transpose + view of the operator (for COO, just swap the `rows`/`cols` arrays) and recurses + as `NoTrans`. Transposing a sketching operator costs nothing. + +After those reductions, a COO operator goes to `apply_coo_via_csc`, which: + +1. Determines the COO sort order by scanning (set in the view constructor via + `coo_arrays_determine_sort`). +2. If not already CSC-sorted, **deepcopies and re-sorts to CSC on every call**. +3. Calls `apply_csc_kib_1p1_rowmajor` (when `layout_opB == layout_C == RowMajor`) + or `apply_csc_jki_p11` (otherwise). + +The only place structure is exploited is `apply_csc_jki_p11`: it scans `colptr` +to detect fixed-nnz-per-column and, if so, calls `apply_regular_csc_to_vector_ki`. +Note this detection lives **only** in the `jki` path — the RowMajor axpy kernel +`apply_csc_kib_1p1_rowmajor` has no regular specialization. + +## The right way to think about it: the *structured axis*, not tall vs wide + +Because transposition is free and `right_spmm` already flips layout, **"tall vs +wide" is not an independent design axis** — it is absorbed by free transpose plus +a dense-layout flip. What survives transposition is whether the operator is +**column-structured** (feeds the CSC *scatter* kernel) or **row-structured** +(wants a CSR *gather* kernel). A transpose swaps a row-structured operator into a +column-structured one, but it simultaneously swaps the roles/layouts of the dense +operands — so it only helps when the dispatch is *already* transposing for another +reason. + +Re-cast the four shapes accordingly: + +| Operator | Usage | Structured axis after reductions | Reaches which kernel | +|---|---|---|---| +| **Wide SASO** (d×m) | left-sketch | exactly `vec_nnz` per **column** (CSC-regular) | ✅ `apply_regular_csc...` | +| **Tall SASO** (m×d) | right-sketch | `right_spmm` transposes → **column-regular** (= wide SASO) | ✅ same CSC-regular machinery — **no new kernel needed** | +| **CountSketch** | left-sketch | column-regular with `vec_nnz = 1` | ✅ regular-CSC, but very under-specialized | +| **Wide LASO** (d×m) | left-sketch | **≤** `vec_nnz` per **row**, mostly-empty cols (CSR) | ❌ no auto-transpose → general CSC scatter over all `m` cols | +| **Tall LASO** (m×d) | right-sketch | transposes → **≤** per column (CSC, not exact) | ⚠️ general CSC scatter; regular detector can't fire (counts vary) | + +The two takeaways: + +1. **Tall SASO needs no new *kernel*.** Its row structure is converted to the + existing column-regular kernel by the transpose reduction. (Measurement caveat: + the COO view's sort flag comes back `None` for `vec_nnz ≥ 2` — within-vector + indices aren't fully ordered — so `apply_coo_via_csc` still re-sorts on every + apply; only CountSketch, `vec_nnz = 1`, is detected as CSC-sorted. See the + benchmark findings in `sparsesketching_bench_plan.md`. This is a dispatch cost, + not a missing kernel.) +2. The genuinely uncovered case is the **row-structured operator that reaches a + kernel *without* an auto-transpose** — i.e., **wide LASO in left-sketching**. + There the transpose trick doesn't help (it would just swap the dense roles + back), so we land on the general CSC scatter, which iterates over all `m` + columns (most empty) and sees no regularity. + +A caveat that matters for *invocation* (not just structure): the regular-CSC fast +path lives only in `apply_csc_jki_p11`. A common **ColMajor right-sketch with a +tall SASO** flips to RowMajor and routes to `apply_csc_kib_1p1_rowmajor` (the +axpy kernel), which does **not** specialize on regularity. So "structure covered" +≠ "fast path invoked" — which kernel fires is layout-dependent. + +## Proposed kernels + +**1. CountSketch kernels (`vec_nnz == 1`)** — highest value and most +self-contained. A wide CountSketch maps each column `c` to one row `h(c)` with +sign `s(c)`: `C[h(c),:] += s(c)·B[c,:]`. Today this runs through regular-CSC with +`col_nnz=1` — correct but carrying full loop/index overhead for a length-1 inner +loop. Dedicated variants: + +- **Bucketed / CSR-by-output-row form**: invert the hash once (group source + columns by target row). Each of the `d` output rows is then an independent + signed sum of B-rows → embarrassingly parallel over `d` with **no races and no + per-thread rescan** (unlike `apply_csc_kib_1p1_rowmajor`, which loops all `m` + per thread and filters by row range). This is the right structure when `n` is + small (SpMV / few RHS), where parallel-over-`n` starves threads. +- Collapse the length-1 inner loop entirely. + +**2. Bounded-CSR gather kernel for wide LASO (left-sketch).** This is the case the +transpose reduction does *not* rescue. Build a CSR/gather kernel that iterates +only over the `d` nonempty rows (≤ `vec_nnz` each) instead of scattering over all +`m` columns. Realizing this means routing the operator COO→**CSR** instead of CSC +— which *also* eliminates the per-apply re-sort, since a wide LASO's natural COO +order is already CSR. A `≤`-aware variant (explicit short per-row counts, or +padding) handles the merge-induced count variation that keeps LASOs out of the +*exact* regular path. + +**3. Implicit ±1-value kernels.** SASO values are exactly ±1; the isometry scale +is applied separately via `alpha`. If the prior profiling's memory-bound +hypothesis holds, **not loading the `vals` array** (encode sign in the index, or +split into +/− index lists and do two sign-free accumulations) cuts the +sparse-operand traffic by ~⅓. Machine-independent in the regime the M3 profiling +suggested, and it composes with kernels 1–2. + +**4. Close the regular-path layout gap.** Add a regular-aware specialization to the +RowMajor axpy kernel (`apply_csc_kib_1p1_rowmajor`), and/or a dedicated ColMajor +row-axpy ("BLAS-3-flavored") path. Per the perf notes, ColMajor is the BLAS +default yet routes to the slow scalar scatter/gather; RowMajor gets the fast +axpy kernels and is up to ~1.9× faster. +A structured ±1 operator makes a well-vectorized ColMajor kernel far easier than +for a general sparse matrix, and it ensures the regular fast path is reachable +regardless of which layout the reductions land in. + +## Cross-cutting infrastructure (needed for the above to pay off) + +- **Structure-aware dispatch carrying the operator's metadata.** The skop already + knows `major_axis`, its shape, and `vec_nnz`, so it knows its structured axis + and whether counts are exact or bounded. Thread that through `lskges`/`rskges` + and **choose the canonical orientation** (transpose for free so the operator is + column-structured whenever that lands on the CSC-regular kernel; route to CSR + when it's genuinely row-structured and not auto-transposed). This replaces the + re-derive-by-scanning-`colptr` heuristic and removes the wrong-format re-sort. +- **A `≤`-regular variant** so LASO operators get a fast path despite + merge-induced count variation. + +## Recommendation: gather targeted data first + +I'd **benchmark before writing kernels**, for concrete reasons grounded in the +prior investigation: + +1. **The existing `spmm_performance` benchmark already covers both layouts and + sides, but only on generic uniform-random matrices — not these operator + shapes.** It drives `left_spmm` in ColMajor *and* RowMajor (plus `op(B)=Trans`) + across CSR/CSC/COO, and its header documents that this reaches what `left_spmm` + and `right_spmm` together hit via the transpose reduction. But it builds + matrices with `random_coo` (uniform density), so it never produces the + *structured* operators we care about: fixed-nnz-per-column/row, ±1 values, or + mostly-empty columns. Consequently it **never exercises the regular-CSC fast + path** (random density → variable nnz per column → the detector in + `apply_csc_jki_p11` fails) and never goes through `sketch_general`. No baseline + isolates wide-LASO, CountSketch, or the regular-structure kernels — that's the + gap to close. +2. **Which kernel actually fires is the product of (operator shape × side × + layout)** — e.g. tall-SASO right-sketch reaches the regular fast path only when + the original layout is RowMajor (ColMajor right-sketch flips to RowMajor and + lands on the axpy kernel, which has no regular specialization). The benchmark + already crosses layout and side; what it must add is the *structured* operators, + so the regular path and the ±1/mostly-empty structure are actually represented. +3. **The `n`-regime decides the kernel design.** Wide `n` (parallel-over-`n` is + fine) vs narrow `n`/SpMV (needs operator-dimension parallelism, which reshapes + kernel 1) is the biggest design fork. Measure the shapes RandLAPACK actually + drives. +4. **The memory-bound hypothesis is inferred from stack shape, not counters** (SIP + gates perf counters on the M3). If it holds, kernel 3 (drop `vals`) and + index-width (int32 vs int64) dominate arithmetic restructuring — reordering the + priority list. +5. **The per-apply re-sort is a real cost worth quantifying** — and, per the + benchmark, it hits *every* SASO with `vec_nnz ≥ 2` (not just wide LASO), since + only `vec_nnz = 1` is detected as CSC-sorted. It scales with nnz and argues for + fixing dispatch (mark/skip the sort) before chasing kernel-level gains. + +Concretely: extend the harness to sweep {wide SASO, CountSketch, wide LASO} × +{left, right} × {RowMajor, ColMajor} × a range of `n` (including `n=1` SpMV) × +`vec_nnz ∈ {1,2,4,8}`, on both the M3 and an Arm SVE box (NEON/SVE is a +first-class target). That isolates which kernel actually moves the needle before +we hand-tune any of them. + +## Suggested next step + +Either: + +- **Extend the benchmark** to cover these shapes/sides/layouts and produce the + baseline, or +- **Prototype one kernel first** — CountSketch is the highest-leverage and most + self-contained. From cb84355624237122c2c53d5548e3bf58c587ce49 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 23:21:38 -0700 Subject: [PATCH 04/20] Sort LASO nonzeros within each major-axis vector during sampling Completes the construction-time sort started for SASO. The LASO branch of fill_sparse_unpacked emits each major-axis vector's (<= vec_nnz) merged survivors in hash/sample order, so the COO is grouped by the minor axis but unordered within each vector (coo-sort = None). Sorting each merged block by its major coordinate makes a wide LASO CSR-sorted (cols ascending within each row) and a tall LASO CSC-sorted. For a wide LASO this natural order also matches the ColMajor dispatch preference (CSR for wide operators), so apply_coo_via_csc skips its per-apply re-sort -- which also makes the ColMajor CSR routing an unconditional win even at tiny nnz, where the re-sort had previously cost more than the kernel gain. Operator is mathematically unchanged; only within-vector storage order changes. All 435 ctest pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- RandBLAS/sparse_skops.hh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index e140ed8e..0a52f13a 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -680,6 +680,24 @@ state_t fill_sparse_unpacked( end_state = sample_indices_iid_uniform(dim_major, vec_nnz, im, v, end_state); laso_merge_long_axis_vector_coo_data(vec_nnz, v, im, in, i, loc2count, loc2scale); int64_t count = (int64_t) loc2count.size(); + // Emit this major-axis vector in ascending major-coordinate order, exactly as + // the SASO branch does. The merge leaves the (<= vec_nnz) survivors in hash / + // sample order; sorting the block by its major coordinate makes a wide LASO + // CSR-sorted (cols ascending within each row) and a tall LASO CSC-sorted. That + // natural order also matches the ColMajor dispatch preference (CSR for wide), + // so apply_coo_via_csc skips its per-apply re-sort. idxs_minor is constant + // within the block. The block length varies, so insertion-sort (count is small). + for (int64_t a = 1; a < count; ++a) { + sint_t key = im[a]; + T vv = v[a]; + int64_t c = a - 1; + for (; c >= 0 && im[c] > key; --c) { + im[c+1] = im[c]; + v[c+1] = v[c]; + } + im[c+1] = key; + v[c+1] = vv; + } im += count; in += count; v += count; total += count; } } From cd9832f015efa6bb527902e791e0e26f99351595 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 23:21:49 -0700 Subject: [PATCH 05/20] Route ColMajor wide sparse operators through the CSR jik kernel apply_coo_via_csc previously always converted COO to CSC. The per-RHS-column kernels loop the operator's outer dimension once per output column: CSC jki's outer loop is n_cols (= m), CSR jik's is n_rows (= d). For a WIDE operator (d < m) in the ColMajor path, the shorter CSR outer loop measured 1.1-2x faster than CSC jki for both SASO and LASO, with no regression (see the --csr-probe benchmark mode). So we now sort the COO to CSR and dispatch apply_csr_jik_p11 when ColMajor and d < m; RowMajor keeps the bandwidth-saturating CSC kib axpy kernel, and tall operators keep CSC. A ColMajor strided-axpy 'CSC kib' kernel was also prototyped and rejected: it regressed dense-column SASO ~3x because ColMajor rows are strided. Conversion cost is unchanged (the COO is sorted either way). A wide SASO that sampling sorted to CSC is still re-sorted to CSR here; a follow-up will build the CSR rowptr via an O(nnz) counting-sort transpose to remove that. All 435 ctest pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- RandBLAS/sparse_data/coo_spmm_impl.hh | 30 +++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/RandBLAS/sparse_data/coo_spmm_impl.hh b/RandBLAS/sparse_data/coo_spmm_impl.hh index e2781453..3d2df7ea 100644 --- a/RandBLAS/sparse_data/coo_spmm_impl.hh +++ b/RandBLAS/sparse_data/coo_spmm_impl.hh @@ -34,6 +34,7 @@ #include "RandBLAS/sparse_data/base.hh" #include "RandBLAS/sparse_data/coo_matrix.hh" #include "RandBLAS/sparse_data/csc_spmm_impl.hh" +#include "RandBLAS/sparse_data/csr_spmm_impl.hh" #include #include #if defined(RandBLAS_HAS_OpenMP) @@ -67,8 +68,21 @@ static void apply_coo_via_csc( ) { randblas_require(A0.index_base == IndexBase::Zero); + // Format choice (PROTOTYPE). The per-RHS-column kernels loop the operator's OUTER + // dimension once per output column: CSC jki's outer loop is n_cols (= m), CSR jik's + // is n_rows (= d). In the ColMajor path, routing a WIDE operator (d < m) through the + // CSR jik kernel (shorter outer loop) measured 1.1-2x faster than CSC jki for both + // SASO and LASO, with no regression -- so we prefer CSR there. The RowMajor path + // keeps the bandwidth-saturating CSC kib axpy kernel. The COO must be sorted either + // way, so the format choice is otherwise free (though a SASO that sampling sorted to + // CSC will be re-sorted to CSR here; the kernel win exceeds that cost). See the + // --csr-probe mode of examples/.../sketch_general_performance.cc for measurements. + bool col_major = (layout_B == blas::Layout::ColMajor && layout_C == blas::Layout::ColMajor); + bool prefer_csr = col_major && (d < m); + NonzeroSort target = prefer_csr ? NonzeroSort::CSR : NonzeroSort::CSC; + bool submatrix = (A0.n_rows != d) || (A0.n_cols != m); - if (submatrix || A0.sort != NonzeroSort::CSC) { + if (submatrix || A0.sort != target) { auto A1 = A0.deepcopy(); auto new_nnz = A1.nnz; if (submatrix) { @@ -86,10 +100,22 @@ static void apply_coo_via_csc( new_nnz = write; } COOMatrix A2(d, m, new_nnz, A1.vals, A1.rows, A1.cols, false); - A2.sort_arrays(NonzeroSort::CSC); + A2.sort_arrays(target); apply_coo_via_csc(alpha, layout_B, layout_C, d, n, m, A2, 0, 0, B, ldb, C, ldc); return; } + + if (prefer_csr) { + // ColMajor + wide: CSR jik (outer loop over the d rows). + auto rowptr = new sint_t[d+1]; + sorted_idxs_to_compressed_ptr( A0.nnz, A0.rows, d, rowptr ); + CSRMatrix A_csr( d, m, A0.nnz, A0.vals, rowptr, A0.cols ); + using RandBLAS::sparse_data::csr::apply_csr_jik_p11; + apply_csr_jik_p11(alpha, layout_B, layout_C, d, n, m, A_csr, B, ldb, C, ldc); + delete [] rowptr; + return; + } + auto colptr = new sint_t[m+1]; sorted_idxs_to_compressed_ptr( A0.nnz, A0.cols, m, colptr ); CSCMatrix A_csc( d, m, A0.nnz, A0.vals, A0.rows, colptr ); From ecb76c18b8efa5e1ef9ae63ec0dce00832726e0d Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 23:21:55 -0700 Subject: [PATCH 06/20] Add --csr-probe mode to the sketch_general benchmark Builds each operator's CSC and CSR representations once and times left_spmm directly on each (no per-call COO re-sort), in both layouts, to isolate the kernel: does CSR jik (ColMajor outer loop over d rows) beat CSC jki (outer loop over m mostly-empty columns)? CSC kib (RowMajor) is the fast baseline. This is the harness that justified the ColMajor CSR routing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sketch_general_performance.cc | 101 +++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/examples/simple-kernel-benchmarks/sketch_general_performance.cc b/examples/simple-kernel-benchmarks/sketch_general_performance.cc index 24e06d53..a6b32563 100644 --- a/examples/simple-kernel-benchmarks/sketch_general_performance.cc +++ b/examples/simple-kernel-benchmarks/sketch_general_performance.cc @@ -467,6 +467,96 @@ void run_scaling(int64_t d, int64_t m, int64_t n, const std::vector& spe } } +// --------------------------------------------------------------------------- +// CSR-vs-CSC KERNEL PROBE (left-sketch). Builds the operator's CSC and CSR +// representations once and times left_spmm DIRECTLY on each (no per-call COO +// re-sort), in both layouts, to isolate the kernel. The question: does the CSR +// jik path -- whose ColMajor outer loop runs over d rows (all non-empty for a +// wide LASO) -- recover the gap vs the CSC jki path, whose outer loop runs over +// m mostly-empty columns? CSC kib (RowMajor) is the already-fast baseline. +// +// Finding: CSR jik (ColMajor) beats CSC jki (ColMajor) 1.1-2x for BOTH SASO and +// LASO with no regression, motivating the ColMajor "prefer CSR for wide +// operators" routing now in apply_coo_via_csc (coo_spmm_impl.hh). (A strided-axpy +// "ColMajor kib" kernel was also tried and rejected -- it regressed dense-column +// SASO ~3x.) The probe times the kernels directly, so it is unaffected by that +// routing change and remains the A/B harness for it. +// --------------------------------------------------------------------------- +void run_csr_probe(int64_t d, int64_t m, int64_t n, + const std::vector& specs, int num_trials) { + using T = double; + namespace rb = RandBLAS; + uint64_t seed = 12345; + + std::cout << "=== CSR-vs-CSC KERNEL PROBE (left-sketch) d=" << d << " m=" << m + << " n=" << n << ", trials=" << num_trials << " ===\n"; + std::cout << " kernels timed directly via left_spmm on prebuilt CSC/CSR (no COO re-sort)\n\n"; + + std::vector A_cm(m * n), A_rm(m * n); + rb::DenseDist DA(m, n); + auto st = rb::fill_dense(DA, A_cm.data(), rb::RNGState<>(seed)); + for (int64_t i = 0; i < m; ++i) + for (int64_t j = 0; j < n; ++j) + A_rm[i * n + j] = A_cm[i + j * m]; + std::vector C_cm(d * n), C_rm(d * n), B_ref(d * n), S_dense(d * m), result(d * n); + + for (const auto& spec : specs) { + SparseDist dist(d, m, spec.vec_nnz, spec.axis); + SparseSkOp S(dist, st); + rb::fill_sparse(S); + auto view = rb::sparse::coo_view_of_skop(S); + int64_t nnz = S.nnz; + auto S_csc = view.as_owning_csc(); + auto S_csr = view.as_owning_csr(); + int64_t a_read = std::min(nnz, m) * n; + + // Oracle: densify(S) + GEMM (also the correctness reference). + rb::sparse_data::coo::coo_to_dense(view, Layout::ColMajor, S_dense.data()); + std::fill(B_ref.begin(), B_ref.end(), 0.0); + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, S_dense.data(), d, A_cm.data(), m, 0.0, B_ref.data(), d); + + std::vector rows; + auto probe = [&](const std::string& label, const auto& Sp, Layout lay) { + const T* Bp = (lay == Layout::ColMajor) ? A_cm.data() : A_rm.data(); + int64_t ldb = (lay == Layout::ColMajor) ? m : n; + T* Cp = (lay == Layout::ColMajor) ? C_cm.data() : C_rm.data(); + int64_t ldc = (lay == Layout::ColMajor) ? d : n; + auto [mn, md] = run_trials([&]() { + std::fill(Cp, Cp + d * n, 0.0); + rb::sparse_data::left_spmm(lay, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, Sp, 0, 0, Bp, ldb, 0.0, Cp, ldc); + }, num_trials); + std::fill(result.begin(), result.end(), 0.0); + rb::sparse_data::left_spmm(lay, Op::NoTrans, Op::NoTrans, d, n, m, + 1.0, Sp, 0, 0, Bp, ldb, 0.0, result.data(), ldc); + double maxdiff = 0; + if (lay == Layout::ColMajor) + for (int64_t i = 0; i < d * n; ++i) + maxdiff = std::max(maxdiff, std::abs(result[i] - B_ref[i])); + else + for (int64_t i = 0; i < d; ++i) + for (int64_t j = 0; j < n; ++j) + maxdiff = std::max(maxdiff, std::abs(result[i * n + j] - B_ref[i + j * d])); + if (maxdiff > 1e-9) + std::cout << " FAIL " << label << " max|diff|=" << std::scientific + << maxdiff << std::fixed << "\n"; + Row r; r.label = label; r.min_us = mn; r.med_us = md; + fill_metrics(r, mn, nnz, n, a_read, d * n); + r.notes = std::string(sort_name(view.sort)) + " nnz=" + std::to_string(nnz); + rows.push_back(r); + }; + + print_metric_header(spec.label); + probe("CSC jki ColMajor", S_csc, Layout::ColMajor); + probe("CSR jik ColMajor", S_csr, Layout::ColMajor); + probe("CSC kib RowMajor", S_csc, Layout::RowMajor); + probe("CSR ikb RowMajor", S_csr, Layout::RowMajor); + for (auto& r : rows) print_metric_row(r); + std::cout << "\n"; + } +} + std::vector sweep_specs() { return { {"Wide SASO k=2", 2, Axis::Short}, @@ -491,13 +581,14 @@ static std::vector parse_threads(const std::string& csv) { } int main(int argc, char** argv) { - bool no_stream = false, scaling = false; + bool no_stream = false, scaling = false, csr_probe = false; std::vector threads = {1, 2, 4, 8}; std::vector pos; for (int i = 1; i < argc; ++i) { std::string s = argv[i]; if (s == "--no-stream") no_stream = true; else if (s == "--scaling") scaling = true; + else if (s == "--csr-probe") csr_probe = true; else if (s.rfind("--threads=", 0) == 0) threads = parse_threads(s.substr(10)); else pos.push_back(s); } @@ -535,6 +626,14 @@ int main(int argc, char** argv) { } std::cout << "\n"; + if (csr_probe) { + std::vector specs = have_cfg + ? std::vector{{"single", k, axis}} : sweep_specs(); + int64_t sd = have_cfg ? d : 200, sm = have_cfg ? m : 2000, sn = have_cfg ? n : 2000; + run_csr_probe(sd, sm, sn, specs, trials); + return 0; + } + if (have_cfg) { std::vector specs = {{"single", k, axis}}; run_left(d, m, n, specs, trials); From cd95848dd25da83b309e0a46251453542e65cd5f Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 23:26:00 -0700 Subject: [PATCH 07/20] Transpose to the wanted compressed format via O(nnz) counting sort When apply_coo_via_csc needs a format the COO isn't sorted for (e.g. a wide SASO sampled CSC-sorted, but the ColMajor path wants CSR), it previously deep-copied and ran an O(nnz log nnz) comparison re-sort. For a full operator already sorted in the OPPOSITE compressed order, build the wanted format directly with an O(nnz + dim) counting-sort transpose instead. Iterating the source in its sorted order leaves the companion index ascending within each group, so the result is properly sorted in the target order. This removes the per-apply re-sort tax that had capped wide-SASO ColMajor at ~1.1x: it now reaches ~1.45x (matching the CSR jik kernel's intrinsic advantage over CSC jki). The submatrix and unsorted (None) cases still take the general deepcopy + sort path. All 435 ctest pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- RandBLAS/sparse_data/coo_spmm_impl.hh | 67 +++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/RandBLAS/sparse_data/coo_spmm_impl.hh b/RandBLAS/sparse_data/coo_spmm_impl.hh index 3d2df7ea..341e50f8 100644 --- a/RandBLAS/sparse_data/coo_spmm_impl.hh +++ b/RandBLAS/sparse_data/coo_spmm_impl.hh @@ -50,6 +50,33 @@ using RandBLAS::SignedInteger; #endif +// Build a compressed layout (CSR or CSC) from a COO that is already sorted in the +// OPPOSITE compressed order, via an O(nnz + dim) counting-sort transpose -- cheaper +// than the O(nnz log nnz) comparison re-sort that COOMatrix::sort_arrays would do. +// Because the source is iterated in its (oppositely-)sorted order, the companion +// index comes out ascending within each group, so the result is properly sorted in +// the target order too. +// group_idx : axis to compress on (rows for a CSR target, cols for CSC); length nnz +// other_idx : the companion axis (cols for CSR, rows for CSC); length nnz +// dim : number of groups (n_rows for CSR, n_cols for CSC) +// Outputs (caller-allocated): ptr[dim+1], out_other[nnz], out_vals[nnz]. +template +static void counting_sort_transpose( + int64_t nnz, const sint_t* group_idx, const sint_t* other_idx, const T* vals, + int64_t dim, sint_t* ptr, sint_t* out_other, T* out_vals +) { + for (int64_t g = 0; g <= dim; ++g) ptr[g] = 0; + for (int64_t e = 0; e < nnz; ++e) ptr[group_idx[e] + 1] += 1; + for (int64_t g = 0; g < dim; ++g) ptr[g + 1] += ptr[g]; + std::vector cursor(ptr, ptr + dim); // running write position per group + for (int64_t e = 0; e < nnz; ++e) { + sint_t g = group_idx[e]; + sint_t pos = cursor[g]++; + out_other[pos] = other_idx[e]; + out_vals[pos] = vals[e]; + } +} + template static void apply_coo_via_csc( T alpha, @@ -73,15 +100,47 @@ static void apply_coo_via_csc( // is n_rows (= d). In the ColMajor path, routing a WIDE operator (d < m) through the // CSR jik kernel (shorter outer loop) measured 1.1-2x faster than CSC jki for both // SASO and LASO, with no regression -- so we prefer CSR there. The RowMajor path - // keeps the bandwidth-saturating CSC kib axpy kernel. The COO must be sorted either - // way, so the format choice is otherwise free (though a SASO that sampling sorted to - // CSC will be re-sorted to CSR here; the kernel win exceeds that cost). See the - // --csr-probe mode of examples/.../sketch_general_performance.cc for measurements. + // keeps the bandwidth-saturating CSC kib axpy kernel. See the --csr-probe mode of + // examples/.../sketch_general_performance.cc for measurements. bool col_major = (layout_B == blas::Layout::ColMajor && layout_C == blas::Layout::ColMajor); bool prefer_csr = col_major && (d < m); NonzeroSort target = prefer_csr ? NonzeroSort::CSR : NonzeroSort::CSC; bool submatrix = (A0.n_rows != d) || (A0.n_cols != m); + + // Fast transpose path: full operator already sorted in the OPPOSITE compressed + // order (e.g. a wide SASO sampled CSC-sorted, but the ColMajor path wants CSR). + // Build the wanted format via an O(nnz) counting-sort transpose instead of the + // O(nnz log nnz) comparison re-sort + recurse below. This removes the per-apply + // re-sort tax for operators whose sampled order doesn't match the chosen format. + if (!submatrix && A0.sort != target && A0.sort != NonzeroSort::None) { + if (prefer_csr) { // A0 is CSC-sorted; transpose to CSR (group by row). + sint_t* rowptr = new sint_t[d + 1]; + sint_t* colidxs = new sint_t[A0.nnz]; + T* csrvals = new T[A0.nnz]; + counting_sort_transpose(A0.nnz, A0.rows, A0.cols, A0.vals, d, rowptr, colidxs, csrvals); + CSRMatrix A_csr(d, m, A0.nnz, csrvals, rowptr, colidxs); + using RandBLAS::sparse_data::csr::apply_csr_jik_p11; + apply_csr_jik_p11(alpha, layout_B, layout_C, d, n, m, A_csr, B, ldb, C, ldc); + delete [] rowptr; delete [] colidxs; delete [] csrvals; + } else { // A0 is CSR-sorted; transpose to CSC (group by col). + sint_t* colptr = new sint_t[m + 1]; + sint_t* rowidxs = new sint_t[A0.nnz]; + T* cscvals = new T[A0.nnz]; + counting_sort_transpose(A0.nnz, A0.cols, A0.rows, A0.vals, m, colptr, rowidxs, cscvals); + CSCMatrix A_csc(d, m, A0.nnz, cscvals, rowidxs, colptr); + if (layout_B == layout_C && layout_B == blas::Layout::RowMajor) { + using RandBLAS::sparse_data::csc::apply_csc_kib_1p1_rowmajor; + apply_csc_kib_1p1_rowmajor(alpha, n, A_csc, B, ldb, C, ldc); + } else { + using RandBLAS::sparse_data::csc::apply_csc_jki_p11; + apply_csc_jki_p11(alpha, layout_B, layout_C, n, A_csc, B, ldb, C, ldc); + } + delete [] colptr; delete [] rowidxs; delete [] cscvals; + } + return; + } + if (submatrix || A0.sort != target) { auto A1 = A0.deepcopy(); auto new_nnz = A1.nnz; From b57a70ffc199e1ee6a66c0ff11d74a8436a20b9f Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Wed, 17 Jun 2026 23:26:45 -0700 Subject: [PATCH 08/20] Refresh post-change benchmark results (LASO sort + CSR dispatch + transpose) Regenerate sketch_general_results_postfix.txt to capture the combined branch state: construction-time SASO/LASO sorting, ColMajor wide -> CSR jik routing, and the O(nnz) counting-sort transpose. ColMajor warm apply vs the pre-change baseline: wide SASO ~1.45x, wide LASO 1.2-2.1x, no RowMajor regression, 0 FAIL. sketch_general_results.txt is left frozen as the pre-change baseline. Co-Authored-By: Claude Opus 4.8 (1M context) --- sketch_general_results_postfix.txt | 255 ++++++++++++++--------------- 1 file changed, 127 insertions(+), 128 deletions(-) diff --git a/sketch_general_results_postfix.txt b/sketch_general_results_postfix.txt index 5ddeb277..c6ce9297 100644 --- a/sketch_general_results_postfix.txt +++ b/sketch_general_results_postfix.txt @@ -1,20 +1,19 @@ -# POST-FIX run with work-normalized metrics (ns/elt, GB/s, %STR). -# Sampling fix: within-vector sort in fill_sparse_unpacked SASO branch -# (RandBLAS/sparse_skops.hh) -> wide SASO vec_nnz>=2 emits CSC-sorted COO -# (tall SASO -> CSR), so apply_coo_via_csc skips its per-apply re-sort. -# ns/elt = us*1000/(nnz*work_mult); work_mult = n (left) or m (right). -# GB/s = model bytes / time; %STR = GB/s as a fraction of the STREAM triad -# below. Byte model bounds A traffic by min(nnz, contract_dim); STREAM is a -# DRAM ceiling, so tiny cache-resident LASO cases can exceed 100% %STR. -# Pre-fix baseline (no metric columns): sketch_general_results.txt -# Environment: Apple M3, OMP_NUM_THREADS=4, Release build (MKL off) -# RandBLAS commit: 705f09e + uncommitted sampling fix | Correctness: 0 FAIL +# POST-CHANGES run. Combined state of the sparse-sketching branch: +# 1. Construction-time within-vector sort for SASO and LASO (sparse_skops.hh): +# wide SASO -> CSC-sorted COO, wide LASO -> CSR-sorted COO. +# 2. ColMajor wide operators routed through the CSR jik kernel (coo_spmm_impl.hh). +# 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen +# format (closes the wide-SASO ColMajor re-sort tax). +# Net vs the pre-change baseline (sketch_general_results.txt), ColMajor warm apply: +# wide SASO ~1.45x, wide LASO 1.2-2.1x, no RowMajor regression. +# ns/elt = us*1000/(nnz*work_mult); GB/s = model bytes/time; %STR vs STREAM below. +# Environment: Apple M3, OMP_NUM_THREADS=4, Release (MKL off) | Correctness: 0 FAIL ============================================================ SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) ============================================================ Threads (OMP_NUM_THREADS/max): 4 -Calibrating STREAM triad... 85.4 GB/s (= 100% on the %STR column) +Calibrating STREAM triad... 88.6 GB/s (= 100% on the %STR column) ########## SECTION 1: vary sketch size d (m=n=2000) ########## @@ -23,147 +22,147 @@ Calibrating STREAM triad... 85.4 GB/s (= 100% on the %STR column) warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1097 1111 0.137 30.4 36 CSC nnz=4000 cold=1153 cvt=7 - Wide SASO k=4 2223 2332 0.139 15.0 18 CSC nnz=8000 cold=2308 cvt=14 - Wide SASO k=8 3934 3988 0.123 8.5 10 CSC nnz=16000 cold=4423 cvt=26 - CountSketch k=1 834 844 0.208 39.9 47 CSC nnz=2000 cold=852 cvt=3 - Wide LASO k=2 420 597 2.625 6.1 7 None nnz=80 cold=689 cvt=0 - Wide LASO k=4 401 414 1.253 9.6 11 None nnz=160 cold=464 cvt=1 - Wide LASO k=8 485 501 0.763 13.1 15 None nnz=318 cold=517 cvt=3 - densify(S)+GEMM 1574 1599 - - - dense oracle + Wide SASO k=2 676 691 0.085 49.3 56 CSC nnz=4000 cold=867 cvt=6 + Wide SASO k=4 1425 1732 0.089 23.4 26 CSC nnz=8000 cold=1742 cvt=14 + Wide SASO k=8 2594 3294 0.081 12.9 15 CSC nnz=16000 cold=3250 cvt=27 + CountSketch k=1 582 717 0.145 57.2 65 CSC nnz=2000 cold=642 cvt=3 + Wide LASO k=2 221 274 1.381 11.6 13 CSR nnz=80 cold=220 cvt=0 + Wide LASO k=4 351 380 1.097 10.9 12 CSR nnz=160 cold=364 cvt=1 + Wide LASO k=8 383 392 0.602 16.6 19 CSR nnz=318 cold=390 cvt=3 + densify(S)+GEMM 1508 1526 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 740 754 0.092 45.1 53 CSC nnz=4000 - Wide SASO k=4 1073 1112 0.067 31.1 36 CSC nnz=8000 - Wide SASO k=8 1539 1839 0.048 21.8 26 CSC nnz=16000 - CountSketch k=1 497 523 0.124 67.0 78 CSC nnz=2000 - Wide LASO k=2 30 32 0.188 85.4 100 None nnz=80 - Wide LASO k=4 36 38 0.113 106.7 125 None nnz=160 - Wide LASO k=8 57 63 0.090 111.8 131 None nnz=318 + Wide SASO k=2 753 821 0.094 44.3 50 CSC nnz=4000 + Wide SASO k=4 1050 1066 0.066 31.8 36 CSC nnz=8000 + Wide SASO k=8 1770 1795 0.055 18.9 21 CSC nnz=16000 + CountSketch k=1 488 514 0.122 68.3 77 CSC nnz=2000 + Wide LASO k=2 28 31 0.175 91.5 103 CSR nnz=80 + Wide LASO k=4 35 47 0.109 109.8 124 CSR nnz=160 + Wide LASO k=8 48 49 0.075 132.8 150 CSR nnz=318 === RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === warm apply, ColMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 649 719 0.081 51.4 60 CSR nnz=4000 - Wide SASO k=4 1103 1138 0.069 30.3 35 CSR nnz=8000 - Wide SASO k=8 1505 1849 0.047 22.3 26 CSR nnz=16000 - CountSketch k=1 474 509 0.118 70.3 82 CSR nnz=2000 - Wide LASO k=2 28 31 0.175 91.5 107 None nnz=80 - Wide LASO k=4 37 39 0.116 103.9 122 None nnz=160 - Wide LASO k=8 53 55 0.083 120.9 141 None nnz=320 + Wide SASO k=2 701 740 0.088 47.6 54 CSR nnz=4000 + Wide SASO k=4 1090 1166 0.068 30.6 35 CSR nnz=8000 + Wide SASO k=8 1730 1859 0.054 19.4 22 CSR nnz=16000 + CountSketch k=1 508 510 0.127 65.6 74 CSR nnz=2000 + Wide LASO k=2 29 29 0.181 88.3 100 CSC nnz=80 + Wide LASO k=4 35 45 0.109 109.8 124 CSC nnz=160 + Wide LASO k=8 48 52 0.075 133.4 151 CSC nnz=320 warm apply, RowMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1095 1104 0.137 30.5 36 CSR nnz=4000 - Wide SASO k=4 2197 2206 0.137 15.2 18 CSR nnz=8000 - Wide SASO k=8 3938 4008 0.123 8.5 10 CSR nnz=16000 - CountSketch k=1 833 843 0.208 40.0 47 CSR nnz=2000 - Wide LASO k=2 528 857 3.300 4.9 6 None nnz=80 - Wide LASO k=4 458 463 1.431 8.4 10 None nnz=160 - Wide LASO k=8 496 505 0.775 12.9 15 None nnz=320 + Wide SASO k=2 584 615 0.073 57.1 64 CSR nnz=4000 + Wide SASO k=4 1239 1282 0.077 27.0 30 CSR nnz=8000 + Wide SASO k=8 2344 3083 0.073 14.3 16 CSR nnz=16000 + CountSketch k=1 504 582 0.126 66.1 75 CSR nnz=2000 + Wide LASO k=2 150 174 0.938 17.1 19 CSC nnz=80 + Wide LASO k=4 330 367 1.031 11.6 13 CSC nnz=160 + Wide LASO k=8 410 454 0.641 15.6 18 CSC nnz=320 === LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1134 1193 0.142 33.9 40 CSC nnz=4000 cold=1180 cvt=6 - Wide SASO k=4 2134 2349 0.133 18.1 21 CSC nnz=8000 cold=3055 cvt=15 - Wide SASO k=8 3752 3956 0.117 10.3 12 CSC nnz=16000 cold=4091 cvt=26 - CountSketch k=1 875 884 0.219 43.9 51 CSC nnz=2000 cold=893 cvt=3 - Wide LASO k=2 571 607 0.714 22.4 26 None nnz=400 cold=606 cvt=4 - Wide LASO k=4 880 1047 0.550 21.8 26 None nnz=800 cold=847 cvt=10 - Wide LASO k=8 1356 1436 0.425 23.6 28 None nnz=1597 cold=1465 cvt=22 - densify(S)+GEMM 4726 4855 - - - dense oracle + Wide SASO k=2 769 925 0.096 50.0 56 CSC nnz=4000 cold=1233 cvt=7 + Wide SASO k=4 1432 1495 0.089 26.9 30 CSC nnz=8000 cold=1985 cvt=15 + Wide SASO k=8 2508 3257 0.078 15.4 17 CSC nnz=16000 cold=3901 cvt=26 + CountSketch k=1 716 910 0.179 53.7 61 CSC nnz=2000 cold=794 cvt=3 + Wide LASO k=2 490 535 0.613 26.1 29 CSR nnz=400 cold=531 cvt=4 + Wide LASO k=4 533 710 0.333 36.0 41 CSR nnz=800 cold=712 cvt=10 + Wide LASO k=8 593 640 0.186 53.9 61 CSR nnz=1597 cold=662 cvt=22 + densify(S)+GEMM 4698 4732 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 717 859 0.090 53.6 63 CSC nnz=4000 - Wide SASO k=4 1074 1230 0.067 35.9 42 CSC nnz=8000 - Wide SASO k=8 1785 1908 0.056 21.7 25 CSC nnz=16000 - CountSketch k=1 560 573 0.140 68.6 80 CSC nnz=2000 - Wide LASO k=2 101 107 0.126 126.8 148 None nnz=400 - Wide LASO k=4 174 211 0.109 110.4 129 None nnz=800 - Wide LASO k=8 406 414 0.127 78.8 92 None nnz=1597 + Wide SASO k=2 739 783 0.092 52.0 59 CSC nnz=4000 + Wide SASO k=4 1153 1179 0.072 33.4 38 CSC nnz=8000 + Wide SASO k=8 1830 1934 0.057 21.1 24 CSC nnz=16000 + CountSketch k=1 504 550 0.126 76.3 86 CSC nnz=2000 + Wide LASO k=2 98 105 0.122 130.7 147 CSR nnz=400 + Wide LASO k=4 162 173 0.101 118.6 134 CSR nnz=800 + Wide LASO k=8 368 388 0.115 86.9 98 CSR nnz=1597 === RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === warm apply, ColMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 769 782 0.096 50.0 59 CSR nnz=4000 - Wide SASO k=4 1133 1174 0.071 34.0 40 CSR nnz=8000 - Wide SASO k=8 1657 1903 0.052 23.3 27 CSR nnz=16000 - CountSketch k=1 563 656 0.141 68.3 80 CSR nnz=2000 - Wide LASO k=2 105 114 0.131 122.0 143 None nnz=400 - Wide LASO k=4 166 171 0.104 115.5 135 None nnz=798 - Wide LASO k=8 417 458 0.131 76.7 90 None nnz=1597 + Wide SASO k=2 730 799 0.091 52.7 59 CSR nnz=4000 + Wide SASO k=4 1133 1163 0.071 34.0 38 CSR nnz=8000 + Wide SASO k=8 1872 1907 0.059 20.6 23 CSR nnz=16000 + CountSketch k=1 527 567 0.132 72.9 82 CSR nnz=2000 + Wide LASO k=2 97 101 0.121 132.0 149 CSC nnz=400 + Wide LASO k=4 176 185 0.110 109.0 123 CSC nnz=798 + Wide LASO k=8 362 370 0.113 88.3 100 CSC nnz=1597 warm apply, RowMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1134 1153 0.142 33.9 40 CSR nnz=4000 - Wide SASO k=4 2111 2175 0.132 18.3 21 CSR nnz=8000 - Wide SASO k=8 3727 4065 0.116 10.4 12 CSR nnz=16000 - CountSketch k=1 881 913 0.220 43.6 51 CSR nnz=2000 - Wide LASO k=2 584 600 0.730 21.9 26 None nnz=400 - Wide LASO k=4 801 836 0.502 23.9 28 None nnz=798 - Wide LASO k=8 1259 1326 0.394 25.4 30 None nnz=1597 + Wide SASO k=2 849 958 0.106 45.3 51 CSR nnz=4000 + Wide SASO k=4 1414 1473 0.088 27.2 31 CSR nnz=8000 + Wide SASO k=8 2783 3272 0.087 13.9 16 CSR nnz=16000 + CountSketch k=1 746 934 0.186 51.5 58 CSR nnz=2000 + Wide LASO k=2 514 623 0.642 24.9 28 CSC nnz=400 + Wide LASO k=4 614 637 0.385 31.2 35 CSC nnz=798 + Wide LASO k=8 613 727 0.192 52.2 59 CSC nnz=1597 === LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1242 1281 0.155 38.7 45 CSC nnz=4000 cold=1281 cvt=7 - Wide SASO k=4 2193 2391 0.137 21.9 26 CSC nnz=8000 cold=2314 cvt=14 - Wide SASO k=8 3854 4161 0.120 12.5 15 CSC nnz=16000 cold=4149 cvt=26 - CountSketch k=1 997 1400 0.249 48.2 56 CSC nnz=2000 cold=997 cvt=3 - Wide LASO k=2 1032 1104 0.516 31.0 36 None nnz=1000 cold=1089 cvt=13 - Wide LASO k=4 1580 1767 0.395 30.4 36 None nnz=1999 cold=2217 cvt=30 - Wide LASO k=8 2173 2294 0.272 22.1 26 None nnz=3994 cold=2408 cvt=65 - densify(S)+GEMM 11199 11712 - - - dense oracle + Wide SASO k=2 1229 1274 0.154 39.1 44 CSC nnz=4000 cold=1516 cvt=7 + Wide SASO k=4 1799 1838 0.112 26.8 30 CSC nnz=8000 cold=2206 cvt=15 + Wide SASO k=8 2756 3291 0.086 17.5 20 CSC nnz=16000 cold=4653 cvt=26 + CountSketch k=1 949 1027 0.237 50.6 57 CSC nnz=2000 cold=1061 cvt=3 + Wide LASO k=2 798 1093 0.399 40.1 45 CSR nnz=1000 cold=815 cvt=13 + Wide LASO k=4 1019 1052 0.255 47.1 53 CSR nnz=1999 cold=1106 cvt=30 + Wide LASO k=8 999 1100 0.125 48.1 54 CSR nnz=3994 cold=1286 cvt=81 + densify(S)+GEMM 11082 11279 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 945 979 0.118 50.9 60 CSC nnz=4000 - Wide SASO k=4 1307 1493 0.082 36.8 43 CSC nnz=8000 - Wide SASO k=8 1995 2040 0.062 24.2 28 CSC nnz=16000 - CountSketch k=1 708 807 0.177 67.8 79 CSC nnz=2000 - Wide LASO k=2 411 418 0.205 77.9 91 None nnz=1000 - Wide LASO k=4 650 742 0.163 73.9 86 None nnz=1999 - Wide LASO k=8 977 1113 0.122 49.2 58 None nnz=3994 + Wide SASO k=2 895 942 0.112 53.7 61 CSC nnz=4000 + Wide SASO k=4 1258 1265 0.079 38.3 43 CSC nnz=8000 + Wide SASO k=8 2009 2105 0.063 24.0 27 CSC nnz=16000 + CountSketch k=1 673 777 0.168 71.4 81 CSC nnz=2000 + Wide LASO k=2 389 392 0.195 82.3 93 CSR nnz=1000 + Wide LASO k=4 587 597 0.147 81.8 92 CSR nnz=1999 + Wide LASO k=8 854 906 0.107 56.3 64 CSR nnz=3994 === RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === warm apply, ColMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 926 1070 0.116 51.9 61 CSR nnz=4000 - Wide SASO k=4 1272 1357 0.080 37.8 44 CSR nnz=8000 - Wide SASO k=8 1973 2116 0.062 24.5 29 CSR nnz=16000 - CountSketch k=1 773 787 0.193 62.1 73 CSR nnz=2000 - Wide LASO k=2 434 453 0.217 73.7 86 None nnz=998 - Wide LASO k=4 668 699 0.167 71.9 84 None nnz=1998 - Wide LASO k=8 979 1088 0.123 49.1 57 None nnz=3993 + Wide SASO k=2 895 905 0.112 53.7 61 CSR nnz=4000 + Wide SASO k=4 1217 1232 0.076 39.5 45 CSR nnz=8000 + Wide SASO k=8 1922 1980 0.060 25.1 28 CSR nnz=16000 + CountSketch k=1 761 767 0.190 63.1 71 CSR nnz=2000 + Wide LASO k=2 391 405 0.196 81.8 92 CSC nnz=998 + Wide LASO k=4 595 686 0.149 80.7 91 CSC nnz=1998 + Wide LASO k=8 865 898 0.108 55.6 63 CSC nnz=3993 warm apply, RowMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1241 1333 0.155 38.7 45 CSR nnz=4000 - Wide SASO k=4 2179 2255 0.136 22.1 26 CSR nnz=8000 - Wide SASO k=8 3595 3936 0.112 13.4 16 CSR nnz=16000 - CountSketch k=1 1003 1010 0.251 47.9 56 CSR nnz=2000 - Wide LASO k=2 1047 1168 0.525 30.5 36 None nnz=998 - Wide LASO k=4 1575 1607 0.394 30.5 36 None nnz=1998 - Wide LASO k=8 2112 2255 0.264 22.8 27 None nnz=3993 + Wide SASO k=2 1308 1348 0.164 36.7 41 CSR nnz=4000 + Wide SASO k=4 1708 1774 0.107 28.2 32 CSR nnz=8000 + Wide SASO k=8 3022 3513 0.094 16.0 18 CSR nnz=16000 + CountSketch k=1 1008 1052 0.252 47.7 54 CSR nnz=2000 + Wide LASO k=2 919 1021 0.460 34.8 39 CSC nnz=998 + Wide LASO k=4 1126 1150 0.282 42.6 48 CSC nnz=1998 + Wide LASO k=8 1070 1177 0.134 44.9 51 CSC nnz=3993 ########## SECTION 2: narrow n (SpMV-ish) ########## @@ -172,48 +171,48 @@ Calibrating STREAM triad... 85.4 GB/s (= 100% on the %STR column) warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 16 30 4.000 5.2 6 CSC nnz=4000 cold=75 cvt=7 - Wide SASO k=4 23 36 2.875 6.4 7 CSC nnz=8000 cold=119 cvt=14 - Wide SASO k=8 39 40 2.438 7.1 8 CSC nnz=16000 cold=245 cvt=26 - CountSketch k=1 13 15 6.500 3.9 5 CSC nnz=2000 cold=34 cvt=3 - Wide LASO k=2 17 21 42.500 0.8 1 None nnz=400 cold=42 cvt=4 - Wide LASO k=4 25 33 31.250 0.9 1 None nnz=800 cold=75 cvt=10 - Wide LASO k=8 37 47 23.139 1.1 1 None nnz=1599 cold=139 cvt=22 + Wide SASO k=2 18 20 4.500 4.6 5 CSC nnz=4000 cold=184 cvt=7 + Wide SASO k=4 31 34 3.875 4.7 5 CSC nnz=8000 cold=482 cvt=14 + Wide SASO k=8 45 51 2.812 6.1 7 CSC nnz=16000 cold=1121 cvt=25 + CountSketch k=1 13 14 6.500 3.9 4 CSC nnz=2000 cold=60 cvt=3 + Wide LASO k=2 8 9 20.000 1.6 2 CSR nnz=400 cold=36 cvt=5 + Wide LASO k=4 8 9 10.000 2.8 3 CSR nnz=800 cold=65 cvt=11 + Wide LASO k=8 10 12 6.254 4.2 5 CSR nnz=1599 cold=110 cvt=22 densify(S)+GEMM 13 14 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 77 80 19.250 1.1 1 CSC nnz=4000 - Wide SASO k=4 89 91 11.125 1.7 2 CSC nnz=8000 - Wide SASO k=8 172 246 10.750 1.6 2 CSC nnz=16000 - CountSketch k=1 35 36 17.500 1.5 2 CSC nnz=2000 - Wide LASO k=2 22 23 55.000 0.6 1 None nnz=400 - Wide LASO k=4 38 47 47.500 0.6 1 None nnz=800 - Wide LASO k=8 65 81 40.650 0.6 1 None nnz=1599 + Wide SASO k=2 56 57 14.000 1.5 2 CSC nnz=4000 + Wide SASO k=4 100 101 12.500 1.5 2 CSC nnz=8000 + Wide SASO k=8 180 183 11.250 1.5 2 CSC nnz=16000 + CountSketch k=1 35 35 17.500 1.5 2 CSC nnz=2000 + Wide LASO k=2 16 18 40.000 0.8 1 CSR nnz=400 + Wide LASO k=4 21 23 26.250 1.1 1 CSR nnz=800 + Wide LASO k=8 32 34 20.013 1.3 1 CSR nnz=1599 === LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 26 29 0.650 9.8 12 CSC nnz=4000 cold=74 cvt=7 - Wide SASO k=4 41 43 0.512 7.8 9 CSC nnz=8000 cold=138 cvt=14 - Wide SASO k=8 64 65 0.400 7.0 8 CSC nnz=16000 cold=269 cvt=25 - CountSketch k=1 21 22 1.050 10.7 12 CSC nnz=2000 cold=38 cvt=3 - Wide LASO k=2 21 24 5.250 3.4 4 None nnz=400 cold=47 cvt=4 - Wide LASO k=4 29 32 3.625 3.8 4 None nnz=800 cold=78 cvt=10 - Wide LASO k=8 45 55 2.820 4.1 5 None nnz=1596 cold=143 cvt=22 - densify(S)+GEMM 35 35 - - - dense oracle + Wide SASO k=2 22 25 0.550 11.6 13 CSC nnz=4000 cold=172 cvt=6 + Wide SASO k=4 36 41 0.450 8.9 10 CSC nnz=8000 cold=477 cvt=13 + Wide SASO k=8 63 67 0.394 7.1 8 CSC nnz=16000 cold=1114 cvt=26 + CountSketch k=1 19 32 0.950 11.8 13 CSC nnz=2000 cold=82 cvt=4 + Wide LASO k=2 9 11 2.250 7.8 9 CSR nnz=400 cold=36 cvt=4 + Wide LASO k=4 11 13 1.375 9.9 11 CSR nnz=800 cold=67 cvt=11 + Wide LASO k=8 14 15 0.877 13.2 15 CSR nnz=1596 cold=124 cvt=23 + densify(S)+GEMM 34 36 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 54 56 1.350 4.7 6 CSC nnz=4000 - Wide SASO k=4 94 100 1.175 3.4 4 CSC nnz=8000 - Wide SASO k=8 169 174 1.056 2.7 3 CSC nnz=16000 - CountSketch k=1 44 44 2.200 5.1 6 CSC nnz=2000 - Wide LASO k=2 21 23 5.250 3.4 4 None nnz=400 - Wide LASO k=4 36 43 4.500 3.0 4 None nnz=800 - Wide LASO k=8 58 64 3.634 3.2 4 None nnz=1596 + Wide SASO k=2 57 59 1.425 4.5 5 CSC nnz=4000 + Wide SASO k=4 97 100 1.212 3.3 4 CSC nnz=8000 + Wide SASO k=8 168 228 1.050 2.7 3 CSC nnz=16000 + CountSketch k=1 36 50 1.800 6.2 7 CSC nnz=2000 + Wide LASO k=2 17 19 4.250 4.1 5 CSR nnz=400 + Wide LASO k=4 21 23 2.625 5.2 6 CSR nnz=800 + Wide LASO k=8 33 40 2.068 5.6 6 CSR nnz=1596 From a95a5b4c6983dd18daff75a2ee6c6673b11a2ccf Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Fri, 19 Jun 2026 22:01:52 -0700 Subject: [PATCH 09/20] remove old performance log --- sketch_general_results.txt | 313 +++++++++++++++-------------- sketch_general_results_postfix.txt | 218 -------------------- 2 files changed, 161 insertions(+), 370 deletions(-) delete mode 100644 sketch_general_results_postfix.txt diff --git a/sketch_general_results.txt b/sketch_general_results.txt index fe371e78..c6ce9297 100644 --- a/sketch_general_results.txt +++ b/sketch_general_results.txt @@ -1,209 +1,218 @@ -# Environment: Apple M3, OMP_NUM_THREADS=4, Release build (MKL off) -# RandBLAS commit: 705f09e | Date: 2026-05-31 | Binary: build-randblas-examples/sketch_general_performance -# Correctness: all configs PASS (0 FAIL lines) +# POST-CHANGES run. Combined state of the sparse-sketching branch: +# 1. Construction-time within-vector sort for SASO and LASO (sparse_skops.hh): +# wide SASO -> CSC-sorted COO, wide LASO -> CSR-sorted COO. +# 2. ColMajor wide operators routed through the CSR jik kernel (coo_spmm_impl.hh). +# 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen +# format (closes the wide-SASO ColMajor re-sort tax). +# Net vs the pre-change baseline (sketch_general_results.txt), ColMajor warm apply: +# wide SASO ~1.45x, wide LASO 1.2-2.1x, no RowMajor regression. +# ns/elt = us*1000/(nnz*work_mult); GB/s = model bytes/time; %STR vs STREAM below. +# Environment: Apple M3, OMP_NUM_THREADS=4, Release (MKL off) | Correctness: 0 FAIL ============================================================ SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) ============================================================ +Threads (OMP_NUM_THREADS/max): 4 +Calibrating STREAM triad... 88.6 GB/s (= 100% on the %STR column) ########## SECTION 1: vary sketch size d (m=n=2000) ########## === LEFT-SKETCH B(40x2000) = S(40x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 1453 1469 3.29x nnz=4000 coo-sort=None cold=1483 convert=28 - Wide SASO k=4 2279 2337 5.16x nnz=8000 coo-sort=None cold=2357 convert=77 - Wide SASO k=8 4418 4528 10.00x nnz=16000 coo-sort=None cold=4363 convert=180 - CountSketch k=1 832 838 1.88x nnz=2000 coo-sort=CSC cold=859 convert=3 - Wide LASO k=2 553 757 1.25x nnz=78 coo-sort=None cold=483 convert=0 - Wide LASO k=4 442 450 1.00x nnz=158 coo-sort=None cold=460 convert=1 - Wide LASO k=8 477 488 1.08x nnz=318 coo-sort=None cold=506 convert=3 - densify(S)+GEMM 1566 1566 3.54x dense oracle + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 676 691 0.085 49.3 56 CSC nnz=4000 cold=867 cvt=6 + Wide SASO k=4 1425 1732 0.089 23.4 26 CSC nnz=8000 cold=1742 cvt=14 + Wide SASO k=8 2594 3294 0.081 12.9 15 CSC nnz=16000 cold=3250 cvt=27 + CountSketch k=1 582 717 0.145 57.2 65 CSC nnz=2000 cold=642 cvt=3 + Wide LASO k=2 221 274 1.381 11.6 13 CSR nnz=80 cold=220 cvt=0 + Wide LASO k=4 351 380 1.097 10.9 12 CSR nnz=160 cold=364 cvt=1 + Wide LASO k=8 383 392 0.602 16.6 19 CSR nnz=318 cold=390 cvt=3 + densify(S)+GEMM 1508 1526 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 767 779 27.39x nnz=4000 coo-sort=None - Wide SASO k=4 1173 1185 41.89x nnz=8000 coo-sort=None - Wide SASO k=8 2003 2063 71.54x nnz=16000 coo-sort=None - CountSketch k=1 515 518 18.39x nnz=2000 coo-sort=CSC - Wide LASO k=2 28 31 1.00x nnz=78 coo-sort=None - Wide LASO k=4 35 39 1.25x nnz=158 coo-sort=None - Wide LASO k=8 53 57 1.89x nnz=318 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 753 821 0.094 44.3 50 CSC nnz=4000 + Wide SASO k=4 1050 1066 0.066 31.8 36 CSC nnz=8000 + Wide SASO k=8 1770 1795 0.055 18.9 21 CSC nnz=16000 + CountSketch k=1 488 514 0.122 68.3 77 CSC nnz=2000 + Wide LASO k=2 28 31 0.175 91.5 103 CSR nnz=80 + Wide LASO k=4 35 47 0.109 109.8 124 CSR nnz=160 + Wide LASO k=8 48 49 0.075 132.8 150 CSR nnz=318 === RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === warm apply, ColMajor dense - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 733 758 23.65x nnz=4000 coo-sort=None - Wide SASO k=4 1174 1203 37.87x nnz=8000 coo-sort=None - Wide SASO k=8 2107 2217 67.97x nnz=16000 coo-sort=None - CountSketch k=1 472 550 15.23x nnz=2000 coo-sort=CSR - Wide LASO k=2 31 34 1.00x nnz=80 coo-sort=None - Wide LASO k=4 37 39 1.19x nnz=160 coo-sort=None - Wide LASO k=8 57 66 1.84x nnz=320 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 701 740 0.088 47.6 54 CSR nnz=4000 + Wide SASO k=4 1090 1166 0.068 30.6 35 CSR nnz=8000 + Wide SASO k=8 1730 1859 0.054 19.4 22 CSR nnz=16000 + CountSketch k=1 508 510 0.127 65.6 74 CSR nnz=2000 + Wide LASO k=2 29 29 0.181 88.3 100 CSC nnz=80 + Wide LASO k=4 35 45 0.109 109.8 124 CSC nnz=160 + Wide LASO k=8 48 52 0.075 133.4 151 CSC nnz=320 warm apply, RowMajor dense - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 1443 1452 3.06x nnz=4000 coo-sort=None - Wide SASO k=4 2277 2317 4.83x nnz=8000 coo-sort=None - Wide SASO k=8 4158 4459 8.83x nnz=16000 coo-sort=None - CountSketch k=1 852 1287 1.81x nnz=2000 coo-sort=CSR - Wide LASO k=2 471 517 1.00x nnz=80 coo-sort=None - Wide LASO k=4 471 495 1.00x nnz=160 coo-sort=None - Wide LASO k=8 495 500 1.05x nnz=320 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 584 615 0.073 57.1 64 CSR nnz=4000 + Wide SASO k=4 1239 1282 0.077 27.0 30 CSR nnz=8000 + Wide SASO k=8 2344 3083 0.073 14.3 16 CSR nnz=16000 + CountSketch k=1 504 582 0.126 66.1 75 CSR nnz=2000 + Wide LASO k=2 150 174 0.938 17.1 19 CSC nnz=80 + Wide LASO k=4 330 367 1.031 11.6 13 CSC nnz=160 + Wide LASO k=8 410 454 0.641 15.6 18 CSC nnz=320 === LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 1491 1552 2.62x nnz=4000 coo-sort=None cold=1524 convert=28 - Wide SASO k=4 2183 2211 3.84x nnz=8000 coo-sort=None cold=2259 convert=76 - Wide SASO k=8 4189 4240 7.36x nnz=16000 coo-sort=None cold=4202 convert=178 - CountSketch k=1 875 886 1.54x nnz=2000 coo-sort=CSC cold=897 convert=3 - Wide LASO k=2 569 590 1.00x nnz=398 coo-sort=None cold=606 convert=4 - Wide LASO k=4 798 826 1.40x nnz=797 coo-sort=None cold=859 convert=10 - Wide LASO k=8 1247 1364 2.19x nnz=1594 coo-sort=None cold=1400 convert=23 - densify(S)+GEMM 4670 4670 8.21x dense oracle + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 769 925 0.096 50.0 56 CSC nnz=4000 cold=1233 cvt=7 + Wide SASO k=4 1432 1495 0.089 26.9 30 CSC nnz=8000 cold=1985 cvt=15 + Wide SASO k=8 2508 3257 0.078 15.4 17 CSC nnz=16000 cold=3901 cvt=26 + CountSketch k=1 716 910 0.179 53.7 61 CSC nnz=2000 cold=794 cvt=3 + Wide LASO k=2 490 535 0.613 26.1 29 CSR nnz=400 cold=531 cvt=4 + Wide LASO k=4 533 710 0.333 36.0 41 CSR nnz=800 cold=712 cvt=10 + Wide LASO k=8 593 640 0.186 53.9 61 CSR nnz=1597 cold=662 cvt=22 + densify(S)+GEMM 4698 4732 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 772 827 7.64x nnz=4000 coo-sort=None - Wide SASO k=4 1149 1245 11.38x nnz=8000 coo-sort=None - Wide SASO k=8 1780 2121 17.62x nnz=16000 coo-sort=None - CountSketch k=1 514 559 5.09x nnz=2000 coo-sort=CSC - Wide LASO k=2 101 106 1.00x nnz=398 coo-sort=None - Wide LASO k=4 167 180 1.65x nnz=797 coo-sort=None - Wide LASO k=8 400 407 3.96x nnz=1594 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 739 783 0.092 52.0 59 CSC nnz=4000 + Wide SASO k=4 1153 1179 0.072 33.4 38 CSC nnz=8000 + Wide SASO k=8 1830 1934 0.057 21.1 24 CSC nnz=16000 + CountSketch k=1 504 550 0.126 76.3 86 CSC nnz=2000 + Wide LASO k=2 98 105 0.122 130.7 147 CSR nnz=400 + Wide LASO k=4 162 173 0.101 118.6 134 CSR nnz=800 + Wide LASO k=8 368 388 0.115 86.9 98 CSR nnz=1597 === RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === warm apply, ColMajor dense - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 786 814 7.63x nnz=4000 coo-sort=None - Wide SASO k=4 1241 1300 12.05x nnz=8000 coo-sort=None - Wide SASO k=8 2087 2309 20.26x nnz=16000 coo-sort=None - CountSketch k=1 558 560 5.42x nnz=2000 coo-sort=CSR - Wide LASO k=2 103 114 1.00x nnz=400 coo-sort=None - Wide LASO k=4 153 163 1.49x nnz=800 coo-sort=None - Wide LASO k=8 397 448 3.85x nnz=1599 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 730 799 0.091 52.7 59 CSR nnz=4000 + Wide SASO k=4 1133 1163 0.071 34.0 38 CSR nnz=8000 + Wide SASO k=8 1872 1907 0.059 20.6 23 CSR nnz=16000 + CountSketch k=1 527 567 0.132 72.9 82 CSR nnz=2000 + Wide LASO k=2 97 101 0.121 132.0 149 CSC nnz=400 + Wide LASO k=4 176 185 0.110 109.0 123 CSC nnz=798 + Wide LASO k=8 362 370 0.113 88.3 100 CSC nnz=1597 warm apply, RowMajor dense - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 1496 1539 2.62x nnz=4000 coo-sort=None - Wide SASO k=4 2186 2228 3.83x nnz=8000 coo-sort=None - Wide SASO k=8 4064 4135 7.12x nnz=16000 coo-sort=None - CountSketch k=1 876 883 1.53x nnz=2000 coo-sort=CSR - Wide LASO k=2 571 601 1.00x nnz=400 coo-sort=None - Wide LASO k=4 779 802 1.36x nnz=800 coo-sort=None - Wide LASO k=8 1178 1319 2.06x nnz=1599 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 849 958 0.106 45.3 51 CSR nnz=4000 + Wide SASO k=4 1414 1473 0.088 27.2 31 CSR nnz=8000 + Wide SASO k=8 2783 3272 0.087 13.9 16 CSR nnz=16000 + CountSketch k=1 746 934 0.186 51.5 58 CSR nnz=2000 + Wide LASO k=2 514 623 0.642 24.9 28 CSC nnz=400 + Wide LASO k=4 614 637 0.385 31.2 35 CSC nnz=798 + Wide LASO k=8 613 727 0.192 52.2 59 CSC nnz=1597 === LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 1592 1615 1.63x nnz=4000 coo-sort=None cold=1632 convert=29 - Wide SASO k=4 2323 2434 2.38x nnz=8000 coo-sort=None cold=2329 convert=76 - Wide SASO k=8 4163 4279 4.26x nnz=16000 coo-sort=None cold=4201 convert=184 - CountSketch k=1 978 985 1.00x nnz=2000 coo-sort=CSC cold=999 convert=3 - Wide LASO k=2 1035 1084 1.06x nnz=998 coo-sort=None cold=1083 convert=14 - Wide LASO k=4 1652 1735 1.69x nnz=1996 coo-sort=None cold=1759 convert=29 - Wide LASO k=8 2117 2150 2.16x nnz=3990 coo-sort=None cold=2326 convert=66 - densify(S)+GEMM 11075 11075 11.32x dense oracle + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1229 1274 0.154 39.1 44 CSC nnz=4000 cold=1516 cvt=7 + Wide SASO k=4 1799 1838 0.112 26.8 30 CSC nnz=8000 cold=2206 cvt=15 + Wide SASO k=8 2756 3291 0.086 17.5 20 CSC nnz=16000 cold=4653 cvt=26 + CountSketch k=1 949 1027 0.237 50.6 57 CSC nnz=2000 cold=1061 cvt=3 + Wide LASO k=2 798 1093 0.399 40.1 45 CSR nnz=1000 cold=815 cvt=13 + Wide LASO k=4 1019 1052 0.255 47.1 53 CSR nnz=1999 cold=1106 cvt=30 + Wide LASO k=8 999 1100 0.125 48.1 54 CSR nnz=3994 cold=1286 cvt=81 + densify(S)+GEMM 11082 11279 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 923 985 2.19x nnz=4000 coo-sort=None - Wide SASO k=4 1343 1374 3.19x nnz=8000 coo-sort=None - Wide SASO k=8 2141 2220 5.09x nnz=16000 coo-sort=None - CountSketch k=1 776 786 1.84x nnz=2000 coo-sort=CSC - Wide LASO k=2 421 456 1.00x nnz=998 coo-sort=None - Wide LASO k=4 616 657 1.46x nnz=1996 coo-sort=None - Wide LASO k=8 990 1097 2.35x nnz=3990 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 895 942 0.112 53.7 61 CSC nnz=4000 + Wide SASO k=4 1258 1265 0.079 38.3 43 CSC nnz=8000 + Wide SASO k=8 2009 2105 0.063 24.0 27 CSC nnz=16000 + CountSketch k=1 673 777 0.168 71.4 81 CSC nnz=2000 + Wide LASO k=2 389 392 0.195 82.3 93 CSR nnz=1000 + Wide LASO k=4 587 597 0.147 81.8 92 CSR nnz=1999 + Wide LASO k=8 854 906 0.107 56.3 64 CSR nnz=3994 === RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === warm apply, ColMajor dense - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 927 957 2.26x nnz=4000 coo-sort=None - Wide SASO k=4 1352 1398 3.30x nnz=8000 coo-sort=None - Wide SASO k=8 2190 2356 5.34x nnz=16000 coo-sort=None - CountSketch k=1 713 917 1.74x nnz=2000 coo-sort=CSR - Wide LASO k=2 410 452 1.00x nnz=1000 coo-sort=None - Wide LASO k=4 626 683 1.53x nnz=2000 coo-sort=None - Wide LASO k=8 944 1044 2.30x nnz=3993 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 895 905 0.112 53.7 61 CSR nnz=4000 + Wide SASO k=4 1217 1232 0.076 39.5 45 CSR nnz=8000 + Wide SASO k=8 1922 1980 0.060 25.1 28 CSR nnz=16000 + CountSketch k=1 761 767 0.190 63.1 71 CSR nnz=2000 + Wide LASO k=2 391 405 0.196 81.8 92 CSC nnz=998 + Wide LASO k=4 595 686 0.149 80.7 91 CSC nnz=1998 + Wide LASO k=8 865 898 0.108 55.6 63 CSC nnz=3993 warm apply, RowMajor dense - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 1600 1705 1.58x nnz=4000 coo-sort=None - Wide SASO k=4 2258 2333 2.23x nnz=8000 coo-sort=None - Wide SASO k=8 4047 4176 4.00x nnz=16000 coo-sort=None - CountSketch k=1 1012 1446 1.00x nnz=2000 coo-sort=CSR - Wide LASO k=2 1014 1028 1.00x nnz=1000 coo-sort=None - Wide LASO k=4 1557 1635 1.54x nnz=2000 coo-sort=None - Wide LASO k=8 2096 2229 2.07x nnz=3993 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 1308 1348 0.164 36.7 41 CSR nnz=4000 + Wide SASO k=4 1708 1774 0.107 28.2 32 CSR nnz=8000 + Wide SASO k=8 3022 3513 0.094 16.0 18 CSR nnz=16000 + CountSketch k=1 1008 1052 0.252 47.7 54 CSR nnz=2000 + Wide LASO k=2 919 1021 0.460 34.8 39 CSC nnz=998 + Wide LASO k=4 1126 1150 0.282 42.6 48 CSC nnz=1998 + Wide LASO k=8 1070 1177 0.134 44.9 51 CSC nnz=3993 ########## SECTION 2: narrow n (SpMV-ish) ########## === LEFT-SKETCH B(200x1) = S(200x2000) * A(2000x1), trials=10 === warm apply, ColMajor dense (jik/jki kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 46 50 3.29x nnz=4000 coo-sort=None cold=83 convert=29 - Wide SASO k=4 100 115 7.14x nnz=8000 coo-sort=None cold=167 convert=78 - Wide SASO k=8 216 266 15.43x nnz=16000 coo-sort=None cold=339 convert=176 - CountSketch k=1 14 15 1.00x nnz=2000 coo-sort=CSC cold=37 convert=3 - Wide LASO k=2 16 22 1.14x nnz=400 coo-sort=None cold=40 convert=4 - Wide LASO k=4 27 37 1.93x nnz=799 coo-sort=None cold=75 convert=10 - Wide LASO k=8 44 60 3.14x nnz=1595 coo-sort=None cold=135 convert=22 - densify(S)+GEMM 13 13 0.93x dense oracle + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 18 20 4.500 4.6 5 CSC nnz=4000 cold=184 cvt=7 + Wide SASO k=4 31 34 3.875 4.7 5 CSC nnz=8000 cold=482 cvt=14 + Wide SASO k=8 45 51 2.812 6.1 7 CSC nnz=16000 cold=1121 cvt=25 + CountSketch k=1 13 14 6.500 3.9 4 CSC nnz=2000 cold=60 cvt=3 + Wide LASO k=2 8 9 20.000 1.6 2 CSR nnz=400 cold=36 cvt=5 + Wide LASO k=4 8 9 10.000 2.8 3 CSR nnz=800 cold=65 cvt=11 + Wide LASO k=8 10 12 6.254 4.2 5 CSR nnz=1599 cold=110 cvt=22 + densify(S)+GEMM 13 14 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 85 86 3.86x nnz=4000 coo-sort=None - Wide SASO k=4 174 176 7.91x nnz=8000 coo-sort=None - Wide SASO k=8 355 437 16.14x nnz=16000 coo-sort=None - CountSketch k=1 35 35 1.59x nnz=2000 coo-sort=CSC - Wide LASO k=2 22 24 1.00x nnz=400 coo-sort=None - Wide LASO k=4 34 36 1.55x nnz=799 coo-sort=None - Wide LASO k=8 62 73 2.82x nnz=1595 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 56 57 14.000 1.5 2 CSC nnz=4000 + Wide SASO k=4 100 101 12.500 1.5 2 CSC nnz=8000 + Wide SASO k=8 180 183 11.250 1.5 2 CSC nnz=16000 + CountSketch k=1 35 35 17.500 1.5 2 CSC nnz=2000 + Wide LASO k=2 16 18 40.000 0.8 1 CSR nnz=400 + Wide LASO k=4 21 23 26.250 1.1 1 CSR nnz=800 + Wide LASO k=8 32 34 20.013 1.3 1 CSR nnz=1599 === LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === warm apply, ColMajor dense (jik/jki kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 55 59 2.75x nnz=4000 coo-sort=None cold=89 convert=29 - Wide SASO k=4 125 161 6.25x nnz=8000 coo-sort=None cold=182 convert=76 - Wide SASO k=8 261 357 13.05x nnz=16000 coo-sort=None cold=364 convert=182 - CountSketch k=1 20 21 1.00x nnz=2000 coo-sort=CSC cold=42 convert=3 - Wide LASO k=2 21 23 1.05x nnz=400 coo-sort=None cold=47 convert=4 - Wide LASO k=4 30 33 1.50x nnz=800 coo-sort=None cold=77 convert=10 - Wide LASO k=8 45 54 2.25x nnz=1599 coo-sort=None cold=139 convert=22 - densify(S)+GEMM 35 35 1.75x dense oracle + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 22 25 0.550 11.6 13 CSC nnz=4000 cold=172 cvt=6 + Wide SASO k=4 36 41 0.450 8.9 10 CSC nnz=8000 cold=477 cvt=13 + Wide SASO k=8 63 67 0.394 7.1 8 CSC nnz=16000 cold=1114 cvt=26 + CountSketch k=1 19 32 0.950 11.8 13 CSC nnz=2000 cold=82 cvt=4 + Wide LASO k=2 9 11 2.250 7.8 9 CSR nnz=400 cold=36 cvt=4 + Wide LASO k=4 11 13 1.375 9.9 11 CSR nnz=800 cold=67 cvt=11 + Wide LASO k=8 14 15 0.877 13.2 15 CSR nnz=1596 cold=124 cvt=23 + densify(S)+GEMM 34 36 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) - Operator Min (us) Med (us) vs best notes - -------------------------------------------------------------------- - Wide SASO k=2 79 84 3.59x nnz=4000 coo-sort=None - Wide SASO k=4 170 228 7.73x nnz=8000 coo-sort=None - Wide SASO k=8 334 345 15.18x nnz=16000 coo-sort=None - CountSketch k=1 33 35 1.50x nnz=2000 coo-sort=CSC - Wide LASO k=2 22 24 1.00x nnz=400 coo-sort=None - Wide LASO k=4 35 36 1.59x nnz=800 coo-sort=None - Wide LASO k=8 59 60 2.68x nnz=1599 coo-sort=None + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + Wide SASO k=2 57 59 1.425 4.5 5 CSC nnz=4000 + Wide SASO k=4 97 100 1.212 3.3 4 CSC nnz=8000 + Wide SASO k=8 168 228 1.050 2.7 3 CSC nnz=16000 + CountSketch k=1 36 50 1.800 6.2 7 CSC nnz=2000 + Wide LASO k=2 17 19 4.250 4.1 5 CSR nnz=400 + Wide LASO k=4 21 23 2.625 5.2 6 CSR nnz=800 + Wide LASO k=8 33 40 2.068 5.6 6 CSR nnz=1596 diff --git a/sketch_general_results_postfix.txt b/sketch_general_results_postfix.txt deleted file mode 100644 index c6ce9297..00000000 --- a/sketch_general_results_postfix.txt +++ /dev/null @@ -1,218 +0,0 @@ -# POST-CHANGES run. Combined state of the sparse-sketching branch: -# 1. Construction-time within-vector sort for SASO and LASO (sparse_skops.hh): -# wide SASO -> CSC-sorted COO, wide LASO -> CSR-sorted COO. -# 2. ColMajor wide operators routed through the CSR jik kernel (coo_spmm_impl.hh). -# 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen -# format (closes the wide-SASO ColMajor re-sort tax). -# Net vs the pre-change baseline (sketch_general_results.txt), ColMajor warm apply: -# wide SASO ~1.45x, wide LASO 1.2-2.1x, no RowMajor regression. -# ns/elt = us*1000/(nnz*work_mult); GB/s = model bytes/time; %STR vs STREAM below. -# Environment: Apple M3, OMP_NUM_THREADS=4, Release (MKL off) | Correctness: 0 FAIL - -============================================================ -SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) -============================================================ -Threads (OMP_NUM_THREADS/max): 4 -Calibrating STREAM triad... 88.6 GB/s (= 100% on the %STR column) - -########## SECTION 1: vary sketch size d (m=n=2000) ########## - -=== LEFT-SKETCH B(40x2000) = S(40x2000) * A(2000x2000), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 676 691 0.085 49.3 56 CSC nnz=4000 cold=867 cvt=6 - Wide SASO k=4 1425 1732 0.089 23.4 26 CSC nnz=8000 cold=1742 cvt=14 - Wide SASO k=8 2594 3294 0.081 12.9 15 CSC nnz=16000 cold=3250 cvt=27 - CountSketch k=1 582 717 0.145 57.2 65 CSC nnz=2000 cold=642 cvt=3 - Wide LASO k=2 221 274 1.381 11.6 13 CSR nnz=80 cold=220 cvt=0 - Wide LASO k=4 351 380 1.097 10.9 12 CSR nnz=160 cold=364 cvt=1 - Wide LASO k=8 383 392 0.602 16.6 19 CSR nnz=318 cold=390 cvt=3 - densify(S)+GEMM 1508 1526 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 753 821 0.094 44.3 50 CSC nnz=4000 - Wide SASO k=4 1050 1066 0.066 31.8 36 CSC nnz=8000 - Wide SASO k=8 1770 1795 0.055 18.9 21 CSC nnz=16000 - CountSketch k=1 488 514 0.122 68.3 77 CSC nnz=2000 - Wide LASO k=2 28 31 0.175 91.5 103 CSR nnz=80 - Wide LASO k=4 35 47 0.109 109.8 124 CSR nnz=160 - Wide LASO k=8 48 49 0.075 132.8 150 CSR nnz=318 - -=== RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === - - warm apply, ColMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 701 740 0.088 47.6 54 CSR nnz=4000 - Wide SASO k=4 1090 1166 0.068 30.6 35 CSR nnz=8000 - Wide SASO k=8 1730 1859 0.054 19.4 22 CSR nnz=16000 - CountSketch k=1 508 510 0.127 65.6 74 CSR nnz=2000 - Wide LASO k=2 29 29 0.181 88.3 100 CSC nnz=80 - Wide LASO k=4 35 45 0.109 109.8 124 CSC nnz=160 - Wide LASO k=8 48 52 0.075 133.4 151 CSC nnz=320 - - warm apply, RowMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 584 615 0.073 57.1 64 CSR nnz=4000 - Wide SASO k=4 1239 1282 0.077 27.0 30 CSR nnz=8000 - Wide SASO k=8 2344 3083 0.073 14.3 16 CSR nnz=16000 - CountSketch k=1 504 582 0.126 66.1 75 CSR nnz=2000 - Wide LASO k=2 150 174 0.938 17.1 19 CSC nnz=80 - Wide LASO k=4 330 367 1.031 11.6 13 CSC nnz=160 - Wide LASO k=8 410 454 0.641 15.6 18 CSC nnz=320 - -=== LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 769 925 0.096 50.0 56 CSC nnz=4000 cold=1233 cvt=7 - Wide SASO k=4 1432 1495 0.089 26.9 30 CSC nnz=8000 cold=1985 cvt=15 - Wide SASO k=8 2508 3257 0.078 15.4 17 CSC nnz=16000 cold=3901 cvt=26 - CountSketch k=1 716 910 0.179 53.7 61 CSC nnz=2000 cold=794 cvt=3 - Wide LASO k=2 490 535 0.613 26.1 29 CSR nnz=400 cold=531 cvt=4 - Wide LASO k=4 533 710 0.333 36.0 41 CSR nnz=800 cold=712 cvt=10 - Wide LASO k=8 593 640 0.186 53.9 61 CSR nnz=1597 cold=662 cvt=22 - densify(S)+GEMM 4698 4732 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 739 783 0.092 52.0 59 CSC nnz=4000 - Wide SASO k=4 1153 1179 0.072 33.4 38 CSC nnz=8000 - Wide SASO k=8 1830 1934 0.057 21.1 24 CSC nnz=16000 - CountSketch k=1 504 550 0.126 76.3 86 CSC nnz=2000 - Wide LASO k=2 98 105 0.122 130.7 147 CSR nnz=400 - Wide LASO k=4 162 173 0.101 118.6 134 CSR nnz=800 - Wide LASO k=8 368 388 0.115 86.9 98 CSR nnz=1597 - -=== RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === - - warm apply, ColMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 730 799 0.091 52.7 59 CSR nnz=4000 - Wide SASO k=4 1133 1163 0.071 34.0 38 CSR nnz=8000 - Wide SASO k=8 1872 1907 0.059 20.6 23 CSR nnz=16000 - CountSketch k=1 527 567 0.132 72.9 82 CSR nnz=2000 - Wide LASO k=2 97 101 0.121 132.0 149 CSC nnz=400 - Wide LASO k=4 176 185 0.110 109.0 123 CSC nnz=798 - Wide LASO k=8 362 370 0.113 88.3 100 CSC nnz=1597 - - warm apply, RowMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 849 958 0.106 45.3 51 CSR nnz=4000 - Wide SASO k=4 1414 1473 0.088 27.2 31 CSR nnz=8000 - Wide SASO k=8 2783 3272 0.087 13.9 16 CSR nnz=16000 - CountSketch k=1 746 934 0.186 51.5 58 CSR nnz=2000 - Wide LASO k=2 514 623 0.642 24.9 28 CSC nnz=400 - Wide LASO k=4 614 637 0.385 31.2 35 CSC nnz=798 - Wide LASO k=8 613 727 0.192 52.2 59 CSC nnz=1597 - -=== LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 1229 1274 0.154 39.1 44 CSC nnz=4000 cold=1516 cvt=7 - Wide SASO k=4 1799 1838 0.112 26.8 30 CSC nnz=8000 cold=2206 cvt=15 - Wide SASO k=8 2756 3291 0.086 17.5 20 CSC nnz=16000 cold=4653 cvt=26 - CountSketch k=1 949 1027 0.237 50.6 57 CSC nnz=2000 cold=1061 cvt=3 - Wide LASO k=2 798 1093 0.399 40.1 45 CSR nnz=1000 cold=815 cvt=13 - Wide LASO k=4 1019 1052 0.255 47.1 53 CSR nnz=1999 cold=1106 cvt=30 - Wide LASO k=8 999 1100 0.125 48.1 54 CSR nnz=3994 cold=1286 cvt=81 - densify(S)+GEMM 11082 11279 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 895 942 0.112 53.7 61 CSC nnz=4000 - Wide SASO k=4 1258 1265 0.079 38.3 43 CSC nnz=8000 - Wide SASO k=8 2009 2105 0.063 24.0 27 CSC nnz=16000 - CountSketch k=1 673 777 0.168 71.4 81 CSC nnz=2000 - Wide LASO k=2 389 392 0.195 82.3 93 CSR nnz=1000 - Wide LASO k=4 587 597 0.147 81.8 92 CSR nnz=1999 - Wide LASO k=8 854 906 0.107 56.3 64 CSR nnz=3994 - -=== RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === - - warm apply, ColMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 895 905 0.112 53.7 61 CSR nnz=4000 - Wide SASO k=4 1217 1232 0.076 39.5 45 CSR nnz=8000 - Wide SASO k=8 1922 1980 0.060 25.1 28 CSR nnz=16000 - CountSketch k=1 761 767 0.190 63.1 71 CSR nnz=2000 - Wide LASO k=2 391 405 0.196 81.8 92 CSC nnz=998 - Wide LASO k=4 595 686 0.149 80.7 91 CSC nnz=1998 - Wide LASO k=8 865 898 0.108 55.6 63 CSC nnz=3993 - - warm apply, RowMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 1308 1348 0.164 36.7 41 CSR nnz=4000 - Wide SASO k=4 1708 1774 0.107 28.2 32 CSR nnz=8000 - Wide SASO k=8 3022 3513 0.094 16.0 18 CSR nnz=16000 - CountSketch k=1 1008 1052 0.252 47.7 54 CSR nnz=2000 - Wide LASO k=2 919 1021 0.460 34.8 39 CSC nnz=998 - Wide LASO k=4 1126 1150 0.282 42.6 48 CSC nnz=1998 - Wide LASO k=8 1070 1177 0.134 44.9 51 CSC nnz=3993 - -########## SECTION 2: narrow n (SpMV-ish) ########## - -=== LEFT-SKETCH B(200x1) = S(200x2000) * A(2000x1), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 18 20 4.500 4.6 5 CSC nnz=4000 cold=184 cvt=7 - Wide SASO k=4 31 34 3.875 4.7 5 CSC nnz=8000 cold=482 cvt=14 - Wide SASO k=8 45 51 2.812 6.1 7 CSC nnz=16000 cold=1121 cvt=25 - CountSketch k=1 13 14 6.500 3.9 4 CSC nnz=2000 cold=60 cvt=3 - Wide LASO k=2 8 9 20.000 1.6 2 CSR nnz=400 cold=36 cvt=5 - Wide LASO k=4 8 9 10.000 2.8 3 CSR nnz=800 cold=65 cvt=11 - Wide LASO k=8 10 12 6.254 4.2 5 CSR nnz=1599 cold=110 cvt=22 - densify(S)+GEMM 13 14 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 56 57 14.000 1.5 2 CSC nnz=4000 - Wide SASO k=4 100 101 12.500 1.5 2 CSC nnz=8000 - Wide SASO k=8 180 183 11.250 1.5 2 CSC nnz=16000 - CountSketch k=1 35 35 17.500 1.5 2 CSC nnz=2000 - Wide LASO k=2 16 18 40.000 0.8 1 CSR nnz=400 - Wide LASO k=4 21 23 26.250 1.1 1 CSR nnz=800 - Wide LASO k=8 32 34 20.013 1.3 1 CSR nnz=1599 - -=== LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 22 25 0.550 11.6 13 CSC nnz=4000 cold=172 cvt=6 - Wide SASO k=4 36 41 0.450 8.9 10 CSC nnz=8000 cold=477 cvt=13 - Wide SASO k=8 63 67 0.394 7.1 8 CSC nnz=16000 cold=1114 cvt=26 - CountSketch k=1 19 32 0.950 11.8 13 CSC nnz=2000 cold=82 cvt=4 - Wide LASO k=2 9 11 2.250 7.8 9 CSR nnz=400 cold=36 cvt=4 - Wide LASO k=4 11 13 1.375 9.9 11 CSR nnz=800 cold=67 cvt=11 - Wide LASO k=8 14 15 0.877 13.2 15 CSR nnz=1596 cold=124 cvt=23 - densify(S)+GEMM 34 36 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 57 59 1.425 4.5 5 CSC nnz=4000 - Wide SASO k=4 97 100 1.212 3.3 4 CSC nnz=8000 - Wide SASO k=8 168 228 1.050 2.7 3 CSC nnz=16000 - CountSketch k=1 36 50 1.800 6.2 7 CSC nnz=2000 - Wide LASO k=2 17 19 4.250 4.1 5 CSR nnz=400 - Wide LASO k=4 21 23 2.625 5.2 6 CSR nnz=800 - Wide LASO k=8 33 40 2.068 5.6 6 CSR nnz=1596 - From 990e898713d3c77b1b7e7225c171bfc6196df5ba Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Fri, 19 Jun 2026 23:47:51 -0700 Subject: [PATCH 10/20] Ill-conceived attempted optimization that I'll probably revert. --------------------------------------------------------------- Store LASO operators "regular"; add a gated regular-CSR kernel Regularize long-axis-major (LASO) sketching operators so every major-axis vector is stored with exactly vec_nnz records, matching the fixed-nnz structure short-axis-major (SASO) operators already have. This is a storage-only change: a coordinate sampled c times is kept as c duplicate records each scaled by 1/sqrt(c), which sum to the merged value sqrt(c)*s, so the operator is bit-for-bit the same matrix. RNG draws are unchanged, so thread-count reproducibility holds. nnz becomes exactly full_nnz. Because duplicate (row,col) records can now appear, coo_to_dense (and the test densifier) accumulate with += instead of overwriting; SpMM kernels and COO->CSR/CSC conversions already accumulate / preserve duplicates. trsm does not accept duplicates -- noted in DevNotes. Adds a regular-CSR SpMV kernel (apply_regular_csr_to_vector_ik), the CSR analogue of apply_regular_csc_to_vector_ki, plus detection in apply_csr_jik_p11. A back-to-back regular-vs-general A/B (new rows in the benchmark's --csr-probe) shows it is NEUTRAL-to-SLOWER on Apple M3 / NEON (the eliminated rowptr loads were never a bottleneck; 2-wide doubles can't exploit the fixed inner trip count). The dispatch is therefore GATED OFF by default behind RandBLAS_ENABLE_REGULAR_CSR_KERNEL; the default build keeps the general kernel (no regression). The kernel is kept in-tree for evaluation on wide-vector Arm (SVE), where a known-length predicated inner loop may pay off. Also corrects an assumption: wide SASO is CSC-regular (fixed nnz per column), not CSR-regular, so its CSR form has variable row nnz and does not reach the regular CSR kernel; only wide LASO (and tall SASO) are CSR-regular. Tests: operator-preservation (regular vs merged densify identically), regular structure / nnz==full_nnz, coo_to_dense summing, and the regular CSR kernel directly (incl. a duplicated colidx). 439/439 pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- RandBLAS/sparse_data/DevNotes.md | 25 ++ RandBLAS/sparse_data/coo_matrix.hh | 7 +- RandBLAS/sparse_data/csr_spmm_impl.hh | 86 ++++- RandBLAS/sparse_skops.hh | 51 ++- .../sketch_general_performance.cc | 40 +++ sketch_general_results.txt | 329 +++++++++++------- test/datastructures/test_coo_matrix.cc | 36 +- test/datastructures/test_sparseskop.cc | 67 ++++ test/linops/test_spmm/test_spmm_csr.cc | 84 +++++ 9 files changed, 585 insertions(+), 140 deletions(-) diff --git a/RandBLAS/sparse_data/DevNotes.md b/RandBLAS/sparse_data/DevNotes.md index 24d52336..6804bb07 100644 --- a/RandBLAS/sparse_data/DevNotes.md +++ b/RandBLAS/sparse_data/DevNotes.md @@ -73,3 +73,28 @@ From there, we'll do the following. get matched to opposite sides for ``[left/right]_spmm``! This is because all the fancy abstractions in ``S`` have been stripped away by this point in the call sequence, so the "side" that we emphasize in function names changes from emphasizing ``S`` to emphasizing ``A``. + + +## Regular (fixed-nnz) operators and duplicate COO records + +Both short-axis-major (SASO) and long-axis-major (LASO) sketching operators are stored with +exactly ``vec_nnz`` structural records per major-axis vector ("regular" layout). For SASO the +``vec_nnz`` nonzeros are distinct. For LASO, with-replacement sampling can draw a coordinate +several times; rather than merge those into one entry, we keep them as duplicate ``(row, col)`` +records, each scaled so they sum to the merged value (see ``laso_merge_long_axis_vector_coo_data`` +in ``sparse_skops.hh``). This regular structure lets the wide ColMajor dispatch reach the +fixed-nnz fast-path kernels (``apply_regular_csr_to_vector_ik`` for CSR, ``apply_regular_csc_to_vector_ki`` +for CSC), which skip the ``rowptr``/``colptr`` loads. + +Consequences for anyone touching these code paths: + + * **SpMM and densify accumulate duplicates.** The CSR/CSC kernels sum ``vals[ell]*v[...]`` over a + row/column, so duplicate column/row indices add correctly. ``coo_to_dense`` was made to use + ``+=`` (not ``=``) for the same reason. Any new densification path must accumulate, not overwrite. + * **Conversions preserve duplicates.** ``coo_to_csr`` / ``coo_to_csc`` and the counting-sort + transpose in ``coo_spmm_impl.hh`` keep all ``nnz`` records (no de-duplication), which is what the + kernels expect. + * **Do not route a regular operator through ``trsm``.** ``trsm_dispatch.hh`` validates structure + with ``compressed_indices_are_increasing``, which rejects duplicate indices within a major-axis + vector. Triangular solve is unrelated to sketching, so this is not a real constraint in practice, + but it is a trap worth noting. diff --git a/RandBLAS/sparse_data/coo_matrix.hh b/RandBLAS/sparse_data/coo_matrix.hh index 47eded5f..6c8e2123 100644 --- a/RandBLAS/sparse_data/coo_matrix.hh +++ b/RandBLAS/sparse_data/coo_matrix.hh @@ -395,7 +395,12 @@ void coo_to_dense(const COOMatrix &spmat, int64_t stride_row, int64_t stride_ i -= 1; j -= 1; } - MAT(i, j) = spmat.vals[ell]; + // Accumulate (not overwrite): a COO may carry several records at the same + // (i, j) -- e.g. a "regular" long-axis-major sketching operator stores a + // collided coordinate as duplicate entries that must sum to its true value. + // The matrix was zeroed above, so += is correct, and for a deduplicated COO + // (each (i,j) once) it is identical to an assignment. + MAT(i, j) += spmat.vals[ell]; } return; } diff --git a/RandBLAS/sparse_data/csr_spmm_impl.hh b/RandBLAS/sparse_data/csr_spmm_impl.hh index 0705bd4c..afd466a2 100644 --- a/RandBLAS/sparse_data/csr_spmm_impl.hh +++ b/RandBLAS/sparse_data/csr_spmm_impl.hh @@ -107,6 +107,55 @@ static void apply_csr_to_vector_ik( } } +// Regular-CSR SpMV: every row has the same nnz (row_nnz), so rowptr is implicit +// (rowptr[i] == i*row_nnz) and need not be read. This is the CSR analogue of +// apply_regular_csc_to_vector_ki and is reached by short-axis-major operators (wide +// SASO, exactly vec_nnz per row) and by "regular" long-axis-major operators. Mirrors +// apply_csr_to_vector_ik_impl: UnitStride drops index multiplies in the hot ColMajor +// case and the simd reduction breaks the serial accumulator dependence. +template +static void apply_regular_csr_to_vector_ik_impl( + T alpha, + const T *vals, const int64_t row_nnz, const sint_t *colidxs, + const T *v, int64_t incv, + int64_t len_Av, T *Av, int64_t incAv +) { + UNUSED(incv); UNUSED(incAv); + for (int64_t i = 0; i < len_Av; ++i) { + T Av_i_diff = 0.0; + #pragma omp simd reduction(+:Av_i_diff) + for (int64_t ell = i * row_nnz; ell < (i + 1) * row_nnz; ++ell) { + int64_t j = colidxs[ell]; + if constexpr (UnitStride) { + Av_i_diff += vals[ell] * v[j]; + } else { + Av_i_diff += vals[ell] * v[j*incv]; + } + } + if constexpr (UnitStride) { + Av[i] += alpha * Av_i_diff; + } else { + Av[i*incAv] += alpha * Av_i_diff; + } + } +} + +template +static void apply_regular_csr_to_vector_ik( + T alpha, + const T *vals, int64_t row_nnz, const sint_t *colidxs, + const T *v, int64_t incv, + int64_t len_Av, T *Av, int64_t incAv +) { + if (incv == 1 && incAv == 1) { + apply_regular_csr_to_vector_ik_impl( + alpha, vals, row_nnz, colidxs, v, incv, len_Av, Av, incAv); + } else { + apply_regular_csr_to_vector_ik_impl( + alpha, vals, row_nnz, colidxs, v, incv, len_Av, Av, incAv); + } +} + template static void apply_csr_jik_p11( T alpha, @@ -126,6 +175,22 @@ static void apply_csr_jik_p11( randblas_require(d == A.n_rows); randblas_require(m == A.n_cols); +#if defined(RandBLAS_ENABLE_REGULAR_CSR_KERNEL) + // OPT-IN regular-CSR fast path (OFF by default). rowptr is an arithmetic progression + // iff every row has the same nnz; then we index nonzeros as i*row_nnz and skip the + // rowptr loads (mirrors fixed_nnz_per_col in apply_csc_jki_p11). Benchmarked + // NEUTRAL-to-SLOWER on Apple M3 / NEON -- the eliminated rowptr loads are sequential + // and cache-resident (never the bottleneck), and 2-wide doubles can't exploit the + // fixed inner trip count, so it is gated off. Left in-tree (define this macro to + // enable) for evaluation on wide-vector Arm (SVE), where a known-length predicated + // inner loop may pay off. See apply_regular_csr_to_vector_ik above and the --csr-probe + // mode of examples/.../sketch_general_performance.cc for the A/B harness. + bool fixed_nnz_per_row = (d > 0); + for (int64_t i = 2; (i < d + 1) && fixed_nnz_per_row; ++i) + fixed_nnz_per_row = (A.rowptr[1] + A.rowptr[i-1]) == A.rowptr[i]; + int64_t row_nnz = (d > 0) ? A.rowptr[1] : 0; +#endif + auto s = layout_to_strides(layout_B, ldb); auto B_inter_col_stride = s.inter_col_stride; auto B_inter_row_stride = s.inter_row_stride; @@ -138,11 +203,22 @@ static void apply_csr_jik_p11( for (int64_t j = 0; j < n; j++) { const T *B_col = &B[B_inter_col_stride * j]; T *C_col = &C[C_inter_col_stride * j]; - apply_csr_to_vector_ik(alpha, - vals, A.rowptr, A.colidxs, - B_col, B_inter_row_stride, - d, C_col, C_inter_row_stride - ); +#if defined(RandBLAS_ENABLE_REGULAR_CSR_KERNEL) + if (fixed_nnz_per_row) { + apply_regular_csr_to_vector_ik(alpha, + vals, row_nnz, A.colidxs, + B_col, B_inter_row_stride, + d, C_col, C_inter_row_stride + ); + } else +#endif + { + apply_csr_to_vector_ik(alpha, + vals, A.rowptr, A.colidxs, + B_col, B_inter_row_stride, + d, C_col, C_inter_row_stride + ); + } } return; } diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index 0a52f13a..75235f13 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -194,10 +194,18 @@ struct SparseDist { /// If \math{\ttt{major_axis} = \ttt{Long}}, then \math{\mtxx} has *at most* \math{\vecnnz} nonzero /// entries. The locations of the nonzeros are determined by sampling uniformly /// with replacement from \math{\\{0,\ldots,k-1\\}.} - /// If index \math{j} occurs in the sample \math{\ell} times, then + /// If index \math{j} occurs in the sample \math{\ell} times, then /// \math{\mtxx_j} will equal \math{\sqrt{\ell}} with probability 1/2 and /// \math{-\sqrt{\ell}} with probability 1/2. /// + /// Note: while \math{\mtxx} has at most \math{\vecnnz} *distinct* nonzero coordinates, it is + /// *stored* with exactly \math{\vecnnz} structural entries per major-axis vector. A coordinate + /// \math{j} sampled \math{\ell} times is held as \math{\ell} duplicate records each equal to + /// \math{\mtxx_j/\ell = \pm 1/\sqrt{\ell}}, which sum to \math{\mtxx_j.} This "regular" layout + /// (a fixed number of records per major-axis vector) lets long-axis-major operators reach the + /// same fixed-nnz fast-path kernels as short-axis-major operators, without changing the operator + /// that \math{\mtxx} represents. + /// const Axis major_axis; // --------------------------------------------------------------------------- @@ -483,7 +491,8 @@ template void laso_merge_long_axis_vector_coo_data( int64_t vec_nnz, T* vals, sint_t* idxs_lax, sint_t *idxs_sax, int64_t i, std::unordered_map &loc2count, - std::unordered_map &loc2scale + std::unordered_map &loc2scale, + bool regular ) { loc2count.clear(); // ^ Used to count the number of times each long-axis index @@ -505,7 +514,19 @@ void laso_merge_long_axis_vector_coo_data( loc2count[ell] = 1.0; } } - if ((int64_t) loc2scale.size() < vec_nnz) { + if (regular) { + // Operator-preserving regularization (storage only): keep all vec_nnz draws, + // but rescale so the c copies stored at a collided location ell each carry + // s/sqrt(c) (s = loc2scale[ell], the canonical sign). The c copies then sum to + // c * (s/sqrt(c)) = sqrt(c)*s -- the exact value the merge branch below would + // store as a single entry. So the densified operator is identical; we simply + // leave duplicate (idxs_lax, idxs_sax) records in place to give a fixed-vec_nnz + // "regular" layout (every major-axis vector has exactly vec_nnz records). + for (int64_t j = 0; j < vec_nnz; ++j) { + sint_t ell = idxs_lax[j]; + vals[j] = loc2scale[ell] / std::sqrt(loc2count[ell]); + } + } else if ((int64_t) loc2scale.size() < vec_nnz) { // Then we have duplicates. We need to overwrite some of the values // of (idxs_lax, vals, idxs_sax) and implicitly // shift them backward to remove duplicates; @@ -571,7 +592,8 @@ state_t fill_sparse_unpacked( int64_t n_rows_sub, int64_t n_cols_sub, int64_t ro_s, int64_t co_s, int64_t &nnz, T* vals, sint_t* rows, sint_t* cols, - const state_t &seed_state + const state_t &seed_state, + bool regular = true ) { randblas_require(D.n_rows >= n_rows_sub + ro_s); randblas_require(D.n_cols >= n_cols_sub + co_s); @@ -678,15 +700,20 @@ state_t fill_sparse_unpacked( end_state = work_state; for (int64_t i = 0; i < num_major_sub; ++i) { end_state = sample_indices_iid_uniform(dim_major, vec_nnz, im, v, end_state); - laso_merge_long_axis_vector_coo_data(vec_nnz, v, im, in, i, loc2count, loc2scale); - int64_t count = (int64_t) loc2count.size(); + laso_merge_long_axis_vector_coo_data(vec_nnz, v, im, in, i, loc2count, loc2scale, regular); + // In regular mode every major-axis vector keeps exactly vec_nnz records (with + // duplicates rescaled to preserve the operator); otherwise the merge compacts to + // the (<= vec_nnz) distinct survivors. + int64_t count = regular ? vec_nnz : (int64_t) loc2count.size(); // Emit this major-axis vector in ascending major-coordinate order, exactly as - // the SASO branch does. The merge leaves the (<= vec_nnz) survivors in hash / - // sample order; sorting the block by its major coordinate makes a wide LASO - // CSR-sorted (cols ascending within each row) and a tall LASO CSC-sorted. That - // natural order also matches the ColMajor dispatch preference (CSR for wide), - // so apply_coo_via_csc skips its per-apply re-sort. idxs_minor is constant - // within the block. The block length varies, so insertion-sort (count is small). + // the SASO branch does. The (merge or regular rescale) leaves the survivors in + // hash / sample order; sorting the block by its major coordinate makes a wide + // LASO CSR-sorted (cols ascending within each row) and a tall LASO CSC-sorted. + // That natural order also matches the ColMajor dispatch preference (CSR for + // wide), so apply_coo_via_csc skips its per-apply re-sort. idxs_minor is constant + // within the block. In regular mode duplicate major coordinates stay adjacent + // (the comparison is strict), so the block is still CSC-/CSR-sorted with + // duplicates allowed. The block length is small, so insertion-sort is best. for (int64_t a = 1; a < count; ++a) { sint_t key = im[a]; T vv = v[a]; diff --git a/examples/simple-kernel-benchmarks/sketch_general_performance.cc b/examples/simple-kernel-benchmarks/sketch_general_performance.cc index a6b32563..8183c658 100644 --- a/examples/simple-kernel-benchmarks/sketch_general_performance.cc +++ b/examples/simple-kernel-benchmarks/sketch_general_performance.cc @@ -547,11 +547,51 @@ void run_csr_probe(int64_t d, int64_t m, int64_t n, rows.push_back(r); }; + // Regular-vs-general INNER-kernel A/B on the SAME (regular) CSR, ColMajor only. + // Every sketching operator is now stored regular (fixed nnz per major-axis vector), + // so this isolates the ONLY difference between the two CSR SpMV kernels: implicit + // row indexing (i*row_nnz) vs rowptr[i]/rowptr[i+1] loads. Run back-to-back to + // cancel cross-run noise -- this is the definitive "does the regular kernel pay + // off?" measurement. Both mirror apply_csr_jik_p11's parallel-over-columns shape. + int64_t row_nnz = (d > 0) ? S_csr.rowptr[1] : 0; + bool is_regular = (d > 0) && (nnz == row_nnz * d); + auto probe_inner = [&](const std::string& label, bool regular_kernel) { + auto [mn, md] = run_trials([&]() { + std::fill(C_cm.begin(), C_cm.end(), 0.0); + #pragma omp parallel for schedule(static) + for (int64_t j = 0; j < n; ++j) { + const T* B_col = &A_cm[(size_t) j * m]; + T* C_col = &C_cm[(size_t) j * d]; + if (regular_kernel) + rb::sparse_data::csr::apply_regular_csr_to_vector_ik( + 1.0, S_csr.vals, row_nnz, S_csr.colidxs, B_col, 1, d, C_col, 1); + else + rb::sparse_data::csr::apply_csr_to_vector_ik( + 1.0, S_csr.vals, S_csr.rowptr, S_csr.colidxs, B_col, 1, d, C_col, 1); + } + }, num_trials); + double maxdiff = 0; + for (int64_t i = 0; i < d * n; ++i) + maxdiff = std::max(maxdiff, std::abs(C_cm[i] - B_ref[i])); + if (maxdiff > 1e-9) + std::cout << " FAIL " << label << " max|diff|=" << std::scientific + << maxdiff << std::fixed << "\n"; + Row r; r.label = label; r.min_us = mn; r.med_us = md; + fill_metrics(r, mn, nnz, n, a_read, d * n); + r.notes = std::string("regular=") + (regular_kernel ? "Y" : "N") + + " row_nnz=" + std::to_string(row_nnz); + rows.push_back(r); + }; + print_metric_header(spec.label); probe("CSC jki ColMajor", S_csc, Layout::ColMajor); probe("CSR jik ColMajor", S_csr, Layout::ColMajor); probe("CSC kib RowMajor", S_csc, Layout::RowMajor); probe("CSR ikb RowMajor", S_csr, Layout::RowMajor); + if (is_regular) { + probe_inner("CSR REG ColMajor*", true); + probe_inner("CSR GEN ColMajor*", false); + } for (auto& r : rows) print_metric_row(r); std::cout << "\n"; } diff --git a/sketch_general_results.txt b/sketch_general_results.txt index c6ce9297..e49d27d2 100644 --- a/sketch_general_results.txt +++ b/sketch_general_results.txt @@ -2,10 +2,17 @@ # 1. Construction-time within-vector sort for SASO and LASO (sparse_skops.hh): # wide SASO -> CSC-sorted COO, wide LASO -> CSR-sorted COO. # 2. ColMajor wide operators routed through the CSR jik kernel (coo_spmm_impl.hh). -# 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen -# format (closes the wide-SASO ColMajor re-sort tax). -# Net vs the pre-change baseline (sketch_general_results.txt), ColMajor warm apply: -# wide SASO ~1.45x, wide LASO 1.2-2.1x, no RowMajor regression. +# 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen format. +# 4. LASO now stored "regular": exactly vec_nnz records per major-axis vector, with +# collided coordinates split into duplicates scaled to preserve the operator +# (sparse_skops.hh). Correctness-neutral; nnz becomes full_nnz (e.g. wide LASO k=8 +# 318 -> 320 here -- collisions are rare at large dim_major). +# 5. A regular-CSR SpMV kernel (apply_regular_csr_to_vector_ik) was added but is GATED +# OFF by default (#if RandBLAS_ENABLE_REGULAR_CSR_KERNEL). The back-to-back A/B at +# the bottom of this file shows it is NEUTRAL-to-SLOWER than the general CSR kernel +# on this M3 (NEON), so the default build still uses the general kernel here -- the +# numbers below are therefore unchanged from item 1-3 state (deltas are run-to-run +# noise). The kernel is kept in-tree for evaluation on wide-vector Arm (SVE). # ns/elt = us*1000/(nnz*work_mult); GB/s = model bytes/time; %STR vs STREAM below. # Environment: Apple M3, OMP_NUM_THREADS=4, Release (MKL off) | Correctness: 0 FAIL @@ -13,7 +20,7 @@ SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) ============================================================ Threads (OMP_NUM_THREADS/max): 4 -Calibrating STREAM triad... 88.6 GB/s (= 100% on the %STR column) +Calibrating STREAM triad... 88.7 GB/s (= 100% on the %STR column) ########## SECTION 1: vary sketch size d (m=n=2000) ########## @@ -22,147 +29,147 @@ Calibrating STREAM triad... 88.6 GB/s (= 100% on the %STR column) warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 676 691 0.085 49.3 56 CSC nnz=4000 cold=867 cvt=6 - Wide SASO k=4 1425 1732 0.089 23.4 26 CSC nnz=8000 cold=1742 cvt=14 - Wide SASO k=8 2594 3294 0.081 12.9 15 CSC nnz=16000 cold=3250 cvt=27 - CountSketch k=1 582 717 0.145 57.2 65 CSC nnz=2000 cold=642 cvt=3 - Wide LASO k=2 221 274 1.381 11.6 13 CSR nnz=80 cold=220 cvt=0 - Wide LASO k=4 351 380 1.097 10.9 12 CSR nnz=160 cold=364 cvt=1 - Wide LASO k=8 383 392 0.602 16.6 19 CSR nnz=318 cold=390 cvt=3 - densify(S)+GEMM 1508 1526 - - - dense oracle + Wide SASO k=2 701 722 0.088 47.6 54 CSC nnz=4000 cold=794 cvt=7 + Wide SASO k=4 1318 1370 0.082 25.3 29 CSC nnz=8000 cold=1783 cvt=15 + Wide SASO k=8 2317 3126 0.072 14.5 16 CSC nnz=16000 cold=3474 cvt=26 + CountSketch k=1 573 587 0.143 58.1 66 CSC nnz=2000 cold=628 cvt=3 + Wide LASO k=2 212 260 1.325 12.1 14 CSR nnz=80 cold=153 cvt=1 + Wide LASO k=4 389 455 1.216 9.9 11 CSR nnz=160 cold=344 cvt=1 + Wide LASO k=8 400 405 0.625 16.0 18 CSR nnz=320 cold=404 cvt=4 + densify(S)+GEMM 1566 1592 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 753 821 0.094 44.3 50 CSC nnz=4000 - Wide SASO k=4 1050 1066 0.066 31.8 36 CSC nnz=8000 - Wide SASO k=8 1770 1795 0.055 18.9 21 CSC nnz=16000 - CountSketch k=1 488 514 0.122 68.3 77 CSC nnz=2000 - Wide LASO k=2 28 31 0.175 91.5 103 CSR nnz=80 - Wide LASO k=4 35 47 0.109 109.8 124 CSR nnz=160 - Wide LASO k=8 48 49 0.075 132.8 150 CSR nnz=318 + Wide SASO k=2 770 782 0.096 43.3 49 CSC nnz=4000 + Wide SASO k=4 1265 1377 0.079 26.4 30 CSC nnz=8000 + Wide SASO k=8 1775 1829 0.055 18.9 21 CSC nnz=16000 + CountSketch k=1 510 522 0.128 65.3 74 CSC nnz=2000 + Wide LASO k=2 28 30 0.175 91.5 103 CSR nnz=80 + Wide LASO k=4 34 37 0.106 113.0 127 CSR nnz=160 + Wide LASO k=8 48 49 0.075 133.4 150 CSR nnz=320 === RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === warm apply, ColMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 701 740 0.088 47.6 54 CSR nnz=4000 - Wide SASO k=4 1090 1166 0.068 30.6 35 CSR nnz=8000 - Wide SASO k=8 1730 1859 0.054 19.4 22 CSR nnz=16000 - CountSketch k=1 508 510 0.127 65.6 74 CSR nnz=2000 - Wide LASO k=2 29 29 0.181 88.3 100 CSC nnz=80 - Wide LASO k=4 35 45 0.109 109.8 124 CSC nnz=160 - Wide LASO k=8 48 52 0.075 133.4 151 CSC nnz=320 + Wide SASO k=2 652 661 0.082 51.1 58 CSR nnz=4000 + Wide SASO k=4 1039 1056 0.065 32.2 36 CSR nnz=8000 + Wide SASO k=8 1665 1712 0.052 20.1 23 CSR nnz=16000 + CountSketch k=1 464 514 0.116 71.8 81 CSR nnz=2000 + Wide LASO k=2 29 30 0.181 88.3 100 CSC nnz=80 + Wide LASO k=4 35 37 0.109 109.8 124 CSC nnz=160 + Wide LASO k=8 48 50 0.075 133.4 150 CSC nnz=320 warm apply, RowMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 584 615 0.073 57.1 64 CSR nnz=4000 - Wide SASO k=4 1239 1282 0.077 27.0 30 CSR nnz=8000 - Wide SASO k=8 2344 3083 0.073 14.3 16 CSR nnz=16000 - CountSketch k=1 504 582 0.126 66.1 75 CSR nnz=2000 - Wide LASO k=2 150 174 0.938 17.1 19 CSC nnz=80 - Wide LASO k=4 330 367 1.031 11.6 13 CSC nnz=160 - Wide LASO k=8 410 454 0.641 15.6 18 CSC nnz=320 + Wide SASO k=2 628 648 0.079 53.1 60 CSR nnz=4000 + Wide SASO k=4 1216 1314 0.076 27.5 31 CSR nnz=8000 + Wide SASO k=8 2306 2427 0.072 14.5 16 CSR nnz=16000 + CountSketch k=1 457 508 0.114 72.9 82 CSR nnz=2000 + Wide LASO k=2 152 183 0.950 16.9 19 CSC nnz=80 + Wide LASO k=4 317 346 0.991 12.1 14 CSC nnz=160 + Wide LASO k=8 385 470 0.602 16.6 19 CSC nnz=320 === LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 769 925 0.096 50.0 56 CSC nnz=4000 cold=1233 cvt=7 - Wide SASO k=4 1432 1495 0.089 26.9 30 CSC nnz=8000 cold=1985 cvt=15 - Wide SASO k=8 2508 3257 0.078 15.4 17 CSC nnz=16000 cold=3901 cvt=26 - CountSketch k=1 716 910 0.179 53.7 61 CSC nnz=2000 cold=794 cvt=3 - Wide LASO k=2 490 535 0.613 26.1 29 CSR nnz=400 cold=531 cvt=4 - Wide LASO k=4 533 710 0.333 36.0 41 CSR nnz=800 cold=712 cvt=10 - Wide LASO k=8 593 640 0.186 53.9 61 CSR nnz=1597 cold=662 cvt=22 - densify(S)+GEMM 4698 4732 - - - dense oracle + Wide SASO k=2 844 921 0.105 45.6 51 CSC nnz=4000 cold=1003 cvt=7 + Wide SASO k=4 1505 1563 0.094 25.6 29 CSC nnz=8000 cold=1955 cvt=14 + Wide SASO k=8 2424 3260 0.076 15.9 18 CSC nnz=16000 cold=3727 cvt=26 + CountSketch k=1 728 756 0.182 52.8 60 CSC nnz=2000 cold=740 cvt=3 + Wide LASO k=2 493 566 0.616 26.0 29 CSR nnz=400 cold=524 cvt=5 + Wide LASO k=4 600 774 0.375 32.0 36 CSR nnz=800 cold=694 cvt=11 + Wide LASO k=8 650 661 0.203 49.3 56 CSR nnz=1600 cold=787 cvt=25 + densify(S)+GEMM 4648 4716 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 739 783 0.092 52.0 59 CSC nnz=4000 - Wide SASO k=4 1153 1179 0.072 33.4 38 CSC nnz=8000 - Wide SASO k=8 1830 1934 0.057 21.1 24 CSC nnz=16000 - CountSketch k=1 504 550 0.126 76.3 86 CSC nnz=2000 - Wide LASO k=2 98 105 0.122 130.7 147 CSR nnz=400 - Wide LASO k=4 162 173 0.101 118.6 134 CSR nnz=800 - Wide LASO k=8 368 388 0.115 86.9 98 CSR nnz=1597 + Wide SASO k=2 751 837 0.094 51.2 58 CSC nnz=4000 + Wide SASO k=4 1089 1108 0.068 35.4 40 CSC nnz=8000 + Wide SASO k=8 1828 1954 0.057 21.1 24 CSC nnz=16000 + CountSketch k=1 560 563 0.140 68.6 77 CSC nnz=2000 + Wide LASO k=2 111 116 0.139 115.4 130 CSR nnz=400 + Wide LASO k=4 157 166 0.098 122.4 138 CSR nnz=800 + Wide LASO k=8 373 383 0.117 85.9 97 CSR nnz=1600 === RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === warm apply, ColMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 730 799 0.091 52.7 59 CSR nnz=4000 - Wide SASO k=4 1133 1163 0.071 34.0 38 CSR nnz=8000 - Wide SASO k=8 1872 1907 0.059 20.6 23 CSR nnz=16000 - CountSketch k=1 527 567 0.132 72.9 82 CSR nnz=2000 - Wide LASO k=2 97 101 0.121 132.0 149 CSC nnz=400 - Wide LASO k=4 176 185 0.110 109.0 123 CSC nnz=798 - Wide LASO k=8 362 370 0.113 88.3 100 CSC nnz=1597 + Wide SASO k=2 738 777 0.092 52.1 59 CSR nnz=4000 + Wide SASO k=4 1071 1083 0.067 36.0 41 CSR nnz=8000 + Wide SASO k=8 1725 1868 0.054 22.4 25 CSR nnz=16000 + CountSketch k=1 559 563 0.140 68.8 78 CSR nnz=2000 + Wide LASO k=2 97 105 0.121 132.0 149 CSC nnz=400 + Wide LASO k=4 179 192 0.112 107.3 121 CSC nnz=800 + Wide LASO k=8 380 396 0.119 84.3 95 CSC nnz=1600 warm apply, RowMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 849 958 0.106 45.3 51 CSR nnz=4000 - Wide SASO k=4 1414 1473 0.088 27.2 31 CSR nnz=8000 - Wide SASO k=8 2783 3272 0.087 13.9 16 CSR nnz=16000 - CountSketch k=1 746 934 0.186 51.5 58 CSR nnz=2000 - Wide LASO k=2 514 623 0.642 24.9 28 CSC nnz=400 - Wide LASO k=4 614 637 0.385 31.2 35 CSC nnz=798 - Wide LASO k=8 613 727 0.192 52.2 59 CSC nnz=1597 + Wide SASO k=2 897 929 0.112 42.9 48 CSR nnz=4000 + Wide SASO k=4 1431 1467 0.089 26.9 30 CSR nnz=8000 + Wide SASO k=8 2363 3260 0.074 16.4 18 CSR nnz=16000 + CountSketch k=1 830 998 0.207 46.3 52 CSR nnz=2000 + Wide LASO k=2 621 718 0.776 20.6 23 CSC nnz=400 + Wide LASO k=4 753 862 0.471 25.5 29 CSC nnz=800 + Wide LASO k=8 711 807 0.222 45.0 51 CSC nnz=1600 === LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1229 1274 0.154 39.1 44 CSC nnz=4000 cold=1516 cvt=7 - Wide SASO k=4 1799 1838 0.112 26.8 30 CSC nnz=8000 cold=2206 cvt=15 - Wide SASO k=8 2756 3291 0.086 17.5 20 CSC nnz=16000 cold=4653 cvt=26 - CountSketch k=1 949 1027 0.237 50.6 57 CSC nnz=2000 cold=1061 cvt=3 - Wide LASO k=2 798 1093 0.399 40.1 45 CSR nnz=1000 cold=815 cvt=13 - Wide LASO k=4 1019 1052 0.255 47.1 53 CSR nnz=1999 cold=1106 cvt=30 - Wide LASO k=8 999 1100 0.125 48.1 54 CSR nnz=3994 cold=1286 cvt=81 - densify(S)+GEMM 11082 11279 - - - dense oracle + Wide SASO k=2 1346 1460 0.168 35.7 40 CSC nnz=4000 cold=1587 cvt=7 + Wide SASO k=4 1770 1831 0.111 27.2 31 CSC nnz=8000 cold=2310 cvt=15 + Wide SASO k=8 2721 3540 0.085 17.7 20 CSC nnz=16000 cold=3980 cvt=26 + CountSketch k=1 1059 1100 0.265 45.4 51 CSC nnz=2000 cold=1117 cvt=3 + Wide LASO k=2 817 903 0.408 39.2 44 CSR nnz=1000 cold=925 cvt=14 + Wide LASO k=4 1063 1130 0.266 45.2 51 CSR nnz=2000 cold=1216 cvt=32 + Wide LASO k=8 1063 1098 0.133 45.2 51 CSR nnz=4000 cold=1346 cvt=75 + densify(S)+GEMM 10935 11033 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 895 942 0.112 53.7 61 CSC nnz=4000 - Wide SASO k=4 1258 1265 0.079 38.3 43 CSC nnz=8000 - Wide SASO k=8 2009 2105 0.063 24.0 27 CSC nnz=16000 - CountSketch k=1 673 777 0.168 71.4 81 CSC nnz=2000 - Wide LASO k=2 389 392 0.195 82.3 93 CSR nnz=1000 - Wide LASO k=4 587 597 0.147 81.8 92 CSR nnz=1999 - Wide LASO k=8 854 906 0.107 56.3 64 CSR nnz=3994 + Wide SASO k=2 877 920 0.110 54.8 62 CSC nnz=4000 + Wide SASO k=4 1173 1186 0.073 41.0 46 CSC nnz=8000 + Wide SASO k=8 1817 1845 0.057 26.6 30 CSC nnz=16000 + CountSketch k=1 764 773 0.191 62.9 71 CSC nnz=2000 + Wide LASO k=2 392 402 0.196 81.7 92 CSR nnz=1000 + Wide LASO k=4 593 603 0.148 81.0 91 CSR nnz=2000 + Wide LASO k=8 826 858 0.103 58.2 66 CSR nnz=4000 === RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === warm apply, ColMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 895 905 0.112 53.7 61 CSR nnz=4000 - Wide SASO k=4 1217 1232 0.076 39.5 45 CSR nnz=8000 - Wide SASO k=8 1922 1980 0.060 25.1 28 CSR nnz=16000 - CountSketch k=1 761 767 0.190 63.1 71 CSR nnz=2000 - Wide LASO k=2 391 405 0.196 81.8 92 CSC nnz=998 - Wide LASO k=4 595 686 0.149 80.7 91 CSC nnz=1998 - Wide LASO k=8 865 898 0.108 55.6 63 CSC nnz=3993 + Wide SASO k=2 826 842 0.103 58.2 66 CSR nnz=4000 + Wide SASO k=4 1193 1312 0.075 40.3 45 CSR nnz=8000 + Wide SASO k=8 1836 1871 0.057 26.3 30 CSR nnz=16000 + CountSketch k=1 761 774 0.190 63.1 71 CSR nnz=2000 + Wide LASO k=2 404 407 0.202 79.2 89 CSC nnz=1000 + Wide LASO k=4 563 567 0.141 85.3 96 CSC nnz=2000 + Wide LASO k=8 803 860 0.100 59.9 68 CSC nnz=4000 warm apply, RowMajor dense Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 1308 1348 0.164 36.7 41 CSR nnz=4000 - Wide SASO k=4 1708 1774 0.107 28.2 32 CSR nnz=8000 - Wide SASO k=8 3022 3513 0.094 16.0 18 CSR nnz=16000 - CountSketch k=1 1008 1052 0.252 47.7 54 CSR nnz=2000 - Wide LASO k=2 919 1021 0.460 34.8 39 CSC nnz=998 - Wide LASO k=4 1126 1150 0.282 42.6 48 CSC nnz=1998 - Wide LASO k=8 1070 1177 0.134 44.9 51 CSC nnz=3993 + Wide SASO k=2 1270 1294 0.159 37.8 43 CSR nnz=4000 + Wide SASO k=4 1781 1831 0.111 27.0 30 CSR nnz=8000 + Wide SASO k=8 2696 3514 0.084 17.9 20 CSR nnz=16000 + CountSketch k=1 1064 1107 0.266 45.1 51 CSR nnz=2000 + Wide LASO k=2 847 945 0.423 37.8 43 CSC nnz=1000 + Wide LASO k=4 1135 1167 0.284 42.3 48 CSC nnz=2000 + Wide LASO k=8 1130 1157 0.141 42.5 48 CSC nnz=4000 ########## SECTION 2: narrow n (SpMV-ish) ########## @@ -171,48 +178,128 @@ Calibrating STREAM triad... 88.6 GB/s (= 100% on the %STR column) warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 18 20 4.500 4.6 5 CSC nnz=4000 cold=184 cvt=7 - Wide SASO k=4 31 34 3.875 4.7 5 CSC nnz=8000 cold=482 cvt=14 - Wide SASO k=8 45 51 2.812 6.1 7 CSC nnz=16000 cold=1121 cvt=25 - CountSketch k=1 13 14 6.500 3.9 4 CSC nnz=2000 cold=60 cvt=3 - Wide LASO k=2 8 9 20.000 1.6 2 CSR nnz=400 cold=36 cvt=5 - Wide LASO k=4 8 9 10.000 2.8 3 CSR nnz=800 cold=65 cvt=11 - Wide LASO k=8 10 12 6.254 4.2 5 CSR nnz=1599 cold=110 cvt=22 + Wide SASO k=2 19 21 4.750 4.4 5 CSC nnz=4000 cold=182 cvt=7 + Wide SASO k=4 31 32 3.875 4.7 5 CSC nnz=8000 cold=448 cvt=14 + Wide SASO k=8 50 51 3.125 5.5 6 CSC nnz=16000 cold=1098 cvt=25 + CountSketch k=1 13 14 6.500 3.9 4 CSC nnz=2000 cold=63 cvt=3 + Wide LASO k=2 8 8 20.000 1.6 2 CSR nnz=400 cold=39 cvt=5 + Wide LASO k=4 8 10 10.000 2.8 3 CSR nnz=800 cold=72 cvt=11 + Wide LASO k=8 10 12 6.250 4.2 5 CSR nnz=1600 cold=131 cvt=24 densify(S)+GEMM 13 14 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 56 57 14.000 1.5 2 CSC nnz=4000 - Wide SASO k=4 100 101 12.500 1.5 2 CSC nnz=8000 - Wide SASO k=8 180 183 11.250 1.5 2 CSC nnz=16000 - CountSketch k=1 35 35 17.500 1.5 2 CSC nnz=2000 - Wide LASO k=2 16 18 40.000 0.8 1 CSR nnz=400 - Wide LASO k=4 21 23 26.250 1.1 1 CSR nnz=800 - Wide LASO k=8 32 34 20.013 1.3 1 CSR nnz=1599 + Wide SASO k=2 57 58 14.250 1.5 2 CSC nnz=4000 + Wide SASO k=4 99 106 12.375 1.5 2 CSC nnz=8000 + Wide SASO k=8 182 184 11.375 1.5 2 CSC nnz=16000 + CountSketch k=1 36 36 18.000 1.4 2 CSC nnz=2000 + Wide LASO k=2 17 18 42.500 0.8 1 CSR nnz=400 + Wide LASO k=4 21 22 26.250 1.1 1 CSR nnz=800 + Wide LASO k=8 33 34 20.625 1.3 1 CSR nnz=1600 === LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === warm apply, ColMajor dense (jik/jki kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 22 25 0.550 11.6 13 CSC nnz=4000 cold=172 cvt=6 - Wide SASO k=4 36 41 0.450 8.9 10 CSC nnz=8000 cold=477 cvt=13 - Wide SASO k=8 63 67 0.394 7.1 8 CSC nnz=16000 cold=1114 cvt=26 - CountSketch k=1 19 32 0.950 11.8 13 CSC nnz=2000 cold=82 cvt=4 - Wide LASO k=2 9 11 2.250 7.8 9 CSR nnz=400 cold=36 cvt=4 - Wide LASO k=4 11 13 1.375 9.9 11 CSR nnz=800 cold=67 cvt=11 - Wide LASO k=8 14 15 0.877 13.2 15 CSR nnz=1596 cold=124 cvt=23 - densify(S)+GEMM 34 36 - - - dense oracle + Wide SASO k=2 24 26 0.600 10.7 12 CSC nnz=4000 cold=147 cvt=7 + Wide SASO k=4 36 37 0.450 8.9 10 CSC nnz=8000 cold=395 cvt=13 + Wide SASO k=8 66 67 0.412 6.8 8 CSC nnz=16000 cold=1100 cvt=26 + CountSketch k=1 17 18 0.850 13.2 15 CSC nnz=2000 cold=67 cvt=3 + Wide LASO k=2 10 11 2.500 7.0 8 CSR nnz=400 cold=40 cvt=5 + Wide LASO k=4 11 12 1.375 9.9 11 CSR nnz=800 cold=74 cvt=11 + Wide LASO k=8 14 15 0.875 13.3 15 CSR nnz=1600 cold=135 cvt=24 + densify(S)+GEMM 35 35 - - - dense oracle warm apply, RowMajor dense (ikb/kib kernels) Operator Min(us) Med(us) ns/elt GB/s %STR notes ---------------------------------------------------------------------------- - Wide SASO k=2 57 59 1.425 4.5 5 CSC nnz=4000 - Wide SASO k=4 97 100 1.212 3.3 4 CSC nnz=8000 - Wide SASO k=8 168 228 1.050 2.7 3 CSC nnz=16000 - CountSketch k=1 36 50 1.800 6.2 7 CSC nnz=2000 - Wide LASO k=2 17 19 4.250 4.1 5 CSR nnz=400 - Wide LASO k=4 21 23 2.625 5.2 6 CSR nnz=800 - Wide LASO k=8 33 40 2.068 5.6 6 CSR nnz=1596 + Wide SASO k=2 56 58 1.400 4.6 5 CSC nnz=4000 + Wide SASO k=4 97 98 1.212 3.3 4 CSC nnz=8000 + Wide SASO k=8 170 178 1.062 2.6 3 CSC nnz=16000 + CountSketch k=1 35 36 1.750 6.4 7 CSC nnz=2000 + Wide LASO k=2 17 18 4.250 4.1 5 CSR nnz=400 + Wide LASO k=4 22 25 2.750 4.9 6 CSR nnz=800 + Wide LASO k=8 34 34 2.125 5.5 6 CSR nnz=1600 + + + +############################################################ +# CSR REGULAR-vs-GENERAL KERNEL A/B (--csr-probe) +# 'CSR REG ColMajor*' = apply_regular_csr_to_vector_ik (gated off by default) +# 'CSR GEN ColMajor*' = apply_csr_to_vector_ik (the default). Same regular operator, +# same process, back-to-back -> isolates the kernel. REG is NOT faster on M3/NEON. +############################################################ + +============================================================ +SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) +============================================================ +Threads (OMP_NUM_THREADS/max): 4 +Calibrating STREAM triad... 88.1 GB/s (= 100% on the %STR column) + +=== CSR-vs-CSC KERNEL PROBE (left-sketch) d=200 m=2000 n=2000, trials=10 === + kernels timed directly via left_spmm on prebuilt CSC/CSR (no COO re-sort) + + Wide SASO k=2 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 1121 1125 0.140 34.3 39 CSC nnz=4000 + CSR jik ColMajor 823 953 0.103 46.7 53 CSC nnz=4000 + CSC kib RowMajor 772 787 0.097 49.8 57 CSC nnz=4000 + CSR ikb RowMajor 1005 1013 0.126 38.3 43 CSC nnz=4000 + + Wide SASO k=4 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 2092 2108 0.131 18.4 21 CSC nnz=8000 + CSR jik ColMajor 1429 1483 0.089 27.0 31 CSC nnz=8000 + CSC kib RowMajor 1079 1141 0.067 35.7 41 CSC nnz=8000 + CSR ikb RowMajor 1558 1817 0.097 24.7 28 CSC nnz=8000 + + Wide SASO k=8 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 3699 3959 0.116 10.5 12 CSC nnz=16000 + CSR jik ColMajor 2221 2665 0.069 17.4 20 CSC nnz=16000 + CSC kib RowMajor 1845 1867 0.058 21.0 24 CSC nnz=16000 + CSR ikb RowMajor 2836 3354 0.089 13.6 15 CSC nnz=16000 + + CountSketch k=1 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 873 886 0.218 44.0 50 CSC nnz=2000 + CSR jik ColMajor 719 732 0.180 53.5 61 CSC nnz=2000 + CSC kib RowMajor 498 569 0.124 77.2 88 CSC nnz=2000 + CSR ikb RowMajor 563 641 0.141 68.3 78 CSC nnz=2000 + + Wide LASO k=2 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 583 683 0.729 22.0 25 CSR nnz=400 + CSR jik ColMajor 585 652 0.731 21.9 25 CSR nnz=400 + CSC kib RowMajor 98 122 0.122 130.7 148 CSR nnz=400 + CSR ikb RowMajor 96 97 0.120 133.4 151 CSR nnz=400 + CSR REG ColMajor* 468 568 0.585 27.4 31 regular=Y row_nnz=2 + CSR GEN ColMajor* 441 463 0.551 29.0 33 regular=N row_nnz=2 + + Wide LASO k=4 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 788 825 0.492 24.4 28 CSR nnz=800 + CSR jik ColMajor 543 575 0.339 35.4 40 CSR nnz=800 + CSC kib RowMajor 153 163 0.096 125.6 143 CSR nnz=800 + CSR ikb RowMajor 146 168 0.091 131.6 149 CSR nnz=800 + CSR REG ColMajor* 602 626 0.376 31.9 36 regular=Y row_nnz=4 + CSR GEN ColMajor* 535 562 0.334 35.9 41 regular=N row_nnz=4 + + Wide LASO k=8 + Operator Min(us) Med(us) ns/elt GB/s %STR notes + ---------------------------------------------------------------------------- + CSC jki ColMajor 1251 1267 0.391 25.6 29 CSR nnz=1600 + CSR jik ColMajor 542 653 0.169 59.1 67 CSR nnz=1600 + CSC kib RowMajor 380 388 0.119 84.3 96 CSR nnz=1600 + CSR ikb RowMajor 410 422 0.128 78.1 89 CSR nnz=1600 + CSR REG ColMajor* 592 624 0.185 54.1 61 regular=Y row_nnz=8 + CSR GEN ColMajor* 528 555 0.165 60.7 69 regular=N row_nnz=8 diff --git a/test/datastructures/test_coo_matrix.cc b/test/datastructures/test_coo_matrix.cc index 4b83b6df..f135982d 100644 --- a/test/datastructures/test_coo_matrix.cc +++ b/test/datastructures/test_coo_matrix.cc @@ -64,7 +64,10 @@ void sparseskop_to_dense( sint_t row = S0.rows[i]; sint_t col = S0.cols[i]; T val = S0.vals[i]; - mat[idx(row, col)] = val; + // Accumulate: a regular long-axis-major operator stores collided coordinates as + // duplicate records that must sum (matches coo_to_dense). The buffer is zeroed + // above, so += equals = for a deduplicated operator. + mat[idx(row, col)] += val; } } @@ -98,6 +101,32 @@ class TestCOO : public ::testing::Test { return; } + template + void test_dense_sums_duplicates() { + // coo_to_dense must ACCUMULATE duplicate (row, col) records (not overwrite), so + // that a "regular" sketching operator -- which stores a collided coordinate as + // several records that sum to its true value -- densifies correctly. + COOMatrix A(3, 3); + A.reserve(4); + // Two records at (0, 1) that must sum: 1.5 + (-0.5) = 1.0. + A.rows[0] = 0; A.cols[0] = 1; A.vals[0] = (T) 1.5; + A.rows[1] = 0; A.cols[1] = 1; A.vals[1] = (T) -0.5; + // Three-way duplicate at (2, 2): 0.25 + 0.25 + 0.5 = 1.0 (only 2 here + 1 below). + A.rows[2] = 2; A.cols[2] = 2; A.vals[2] = (T) 0.25; + A.rows[3] = 2; A.cols[3] = 2; A.vals[3] = (T) 0.75; + + std::vector dense(9, (T) -7); // sentinel; must be overwritten to 0 then summed. + coo_to_dense(A, Layout::RowMajor, dense.data()); + EXPECT_EQ(dense[0*3 + 1], (T) 1.0); // (0,1): 1.5 - 0.5 + EXPECT_EQ(dense[2*3 + 2], (T) 1.0); // (2,2): 0.25 + 0.75 + // Every other cell is exactly zero (sentinel cleared, no records). + for (int64_t i = 0; i < 3; ++i) + for (int64_t j = 0; j < 3; ++j) + if (!((i==0 && j==1) || (i==2 && j==2))) + EXPECT_EQ((dense[i*3 + j]), (T) 0.0) << "cell (" << i << "," << j << ")"; + return; + } + template void test_sort_order(int64_t n) { COOMatrix A(n, n); @@ -251,6 +280,11 @@ TEST_F(TestCOO, to_from_dense) { test_to_from_dense(10); } +TEST_F(TestCOO, dense_sums_duplicates) { + test_dense_sums_duplicates(); + test_dense_sums_duplicates(); +} + TEST_F(TestCOO, sort_order) { test_sort_order(3); test_sort_order(7); diff --git a/test/datastructures/test_sparseskop.cc b/test/datastructures/test_sparseskop.cc index 5d4a87eb..1a4b4914 100644 --- a/test/datastructures/test_sparseskop.cc +++ b/test/datastructures/test_sparseskop.cc @@ -33,6 +33,8 @@ #include "RandBLAS/testing/comparison.hh" #include #include +#include +#include using std::vector; using RandBLAS::RNGState; @@ -297,6 +299,55 @@ class TestSparseSkOpConstruction : public ::testing::Test } return; } + + // Regularizing a LASO is a storage-only change: for a given seed, the regular form + // (exactly vec_nnz records per major-axis vector, with collided coordinates split into + // duplicate records) must densify to the SAME operator as the merged form. This checks + // (a) the regular structure -- nnz == full_nnz and every minor-axis vector index occurs + // exactly vec_nnz times -- and (b) operator preservation via densify-and-compare. + template + void regular_laso_preserves_operator(int64_t d, int64_t m, int64_t vec_nnz, uint32_t key) { + SparseDist D {d, m, vec_nnz, Axis::Long}; + RNGState seed(key); + int64_t cap = D.full_nnz; + + // Merged form (regular=false) and regular form (regular=true), SAME seed. + vector mvals(cap); vector mrows(cap), mcols(cap); int64_t mnnz = -1; + RandBLAS::fill_sparse_unpacked( + D, d, m, 0, 0, mnnz, mvals.data(), mrows.data(), mcols.data(), seed, false + ); + vector rvals(cap); vector rrows(cap), rcols(cap); int64_t rnnz = -1; + RandBLAS::fill_sparse_unpacked( + D, d, m, 0, 0, rnnz, rvals.data(), rrows.data(), rcols.data(), seed, true + ); + + // (a) Regular structure. + ASSERT_EQ(rnnz, D.full_nnz); + ASSERT_LE(mnnz, D.full_nnz); + bool short_is_rows = (d <= m); + bool major_is_rows = !short_is_rows; // Axis::Long + const sint_t* minor = major_is_rows ? rcols.data() : rrows.data(); + int64_t num_major = D.dim_minor; // count of major-axis vectors + vector hist(num_major, 0); + for (int64_t i = 0; i < rnnz; ++i) { + ASSERT_GE(minor[i], (sint_t) 0); ASSERT_LT(minor[i], (sint_t) num_major); + hist[minor[i]] += 1; + } + for (int64_t g = 0; g < num_major; ++g) + ASSERT_EQ(hist[g], vec_nnz) << "minor vector " << g << " holds " << hist[g] << " records"; + + // (b) Operator preservation: densify both (summing duplicates) and compare. + vector dm(d * m, (T) 0), dr(d * m, (T) 0); + for (int64_t i = 0; i < mnnz; ++i) dm[mrows[i] * m + mcols[i]] += mvals[i]; + for (int64_t i = 0; i < rnnz; ++i) dr[rrows[i] * m + rcols[i]] += rvals[i]; + T atol = (T) (vec_nnz * 16) * std::numeric_limits::epsilon(); + T rtol = (T) (vec_nnz) * std::numeric_limits::epsilon(); + auto msg = RandBLAS::testing::buffs_approx_equal( + dr.data(), dm.data(), d * m, __PRETTY_FUNCTION__, __FILE__, __LINE__, atol, rtol + ); + if (msg.size() > 0) FAIL() << msg; + return; + } }; TEST_F(TestSparseSkOpConstruction, respect_ownership) { @@ -349,6 +400,22 @@ TEST_F(TestSparseSkOpConstruction, fill_unpacked_sub_laso_int32) { } } +TEST_F(TestSparseSkOpConstruction, regular_laso_preserves_operator) { + // Small dim_major (7) with vec_nnz up to 7 guarantees frequent collisions, so the + // regular split path (duplicate records) is heavily exercised, not just the no-op + // vec_nnz==1 case. Cover wide and tall, double and float, int64 and int32 indices. + for (auto key : keys) { + for (int64_t vnz : {(int64_t) 2, (int64_t) 3, (int64_t) 7}) { + regular_laso_preserves_operator(7, 20, vnz, key); // wide + regular_laso_preserves_operator(20, 7, vnz, key); // tall + regular_laso_preserves_operator(7, 20, vnz, key); + regular_laso_preserves_operator( 20, 7, vnz, key); + } + // vec_nnz==1 degenerate case: regular and merged must be identical. + regular_laso_preserves_operator(7, 20, 1, key); + } +} + //////////////////////////////////////////////////////////////////////// // // diff --git a/test/linops/test_spmm/test_spmm_csr.cc b/test/linops/test_spmm/test_spmm_csr.cc index 2071818c..19411ae6 100644 --- a/test/linops/test_spmm/test_spmm_csr.cc +++ b/test/linops/test_spmm/test_spmm_csr.cc @@ -469,3 +469,87 @@ TEST_F(TestPublicAPI_DenseTimesSparse_double, nontrivial_alpha_beta_rowmajor) { test_dense_times_sparse(50, 30, 40, 2.5, -1.0, Layout::RowMajor, 0.80); } + +//////////////////////////////////////////////////////////////////////// +// +// +// Regular-CSR fast path (fixed nnz per row), including duplicate colidxs +// +// +//////////////////////////////////////////////////////////////////////// + +// Compare C = alpha * A @ B against a hand-computed dense reference, where a duplicated +// column index within a row must be accumulated. We check both (1) the public dispatch +// apply_csr_jik_p11 (which uses the general kernel by default; the regular kernel only +// when RandBLAS_ENABLE_REGULAR_CSR_KERNEL is defined) and, (2) when the matrix is regular, +// apply_regular_csr_to_vector_ik directly -- so the regular kernel stays covered regardless +// of the off-by-default dispatch gate. +static void check_csr_jik_against_reference( + int64_t d, int64_t m, int64_t n, + const std::vector& rowptr, + const std::vector& colidxs, + const std::vector& vals +) { + CSRMatrix A(d, m); + A.reserve((int64_t) vals.size()); + for (int64_t i = 0; i <= d; ++i) A.rowptr[i] = rowptr[i]; + for (size_t e = 0; e < vals.size(); ++e) { A.colidxs[e] = colidxs[e]; A.vals[e] = vals[e]; } + + std::vector B(m * n), C(d * n, 0.0), Cref(d * n, 0.0); + for (int64_t k = 0; k < m; ++k) + for (int64_t j = 0; j < n; ++j) + B[k + j*m] = (double)(k + 1) - 0.3 * (double) j; + + double alpha = 2.0; + for (int64_t i = 0; i < d; ++i) + for (int64_t e = rowptr[i]; e < rowptr[i+1]; ++e) + for (int64_t j = 0; j < n; ++j) + Cref[i + j*d] += alpha * vals[e] * B[colidxs[e] + j*m]; + + auto compare = [&](const char* which) { + auto msg = RandBLAS::testing::buffs_approx_equal( + C.data(), Cref.data(), d * n, which, __FILE__, __LINE__ + ); + if (msg.size() > 0) FAIL() << msg; + }; + + // (1) Public dispatch (general kernel unless the regular-CSR macro is enabled). + std::fill(C.begin(), C.end(), 0.0); + apply_csr_jik_p11(alpha, Layout::ColMajor, Layout::ColMajor, + d, n, m, A, B.data(), m, C.data(), d); + compare("apply_csr_jik_p11"); + + // (2) Directly exercise the regular kernel when the matrix has a constant nnz-per-row. + bool regular = (d > 0); + int64_t row_nnz = (d > 0) ? (rowptr[1] - rowptr[0]) : 0; + for (int64_t i = 1; i <= d && regular; ++i) regular = (rowptr[i] - rowptr[i-1] == row_nnz); + if (regular) { + std::fill(C.begin(), C.end(), 0.0); + for (int64_t j = 0; j < n; ++j) + apply_regular_csr_to_vector_ik(alpha, A.vals, row_nnz, A.colidxs, + &B[(size_t) j * m], 1, d, &C[(size_t) j * d], 1); + compare("apply_regular_csr_to_vector_ik"); + } +} + +TEST(TestRegularCSR, fixed_nnz_per_row_with_duplicate_colidx) { + // d=4 rows, each with exactly 2 nonzeros -> regular branch. Row 1 has a duplicated + // column index (2, 2): the two records (0.5, 0.5) must accumulate to 1.0 at column 2. + check_csr_jik_against_reference( + 4, 5, 3, + {0, 2, 4, 6, 8}, + {0, 3, 2, 2, 1, 4, 0, 2}, + {1.0, -2.0, 0.5, 0.5, 3.0, -1.0, 2.0, 1.0} + ); +} + +TEST(TestRegularCSR, variable_nnz_per_row_falls_back) { + // Rows have 1, 2, 1, 2 nonzeros -> NOT regular, so the general branch runs. Same + // dense reference must hold (guards the detection's fallback path). + check_csr_jik_against_reference( + 4, 5, 3, + {0, 1, 3, 4, 6}, + {2, 0, 4, 3, 1, 2}, + {1.5, -0.5, 2.0, 4.0, -1.0, 0.25} + ); +} From ec96cc18a4e2ca5a8e59e3513ee77d13c1472db5 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Sun, 28 Jun 2026 14:50:30 -0700 Subject: [PATCH 11/20] simple clean ups --- CLAUDE.md => AGENTS.md | 6 +- RandBLAS/sparse_data/DevNotes.md | 26 ----- RandBLAS/sparse_skops.hh | 45 ++------- test/datastructures/test_coo_matrix.cc | 7 +- test/datastructures/test_sparseskop.cc | 131 +++---------------------- 5 files changed, 24 insertions(+), 191 deletions(-) rename CLAUDE.md => AGENTS.md (98%) diff --git a/CLAUDE.md b/AGENTS.md similarity index 98% rename from CLAUDE.md rename to AGENTS.md index 17a47240..7ed2d3ac 100644 --- a/CLAUDE.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -# RandBLAS Project Guide for Claude +# RandBLAS Project Guide for Codex ## Project Overview @@ -244,7 +244,7 @@ GitHub Actions workflows test: All CI tests must pass before merging. -## Working with Claude on RandBLAS +## Working with Codex on RandBLAS ### Preferred Workflow @@ -319,4 +319,4 @@ All CI tests must pass before merging. --- -*This CLAUDE.md file was created to help Claude Code understand the RandBLAS project structure, conventions, and workflows. Update it as the project evolves.* +*This AGENTS.md file was created to help Codex understand the RandBLAS project structure, conventions, and workflows. Update it as the project evolves.* diff --git a/RandBLAS/sparse_data/DevNotes.md b/RandBLAS/sparse_data/DevNotes.md index 6804bb07..900fbe6b 100644 --- a/RandBLAS/sparse_data/DevNotes.md +++ b/RandBLAS/sparse_data/DevNotes.md @@ -72,29 +72,3 @@ From there, we'll do the following. Note that the ``l`` and ``r`` in the ``[l/r]sksp3`` function names get matched to opposite sides for ``[left/right]_spmm``! This is because all the fancy abstractions in ``S`` have been stripped away by this point in the call sequence, so the "side" that we emphasize in function names changes from emphasizing ``S`` to emphasizing ``A``. - - - -## Regular (fixed-nnz) operators and duplicate COO records - -Both short-axis-major (SASO) and long-axis-major (LASO) sketching operators are stored with -exactly ``vec_nnz`` structural records per major-axis vector ("regular" layout). For SASO the -``vec_nnz`` nonzeros are distinct. For LASO, with-replacement sampling can draw a coordinate -several times; rather than merge those into one entry, we keep them as duplicate ``(row, col)`` -records, each scaled so they sum to the merged value (see ``laso_merge_long_axis_vector_coo_data`` -in ``sparse_skops.hh``). This regular structure lets the wide ColMajor dispatch reach the -fixed-nnz fast-path kernels (``apply_regular_csr_to_vector_ik`` for CSR, ``apply_regular_csc_to_vector_ki`` -for CSC), which skip the ``rowptr``/``colptr`` loads. - -Consequences for anyone touching these code paths: - - * **SpMM and densify accumulate duplicates.** The CSR/CSC kernels sum ``vals[ell]*v[...]`` over a - row/column, so duplicate column/row indices add correctly. ``coo_to_dense`` was made to use - ``+=`` (not ``=``) for the same reason. Any new densification path must accumulate, not overwrite. - * **Conversions preserve duplicates.** ``coo_to_csr`` / ``coo_to_csc`` and the counting-sort - transpose in ``coo_spmm_impl.hh`` keep all ``nnz`` records (no de-duplication), which is what the - kernels expect. - * **Do not route a regular operator through ``trsm``.** ``trsm_dispatch.hh`` validates structure - with ``compressed_indices_are_increasing``, which rejects duplicate indices within a major-axis - vector. Triangular solve is unrelated to sketching, so this is not a real constraint in practice, - but it is a trap worth noting. diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index 75235f13..a6ecfd8a 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -198,14 +198,6 @@ struct SparseDist { /// \math{\mtxx_j} will equal \math{\sqrt{\ell}} with probability 1/2 and /// \math{-\sqrt{\ell}} with probability 1/2. /// - /// Note: while \math{\mtxx} has at most \math{\vecnnz} *distinct* nonzero coordinates, it is - /// *stored* with exactly \math{\vecnnz} structural entries per major-axis vector. A coordinate - /// \math{j} sampled \math{\ell} times is held as \math{\ell} duplicate records each equal to - /// \math{\mtxx_j/\ell = \pm 1/\sqrt{\ell}}, which sum to \math{\mtxx_j.} This "regular" layout - /// (a fixed number of records per major-axis vector) lets long-axis-major operators reach the - /// same fixed-nnz fast-path kernels as short-axis-major operators, without changing the operator - /// that \math{\mtxx} represents. - /// const Axis major_axis; // --------------------------------------------------------------------------- @@ -491,8 +483,7 @@ template void laso_merge_long_axis_vector_coo_data( int64_t vec_nnz, T* vals, sint_t* idxs_lax, sint_t *idxs_sax, int64_t i, std::unordered_map &loc2count, - std::unordered_map &loc2scale, - bool regular + std::unordered_map &loc2scale ) { loc2count.clear(); // ^ Used to count the number of times each long-axis index @@ -514,19 +505,7 @@ void laso_merge_long_axis_vector_coo_data( loc2count[ell] = 1.0; } } - if (regular) { - // Operator-preserving regularization (storage only): keep all vec_nnz draws, - // but rescale so the c copies stored at a collided location ell each carry - // s/sqrt(c) (s = loc2scale[ell], the canonical sign). The c copies then sum to - // c * (s/sqrt(c)) = sqrt(c)*s -- the exact value the merge branch below would - // store as a single entry. So the densified operator is identical; we simply - // leave duplicate (idxs_lax, idxs_sax) records in place to give a fixed-vec_nnz - // "regular" layout (every major-axis vector has exactly vec_nnz records). - for (int64_t j = 0; j < vec_nnz; ++j) { - sint_t ell = idxs_lax[j]; - vals[j] = loc2scale[ell] / std::sqrt(loc2count[ell]); - } - } else if ((int64_t) loc2scale.size() < vec_nnz) { + if ((int64_t) loc2scale.size() < vec_nnz) { // Then we have duplicates. We need to overwrite some of the values // of (idxs_lax, vals, idxs_sax) and implicitly // shift them backward to remove duplicates; @@ -592,8 +571,7 @@ state_t fill_sparse_unpacked( int64_t n_rows_sub, int64_t n_cols_sub, int64_t ro_s, int64_t co_s, int64_t &nnz, T* vals, sint_t* rows, sint_t* cols, - const state_t &seed_state, - bool regular = true + const state_t &seed_state ) { randblas_require(D.n_rows >= n_rows_sub + ro_s); randblas_require(D.n_cols >= n_cols_sub + co_s); @@ -700,20 +678,11 @@ state_t fill_sparse_unpacked( end_state = work_state; for (int64_t i = 0; i < num_major_sub; ++i) { end_state = sample_indices_iid_uniform(dim_major, vec_nnz, im, v, end_state); - laso_merge_long_axis_vector_coo_data(vec_nnz, v, im, in, i, loc2count, loc2scale, regular); - // In regular mode every major-axis vector keeps exactly vec_nnz records (with - // duplicates rescaled to preserve the operator); otherwise the merge compacts to - // the (<= vec_nnz) distinct survivors. - int64_t count = regular ? vec_nnz : (int64_t) loc2count.size(); + laso_merge_long_axis_vector_coo_data(vec_nnz, v, im, in, i, loc2count, loc2scale); + // The merge compacts to the (<= vec_nnz) distinct survivors. + int64_t count = (int64_t) loc2count.size(); // Emit this major-axis vector in ascending major-coordinate order, exactly as - // the SASO branch does. The (merge or regular rescale) leaves the survivors in - // hash / sample order; sorting the block by its major coordinate makes a wide - // LASO CSR-sorted (cols ascending within each row) and a tall LASO CSC-sorted. - // That natural order also matches the ColMajor dispatch preference (CSR for - // wide), so apply_coo_via_csc skips its per-apply re-sort. idxs_minor is constant - // within the block. In regular mode duplicate major coordinates stay adjacent - // (the comparison is strict), so the block is still CSC-/CSR-sorted with - // duplicates allowed. The block length is small, so insertion-sort is best. + // the SASO branch does. The blocks are handled with insertion-sort. for (int64_t a = 1; a < count; ++a) { sint_t key = im[a]; T vv = v[a]; diff --git a/test/datastructures/test_coo_matrix.cc b/test/datastructures/test_coo_matrix.cc index f135982d..d23ea142 100644 --- a/test/datastructures/test_coo_matrix.cc +++ b/test/datastructures/test_coo_matrix.cc @@ -64,9 +64,6 @@ void sparseskop_to_dense( sint_t row = S0.rows[i]; sint_t col = S0.cols[i]; T val = S0.vals[i]; - // Accumulate: a regular long-axis-major operator stores collided coordinates as - // duplicate records that must sum (matches coo_to_dense). The buffer is zeroed - // above, so += equals = for a deduplicated operator. mat[idx(row, col)] += val; } } @@ -103,9 +100,7 @@ class TestCOO : public ::testing::Test { template void test_dense_sums_duplicates() { - // coo_to_dense must ACCUMULATE duplicate (row, col) records (not overwrite), so - // that a "regular" sketching operator -- which stores a collided coordinate as - // several records that sum to its true value -- densifies correctly. + // coo_to_dense must ACCUMULATE duplicate (row, col) records (not overwrite). COOMatrix A(3, 3); A.reserve(4); // Two records at (0, 1) that must sum: 1.5 + (-0.5) = 1.0. diff --git a/test/datastructures/test_sparseskop.cc b/test/datastructures/test_sparseskop.cc index 1a4b4914..c0c5d5a6 100644 --- a/test/datastructures/test_sparseskop.cc +++ b/test/datastructures/test_sparseskop.cc @@ -106,9 +106,10 @@ class TestSparseSkOpConstruction : public ::testing::Test } template - void proper_laso_construction(int64_t d, int64_t m, int64_t key_index, int64_t nnz_index) { + void proper_laso_construction(int64_t d, int64_t m, int64_t key_index) { using RNG = SparseSkOp::state_t::generator; - SparseDist D0(d, m, vec_nnzs[nnz_index], Axis::Long); + int64_t vec_nnz = 1; + SparseDist D0(d, m, vec_nnz, Axis::Long); SparseSkOp S0(D0, keys[key_index]); fill_sparse(S0); if (d < m) { @@ -300,54 +301,6 @@ class TestSparseSkOpConstruction : public ::testing::Test return; } - // Regularizing a LASO is a storage-only change: for a given seed, the regular form - // (exactly vec_nnz records per major-axis vector, with collided coordinates split into - // duplicate records) must densify to the SAME operator as the merged form. This checks - // (a) the regular structure -- nnz == full_nnz and every minor-axis vector index occurs - // exactly vec_nnz times -- and (b) operator preservation via densify-and-compare. - template - void regular_laso_preserves_operator(int64_t d, int64_t m, int64_t vec_nnz, uint32_t key) { - SparseDist D {d, m, vec_nnz, Axis::Long}; - RNGState seed(key); - int64_t cap = D.full_nnz; - - // Merged form (regular=false) and regular form (regular=true), SAME seed. - vector mvals(cap); vector mrows(cap), mcols(cap); int64_t mnnz = -1; - RandBLAS::fill_sparse_unpacked( - D, d, m, 0, 0, mnnz, mvals.data(), mrows.data(), mcols.data(), seed, false - ); - vector rvals(cap); vector rrows(cap), rcols(cap); int64_t rnnz = -1; - RandBLAS::fill_sparse_unpacked( - D, d, m, 0, 0, rnnz, rvals.data(), rrows.data(), rcols.data(), seed, true - ); - - // (a) Regular structure. - ASSERT_EQ(rnnz, D.full_nnz); - ASSERT_LE(mnnz, D.full_nnz); - bool short_is_rows = (d <= m); - bool major_is_rows = !short_is_rows; // Axis::Long - const sint_t* minor = major_is_rows ? rcols.data() : rrows.data(); - int64_t num_major = D.dim_minor; // count of major-axis vectors - vector hist(num_major, 0); - for (int64_t i = 0; i < rnnz; ++i) { - ASSERT_GE(minor[i], (sint_t) 0); ASSERT_LT(minor[i], (sint_t) num_major); - hist[minor[i]] += 1; - } - for (int64_t g = 0; g < num_major; ++g) - ASSERT_EQ(hist[g], vec_nnz) << "minor vector " << g << " holds " << hist[g] << " records"; - - // (b) Operator preservation: densify both (summing duplicates) and compare. - vector dm(d * m, (T) 0), dr(d * m, (T) 0); - for (int64_t i = 0; i < mnnz; ++i) dm[mrows[i] * m + mcols[i]] += mvals[i]; - for (int64_t i = 0; i < rnnz; ++i) dr[rrows[i] * m + rcols[i]] += rvals[i]; - T atol = (T) (vec_nnz * 16) * std::numeric_limits::epsilon(); - T rtol = (T) (vec_nnz) * std::numeric_limits::epsilon(); - auto msg = RandBLAS::testing::buffs_approx_equal( - dr.data(), dm.data(), d * m, __PRETTY_FUNCTION__, __FILE__, __LINE__, atol, rtol - ); - if (msg.size() > 0) FAIL() << msg; - return; - } }; TEST_F(TestSparseSkOpConstruction, respect_ownership) { @@ -400,22 +353,6 @@ TEST_F(TestSparseSkOpConstruction, fill_unpacked_sub_laso_int32) { } } -TEST_F(TestSparseSkOpConstruction, regular_laso_preserves_operator) { - // Small dim_major (7) with vec_nnz up to 7 guarantees frequent collisions, so the - // regular split path (duplicate records) is heavily exercised, not just the no-op - // vec_nnz==1 case. Cover wide and tall, double and float, int64 and int32 indices. - for (auto key : keys) { - for (int64_t vnz : {(int64_t) 2, (int64_t) 3, (int64_t) 7}) { - regular_laso_preserves_operator(7, 20, vnz, key); // wide - regular_laso_preserves_operator(20, 7, vnz, key); // tall - regular_laso_preserves_operator(7, 20, vnz, key); - regular_laso_preserves_operator( 20, 7, vnz, key); - } - // vec_nnz==1 degenerate case: regular and merged must be identical. - regular_laso_preserves_operator(7, 20, 1, key); - } -} - //////////////////////////////////////////////////////////////////////// // // @@ -488,71 +425,29 @@ TEST_F(TestSparseSkOpConstruction, SASO_Dim_15by7_int32) { TEST_F(TestSparseSkOpConstruction, LASO_Dim_7by20) { // vec_nnz=1 - proper_laso_construction(7, 20, 0, 0); - proper_laso_construction(7, 20, 1, 0); - proper_laso_construction(7, 20, 2, 0); - // // vec_nnz=2 - // proper_laso_construction(7, 20, 0, 1); - // proper_laso_construction(7, 20, 1, 1); - // proper_laso_construction(7, 20, 2, 1); - // // vec_nnz=3 - // proper_laso_construction(7, 20, 0, 2); - // proper_laso_construction(7, 20, 1, 2); - // proper_laso_construction(7, 20, 2, 2); - // // vec_nnz=7 - // proper_laso_construction(7, 20, 0, 3); - // proper_laso_construction(7, 20, 1, 3); - // proper_laso_construction(7, 20, 2, 3); + proper_laso_construction(7, 20, 0); + proper_laso_construction(7, 20, 1); + proper_laso_construction(7, 20, 2); } TEST_F(TestSparseSkOpConstruction, LASO_Dim_15by7) { // vec_nnz=1 - proper_laso_construction(15, 7, 0, 0); - proper_laso_construction(15, 7, 1, 0); - // // vec_nnz=2 - // proper_laso_construction(15, 7, 0, 1); - // proper_laso_construction(15, 7, 1, 1); - // // vec_nnz=3 - // proper_laso_construction(15, 7, 0, 2); - // proper_laso_construction(15, 7, 1, 2); - // // vec_nnz=7 - // proper_laso_construction(15, 7, 0, 3); - // proper_laso_construction(15, 7, 1, 3); + proper_laso_construction(15, 7, 0); + proper_laso_construction(15, 7, 1); } TEST_F(TestSparseSkOpConstruction, LASO_Dim_7by20_int32) { // vec_nnz=1 - proper_laso_construction(7, 20, 0, 0); - proper_laso_construction(7, 20, 1, 0); - proper_laso_construction(7, 20, 2, 0); - // // vec_nnz=2 - // proper_laso_construction(7, 20, 0, 1); - // proper_laso_construction(7, 20, 1, 1); - // proper_laso_construction(7, 20, 2, 1); - // // vec_nnz=3 - // proper_laso_construction(7, 20, 0, 2); - // proper_laso_construction(7, 20, 1, 2); - // proper_laso_construction(7, 20, 2, 2); - // // vec_nnz=7 - // proper_laso_construction(7, 20, 0, 3); - // proper_laso_construction(7, 20, 1, 3); - // proper_laso_construction(7, 20, 2, 3); + proper_laso_construction(7, 20, 0); + proper_laso_construction(7, 20, 1); + proper_laso_construction(7, 20, 2); } TEST_F(TestSparseSkOpConstruction, LASO_Dim_15by7_int32) { // vec_nnz=1 - proper_laso_construction(15, 7, 0, 0); - proper_laso_construction(15, 7, 1, 0); - // // vec_nnz=2 - // proper_laso_construction(15, 7, 0, 1); - // proper_laso_construction(15, 7, 1, 1); - // // vec_nnz=3 - // proper_laso_construction(15, 7, 0, 2); - // proper_laso_construction(15, 7, 1, 2); - // // vec_nnz=7 - // proper_laso_construction(15, 7, 0, 3); - // proper_laso_construction(15, 7, 1, 3); + proper_laso_construction(15, 7, 0); + proper_laso_construction(15, 7, 1); } From 30b9a55349b016b7bfa2f108f60108218a948dbf Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Sun, 28 Jun 2026 20:54:03 -0700 Subject: [PATCH 12/20] add a tiny helper to RandBLAS/base.hh --- RandBLAS/base.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RandBLAS/base.hh b/RandBLAS/base.hh index 4cc2ff94..f7f60c21 100644 --- a/RandBLAS/base.hh +++ b/RandBLAS/base.hh @@ -188,6 +188,11 @@ std::ostream &operator<<( return out; } +inline blas::Layout flipped_layout(const blas::Layout &layout_before) { + using blas::Layout; + return (layout_before == Layout::RowMajor) ? Layout::ColMajor : Layout::RowMajor; +} + /** * Stores stride information for a matrix represented as a buffer. * The intended semantics for a buffer "A" and the conceptualized From 1f2bdc19beadd46ea859563a00d2a18f439d3fce Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Sun, 28 Jun 2026 21:00:33 -0700 Subject: [PATCH 13/20] Have LSKGES and RSKGES do custom dispatching to CSC or CSR kernels (need to optimize this). Have "COO kernel" dispatch to either CSC or CSR (not very important to optimize). --- RandBLAS/skge.hh | 119 +++++++++--- RandBLAS/sparse_data/base.hh | 38 ++++ RandBLAS/sparse_data/conversions.hh | 51 +++++ RandBLAS/sparse_data/coo_spmm_impl.hh | 175 +++++++----------- RandBLAS/sparse_data/spmm_dispatch.hh | 4 +- RandBLAS/sparse_skops.hh | 7 +- .../sketch_general_performance.cc | 4 +- .../spmm_performance.cc | 2 +- test/linops/test_spmm/DevNotes.md | 5 +- 9 files changed, 263 insertions(+), 142 deletions(-) diff --git a/RandBLAS/skge.hh b/RandBLAS/skge.hh index 9bbd4e37..842888d2 100644 --- a/RandBLAS/skge.hh +++ b/RandBLAS/skge.hh @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -359,6 +360,80 @@ void rskge3( namespace RandBLAS::sparse { +// Apply the ENTIRETY of a sparse sketching operator on the LEFT by explicitly instantiating +// it in the compressed format (CSR or CSC) chosen by a heuristic, then handing that operand +// to left_spmm. op(submat(S)) is S if opS == NoTrans, else S^T; we normalize opS away with a +// share-memory transpose view (which flips the CSR<->CSC sort) so the operator is always a +// d-by-m NoTrans operand. +// +// Assumes S has no residual offsets and is in CSC or CSR sort order (never None). +// +template +void _lskges_compress_and_apply( + blas::Layout layout, blas::Op opS, blas::Op opA, int64_t d, int64_t n, int64_t m, + T alpha, const sparse_data::COOMatrix &S, + const T *A, int64_t lda, T beta, T *B, int64_t ldb +) { + using namespace RandBLAS::sparse_data; + using blas::Layout; using blas::Op; + if (opS == Op::Trans) { + // op(submat(S)) = S^T (d-by-m); the transpose view shares S's buffers. + auto St = transpose_as_coo(S, true); + _lskges_compress_and_apply(layout, Op::NoTrans, opA, d, n, m, alpha, St, A, lda, beta, B, ldb); + return; + } + // left_spmm sees opB = opA (the dense operand's op); recover its post-op layout. + Layout layout_opB = (opA == Op::NoTrans) ? layout : flipped_layout(layout); + NonzeroSort fmt = coo::coo_spmm_target_format(layout_opB, layout, d, m); + // The view borrows S's buffers; S and "ptr" outlive the left_spmm call. + std::vector ptr; + if (fmt == NonzeroSort::CSR) { + auto M = coo_to_csr_view_or_copy(S, ptr); + left_spmm(layout, Op::NoTrans, opA, d, n, m, alpha, M, 0, 0, A, lda, beta, B, ldb); + } else { + auto M = coo_to_csc_view_or_copy(S, ptr); + left_spmm(layout, Op::NoTrans, opA, d, n, m, alpha, M, 0, 0, A, lda, beta, B, ldb); + } +} + +// Apply the ENTIRETY of a sparse sketching operator on the RIGHT, analogous to +// _lskges_compress_and_apply. op(submat(S)) is the n-by-d operator (S if opS == NoTrans, else +// S^T). right_spmm reduces to a transposed left_spmm, which flips the operand CSR<->CSC; so to +// make that transpose land on the format the heuristic wants (computed in the reduced NoTrans +// frame, where the operator is S^T of shape d-by-n under the transposed layout) we instantiate +// in the OPPOSITE format. +// +// Assumes S has no residual offsets and is in CSC or CSR sort order (never None). +// +template +void _rskges_compress_and_apply( + blas::Layout layout, blas::Op opS, blas::Op opA, int64_t m, int64_t d, int64_t n, + T alpha, const T *A, int64_t lda, const sparse_data::COOMatrix &S, + T beta, T *B, int64_t ldb +) { + using namespace RandBLAS::sparse_data; + using blas::Layout; using blas::Op; + if (opS == Op::Trans) { + // op(submat(S)) = S^T (n-by-d); the transpose view shares S's buffers. + auto St = transpose_as_coo(S, true); + _rskges_compress_and_apply(layout, Op::NoTrans, opA, m, d, n, alpha, A, lda, St, beta, B, ldb); + return; + } + Layout trans_layout = flipped_layout(layout); + Layout layout_opB = (opA == Op::NoTrans) ? trans_layout : layout; + NonzeroSort reduced = coo::coo_spmm_target_format(layout_opB, trans_layout, d, n); + NonzeroSort fmt = (reduced == NonzeroSort::CSR) ? NonzeroSort::CSC : NonzeroSort::CSR; + // The view borrows S's buffers; S and "ptr" outlive the right_spmm call. + std::vector ptr; + if (fmt == NonzeroSort::CSR) { + auto M = coo_to_csr_view_or_copy(S, ptr); + right_spmm(layout, opA, Op::NoTrans, m, d, n, alpha, A, lda, M, 0, 0, beta, B, ldb); + } else { + auto M = coo_to_csc_view_or_copy(S, ptr); + right_spmm(layout, opA, Op::NoTrans, m, d, n, alpha, A, lda, M, 0, 0, beta, B, ldb); + } +} + // MARK: LSKGES // ============================================================================= @@ -469,7 +544,7 @@ void lskges( blas::Op opA, int64_t d, // B is d-by-n int64_t n, // \op(A) is m-by-n - int64_t m, // \op(\mtxS) is d-by-m + int64_t m, // \op(submat(S)) is d-by-m T alpha, const SparseSkOp &S, int64_t ro_s, @@ -480,19 +555,20 @@ void lskges( T *B, int64_t ldb ) { + auto [n_rows, n_cols] = dims_before_op(d, m, opS); if (S.nnz < 0) { - // The operator hasn't been sampled yet. Generate ONLY the needed submatrix of S - // (op(submat(S)) is d-by-m). submat(S) is d-by-m if opS == NoTrans, else m-by-d. - // We then call left_spmm with offsets (0,0); it applies opS itself. - auto Ssub = submatrix_as_coo(S, - (opS == blas::Op::NoTrans) ? d : m, - (opS == blas::Op::NoTrans) ? m : d, - ro_s, co_s); - left_spmm(layout, opS, opA, d, n, m, alpha, Ssub, 0, 0, A, lda, beta, B, ldb); + auto Ssub = submatrix_as_coo(S, n_rows, n_cols, ro_s, co_s); + _lskges_compress_and_apply(layout, opS, opA, d, n, m, alpha, Ssub, A, lda, beta, B, ldb); return; } + bool full_operator = (S.n_rows == n_rows && S.n_cols == n_cols); auto Scoo = coo_view_of_skop(S); - left_spmm(layout, opS, opA, d, n, m, alpha, Scoo, ro_s, co_s, A, lda, beta, B, ldb); + if (full_operator) { + _lskges_compress_and_apply(layout, opS, opA, d, n, m, alpha, Scoo, A, lda, beta, B, ldb); + } else { + // A proper submatrix of an already-sampled operator: let left_spmm handle the offsets. + left_spmm(layout, opS, opA, d, n, m, alpha, Scoo, ro_s, co_s, A, lda, beta, B, ldb); + } return; } @@ -606,7 +682,7 @@ inline void rskges( blas::Op opA, blas::Op opS, int64_t m, // B is m-by-d - int64_t d, // op(S) is n-by-d + int64_t d, // op(submat(S)) is n-by-d int64_t n, // op(A) is m-by-n T alpha, const T *A, @@ -618,21 +694,20 @@ inline void rskges( T *B, int64_t ldb ) { + auto [n_rows, n_cols] = dims_before_op(n, d, opS); if (S.nnz < 0) { - // The operator hasn't been sampled yet. Generate ONLY the needed submatrix of S - // (op(submat(S)) is n-by-d). submat(S) is n-by-d if opS == NoTrans, else d-by-n. - // We then call right_spmm with offsets (0,0); it applies opS itself. - auto Ssub = submatrix_as_coo(S, - (opS == blas::Op::NoTrans) ? n : d, - (opS == blas::Op::NoTrans) ? d : n, - ro_s, co_s); - right_spmm(layout, opA, opS, m, d, n, alpha, A, lda, Ssub, 0, 0, beta, B, ldb); + auto Ssub = submatrix_as_coo(S, n_rows, n_cols, ro_s, co_s); + _rskges_compress_and_apply(layout, opS, opA, m, d, n, alpha, A, lda, Ssub, beta, B, ldb); return; } + bool full_operator = (S.n_rows == n_rows && S.n_cols == n_cols); auto Scoo = coo_view_of_skop(S); - right_spmm( - layout, opA, opS, m, d, n, alpha, A, lda, Scoo, ro_s, co_s, beta, B, ldb - ); + if (full_operator) { + _rskges_compress_and_apply(layout, opS, opA, m, d, n, alpha, A, lda, Scoo, beta, B, ldb); + } else { + // A proper submatrix of an already-sampled operator: let right_spmm handle the offsets. + right_spmm(layout, opA, opS, m, d, n, alpha, A, lda, Scoo, ro_s, co_s, beta, B, ldb); + } return; } diff --git a/RandBLAS/sparse_data/base.hh b/RandBLAS/sparse_data/base.hh index 7eb4df2a..40cf8322 100644 --- a/RandBLAS/sparse_data/base.hh +++ b/RandBLAS/sparse_data/base.hh @@ -31,6 +31,7 @@ #include "RandBLAS/config.h" #include "RandBLAS/base.hh" #include +#include #ifdef __cpp_concepts #include @@ -117,6 +118,16 @@ enum class NonzeroSort : char { None = 'N' }; +inline NonzeroSort flipped_nonzerosort(const NonzeroSort &s) { + if (s == NonzeroSort::CSC) { + return NonzeroSort::CSR; + } else if (s == NonzeroSort::CSR) { + return NonzeroSort::CSC; + } else { + return NonzeroSort::None; + } +} + template static inline bool increasing_by_csr(sint_t i0, sint_t j0, sint_t i1, sint_t j1) { if (i0 > i1) { @@ -300,6 +311,33 @@ void compressed_ptr_to_sorted_idxs(int64_t num_comp, sint_t1* ptr, int64_t len_i } } +// Build a compressed layout (CSR or CSC) from a COO that is already sorted in the +// OPPOSITE compressed order, via an O(nnz + dim) counting-sort transpose -- cheaper +// than the O(nnz log nnz) comparison re-sort that COOMatrix::sort_arrays would do. +// Because the source is iterated in its (oppositely-)sorted order, the companion +// index comes out ascending within each group, so the result is properly sorted in +// the target order too. +// group_idx : axis to compress on (rows for a CSR target, cols for CSC); length nnz +// other_idx : the companion axis (cols for CSR, rows for CSC); length nnz +// dim : number of groups (n_rows for CSR, n_cols for CSC) +// Outputs (caller-allocated): ptr[dim+1], out_other[nnz], out_vals[nnz]. +template +void counting_sort_transpose( + int64_t nnz, const sint_t* group_idx, const sint_t* other_idx, const T* vals, + int64_t dim, sint_t* ptr, sint_t* out_other, T* out_vals +) { + for (int64_t g = 0; g <= dim; ++g) ptr[g] = 0; + for (int64_t e = 0; e < nnz; ++e) ptr[group_idx[e] + 1] += 1; + for (int64_t g = 0; g < dim; ++g) ptr[g + 1] += ptr[g]; + std::vector cursor(ptr, ptr + dim); // running write position per group + for (int64_t e = 0; e < nnz; ++e) { + sint_t g = group_idx[e]; + sint_t pos = cursor[g]++; + out_other[pos] = other_idx[e]; + out_vals[pos] = vals[e]; + } +} + // MARK: SparseMatrix #ifdef __cpp_concepts diff --git a/RandBLAS/sparse_data/conversions.hh b/RandBLAS/sparse_data/conversions.hh index 77a5a7c6..7353709d 100644 --- a/RandBLAS/sparse_data/conversions.hh +++ b/RandBLAS/sparse_data/conversions.hh @@ -34,6 +34,7 @@ #include "RandBLAS/sparse_data/coo_matrix.hh" #include "RandBLAS/sparse_data/csr_matrix.hh" #include "RandBLAS/sparse_data/csc_matrix.hh" +#include namespace RandBLAS::sparse_data { @@ -90,6 +91,14 @@ auto coo_to_csc( const COOMatrix &coo, CSCMatrix &csc ) { std::copy( coo.vals, coo.vals + coo.nnz, csc.vals ); std::copy( coo.rows, coo.rows + coo.nnz, csc.rowidxs ); return; + } else if (coo.sort == NonzeroSort::CSR) { + // Opposing order: group the CSR-sorted records by column with an O(nnz) + // counting-sort transpose instead of a comparison re-sort. + csc.reserve(coo.nnz); + counting_sort_transpose( + coo.nnz, coo.cols, coo.rows, coo.vals, coo.n_cols, csc.colptr, csc.rowidxs, csc.vals + ); + return; } else { auto coo_copy = coo.deepcopy(); coo_copy.sort_arrays(NonzeroSort::CSC); @@ -112,6 +121,14 @@ void coo_to_csr( const COOMatrix &coo, CSRMatrix &csr ) { std::copy( coo.vals, coo.vals + coo.nnz, csr.vals ); std::copy( coo.cols, coo.cols + coo.nnz, csr.colidxs ); return; + } else if (coo.sort == NonzeroSort::CSC) { + // Opposing order: group the CSC-sorted records by row with an O(nnz) + // counting-sort transpose instead of a comparison re-sort. + csr.reserve(coo.nnz); + counting_sort_transpose( + coo.nnz, coo.rows, coo.cols, coo.vals, coo.n_rows, csr.rowptr, csr.colidxs, csr.vals + ); + return; } else { auto coo_copy = coo.deepcopy(); coo_copy.sort_arrays(NonzeroSort::CSR); @@ -120,6 +137,40 @@ void coo_to_csr( const COOMatrix &coo, CSRMatrix &csr ) { } } +// Represent "coo" as a CSR matrix without copying its structural nonzeros when possible. +// If coo is already CSR-sorted (or empty), the result is a ZERO-COPY view that shares coo's +// vals and cols arrays; its rowptr is written into the caller-provided "rowptr" scratch buffer +// (resized as needed). In that case the returned matrix borrows memory: it is valid only while +// BOTH coo and rowptr stay alive, and should be treated as const (like the transpose_as_* views +// above). Otherwise -- opposite (CSC) sort order, taking the O(nnz) counting-sort transpose, or +// an unsorted COO -- the result is a freshly converted, memory-owning matrix and rowptr is +// left untouched. +template +CSRMatrix coo_to_csr_view_or_copy(const COOMatrix &coo, std::vector &rowptr) { + if (coo.nnz == 0 || coo.sort == NonzeroSort::CSR) { + rowptr.resize(coo.n_rows + 1); + sorted_idxs_to_compressed_ptr(coo.nnz, coo.rows, coo.n_rows, rowptr.data()); + return CSRMatrix(coo.n_rows, coo.n_cols, coo.nnz, coo.vals, rowptr.data(), coo.cols, coo.index_base); + } + CSRMatrix csr(coo.n_rows, coo.n_cols); + coo_to_csr(coo, csr); + return csr; +} + +// CSC analog of coo_to_csr_view_or_copy: the zero-copy view shares coo's vals and rows arrays, +// and writes its colptr into the caller-provided "colptr" scratch buffer. Same lifetime contract. +template +CSCMatrix coo_to_csc_view_or_copy(const COOMatrix &coo, std::vector &colptr) { + if (coo.nnz == 0 || coo.sort == NonzeroSort::CSC) { + colptr.resize(coo.n_cols + 1); + sorted_idxs_to_compressed_ptr(coo.nnz, coo.cols, coo.n_cols, colptr.data()); + return CSCMatrix(coo.n_rows, coo.n_cols, coo.nnz, coo.vals, coo.rows, colptr.data(), coo.index_base); + } + CSCMatrix csc(coo.n_rows, coo.n_cols); + coo_to_csc(coo, csc); + return csc; +} + // MARK: transposes // // These functions' return values should be treated as const if share_memory = true. diff --git a/RandBLAS/sparse_data/coo_spmm_impl.hh b/RandBLAS/sparse_data/coo_spmm_impl.hh index 341e50f8..aad21b18 100644 --- a/RandBLAS/sparse_data/coo_spmm_impl.hh +++ b/RandBLAS/sparse_data/coo_spmm_impl.hh @@ -50,35 +50,51 @@ using RandBLAS::SignedInteger; #endif -// Build a compressed layout (CSR or CSC) from a COO that is already sorted in the -// OPPOSITE compressed order, via an O(nnz + dim) counting-sort transpose -- cheaper -// than the O(nnz log nnz) comparison re-sort that COOMatrix::sort_arrays would do. -// Because the source is iterated in its (oppositely-)sorted order, the companion -// index comes out ascending within each group, so the result is properly sorted in -// the target order too. -// group_idx : axis to compress on (rows for a CSR target, cols for CSC); length nnz -// other_idx : the companion axis (cols for CSR, rows for CSC); length nnz -// dim : number of groups (n_rows for CSR, n_cols for CSC) -// Outputs (caller-allocated): ptr[dim+1], out_other[nnz], out_vals[nnz]. +// Format choice for the COO->compressed SpMM dispatch. The per-RHS-column kernels loop +// the operator's OUTER dimension once per output column: CSC jki's outer loop is n_cols +// (= m), CSR jik's is n_rows (= d). In the ColMajor path, routing a WIDE operator (d < m) +// through the CSR jik kernel (shorter outer loop) measured 1.1-2x faster than CSC jki for +// both SASO and LASO, with no regression -- so we prefer CSR there. The RowMajor path keeps +// the bandwidth-saturating CSC kib axpy kernel. See the --csr-probe mode of +// examples/.../sketch_general_performance.cc for measurements. +inline NonzeroSort coo_spmm_target_format( + blas::Layout layout_opB, blas::Layout layout_C, int64_t d, int64_t m +) { + bool col_major = (layout_opB == blas::Layout::ColMajor && layout_C == blas::Layout::ColMajor); + bool prefer_csr = col_major && (d < m); + return prefer_csr ? NonzeroSort::CSR : NonzeroSort::CSC; +} + +// Restrict A0 to the requested d-by-m window at (ro_a, co_a): copy the in-window nonzeros into +// a fresh, memory-owning COO, shifted to local coordinates. We over-allocate to A0.nnz (the +// worst case) so the compaction is a single pass; the result's logical nnz is the in-window +// count, and its sort label is recomputed so the conversion below can hit a fast path. template -static void counting_sort_transpose( - int64_t nnz, const sint_t* group_idx, const sint_t* other_idx, const T* vals, - int64_t dim, sint_t* ptr, sint_t* out_other, T* out_vals +static COOMatrix restrict_to_window( + const COOMatrix &A0, int64_t d, int64_t m, int64_t ro_a, int64_t co_a ) { - for (int64_t g = 0; g <= dim; ++g) ptr[g] = 0; - for (int64_t e = 0; e < nnz; ++e) ptr[group_idx[e] + 1] += 1; - for (int64_t g = 0; g < dim; ++g) ptr[g + 1] += ptr[g]; - std::vector cursor(ptr, ptr + dim); // running write position per group - for (int64_t e = 0; e < nnz; ++e) { - sint_t g = group_idx[e]; - sint_t pos = cursor[g]++; - out_other[pos] = other_idx[e]; - out_vals[pos] = vals[e]; + COOMatrix sub(d, m); + if (A0.nnz == 0) + return sub; + sub.reserve(A0.nnz); + int64_t write = 0; + for (int64_t i = 0; i < A0.nnz; ++i) { + auto r = A0.rows[i] - ro_a; + auto c = A0.cols[i] - co_a; + if (0 <= r && r < d && 0 <= c && c < m) { + sub.rows[write] = r; + sub.cols[write] = c; + sub.vals[write] = A0.vals[i]; + write += 1; + } } + sub.nnz = write; + sub.sort = coo_arrays_determine_sort(sub.nnz, sub.rows, sub.cols); + return sub; } template -static void apply_coo_via_csc( +static void apply_coo_via_csx( T alpha, blas::Layout layout_B, blas::Layout layout_C, @@ -95,97 +111,36 @@ static void apply_coo_via_csc( ) { randblas_require(A0.index_base == IndexBase::Zero); - // Format choice (PROTOTYPE). The per-RHS-column kernels loop the operator's OUTER - // dimension once per output column: CSC jki's outer loop is n_cols (= m), CSR jik's - // is n_rows (= d). In the ColMajor path, routing a WIDE operator (d < m) through the - // CSR jik kernel (shorter outer loop) measured 1.1-2x faster than CSC jki for both - // SASO and LASO, with no regression -- so we prefer CSR there. The RowMajor path - // keeps the bandwidth-saturating CSC kib axpy kernel. See the --csr-probe mode of - // examples/.../sketch_general_performance.cc for measurements. - bool col_major = (layout_B == blas::Layout::ColMajor && layout_C == blas::Layout::ColMajor); - bool prefer_csr = col_major && (d < m); - NonzeroSort target = prefer_csr ? NonzeroSort::CSR : NonzeroSort::CSC; - + // The exact d-by-m operator: A0 itself when it already has those dimensions, else an owning + // copy restricted to the window. Binding "op" by reference lets the common full-operator + // path use A0 (with its existing sort) at zero cost; "windowed" is then an empty placeholder. bool submatrix = (A0.n_rows != d) || (A0.n_cols != m); + COOMatrix windowed = submatrix + ? restrict_to_window(A0, d, m, ro_a, co_a) + : COOMatrix(0, 0); + const COOMatrix &op = submatrix ? windowed : A0; + if (op.nnz == 0) + return; // structurally-zero operator; C was already scaled by beta upstream. - // Fast transpose path: full operator already sorted in the OPPOSITE compressed - // order (e.g. a wide SASO sampled CSC-sorted, but the ColMajor path wants CSR). - // Build the wanted format via an O(nnz) counting-sort transpose instead of the - // O(nnz log nnz) comparison re-sort + recurse below. This removes the per-apply - // re-sort tax for operators whose sampled order doesn't match the chosen format. - if (!submatrix && A0.sort != target && A0.sort != NonzeroSort::None) { - if (prefer_csr) { // A0 is CSC-sorted; transpose to CSR (group by row). - sint_t* rowptr = new sint_t[d + 1]; - sint_t* colidxs = new sint_t[A0.nnz]; - T* csrvals = new T[A0.nnz]; - counting_sort_transpose(A0.nnz, A0.rows, A0.cols, A0.vals, d, rowptr, colidxs, csrvals); - CSRMatrix A_csr(d, m, A0.nnz, csrvals, rowptr, colidxs); - using RandBLAS::sparse_data::csr::apply_csr_jik_p11; - apply_csr_jik_p11(alpha, layout_B, layout_C, d, n, m, A_csr, B, ldb, C, ldc); - delete [] rowptr; delete [] colidxs; delete [] csrvals; - } else { // A0 is CSR-sorted; transpose to CSC (group by col). - sint_t* colptr = new sint_t[m + 1]; - sint_t* rowidxs = new sint_t[A0.nnz]; - T* cscvals = new T[A0.nnz]; - counting_sort_transpose(A0.nnz, A0.cols, A0.rows, A0.vals, m, colptr, rowidxs, cscvals); - CSCMatrix A_csc(d, m, A0.nnz, cscvals, rowidxs, colptr); - if (layout_B == layout_C && layout_B == blas::Layout::RowMajor) { - using RandBLAS::sparse_data::csc::apply_csc_kib_1p1_rowmajor; - apply_csc_kib_1p1_rowmajor(alpha, n, A_csc, B, ldb, C, ldc); - } else { - using RandBLAS::sparse_data::csc::apply_csc_jki_p11; - apply_csc_jki_p11(alpha, layout_B, layout_C, n, A_csc, B, ldb, C, ldc); - } - delete [] colptr; delete [] rowidxs; delete [] cscvals; - } - return; - } - - if (submatrix || A0.sort != target) { - auto A1 = A0.deepcopy(); - auto new_nnz = A1.nnz; - if (submatrix) { - int64_t write = 0; - for (int64_t i = 0; i < A1.nnz; ++i) { - auto r = A1.rows[i] - ro_a; - auto c = A1.cols[i] - co_a; - if (0 <= r && r < d && 0 <= c && c < m) { - A1.rows [write] = r; - A1.cols [write] = c; - A1.vals [write] = A1.vals[i]; - write += 1; - } - } - new_nnz = write; - } - COOMatrix A2(d, m, new_nnz, A1.vals, A1.rows, A1.cols, false); - A2.sort_arrays(target); - apply_coo_via_csc(alpha, layout_B, layout_C, d, n, m, A2, 0, 0, B, ldb, C, ldc); - return; - } - - if (prefer_csr) { - // ColMajor + wide: CSR jik (outer loop over the d rows). - auto rowptr = new sint_t[d+1]; - sorted_idxs_to_compressed_ptr( A0.nnz, A0.rows, d, rowptr ); - CSRMatrix A_csr( d, m, A0.nnz, A0.vals, rowptr, A0.cols ); - using RandBLAS::sparse_data::csr::apply_csr_jik_p11; - apply_csr_jik_p11(alpha, layout_B, layout_C, d, n, m, A_csr, B, ldb, C, ldc); - delete [] rowptr; - return; - } - - auto colptr = new sint_t[m+1]; - sorted_idxs_to_compressed_ptr( A0.nnz, A0.cols, m, colptr ); - CSCMatrix A_csc( d, m, A0.nnz, A0.vals, A0.rows, colptr ); - if (layout_B == layout_C && layout_B == blas::Layout::RowMajor) { - using RandBLAS::sparse_data::csc::apply_csc_kib_1p1_rowmajor; - apply_csc_kib_1p1_rowmajor(alpha, n, A_csc, B, ldb, C, ldc); + // Choose the compressed format by the dispatch heuristic, materialize "op" in that format + // (a zero-copy view when its sort already matches, else an O(nnz) conversion), and call the + // matching per-RHS-column kernel -- exactly the kernels left_spmm uses for a native CSR/CSC + // operand. coo_spmm_target_format only selects CSR for ColMajor/ColMajor, which is why only + // apply_csr_jik_p11 appears here; CSC is the catch-all and keeps its RowMajor axpy kernel. + // Keep this in sync with coo_spmm_target_format and with left_spmm's native dispatch. + NonzeroSort target = coo_spmm_target_format(layout_B, layout_C, d, m); + std::vector ptr; // backs a zero-copy view's ptr array; must outlive the kernel call + if (target == NonzeroSort::CSR) { + auto M = RandBLAS::sparse_data::coo_to_csr_view_or_copy(op, ptr); + csr::apply_csr_jik_p11(alpha, layout_B, layout_C, d, n, m, M, B, ldb, C, ldc); } else { - using RandBLAS::sparse_data::csc::apply_csc_jki_p11; - apply_csc_jki_p11(alpha, layout_B, layout_C, n, A_csc, B, ldb, C, ldc); + auto M = RandBLAS::sparse_data::coo_to_csc_view_or_copy(op, ptr); + if (layout_B == layout_C && layout_B == blas::Layout::RowMajor) { + csc::apply_csc_kib_1p1_rowmajor(alpha, n, M, B, ldb, C, ldc); + } else { + csc::apply_csc_jki_p11(alpha, layout_B, layout_C, n, M, B, ldb, C, ldc); + } } - delete [] colptr; return; } diff --git a/RandBLAS/sparse_data/spmm_dispatch.hh b/RandBLAS/sparse_data/spmm_dispatch.hh index 2c6344fe..30bf9a82 100644 --- a/RandBLAS/sparse_data/spmm_dispatch.hh +++ b/RandBLAS/sparse_data/spmm_dispatch.hh @@ -148,8 +148,8 @@ void left_spmm( // Fallback: hand-rolled sparse kernels. if constexpr (is_coo) { - using RandBLAS::sparse_data::coo::apply_coo_via_csc; - apply_coo_via_csc(alpha, layout_opB, layout_C, d, n, m, A, ro_a, co_a, B, ldb, C, ldc); + using RandBLAS::sparse_data::coo::apply_coo_via_csx; + apply_coo_via_csx(alpha, layout_opB, layout_C, d, n, m, A, ro_a, co_a, B, ldb, C, ldc); } else if constexpr (is_csc) { if (layout_opB == Layout::RowMajor && layout_C == Layout::RowMajor) { using RandBLAS::sparse_data::csc::apply_csc_kib_1p1_rowmajor; diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index a6ecfd8a..d86bb0d0 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -648,7 +648,7 @@ state_t fill_sparse_unpacked( // Emit each major-axis vector in ascending major-coordinate order. Fisher-Yates // draws the vec_nnz nonzeros in shuffle order; sorting each contiguous block by // its major coordinate (rows for a wide SASO, cols for a tall one) makes the COO - // natively CSC- (wide) / CSR- (tall) sorted, so apply_coo_via_csc skips its + // natively CSC- (wide) / CSR- (tall) sorted, so apply_coo_via_csx skips its // per-apply deepcopy + re-sort. idxs_minor is constant within a block, so only // (idxs_major, vals) move. vec_nnz is small, so a no-alloc insertion sort is best. for (int64_t b = 0; b < num_major_sub; ++b) { @@ -884,7 +884,10 @@ COOMatrix submatrix_as_coo( A.rows = rows; A.cols = cols; A.nnz = nnz; - A.sort = RandBLAS::sparse_data::NonzeroSort::None; + // fill_sparse_unpacked emits each major-axis vector in sorted order, so the sampled + // submatrix is CSR- or CSC-sorted; label it as such (rather than None) so downstream + // conversions can take an O(nnz) fast path. + A.sort = RandBLAS::sparse_data::coo_arrays_determine_sort(A.nnz, A.rows, A.cols); return A; } diff --git a/examples/simple-kernel-benchmarks/sketch_general_performance.cc b/examples/simple-kernel-benchmarks/sketch_general_performance.cc index 8183c658..d1e17e78 100644 --- a/examples/simple-kernel-benchmarks/sketch_general_performance.cc +++ b/examples/simple-kernel-benchmarks/sketch_general_performance.cc @@ -41,7 +41,7 @@ // // The cost of a sparse sketch has three phases; we time them separately: // 1. SAMPLE fill_sparse(S): populate (rows, cols, vals). -// 2. CONVERT the COO->CSC sort inside apply_coo_via_csc (deepcopy + re-sort, +// 2. CONVERT the COO->CSC sort inside apply_coo_via_csx (deepcopy + re-sort, // incurred every apply when the operator's COO is not CSC-sorted). // 3. KERNEL apply_csc_jki_p11 (ColMajor) / apply_csc_kib_1p1_rowmajor (RowMajor). // WARM apply (S pre-sampled; phases 2+3) is the primary number; COLD (1+2+3) and @@ -477,7 +477,7 @@ void run_scaling(int64_t d, int64_t m, int64_t n, const std::vector& spe // // Finding: CSR jik (ColMajor) beats CSC jki (ColMajor) 1.1-2x for BOTH SASO and // LASO with no regression, motivating the ColMajor "prefer CSR for wide -// operators" routing now in apply_coo_via_csc (coo_spmm_impl.hh). (A strided-axpy +// operators" routing now in apply_coo_via_csx (coo_spmm_impl.hh). (A strided-axpy // "ColMajor kib" kernel was also tried and rejected -- it regressed dense-column // SASO ~3x.) The probe times the kernels directly, so it is unaffected by that // routing change and remains the A/B harness for it. diff --git a/examples/simple-kernel-benchmarks/spmm_performance.cc b/examples/simple-kernel-benchmarks/spmm_performance.cc index 84c74e92..ff3ab37e 100644 --- a/examples/simple-kernel-benchmarks/spmm_performance.cc +++ b/examples/simple-kernel-benchmarks/spmm_performance.cc @@ -12,7 +12,7 @@ // // ColMajor dense -> apply_csr_jik_p11 / apply_csc_jki_p11 // RowMajor dense -> apply_csr_ikb_p1b_rowmajor / apply_csc_kib_1p1_rowmajor -// (COO delegates to the CSC kernels in both layouts, via apply_coo_via_csc) +// (COO delegates to the CSC kernels in both layouts, via apply_coo_via_csx) // // A fourth combination -- op(B)=Trans (computing C = A * B^T, the dense operand // fed transposed) -- also routes to apply_csr_jik_p11 / apply_csc_jki_p11, but diff --git a/test/linops/test_spmm/DevNotes.md b/test/linops/test_spmm/DevNotes.md index 5c675463..e46e888a 100644 --- a/test/linops/test_spmm/DevNotes.md +++ b/test/linops/test_spmm/DevNotes.md @@ -45,8 +45,7 @@ The effective layout for reading the dense operand B (`layout_opB`) is determine From there, our next steps are format-dependent. COO format ([spmm_dispatch.hh:124-126](../../../RandBLAS/sparse_data/spmm_dispatch.hh#L124-L126)): -- Always uses `apply_coo_via_csc` (converts to CSC internally) -- 1 kernel handles all 4 (layout × opB) combinations +- Always uses `apply_coo_via_csx` (converts to CSR or CSC internally, at its own discretion). CSC format ([spmm_dispatch.hh:128-134](../../../RandBLAS/sparse_data/spmm_dispatch.hh#L128-L134)): - If `layout_opB == RowMajor && layout_C == RowMajor`: `apply_csc_kib_1p1_rowmajor` @@ -114,7 +113,7 @@ For completeness, here's a grouping of tests based on the kernel they hit. | Kernel | Direct Tests | Via Transformation | |--------|--------------|-------------------| -| `apply_coo_via_csc` | All COO tests | CSR/CSC transpose_self | +| `apply_coo_via_csx` | All COO tests | CSR/CSC transpose_self | | `apply_csc_jki_p11` | CSC ColMajor (both opB), CSC RowMajor + opB=Trans | CSR transpose_self | | `apply_csc_kib_1p1_rowmajor` | CSC RowMajor + opB=NoTrans | CSR transpose_self + RowMajor | | `apply_csr_jik_p11` | CSR ColMajor (both opB), CSR RowMajor + opB=Trans | CSC transpose_self | From abdda96a31970438a2657d2d32b1fc2c4620d79e Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Sun, 28 Jun 2026 21:37:25 -0700 Subject: [PATCH 14/20] remove poorly-motivated kernels --- RandBLAS/sparse_data/coo_matrix.hh | 7 +- RandBLAS/sparse_data/csc_spmm_impl.hh | 49 ++--------- RandBLAS/sparse_data/csr_spmm_impl.hh | 86 ++----------------- .../sketch_general_performance.cc | 42 +-------- sketch_general_results.txt | 22 +---- test/linops/test_spmm/test_spmm_csr.cc | 35 ++------ 6 files changed, 26 insertions(+), 215 deletions(-) diff --git a/RandBLAS/sparse_data/coo_matrix.hh b/RandBLAS/sparse_data/coo_matrix.hh index 6c8e2123..33b15f88 100644 --- a/RandBLAS/sparse_data/coo_matrix.hh +++ b/RandBLAS/sparse_data/coo_matrix.hh @@ -396,10 +396,9 @@ void coo_to_dense(const COOMatrix &spmat, int64_t stride_row, int64_t stride_ j -= 1; } // Accumulate (not overwrite): a COO may carry several records at the same - // (i, j) -- e.g. a "regular" long-axis-major sketching operator stores a - // collided coordinate as duplicate entries that must sum to its true value. - // The matrix was zeroed above, so += is correct, and for a deduplicated COO - // (each (i,j) once) it is identical to an assignment. + // (i, j), which must sum to that entry's true value. The matrix was zeroed + // above, so += is correct, and for a deduplicated COO (each (i,j) once) it + // is identical to an assignment. MAT(i, j) += spmat.vals[ell]; } return; diff --git a/RandBLAS/sparse_data/csc_spmm_impl.hh b/RandBLAS/sparse_data/csc_spmm_impl.hh index 8d2509f2..1f85e4de 100644 --- a/RandBLAS/sparse_data/csc_spmm_impl.hh +++ b/RandBLAS/sparse_data/csc_spmm_impl.hh @@ -71,30 +71,6 @@ static void apply_csc_to_vector_ki( } } -template -static void apply_regular_csc_to_vector_ki( - T alpha, - // data for "regular CSC": CSC with fixed nnz per col, - // which obviates the requirement for colptr. - const T *vals, - const sint_t *rowidxs, - int64_t col_nnz, - // input-output vector data - int64_t len_v, - const T *v, - int64_t incv, // stride between elements of v - T *Av, // Av += A * v. - int64_t incAv // stride between elements of Av -) { - for (int64_t c = 0; c < len_v; ++c) { - T scale = alpha * v[c * incv]; - for (int64_t j = c * col_nnz; j < (c + 1) * col_nnz; ++j) { - int64_t row = rowidxs[j]; - Av[row * incAv] += (vals[j] * scale); - } - } -} - template static void apply_csc_jki_p11( T alpha, @@ -111,10 +87,6 @@ static void apply_csc_jki_p11( auto m = A.n_cols; - bool fixed_nnz_per_col = true; - for (int64_t ell = 2; (ell < m + 1) && fixed_nnz_per_col; ++ell) - fixed_nnz_per_col = (A.colptr[1] + A.colptr[ell-1]) == A.colptr[ell]; - auto s = layout_to_strides(layout_B, ldb); auto B_inter_col_stride = s.inter_col_stride; auto B_inter_row_stride = s.inter_row_stride; @@ -127,21 +99,12 @@ static void apply_csc_jki_p11( for (int64_t j = 0; j < n; j++) { const T* B_col = &B[B_inter_col_stride * j]; T* C_col = &C[C_inter_col_stride * j]; - if (fixed_nnz_per_col) { - apply_regular_csc_to_vector_ki( - alpha, - A.vals, A.rowidxs, A.colptr[1], - m, B_col, B_inter_row_stride, - C_col, C_inter_row_stride - ); - } else { - apply_csc_to_vector_ki( - alpha, - A.vals, A.rowidxs, A.colptr, - m, B_col, B_inter_row_stride, - C_col, C_inter_row_stride - ); - } + apply_csc_to_vector_ki( + alpha, + A.vals, A.rowidxs, A.colptr, + m, B_col, B_inter_row_stride, + C_col, C_inter_row_stride + ); } return; } diff --git a/RandBLAS/sparse_data/csr_spmm_impl.hh b/RandBLAS/sparse_data/csr_spmm_impl.hh index afd466a2..0705bd4c 100644 --- a/RandBLAS/sparse_data/csr_spmm_impl.hh +++ b/RandBLAS/sparse_data/csr_spmm_impl.hh @@ -107,55 +107,6 @@ static void apply_csr_to_vector_ik( } } -// Regular-CSR SpMV: every row has the same nnz (row_nnz), so rowptr is implicit -// (rowptr[i] == i*row_nnz) and need not be read. This is the CSR analogue of -// apply_regular_csc_to_vector_ki and is reached by short-axis-major operators (wide -// SASO, exactly vec_nnz per row) and by "regular" long-axis-major operators. Mirrors -// apply_csr_to_vector_ik_impl: UnitStride drops index multiplies in the hot ColMajor -// case and the simd reduction breaks the serial accumulator dependence. -template -static void apply_regular_csr_to_vector_ik_impl( - T alpha, - const T *vals, const int64_t row_nnz, const sint_t *colidxs, - const T *v, int64_t incv, - int64_t len_Av, T *Av, int64_t incAv -) { - UNUSED(incv); UNUSED(incAv); - for (int64_t i = 0; i < len_Av; ++i) { - T Av_i_diff = 0.0; - #pragma omp simd reduction(+:Av_i_diff) - for (int64_t ell = i * row_nnz; ell < (i + 1) * row_nnz; ++ell) { - int64_t j = colidxs[ell]; - if constexpr (UnitStride) { - Av_i_diff += vals[ell] * v[j]; - } else { - Av_i_diff += vals[ell] * v[j*incv]; - } - } - if constexpr (UnitStride) { - Av[i] += alpha * Av_i_diff; - } else { - Av[i*incAv] += alpha * Av_i_diff; - } - } -} - -template -static void apply_regular_csr_to_vector_ik( - T alpha, - const T *vals, int64_t row_nnz, const sint_t *colidxs, - const T *v, int64_t incv, - int64_t len_Av, T *Av, int64_t incAv -) { - if (incv == 1 && incAv == 1) { - apply_regular_csr_to_vector_ik_impl( - alpha, vals, row_nnz, colidxs, v, incv, len_Av, Av, incAv); - } else { - apply_regular_csr_to_vector_ik_impl( - alpha, vals, row_nnz, colidxs, v, incv, len_Av, Av, incAv); - } -} - template static void apply_csr_jik_p11( T alpha, @@ -175,22 +126,6 @@ static void apply_csr_jik_p11( randblas_require(d == A.n_rows); randblas_require(m == A.n_cols); -#if defined(RandBLAS_ENABLE_REGULAR_CSR_KERNEL) - // OPT-IN regular-CSR fast path (OFF by default). rowptr is an arithmetic progression - // iff every row has the same nnz; then we index nonzeros as i*row_nnz and skip the - // rowptr loads (mirrors fixed_nnz_per_col in apply_csc_jki_p11). Benchmarked - // NEUTRAL-to-SLOWER on Apple M3 / NEON -- the eliminated rowptr loads are sequential - // and cache-resident (never the bottleneck), and 2-wide doubles can't exploit the - // fixed inner trip count, so it is gated off. Left in-tree (define this macro to - // enable) for evaluation on wide-vector Arm (SVE), where a known-length predicated - // inner loop may pay off. See apply_regular_csr_to_vector_ik above and the --csr-probe - // mode of examples/.../sketch_general_performance.cc for the A/B harness. - bool fixed_nnz_per_row = (d > 0); - for (int64_t i = 2; (i < d + 1) && fixed_nnz_per_row; ++i) - fixed_nnz_per_row = (A.rowptr[1] + A.rowptr[i-1]) == A.rowptr[i]; - int64_t row_nnz = (d > 0) ? A.rowptr[1] : 0; -#endif - auto s = layout_to_strides(layout_B, ldb); auto B_inter_col_stride = s.inter_col_stride; auto B_inter_row_stride = s.inter_row_stride; @@ -203,22 +138,11 @@ static void apply_csr_jik_p11( for (int64_t j = 0; j < n; j++) { const T *B_col = &B[B_inter_col_stride * j]; T *C_col = &C[C_inter_col_stride * j]; -#if defined(RandBLAS_ENABLE_REGULAR_CSR_KERNEL) - if (fixed_nnz_per_row) { - apply_regular_csr_to_vector_ik(alpha, - vals, row_nnz, A.colidxs, - B_col, B_inter_row_stride, - d, C_col, C_inter_row_stride - ); - } else -#endif - { - apply_csr_to_vector_ik(alpha, - vals, A.rowptr, A.colidxs, - B_col, B_inter_row_stride, - d, C_col, C_inter_row_stride - ); - } + apply_csr_to_vector_ik(alpha, + vals, A.rowptr, A.colidxs, + B_col, B_inter_row_stride, + d, C_col, C_inter_row_stride + ); } return; } diff --git a/examples/simple-kernel-benchmarks/sketch_general_performance.cc b/examples/simple-kernel-benchmarks/sketch_general_performance.cc index d1e17e78..4aba663c 100644 --- a/examples/simple-kernel-benchmarks/sketch_general_performance.cc +++ b/examples/simple-kernel-benchmarks/sketch_general_performance.cc @@ -14,7 +14,7 @@ // * Wide LASO SparseDist(d, m, k, Long) -- <= k nnz per row, empty cols // // and (via the right-sketch / tall operators) the transpose reduction that maps -// a tall SASO right-sketch onto the same column-regular kernel as a wide SASO. +// a tall SASO right-sketch onto the same CSC kernel path as a wide SASO. // // WHY WORK-NORMALIZED METRICS: raw wall-time conflates "how much work" with "how // efficiently it's done". A vec_nnz=8 operator has 4x the nonzeros of vec_nnz=2, @@ -547,51 +547,11 @@ void run_csr_probe(int64_t d, int64_t m, int64_t n, rows.push_back(r); }; - // Regular-vs-general INNER-kernel A/B on the SAME (regular) CSR, ColMajor only. - // Every sketching operator is now stored regular (fixed nnz per major-axis vector), - // so this isolates the ONLY difference between the two CSR SpMV kernels: implicit - // row indexing (i*row_nnz) vs rowptr[i]/rowptr[i+1] loads. Run back-to-back to - // cancel cross-run noise -- this is the definitive "does the regular kernel pay - // off?" measurement. Both mirror apply_csr_jik_p11's parallel-over-columns shape. - int64_t row_nnz = (d > 0) ? S_csr.rowptr[1] : 0; - bool is_regular = (d > 0) && (nnz == row_nnz * d); - auto probe_inner = [&](const std::string& label, bool regular_kernel) { - auto [mn, md] = run_trials([&]() { - std::fill(C_cm.begin(), C_cm.end(), 0.0); - #pragma omp parallel for schedule(static) - for (int64_t j = 0; j < n; ++j) { - const T* B_col = &A_cm[(size_t) j * m]; - T* C_col = &C_cm[(size_t) j * d]; - if (regular_kernel) - rb::sparse_data::csr::apply_regular_csr_to_vector_ik( - 1.0, S_csr.vals, row_nnz, S_csr.colidxs, B_col, 1, d, C_col, 1); - else - rb::sparse_data::csr::apply_csr_to_vector_ik( - 1.0, S_csr.vals, S_csr.rowptr, S_csr.colidxs, B_col, 1, d, C_col, 1); - } - }, num_trials); - double maxdiff = 0; - for (int64_t i = 0; i < d * n; ++i) - maxdiff = std::max(maxdiff, std::abs(C_cm[i] - B_ref[i])); - if (maxdiff > 1e-9) - std::cout << " FAIL " << label << " max|diff|=" << std::scientific - << maxdiff << std::fixed << "\n"; - Row r; r.label = label; r.min_us = mn; r.med_us = md; - fill_metrics(r, mn, nnz, n, a_read, d * n); - r.notes = std::string("regular=") + (regular_kernel ? "Y" : "N") - + " row_nnz=" + std::to_string(row_nnz); - rows.push_back(r); - }; - print_metric_header(spec.label); probe("CSC jki ColMajor", S_csc, Layout::ColMajor); probe("CSR jik ColMajor", S_csr, Layout::ColMajor); probe("CSC kib RowMajor", S_csc, Layout::RowMajor); probe("CSR ikb RowMajor", S_csr, Layout::RowMajor); - if (is_regular) { - probe_inner("CSR REG ColMajor*", true); - probe_inner("CSR GEN ColMajor*", false); - } for (auto& r : rows) print_metric_row(r); std::cout << "\n"; } diff --git a/sketch_general_results.txt b/sketch_general_results.txt index e49d27d2..4874f14c 100644 --- a/sketch_general_results.txt +++ b/sketch_general_results.txt @@ -3,16 +3,6 @@ # wide SASO -> CSC-sorted COO, wide LASO -> CSR-sorted COO. # 2. ColMajor wide operators routed through the CSR jik kernel (coo_spmm_impl.hh). # 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen format. -# 4. LASO now stored "regular": exactly vec_nnz records per major-axis vector, with -# collided coordinates split into duplicates scaled to preserve the operator -# (sparse_skops.hh). Correctness-neutral; nnz becomes full_nnz (e.g. wide LASO k=8 -# 318 -> 320 here -- collisions are rare at large dim_major). -# 5. A regular-CSR SpMV kernel (apply_regular_csr_to_vector_ik) was added but is GATED -# OFF by default (#if RandBLAS_ENABLE_REGULAR_CSR_KERNEL). The back-to-back A/B at -# the bottom of this file shows it is NEUTRAL-to-SLOWER than the general CSR kernel -# on this M3 (NEON), so the default build still uses the general kernel here -- the -# numbers below are therefore unchanged from item 1-3 state (deltas are run-to-run -# noise). The kernel is kept in-tree for evaluation on wide-vector Arm (SVE). # ns/elt = us*1000/(nnz*work_mult); GB/s = model bytes/time; %STR vs STREAM below. # Environment: Apple M3, OMP_NUM_THREADS=4, Release (MKL off) | Correctness: 0 FAIL @@ -226,10 +216,8 @@ Calibrating STREAM triad... 88.7 GB/s (= 100% on the %STR column) ############################################################ -# CSR REGULAR-vs-GENERAL KERNEL A/B (--csr-probe) -# 'CSR REG ColMajor*' = apply_regular_csr_to_vector_ik (gated off by default) -# 'CSR GEN ColMajor*' = apply_csr_to_vector_ik (the default). Same regular operator, -# same process, back-to-back -> isolates the kernel. REG is NOT faster on M3/NEON. +# CSR-vs-CSC KERNEL PROBE (--csr-probe): per-format SpMM timed directly +# on prebuilt CSC/CSR operands, isolating the kernel from COO conversion. ############################################################ ============================================================ @@ -280,8 +268,6 @@ Calibrating STREAM triad... 88.1 GB/s (= 100% on the %STR column) CSR jik ColMajor 585 652 0.731 21.9 25 CSR nnz=400 CSC kib RowMajor 98 122 0.122 130.7 148 CSR nnz=400 CSR ikb RowMajor 96 97 0.120 133.4 151 CSR nnz=400 - CSR REG ColMajor* 468 568 0.585 27.4 31 regular=Y row_nnz=2 - CSR GEN ColMajor* 441 463 0.551 29.0 33 regular=N row_nnz=2 Wide LASO k=4 Operator Min(us) Med(us) ns/elt GB/s %STR notes @@ -290,8 +276,6 @@ Calibrating STREAM triad... 88.1 GB/s (= 100% on the %STR column) CSR jik ColMajor 543 575 0.339 35.4 40 CSR nnz=800 CSC kib RowMajor 153 163 0.096 125.6 143 CSR nnz=800 CSR ikb RowMajor 146 168 0.091 131.6 149 CSR nnz=800 - CSR REG ColMajor* 602 626 0.376 31.9 36 regular=Y row_nnz=4 - CSR GEN ColMajor* 535 562 0.334 35.9 41 regular=N row_nnz=4 Wide LASO k=8 Operator Min(us) Med(us) ns/elt GB/s %STR notes @@ -300,6 +284,4 @@ Calibrating STREAM triad... 88.1 GB/s (= 100% on the %STR column) CSR jik ColMajor 542 653 0.169 59.1 67 CSR nnz=1600 CSC kib RowMajor 380 388 0.119 84.3 96 CSR nnz=1600 CSR ikb RowMajor 410 422 0.128 78.1 89 CSR nnz=1600 - CSR REG ColMajor* 592 624 0.185 54.1 61 regular=Y row_nnz=8 - CSR GEN ColMajor* 528 555 0.165 60.7 69 regular=N row_nnz=8 diff --git a/test/linops/test_spmm/test_spmm_csr.cc b/test/linops/test_spmm/test_spmm_csr.cc index 19411ae6..18f11fdf 100644 --- a/test/linops/test_spmm/test_spmm_csr.cc +++ b/test/linops/test_spmm/test_spmm_csr.cc @@ -473,17 +473,14 @@ TEST_F(TestPublicAPI_DenseTimesSparse_double, nontrivial_alpha_beta_rowmajor) { //////////////////////////////////////////////////////////////////////// // // -// Regular-CSR fast path (fixed nnz per row), including duplicate colidxs +// CSR SpMM reference checks (duplicate-colidx accumulation, variable nnz/row) // // //////////////////////////////////////////////////////////////////////// -// Compare C = alpha * A @ B against a hand-computed dense reference, where a duplicated -// column index within a row must be accumulated. We check both (1) the public dispatch -// apply_csr_jik_p11 (which uses the general kernel by default; the regular kernel only -// when RandBLAS_ENABLE_REGULAR_CSR_KERNEL is defined) and, (2) when the matrix is regular, -// apply_regular_csr_to_vector_ik directly -- so the regular kernel stays covered regardless -// of the off-by-default dispatch gate. +// Compare C = alpha * A @ B against a hand-computed dense reference via the public dispatch +// apply_csr_jik_p11, exercising in particular the accumulation of a column index that is +// duplicated within a row. static void check_csr_jik_against_reference( int64_t d, int64_t m, int64_t n, const std::vector& rowptr, @@ -513,28 +510,15 @@ static void check_csr_jik_against_reference( if (msg.size() > 0) FAIL() << msg; }; - // (1) Public dispatch (general kernel unless the regular-CSR macro is enabled). std::fill(C.begin(), C.end(), 0.0); apply_csr_jik_p11(alpha, Layout::ColMajor, Layout::ColMajor, d, n, m, A, B.data(), m, C.data(), d); compare("apply_csr_jik_p11"); - - // (2) Directly exercise the regular kernel when the matrix has a constant nnz-per-row. - bool regular = (d > 0); - int64_t row_nnz = (d > 0) ? (rowptr[1] - rowptr[0]) : 0; - for (int64_t i = 1; i <= d && regular; ++i) regular = (rowptr[i] - rowptr[i-1] == row_nnz); - if (regular) { - std::fill(C.begin(), C.end(), 0.0); - for (int64_t j = 0; j < n; ++j) - apply_regular_csr_to_vector_ik(alpha, A.vals, row_nnz, A.colidxs, - &B[(size_t) j * m], 1, d, &C[(size_t) j * d], 1); - compare("apply_regular_csr_to_vector_ik"); - } } -TEST(TestRegularCSR, fixed_nnz_per_row_with_duplicate_colidx) { - // d=4 rows, each with exactly 2 nonzeros -> regular branch. Row 1 has a duplicated - // column index (2, 2): the two records (0.5, 0.5) must accumulate to 1.0 at column 2. +TEST(TestCSRSpMMReference, duplicate_colidx_accumulates) { + // d=4 rows, each with exactly 2 nonzeros. Row 1 has a duplicated column index (2, 2): + // the two records (0.5, 0.5) must accumulate to 1.0 at column 2. check_csr_jik_against_reference( 4, 5, 3, {0, 2, 4, 6, 8}, @@ -543,9 +527,8 @@ TEST(TestRegularCSR, fixed_nnz_per_row_with_duplicate_colidx) { ); } -TEST(TestRegularCSR, variable_nnz_per_row_falls_back) { - // Rows have 1, 2, 1, 2 nonzeros -> NOT regular, so the general branch runs. Same - // dense reference must hold (guards the detection's fallback path). +TEST(TestCSRSpMMReference, variable_nnz_per_row) { + // Rows have 1, 2, 1, 2 nonzeros. The same hand-computed dense reference must hold. check_csr_jik_against_reference( 4, 5, 3, {0, 1, 3, 4, 6}, From 8b53f620550252e34abbd9b604f39eebd871ccb5 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Mon, 6 Jul 2026 13:57:55 -0700 Subject: [PATCH 15/20] clean up --- AGENTS.md | 2 +- RandBLAS/sparse_data/base.hh | 26 +++++++-------- RandBLAS/sparse_data/conversions.hh | 19 ++++------- RandBLAS/sparse_skops.hh | 52 +++++++++++------------------ 4 files changed, 40 insertions(+), 59 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7ed2d3ac..0249d621 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -# RandBLAS Project Guide for Codex +# RandBLAS Project Guide for coding agents ## Project Overview diff --git a/RandBLAS/sparse_data/base.hh b/RandBLAS/sparse_data/base.hh index 40cf8322..ecf98471 100644 --- a/RandBLAS/sparse_data/base.hh +++ b/RandBLAS/sparse_data/base.hh @@ -312,24 +312,22 @@ void compressed_ptr_to_sorted_idxs(int64_t num_comp, sint_t1* ptr, int64_t len_i } // Build a compressed layout (CSR or CSC) from a COO that is already sorted in the -// OPPOSITE compressed order, via an O(nnz + dim) counting-sort transpose -- cheaper -// than the O(nnz log nnz) comparison re-sort that COOMatrix::sort_arrays would do. -// Because the source is iterated in its (oppositely-)sorted order, the companion -// index comes out ascending within each group, so the result is properly sorted in -// the target order too. -// group_idx : axis to compress on (rows for a CSR target, cols for CSC); length nnz -// other_idx : the companion axis (cols for CSR, rows for CSC); length nnz -// dim : number of groups (n_rows for CSR, n_cols for CSC) -// Outputs (caller-allocated): ptr[dim+1], out_other[nnz], out_vals[nnz]. +// OPPOSITE compressed order, via an O(nnz + n_groups) counting-sort transpose. +// +// group_idx[nnz] : axis to compress (rows for a CSR target, cols for CSC) +// other_idx[nnz] : the companion axis (cols for CSR, rows for CSC) +// n_groups : number of groups (n_rows for CSR, n_cols for CSC) +// +// Outputs (caller-allocated): ptr[n_groups+1], out_other[nnz], out_vals[nnz]. template void counting_sort_transpose( int64_t nnz, const sint_t* group_idx, const sint_t* other_idx, const T* vals, - int64_t dim, sint_t* ptr, sint_t* out_other, T* out_vals + int64_t n_groups, sint_t* ptr, sint_t* out_other, T* out_vals ) { - for (int64_t g = 0; g <= dim; ++g) ptr[g] = 0; - for (int64_t e = 0; e < nnz; ++e) ptr[group_idx[e] + 1] += 1; - for (int64_t g = 0; g < dim; ++g) ptr[g + 1] += ptr[g]; - std::vector cursor(ptr, ptr + dim); // running write position per group + for (int64_t g = 0; g <= n_groups; ++g) ptr[g] = 0; + for (int64_t e = 0; e < nnz; ++e) ptr[group_idx[e] + 1] += 1; + for (int64_t g = 1; g <= n_groups; ++g) ptr[g] += ptr[g-1]; + std::vector cursor(ptr, ptr + n_groups); // running write position per group for (int64_t e = 0; e < nnz; ++e) { sint_t g = group_idx[e]; sint_t pos = cursor[g]++; diff --git a/RandBLAS/sparse_data/conversions.hh b/RandBLAS/sparse_data/conversions.hh index 7353709d..819e2fac 100644 --- a/RandBLAS/sparse_data/conversions.hh +++ b/RandBLAS/sparse_data/conversions.hh @@ -92,8 +92,6 @@ auto coo_to_csc( const COOMatrix &coo, CSCMatrix &csc ) { std::copy( coo.rows, coo.rows + coo.nnz, csc.rowidxs ); return; } else if (coo.sort == NonzeroSort::CSR) { - // Opposing order: group the CSR-sorted records by column with an O(nnz) - // counting-sort transpose instead of a comparison re-sort. csc.reserve(coo.nnz); counting_sort_transpose( coo.nnz, coo.cols, coo.rows, coo.vals, coo.n_cols, csc.colptr, csc.rowidxs, csc.vals @@ -137,14 +135,12 @@ void coo_to_csr( const COOMatrix &coo, CSRMatrix &csr ) { } } -// Represent "coo" as a CSR matrix without copying its structural nonzeros when possible. -// If coo is already CSR-sorted (or empty), the result is a ZERO-COPY view that shares coo's -// vals and cols arrays; its rowptr is written into the caller-provided "rowptr" scratch buffer -// (resized as needed). In that case the returned matrix borrows memory: it is valid only while -// BOTH coo and rowptr stay alive, and should be treated as const (like the transpose_as_* views -// above). Otherwise -- opposite (CSC) sort order, taking the O(nnz) counting-sort transpose, or -// an unsorted COO -- the result is a freshly converted, memory-owning matrix and rowptr is -// left untouched. +// Represent `coo` as a CSR matrix without copying its structural nonzeros when possible. +// +// If coo is already CSR-sorted (or empty), the result `csr` is a ZERO-COPY view that shares +// coo's vals and cols arrays, and has `csr.rowptr` backed by the input `rowptr` std::vector. +// If coo is not CSR-sorted, the result `csr` is a new matrix from coo_to_csr. +// template CSRMatrix coo_to_csr_view_or_copy(const COOMatrix &coo, std::vector &rowptr) { if (coo.nnz == 0 || coo.sort == NonzeroSort::CSR) { @@ -157,8 +153,7 @@ CSRMatrix coo_to_csr_view_or_copy(const COOMatrix &coo, st return csr; } -// CSC analog of coo_to_csr_view_or_copy: the zero-copy view shares coo's vals and rows arrays, -// and writes its colptr into the caller-provided "colptr" scratch buffer. Same lifetime contract. +// CSC analog of coo_to_csr_view_or_copy. template CSCMatrix coo_to_csc_view_or_copy(const COOMatrix &coo, std::vector &colptr) { if (coo.nnz == 0 || coo.sort == NonzeroSort::CSC) { diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index d86bb0d0..390d8e4f 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -634,6 +634,24 @@ state_t fill_sparse_unpacked( sint_t* idxs_major = major_is_rows ? rows : cols; sint_t* idxs_minor = major_is_rows ? cols : rows; + // Sort a contiguous block of "len" nonzeros into ascending major-coordinate order, + // moving the parallel (major, vals) pair together. len is at most vec_nnz, which is + // small, so use a no-alloc insertion sort. The idxs_minor array is constant across + // these blocks, so the helper doesn't need to look at it. + auto sort_block_by_major = [](sint_t* blk_major, T* blk_vals, int64_t len) { + for (int64_t a = 1; a < len; ++a) { + sint_t key = blk_major[a]; + T v = blk_vals[a]; + int64_t c = a - 1; + for (; c >= 0 && blk_major[c] > key; --c) { + blk_major[c+1] = blk_major[c]; + blk_vals[c+1] = blk_vals[c]; + } + blk_major[c+1] = key; + blk_vals[c+1] = v; + } + }; + // Phase 1: sample the num_major_sub requested major-axis vectors directly into the // output buffers, using the same helpers (and hence the same RNG stream) as the full // operator. On exit, the first "total" entries carry full major coordinates and local @@ -645,26 +663,8 @@ state_t fill_sparse_unpacked( work_state, vec_nnz, dim_major, num_major_sub, idxs_major, idxs_minor, vals ); total = vec_nnz * num_major_sub; - // Emit each major-axis vector in ascending major-coordinate order. Fisher-Yates - // draws the vec_nnz nonzeros in shuffle order; sorting each contiguous block by - // its major coordinate (rows for a wide SASO, cols for a tall one) makes the COO - // natively CSC- (wide) / CSR- (tall) sorted, so apply_coo_via_csx skips its - // per-apply deepcopy + re-sort. idxs_minor is constant within a block, so only - // (idxs_major, vals) move. vec_nnz is small, so a no-alloc insertion sort is best. for (int64_t b = 0; b < num_major_sub; ++b) { - sint_t* blk_major = idxs_major + b * vec_nnz; - T* blk_vals = vals + b * vec_nnz; - for (int64_t a = 1; a < vec_nnz; ++a) { - sint_t key = blk_major[a]; - T v = blk_vals[a]; - int64_t c = a - 1; - for (; c >= 0 && blk_major[c] > key; --c) { - blk_major[c+1] = blk_major[c]; - blk_vals[c+1] = blk_vals[c]; - } - blk_major[c+1] = key; - blk_vals[c+1] = v; - } + sort_block_by_major(idxs_major + b * vec_nnz, vals + b * vec_nnz, vec_nnz); } } else { // LASO: each major-axis vector is sampled with replacement and merged in place, @@ -681,19 +681,7 @@ state_t fill_sparse_unpacked( laso_merge_long_axis_vector_coo_data(vec_nnz, v, im, in, i, loc2count, loc2scale); // The merge compacts to the (<= vec_nnz) distinct survivors. int64_t count = (int64_t) loc2count.size(); - // Emit this major-axis vector in ascending major-coordinate order, exactly as - // the SASO branch does. The blocks are handled with insertion-sort. - for (int64_t a = 1; a < count; ++a) { - sint_t key = im[a]; - T vv = v[a]; - int64_t c = a - 1; - for (; c >= 0 && im[c] > key; --c) { - im[c+1] = im[c]; - v[c+1] = v[c]; - } - im[c+1] = key; - v[c+1] = vv; - } + sort_block_by_major(im, v, count); im += count; in += count; v += count; total += count; } } From 224e4e1a4d4e8feff4a0d2b754486317b8d6ab0e Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Mon, 6 Jul 2026 14:04:56 -0700 Subject: [PATCH 16/20] remove benchmark log --- sketch_general_results.txt | 287 ---------------------------------- sparsesketching_bench_plan.md | 184 ---------------------- sparsesketching_opt.md | 185 ---------------------- 3 files changed, 656 deletions(-) delete mode 100644 sketch_general_results.txt delete mode 100644 sparsesketching_bench_plan.md delete mode 100644 sparsesketching_opt.md diff --git a/sketch_general_results.txt b/sketch_general_results.txt deleted file mode 100644 index 4874f14c..00000000 --- a/sketch_general_results.txt +++ /dev/null @@ -1,287 +0,0 @@ -# POST-CHANGES run. Combined state of the sparse-sketching branch: -# 1. Construction-time within-vector sort for SASO and LASO (sparse_skops.hh): -# wide SASO -> CSC-sorted COO, wide LASO -> CSR-sorted COO. -# 2. ColMajor wide operators routed through the CSR jik kernel (coo_spmm_impl.hh). -# 3. O(nnz) counting-sort transpose when the COO's sorted order != the chosen format. -# ns/elt = us*1000/(nnz*work_mult); GB/s = model bytes/time; %STR vs STREAM below. -# Environment: Apple M3, OMP_NUM_THREADS=4, Release (MKL off) | Correctness: 0 FAIL - -============================================================ -SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) -============================================================ -Threads (OMP_NUM_THREADS/max): 4 -Calibrating STREAM triad... 88.7 GB/s (= 100% on the %STR column) - -########## SECTION 1: vary sketch size d (m=n=2000) ########## - -=== LEFT-SKETCH B(40x2000) = S(40x2000) * A(2000x2000), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 701 722 0.088 47.6 54 CSC nnz=4000 cold=794 cvt=7 - Wide SASO k=4 1318 1370 0.082 25.3 29 CSC nnz=8000 cold=1783 cvt=15 - Wide SASO k=8 2317 3126 0.072 14.5 16 CSC nnz=16000 cold=3474 cvt=26 - CountSketch k=1 573 587 0.143 58.1 66 CSC nnz=2000 cold=628 cvt=3 - Wide LASO k=2 212 260 1.325 12.1 14 CSR nnz=80 cold=153 cvt=1 - Wide LASO k=4 389 455 1.216 9.9 11 CSR nnz=160 cold=344 cvt=1 - Wide LASO k=8 400 405 0.625 16.0 18 CSR nnz=320 cold=404 cvt=4 - densify(S)+GEMM 1566 1592 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 770 782 0.096 43.3 49 CSC nnz=4000 - Wide SASO k=4 1265 1377 0.079 26.4 30 CSC nnz=8000 - Wide SASO k=8 1775 1829 0.055 18.9 21 CSC nnz=16000 - CountSketch k=1 510 522 0.128 65.3 74 CSC nnz=2000 - Wide LASO k=2 28 30 0.175 91.5 103 CSR nnz=80 - Wide LASO k=4 34 37 0.106 113.0 127 CSR nnz=160 - Wide LASO k=8 48 49 0.075 133.4 150 CSR nnz=320 - -=== RIGHT-SKETCH B(2000x40) = A(2000x2000) * S(2000x40), trials=10 === - - warm apply, ColMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 652 661 0.082 51.1 58 CSR nnz=4000 - Wide SASO k=4 1039 1056 0.065 32.2 36 CSR nnz=8000 - Wide SASO k=8 1665 1712 0.052 20.1 23 CSR nnz=16000 - CountSketch k=1 464 514 0.116 71.8 81 CSR nnz=2000 - Wide LASO k=2 29 30 0.181 88.3 100 CSC nnz=80 - Wide LASO k=4 35 37 0.109 109.8 124 CSC nnz=160 - Wide LASO k=8 48 50 0.075 133.4 150 CSC nnz=320 - - warm apply, RowMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 628 648 0.079 53.1 60 CSR nnz=4000 - Wide SASO k=4 1216 1314 0.076 27.5 31 CSR nnz=8000 - Wide SASO k=8 2306 2427 0.072 14.5 16 CSR nnz=16000 - CountSketch k=1 457 508 0.114 72.9 82 CSR nnz=2000 - Wide LASO k=2 152 183 0.950 16.9 19 CSC nnz=80 - Wide LASO k=4 317 346 0.991 12.1 14 CSC nnz=160 - Wide LASO k=8 385 470 0.602 16.6 19 CSC nnz=320 - -=== LEFT-SKETCH B(200x2000) = S(200x2000) * A(2000x2000), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 844 921 0.105 45.6 51 CSC nnz=4000 cold=1003 cvt=7 - Wide SASO k=4 1505 1563 0.094 25.6 29 CSC nnz=8000 cold=1955 cvt=14 - Wide SASO k=8 2424 3260 0.076 15.9 18 CSC nnz=16000 cold=3727 cvt=26 - CountSketch k=1 728 756 0.182 52.8 60 CSC nnz=2000 cold=740 cvt=3 - Wide LASO k=2 493 566 0.616 26.0 29 CSR nnz=400 cold=524 cvt=5 - Wide LASO k=4 600 774 0.375 32.0 36 CSR nnz=800 cold=694 cvt=11 - Wide LASO k=8 650 661 0.203 49.3 56 CSR nnz=1600 cold=787 cvt=25 - densify(S)+GEMM 4648 4716 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 751 837 0.094 51.2 58 CSC nnz=4000 - Wide SASO k=4 1089 1108 0.068 35.4 40 CSC nnz=8000 - Wide SASO k=8 1828 1954 0.057 21.1 24 CSC nnz=16000 - CountSketch k=1 560 563 0.140 68.6 77 CSC nnz=2000 - Wide LASO k=2 111 116 0.139 115.4 130 CSR nnz=400 - Wide LASO k=4 157 166 0.098 122.4 138 CSR nnz=800 - Wide LASO k=8 373 383 0.117 85.9 97 CSR nnz=1600 - -=== RIGHT-SKETCH B(2000x200) = A(2000x2000) * S(2000x200), trials=10 === - - warm apply, ColMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 738 777 0.092 52.1 59 CSR nnz=4000 - Wide SASO k=4 1071 1083 0.067 36.0 41 CSR nnz=8000 - Wide SASO k=8 1725 1868 0.054 22.4 25 CSR nnz=16000 - CountSketch k=1 559 563 0.140 68.8 78 CSR nnz=2000 - Wide LASO k=2 97 105 0.121 132.0 149 CSC nnz=400 - Wide LASO k=4 179 192 0.112 107.3 121 CSC nnz=800 - Wide LASO k=8 380 396 0.119 84.3 95 CSC nnz=1600 - - warm apply, RowMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 897 929 0.112 42.9 48 CSR nnz=4000 - Wide SASO k=4 1431 1467 0.089 26.9 30 CSR nnz=8000 - Wide SASO k=8 2363 3260 0.074 16.4 18 CSR nnz=16000 - CountSketch k=1 830 998 0.207 46.3 52 CSR nnz=2000 - Wide LASO k=2 621 718 0.776 20.6 23 CSC nnz=400 - Wide LASO k=4 753 862 0.471 25.5 29 CSC nnz=800 - Wide LASO k=8 711 807 0.222 45.0 51 CSC nnz=1600 - -=== LEFT-SKETCH B(500x2000) = S(500x2000) * A(2000x2000), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 1346 1460 0.168 35.7 40 CSC nnz=4000 cold=1587 cvt=7 - Wide SASO k=4 1770 1831 0.111 27.2 31 CSC nnz=8000 cold=2310 cvt=15 - Wide SASO k=8 2721 3540 0.085 17.7 20 CSC nnz=16000 cold=3980 cvt=26 - CountSketch k=1 1059 1100 0.265 45.4 51 CSC nnz=2000 cold=1117 cvt=3 - Wide LASO k=2 817 903 0.408 39.2 44 CSR nnz=1000 cold=925 cvt=14 - Wide LASO k=4 1063 1130 0.266 45.2 51 CSR nnz=2000 cold=1216 cvt=32 - Wide LASO k=8 1063 1098 0.133 45.2 51 CSR nnz=4000 cold=1346 cvt=75 - densify(S)+GEMM 10935 11033 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 877 920 0.110 54.8 62 CSC nnz=4000 - Wide SASO k=4 1173 1186 0.073 41.0 46 CSC nnz=8000 - Wide SASO k=8 1817 1845 0.057 26.6 30 CSC nnz=16000 - CountSketch k=1 764 773 0.191 62.9 71 CSC nnz=2000 - Wide LASO k=2 392 402 0.196 81.7 92 CSR nnz=1000 - Wide LASO k=4 593 603 0.148 81.0 91 CSR nnz=2000 - Wide LASO k=8 826 858 0.103 58.2 66 CSR nnz=4000 - -=== RIGHT-SKETCH B(2000x500) = A(2000x2000) * S(2000x500), trials=10 === - - warm apply, ColMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 826 842 0.103 58.2 66 CSR nnz=4000 - Wide SASO k=4 1193 1312 0.075 40.3 45 CSR nnz=8000 - Wide SASO k=8 1836 1871 0.057 26.3 30 CSR nnz=16000 - CountSketch k=1 761 774 0.190 63.1 71 CSR nnz=2000 - Wide LASO k=2 404 407 0.202 79.2 89 CSC nnz=1000 - Wide LASO k=4 563 567 0.141 85.3 96 CSC nnz=2000 - Wide LASO k=8 803 860 0.100 59.9 68 CSC nnz=4000 - - warm apply, RowMajor dense - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 1270 1294 0.159 37.8 43 CSR nnz=4000 - Wide SASO k=4 1781 1831 0.111 27.0 30 CSR nnz=8000 - Wide SASO k=8 2696 3514 0.084 17.9 20 CSR nnz=16000 - CountSketch k=1 1064 1107 0.266 45.1 51 CSR nnz=2000 - Wide LASO k=2 847 945 0.423 37.8 43 CSC nnz=1000 - Wide LASO k=4 1135 1167 0.284 42.3 48 CSC nnz=2000 - Wide LASO k=8 1130 1157 0.141 42.5 48 CSC nnz=4000 - -########## SECTION 2: narrow n (SpMV-ish) ########## - -=== LEFT-SKETCH B(200x1) = S(200x2000) * A(2000x1), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 19 21 4.750 4.4 5 CSC nnz=4000 cold=182 cvt=7 - Wide SASO k=4 31 32 3.875 4.7 5 CSC nnz=8000 cold=448 cvt=14 - Wide SASO k=8 50 51 3.125 5.5 6 CSC nnz=16000 cold=1098 cvt=25 - CountSketch k=1 13 14 6.500 3.9 4 CSC nnz=2000 cold=63 cvt=3 - Wide LASO k=2 8 8 20.000 1.6 2 CSR nnz=400 cold=39 cvt=5 - Wide LASO k=4 8 10 10.000 2.8 3 CSR nnz=800 cold=72 cvt=11 - Wide LASO k=8 10 12 6.250 4.2 5 CSR nnz=1600 cold=131 cvt=24 - densify(S)+GEMM 13 14 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 57 58 14.250 1.5 2 CSC nnz=4000 - Wide SASO k=4 99 106 12.375 1.5 2 CSC nnz=8000 - Wide SASO k=8 182 184 11.375 1.5 2 CSC nnz=16000 - CountSketch k=1 36 36 18.000 1.4 2 CSC nnz=2000 - Wide LASO k=2 17 18 42.500 0.8 1 CSR nnz=400 - Wide LASO k=4 21 22 26.250 1.1 1 CSR nnz=800 - Wide LASO k=8 33 34 20.625 1.3 1 CSR nnz=1600 - -=== LEFT-SKETCH B(200x10) = S(200x2000) * A(2000x10), trials=10 === - - warm apply, ColMajor dense (jik/jki kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 24 26 0.600 10.7 12 CSC nnz=4000 cold=147 cvt=7 - Wide SASO k=4 36 37 0.450 8.9 10 CSC nnz=8000 cold=395 cvt=13 - Wide SASO k=8 66 67 0.412 6.8 8 CSC nnz=16000 cold=1100 cvt=26 - CountSketch k=1 17 18 0.850 13.2 15 CSC nnz=2000 cold=67 cvt=3 - Wide LASO k=2 10 11 2.500 7.0 8 CSR nnz=400 cold=40 cvt=5 - Wide LASO k=4 11 12 1.375 9.9 11 CSR nnz=800 cold=74 cvt=11 - Wide LASO k=8 14 15 0.875 13.3 15 CSR nnz=1600 cold=135 cvt=24 - densify(S)+GEMM 35 35 - - - dense oracle - - warm apply, RowMajor dense (ikb/kib kernels) - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - Wide SASO k=2 56 58 1.400 4.6 5 CSC nnz=4000 - Wide SASO k=4 97 98 1.212 3.3 4 CSC nnz=8000 - Wide SASO k=8 170 178 1.062 2.6 3 CSC nnz=16000 - CountSketch k=1 35 36 1.750 6.4 7 CSC nnz=2000 - Wide LASO k=2 17 18 4.250 4.1 5 CSR nnz=400 - Wide LASO k=4 22 25 2.750 4.9 6 CSR nnz=800 - Wide LASO k=8 34 34 2.125 5.5 6 CSR nnz=1600 - - - -############################################################ -# CSR-vs-CSC KERNEL PROBE (--csr-probe): per-format SpMM timed directly -# on prebuilt CSC/CSR operands, isolating the kernel from COO conversion. -############################################################ - -============================================================ -SKETCH_GENERAL PERFORMANCE BENCHMARK (sparse operators) -============================================================ -Threads (OMP_NUM_THREADS/max): 4 -Calibrating STREAM triad... 88.1 GB/s (= 100% on the %STR column) - -=== CSR-vs-CSC KERNEL PROBE (left-sketch) d=200 m=2000 n=2000, trials=10 === - kernels timed directly via left_spmm on prebuilt CSC/CSR (no COO re-sort) - - Wide SASO k=2 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 1121 1125 0.140 34.3 39 CSC nnz=4000 - CSR jik ColMajor 823 953 0.103 46.7 53 CSC nnz=4000 - CSC kib RowMajor 772 787 0.097 49.8 57 CSC nnz=4000 - CSR ikb RowMajor 1005 1013 0.126 38.3 43 CSC nnz=4000 - - Wide SASO k=4 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 2092 2108 0.131 18.4 21 CSC nnz=8000 - CSR jik ColMajor 1429 1483 0.089 27.0 31 CSC nnz=8000 - CSC kib RowMajor 1079 1141 0.067 35.7 41 CSC nnz=8000 - CSR ikb RowMajor 1558 1817 0.097 24.7 28 CSC nnz=8000 - - Wide SASO k=8 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 3699 3959 0.116 10.5 12 CSC nnz=16000 - CSR jik ColMajor 2221 2665 0.069 17.4 20 CSC nnz=16000 - CSC kib RowMajor 1845 1867 0.058 21.0 24 CSC nnz=16000 - CSR ikb RowMajor 2836 3354 0.089 13.6 15 CSC nnz=16000 - - CountSketch k=1 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 873 886 0.218 44.0 50 CSC nnz=2000 - CSR jik ColMajor 719 732 0.180 53.5 61 CSC nnz=2000 - CSC kib RowMajor 498 569 0.124 77.2 88 CSC nnz=2000 - CSR ikb RowMajor 563 641 0.141 68.3 78 CSC nnz=2000 - - Wide LASO k=2 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 583 683 0.729 22.0 25 CSR nnz=400 - CSR jik ColMajor 585 652 0.731 21.9 25 CSR nnz=400 - CSC kib RowMajor 98 122 0.122 130.7 148 CSR nnz=400 - CSR ikb RowMajor 96 97 0.120 133.4 151 CSR nnz=400 - - Wide LASO k=4 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 788 825 0.492 24.4 28 CSR nnz=800 - CSR jik ColMajor 543 575 0.339 35.4 40 CSR nnz=800 - CSC kib RowMajor 153 163 0.096 125.6 143 CSR nnz=800 - CSR ikb RowMajor 146 168 0.091 131.6 149 CSR nnz=800 - - Wide LASO k=8 - Operator Min(us) Med(us) ns/elt GB/s %STR notes - ---------------------------------------------------------------------------- - CSC jki ColMajor 1251 1267 0.391 25.6 29 CSR nnz=1600 - CSR jik ColMajor 542 653 0.169 59.1 67 CSR nnz=1600 - CSC kib RowMajor 380 388 0.119 84.3 96 CSR nnz=1600 - CSR ikb RowMajor 410 422 0.128 78.1 89 CSR nnz=1600 - diff --git a/sparsesketching_bench_plan.md b/sparsesketching_bench_plan.md deleted file mode 100644 index 4c001c8d..00000000 --- a/sparsesketching_bench_plan.md +++ /dev/null @@ -1,184 +0,0 @@ -# Plan: a `sketch_general` benchmark, analogous to `spmm_performance.cc` - -## Goal - -`spmm_performance.cc` benchmarks the low-level `left_spmm` kernels on **generic -uniform-random** sparse matrices. It therefore never exercises the *structured* -sketching operators (fixed nnz per column/row, ±1 values, mostly-empty columns), -never hits the regular-CSC fast path, and never goes through `sketch_general`. -This benchmark closes that gap: it measures the **full sketching call** -(`sketch_general` with a real `SparseSkOp`) across the operator shapes identified -in `sparsesketching_opt.md`, so we get a baseline for the proposed kernels -(CountSketch, bounded-CSR for wide LASO, regular-path layout coverage, ±1 -implicit-value) before any of them is written. - -New executable: `examples/simple-kernel-benchmarks/sketch_general_performance.cc`. - -## What `sketch_general` does, and the cost lifecycle we must measure - -Left-sketch form (the headline case): - -``` -sketch_general(layout, opS, opA, d, n, m, alpha, S, ro_s, co_s, A, lda, beta, B, ldb) - // B(d x n) = alpha * op(S)(d x m) * op(A)(m x n) + beta * B -``` - -with `S` a `SparseSkOp` of distribution `SparseDist(d, m, vec_nnz, major_axis)`. -For a sparse `S` this routes `lskges → coo_view_of_skop → left_spmm`. The cost has -**three distinct phases** that the benchmark must be able to separate, because the -opt-doc proposals target different ones: - -1. **Sample / fill** — `fill_sparse(S)` populates `(rows, cols, vals)`. Happens - once in real build-once/apply-many use; if `S.nnz < 0` at call time, - `sketch_general` materializes a *temporary* representation **every call** and - frees it. We must pre-sample (`fill_sparse(S)`) to measure apply-only cost, and - *separately* measure the cold path. -2. **View + convert/sort** — `apply_coo_via_csc` determines the COO sort order and, - if not already CSC-sorted, **deepcopies + re-sorts to CSC on every call**. This - is the per-apply re-sort flagged for wide-LASO left-sketch; it must be timed in - isolation. -3. **Kernel** — `apply_csc_jki_p11` (ColMajor) / `apply_csc_kib_1p1_rowmajor` - (RowMajor), with the regular-CSC specialization firing only in the `jki` path - when nnz-per-column is exactly fixed. - -> Design choice: report **warm apply** (phases 2+3, `S` pre-sampled) as the primary -> number, plus a **cold** number (phases 1+2+3) and a **convert-only** number, so -> the re-sort cost is visible. This mirrors `spmm_performance`'s -> `run_split_trials` (densify vs compute) but with a sketching-specific split. - -## Relationship to `spmm_performance.cc` (reuse vs. change) - -**Reuse verbatim:** -- `run_trials` / `run_split_trials` (min + median over N trials), `print_row`, - `print_table_header`, the densify+GEMM correctness oracle pattern. -- Both dense layouts (`ColMajor`, `RowMajor`) and both `op` flags — the dispatch - selects the kernel purely from `(layout_opB, layout_C)`, exactly as documented in - the spmm benchmark header. -- The "compute the identical logical product in both layouts and check against one - ColMajor oracle" structure. - -**Change / add:** -- Operand is a `SparseSkOp` built from a `SparseDist`, **not** `random_coo`. So the - regular-CSC path and CountSketch are actually represented. -- New sweep axes: `major_axis ∈ {Short, Long}` and `vec_nnz ∈ {1,2,4,8}`. -- Both **sides** (left-sketch and right-sketch). Right-sketch uses the tall - operator and, per the transpose reduction, should match the corresponding - left-sketch RowMajor/ColMajor kernel — the benchmark lets us confirm this - empirically (a useful cross-check, not just a measurement). -- Phase-split timing (sample / convert / kernel) described above. -- Oracle is `densify(S) + GEMM`, where `densify(S)` comes from the COO view of the - sampled operator (reuse `coo` → dense, or build dense directly from - `rows/cols/vals`). - -## Operator shapes to sweep (the structured taxonomy) - -Map each `SparseDist` onto the opt-doc taxonomy. For left-sketch, `S` is `d×m` -with `d < m` (wide); for right-sketch, `S` is `n×d` with `n > d` (tall). - -| Config | dist | side | structured axis | what it isolates | -|---|---|---|---|---| -| Wide SASO | `SparseDist(d,m,k,Short)` | left | fixed `k`/col (CSC-regular) | regular-CSC fast path | -| CountSketch | `SparseDist(d,m,1,Short)` | left | 1/col | the `vec_nnz=1` special case | -| Wide LASO | `SparseDist(d,m,k,Long)` | left | ≤`k`/row, empty cols | the uncovered CSR-gather case + re-sort cost | -| Tall SASO | `SparseDist(n,d,k,Short)` | right | →CSC-regular via transpose | confirms transpose reduction == wide SASO | -| Tall LASO | `SparseDist(n,d,k,Long)` | right | ≤`k`/col after transpose | general CSC scatter (no regular) | - -For each: `× {ColMajor, RowMajor} × {opS, opA NoTrans/Trans as applicable}`. - -## Parameters and sweep - -CLI mirrors `spmm_performance`: - -``` -./sketch_general_performance # default sweep -./sketch_general_performance d m n vec_nnz major_axis [trials] # single config - major_axis: 0 = Short (SASO), 1 = Long (LASO) -``` - -**Default sweep** (10 trials each), two sections: - -1. **Square-ish data, varying sketch size** — fix `m=n` large, sweep - `d ∈ {m/50, m/10, m/4}`; e.g. `m=n=2000`, `d ∈ {40,200,500}`. Covers the - common "tall data, modest embedding" regime. -2. **`vec_nnz` / `major_axis` cross** — fix one geometry (e.g. `d=200, m=n=2000`) - and sweep `vec_nnz ∈ {1,2,4,8} × major_axis ∈ {Short, Long}`, both sides. This - is the section that exercises CountSketch, wide LASO, and the regular path. - -Also expose, via env (as in the existing benchmark): `OMP_NUM_THREADS`. Per the -perf notes, sweep `{1,4,8}` on the M3 and additionally on an Arm SVE box. - -Include the **`n=1` / narrow-`n`** SpMV regime explicitly (a small extra config), -since that is the design fork for the CountSketch bucketed kernel. - -## Measurement methodology - -For each (shape × layout × side): - -- **Pre-sample** `S` once with `fill_sparse(S)`; then time the warm - `sketch_general` call (re-zeroing `B` each trial), reporting `{min, median}`. -- **Cold** variant: reconstruct an unsampled `SparseSkOp` each trial and time the - full call (captures sample + convert + kernel) — report `min` only. -- **Convert-only**: time `coo_view_of_skop` + the COO→CSC sort path in isolation - (or infer as cold − warm − sample) to expose the per-apply re-sort. -- **References**: `densify(S)+GEMM` (oracle, split densify vs GEMM like spmm), and - optionally a `DenseSkOp` (Gaussian) sketch via `lskge3` at the same `d,m,n` for a - "dense operator" cost anchor. -- **Correctness**: one extra call per (shape×layout×side) compared against the - ColMajor densify+GEMM oracle, `max|diff| < 1e-10`, printed as a PASS/FAIL line. - -Report a SUMMARY per config: best warm (ColMajor vs RowMajor), regular-vs-general -gap where both are reachable, and convert/re-sort as a fraction of warm apply. - -## Output tables (per config) - -Mirror the spmm layout — one table per (side × layout): - -``` -SKETCH_GENERAL left, ColMajor d x n = A applied - Operator Min(us) Med(us) vs best notes - Wide SASO k=4 ... (regular-CSC) - CountSketch k=1 ... (regular, 1/col) - Wide LASO k=4 ... (general CSC + resort) - densify(S)+GEMM ... (densify .. + GEMM ..) -``` - -Plus the RowMajor table, the right-sketch (tall) tables, and a convert-cost line. - -## File / build wiring - -- Add `sketch_general_performance.cc` under - `examples/simple-kernel-benchmarks/`. -- In `examples/CMakeLists.txt`, add an `add_executable` / - `target_include_directories(... ${Random123_DIR})` / - `target_link_libraries(... RandBLAS blaspp lapackpp)` block copied from the - `spmm_performance` block (lines ~95–103). -- Headers: `` is enough for `sketch_general` / `SparseSkOp` / - `SparseDist` / `fill_sparse`; pull `RandBLAS/sparse_data/conversions.hh` only if - densifying via the conversion helpers. - -## Things to confirm while implementing - -1. Whether `sketch_general` with a pre-sampled `S` still re-sorts COO→CSC on each - call (expected yes, via `apply_coo_via_csc`) — the convert-only timing settles - this. -2. Which kernel each (side × layout) actually lands on, to confirm the regular - fast path is/ isn't invoked (e.g. ColMajor right-sketch → RowMajor axpy kernel, - no regular specialization). -3. That a sampled wide SASO's COO is in CSC sort order (so no re-sort) while a - wide LASO's is not — the structural prediction the benchmark is meant to verify. - -## Measured findings (first run, M3, OMP=4) - -The benchmark is implemented as `sketch_general_performance.cc` and runs clean -(all correctness checks pass). First results already correct a prediction: - -- **Only CountSketch (`k=1`) is `coo-sort=CSC`** and skips the re-sort - (convert ≈ 3µs). **Wide SASO `k≥2` is `coo-sort=None`** and re-sorts on *every* - apply (convert ≈ 30/77/171µs for k=2/4/8 at d=40) — i.e. the per-apply re-sort - is **not** limited to wide LASO; SASO pays it too because within-column row - indices aren't sorted. This strengthens the case for structure-aware dispatch. -- **RowMajor is ~2× faster than ColMajor** for these operators (e.g. wide SASO - k=4, d=40: 1144 vs 2276µs), consistent with the prior SpMM findings. -- Wide LASO has very low nnz (≤ `k·d` minus merge collisions) and is the cheapest - case by a wide margin. -``` diff --git a/sparsesketching_opt.md b/sparsesketching_opt.md deleted file mode 100644 index c3248e83..00000000 --- a/sparsesketching_opt.md +++ /dev/null @@ -1,185 +0,0 @@ -# Optimizing SPMM kernels for sparse sketching operators - -A proposal for new structure-aware SPMM kernels in RandBLAS, plus a recommendation -to gather targeted performance data before hand-tuning. - -> **Revision note.** An earlier draft listed *tall SASO (right-sketching)* as a -> motivating case for a new regular-CSR kernel. That was wrong: free operator -> transposition plus the `right_spmm`→`left_spmm` reduction already turn that case -> into the existing wide-SASO / CSC-regular kernel. This draft reframes the -> taxonomy around the operator's *structured axis* (row vs column), which is the -> distinction that survives transposition. - -## How sketching operators reach the kernels today - -Every `SparseSkOp` is stored as **COO** (`rows, cols, vals`). Sketching -(`lskges`/`rskges`) takes a `coo_view_of_skop` and calls `left_spmm`/`right_spmm`. - -Two reductions happen *before* any kernel runs, and they are central to this -analysis: - -- **`right_spmm` reduces to `left_spmm`** by transposing the operation and - **flipping the dense layout** (`trans_layout`: ColMajor↔RowMajor), so the sparse - operand always lands as the left operand of `left_spmm`. -- **`left_spmm` resolves `opA=Trans` for free**: it takes a lightweight transpose - view of the operator (for COO, just swap the `rows`/`cols` arrays) and recurses - as `NoTrans`. Transposing a sketching operator costs nothing. - -After those reductions, a COO operator goes to `apply_coo_via_csc`, which: - -1. Determines the COO sort order by scanning (set in the view constructor via - `coo_arrays_determine_sort`). -2. If not already CSC-sorted, **deepcopies and re-sorts to CSC on every call**. -3. Calls `apply_csc_kib_1p1_rowmajor` (when `layout_opB == layout_C == RowMajor`) - or `apply_csc_jki_p11` (otherwise). - -The only place structure is exploited is `apply_csc_jki_p11`: it scans `colptr` -to detect fixed-nnz-per-column and, if so, calls `apply_regular_csc_to_vector_ki`. -Note this detection lives **only** in the `jki` path — the RowMajor axpy kernel -`apply_csc_kib_1p1_rowmajor` has no regular specialization. - -## The right way to think about it: the *structured axis*, not tall vs wide - -Because transposition is free and `right_spmm` already flips layout, **"tall vs -wide" is not an independent design axis** — it is absorbed by free transpose plus -a dense-layout flip. What survives transposition is whether the operator is -**column-structured** (feeds the CSC *scatter* kernel) or **row-structured** -(wants a CSR *gather* kernel). A transpose swaps a row-structured operator into a -column-structured one, but it simultaneously swaps the roles/layouts of the dense -operands — so it only helps when the dispatch is *already* transposing for another -reason. - -Re-cast the four shapes accordingly: - -| Operator | Usage | Structured axis after reductions | Reaches which kernel | -|---|---|---|---| -| **Wide SASO** (d×m) | left-sketch | exactly `vec_nnz` per **column** (CSC-regular) | ✅ `apply_regular_csc...` | -| **Tall SASO** (m×d) | right-sketch | `right_spmm` transposes → **column-regular** (= wide SASO) | ✅ same CSC-regular machinery — **no new kernel needed** | -| **CountSketch** | left-sketch | column-regular with `vec_nnz = 1` | ✅ regular-CSC, but very under-specialized | -| **Wide LASO** (d×m) | left-sketch | **≤** `vec_nnz` per **row**, mostly-empty cols (CSR) | ❌ no auto-transpose → general CSC scatter over all `m` cols | -| **Tall LASO** (m×d) | right-sketch | transposes → **≤** per column (CSC, not exact) | ⚠️ general CSC scatter; regular detector can't fire (counts vary) | - -The two takeaways: - -1. **Tall SASO needs no new *kernel*.** Its row structure is converted to the - existing column-regular kernel by the transpose reduction. (Measurement caveat: - the COO view's sort flag comes back `None` for `vec_nnz ≥ 2` — within-vector - indices aren't fully ordered — so `apply_coo_via_csc` still re-sorts on every - apply; only CountSketch, `vec_nnz = 1`, is detected as CSC-sorted. See the - benchmark findings in `sparsesketching_bench_plan.md`. This is a dispatch cost, - not a missing kernel.) -2. The genuinely uncovered case is the **row-structured operator that reaches a - kernel *without* an auto-transpose** — i.e., **wide LASO in left-sketching**. - There the transpose trick doesn't help (it would just swap the dense roles - back), so we land on the general CSC scatter, which iterates over all `m` - columns (most empty) and sees no regularity. - -A caveat that matters for *invocation* (not just structure): the regular-CSC fast -path lives only in `apply_csc_jki_p11`. A common **ColMajor right-sketch with a -tall SASO** flips to RowMajor and routes to `apply_csc_kib_1p1_rowmajor` (the -axpy kernel), which does **not** specialize on regularity. So "structure covered" -≠ "fast path invoked" — which kernel fires is layout-dependent. - -## Proposed kernels - -**1. CountSketch kernels (`vec_nnz == 1`)** — highest value and most -self-contained. A wide CountSketch maps each column `c` to one row `h(c)` with -sign `s(c)`: `C[h(c),:] += s(c)·B[c,:]`. Today this runs through regular-CSC with -`col_nnz=1` — correct but carrying full loop/index overhead for a length-1 inner -loop. Dedicated variants: - -- **Bucketed / CSR-by-output-row form**: invert the hash once (group source - columns by target row). Each of the `d` output rows is then an independent - signed sum of B-rows → embarrassingly parallel over `d` with **no races and no - per-thread rescan** (unlike `apply_csc_kib_1p1_rowmajor`, which loops all `m` - per thread and filters by row range). This is the right structure when `n` is - small (SpMV / few RHS), where parallel-over-`n` starves threads. -- Collapse the length-1 inner loop entirely. - -**2. Bounded-CSR gather kernel for wide LASO (left-sketch).** This is the case the -transpose reduction does *not* rescue. Build a CSR/gather kernel that iterates -only over the `d` nonempty rows (≤ `vec_nnz` each) instead of scattering over all -`m` columns. Realizing this means routing the operator COO→**CSR** instead of CSC -— which *also* eliminates the per-apply re-sort, since a wide LASO's natural COO -order is already CSR. A `≤`-aware variant (explicit short per-row counts, or -padding) handles the merge-induced count variation that keeps LASOs out of the -*exact* regular path. - -**3. Implicit ±1-value kernels.** SASO values are exactly ±1; the isometry scale -is applied separately via `alpha`. If the prior profiling's memory-bound -hypothesis holds, **not loading the `vals` array** (encode sign in the index, or -split into +/− index lists and do two sign-free accumulations) cuts the -sparse-operand traffic by ~⅓. Machine-independent in the regime the M3 profiling -suggested, and it composes with kernels 1–2. - -**4. Close the regular-path layout gap.** Add a regular-aware specialization to the -RowMajor axpy kernel (`apply_csc_kib_1p1_rowmajor`), and/or a dedicated ColMajor -row-axpy ("BLAS-3-flavored") path. Per the perf notes, ColMajor is the BLAS -default yet routes to the slow scalar scatter/gather; RowMajor gets the fast -axpy kernels and is up to ~1.9× faster. -A structured ±1 operator makes a well-vectorized ColMajor kernel far easier than -for a general sparse matrix, and it ensures the regular fast path is reachable -regardless of which layout the reductions land in. - -## Cross-cutting infrastructure (needed for the above to pay off) - -- **Structure-aware dispatch carrying the operator's metadata.** The skop already - knows `major_axis`, its shape, and `vec_nnz`, so it knows its structured axis - and whether counts are exact or bounded. Thread that through `lskges`/`rskges` - and **choose the canonical orientation** (transpose for free so the operator is - column-structured whenever that lands on the CSC-regular kernel; route to CSR - when it's genuinely row-structured and not auto-transposed). This replaces the - re-derive-by-scanning-`colptr` heuristic and removes the wrong-format re-sort. -- **A `≤`-regular variant** so LASO operators get a fast path despite - merge-induced count variation. - -## Recommendation: gather targeted data first - -I'd **benchmark before writing kernels**, for concrete reasons grounded in the -prior investigation: - -1. **The existing `spmm_performance` benchmark already covers both layouts and - sides, but only on generic uniform-random matrices — not these operator - shapes.** It drives `left_spmm` in ColMajor *and* RowMajor (plus `op(B)=Trans`) - across CSR/CSC/COO, and its header documents that this reaches what `left_spmm` - and `right_spmm` together hit via the transpose reduction. But it builds - matrices with `random_coo` (uniform density), so it never produces the - *structured* operators we care about: fixed-nnz-per-column/row, ±1 values, or - mostly-empty columns. Consequently it **never exercises the regular-CSC fast - path** (random density → variable nnz per column → the detector in - `apply_csc_jki_p11` fails) and never goes through `sketch_general`. No baseline - isolates wide-LASO, CountSketch, or the regular-structure kernels — that's the - gap to close. -2. **Which kernel actually fires is the product of (operator shape × side × - layout)** — e.g. tall-SASO right-sketch reaches the regular fast path only when - the original layout is RowMajor (ColMajor right-sketch flips to RowMajor and - lands on the axpy kernel, which has no regular specialization). The benchmark - already crosses layout and side; what it must add is the *structured* operators, - so the regular path and the ±1/mostly-empty structure are actually represented. -3. **The `n`-regime decides the kernel design.** Wide `n` (parallel-over-`n` is - fine) vs narrow `n`/SpMV (needs operator-dimension parallelism, which reshapes - kernel 1) is the biggest design fork. Measure the shapes RandLAPACK actually - drives. -4. **The memory-bound hypothesis is inferred from stack shape, not counters** (SIP - gates perf counters on the M3). If it holds, kernel 3 (drop `vals`) and - index-width (int32 vs int64) dominate arithmetic restructuring — reordering the - priority list. -5. **The per-apply re-sort is a real cost worth quantifying** — and, per the - benchmark, it hits *every* SASO with `vec_nnz ≥ 2` (not just wide LASO), since - only `vec_nnz = 1` is detected as CSC-sorted. It scales with nnz and argues for - fixing dispatch (mark/skip the sort) before chasing kernel-level gains. - -Concretely: extend the harness to sweep {wide SASO, CountSketch, wide LASO} × -{left, right} × {RowMajor, ColMajor} × a range of `n` (including `n=1` SpMV) × -`vec_nnz ∈ {1,2,4,8}`, on both the M3 and an Arm SVE box (NEON/SVE is a -first-class target). That isolates which kernel actually moves the needle before -we hand-tune any of them. - -## Suggested next step - -Either: - -- **Extend the benchmark** to cover these shapes/sides/layouts and produce the - baseline, or -- **Prototype one kernel first** — CountSketch is the highest-leverage and most - self-contained. From 3257edb5d699d8bd7ea7bc1bd58b789b22fa95fb Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Mon, 6 Jul 2026 14:51:03 -0700 Subject: [PATCH 17/20] DevNotes and style --- RandBLAS/skge.hh | 48 +++++++++++++++----------------- RandBLAS/sparse_data/DevNotes.md | 16 +++++++---- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/RandBLAS/skge.hh b/RandBLAS/skge.hh index 842888d2..aab9f58c 100644 --- a/RandBLAS/skge.hh +++ b/RandBLAS/skge.hh @@ -360,16 +360,14 @@ void rskge3( namespace RandBLAS::sparse { -// Apply the ENTIRETY of a sparse sketching operator on the LEFT by explicitly instantiating -// it in the compressed format (CSR or CSC) chosen by a heuristic, then handing that operand -// to left_spmm. op(submat(S)) is S if opS == NoTrans, else S^T; we normalize opS away with a -// share-memory transpose view (which flips the CSR<->CSC sort) so the operator is always a -// d-by-m NoTrans operand. +// Apply a COOMatrix-represented (possibly transposed) operator `op(S)` on the LEFT +// by instantiating it in the compressed format chosen by a heuristic, then handing +// that to left_spmm. // -// Assumes S has no residual offsets and is in CSC or CSR sort order (never None). +// Assumes S is in CSC or CSR sort order (never None). // template -void _lskges_compress_and_apply( +void _lskges_compress_and_apply_coo( blas::Layout layout, blas::Op opS, blas::Op opA, int64_t d, int64_t n, int64_t m, T alpha, const sparse_data::COOMatrix &S, const T *A, int64_t lda, T beta, T *B, int64_t ldb @@ -377,9 +375,8 @@ void _lskges_compress_and_apply( using namespace RandBLAS::sparse_data; using blas::Layout; using blas::Op; if (opS == Op::Trans) { - // op(submat(S)) = S^T (d-by-m); the transpose view shares S's buffers. - auto St = transpose_as_coo(S, true); - _lskges_compress_and_apply(layout, Op::NoTrans, opA, d, n, m, alpha, St, A, lda, beta, B, ldb); + auto St = transpose_as_coo(S, /*share_memory=*/true); + _lskges_compress_and_apply_coo(layout, Op::NoTrans, opA, d, n, m, alpha, St, A, lda, beta, B, ldb); return; } // left_spmm sees opB = opA (the dense operand's op); recover its post-op layout. @@ -394,19 +391,20 @@ void _lskges_compress_and_apply( auto M = coo_to_csc_view_or_copy(S, ptr); left_spmm(layout, Op::NoTrans, opA, d, n, m, alpha, M, 0, 0, A, lda, beta, B, ldb); } + return; } -// Apply the ENTIRETY of a sparse sketching operator on the RIGHT, analogous to -// _lskges_compress_and_apply. op(submat(S)) is the n-by-d operator (S if opS == NoTrans, else -// S^T). right_spmm reduces to a transposed left_spmm, which flips the operand CSR<->CSC; so to -// make that transpose land on the format the heuristic wants (computed in the reduced NoTrans -// frame, where the operator is S^T of shape d-by-n under the transposed layout) we instantiate -// in the OPPOSITE format. +// Apply a COOMatrix-represented (possibly transposed) operator `op(S)` on the RIGHT, +// analogously to _lskges_compress_and_apply_coo. +// +// right_spmm reduces to a transposed left_spmm, which flips the operand CSR<->CSC. +// To get the desired format after that transposition we instantiate S's compressed +// representation in the OPPOSITE of the hueristic's requested format. // -// Assumes S has no residual offsets and is in CSC or CSR sort order (never None). +// Assumes S is in CSC or CSR sort order (never None). // template -void _rskges_compress_and_apply( +void _rskges_compress_and_apply_coo( blas::Layout layout, blas::Op opS, blas::Op opA, int64_t m, int64_t d, int64_t n, T alpha, const T *A, int64_t lda, const sparse_data::COOMatrix &S, T beta, T *B, int64_t ldb @@ -414,9 +412,8 @@ void _rskges_compress_and_apply( using namespace RandBLAS::sparse_data; using blas::Layout; using blas::Op; if (opS == Op::Trans) { - // op(submat(S)) = S^T (n-by-d); the transpose view shares S's buffers. - auto St = transpose_as_coo(S, true); - _rskges_compress_and_apply(layout, Op::NoTrans, opA, m, d, n, alpha, A, lda, St, beta, B, ldb); + auto St = transpose_as_coo(S, /*share_memory=*/true); + _rskges_compress_and_apply_coo(layout, Op::NoTrans, opA, m, d, n, alpha, A, lda, St, beta, B, ldb); return; } Layout trans_layout = flipped_layout(layout); @@ -432,6 +429,7 @@ void _rskges_compress_and_apply( auto M = coo_to_csc_view_or_copy(S, ptr); right_spmm(layout, opA, Op::NoTrans, m, d, n, alpha, A, lda, M, 0, 0, beta, B, ldb); } + return; } // MARK: LSKGES @@ -558,13 +556,13 @@ void lskges( auto [n_rows, n_cols] = dims_before_op(d, m, opS); if (S.nnz < 0) { auto Ssub = submatrix_as_coo(S, n_rows, n_cols, ro_s, co_s); - _lskges_compress_and_apply(layout, opS, opA, d, n, m, alpha, Ssub, A, lda, beta, B, ldb); + _lskges_compress_and_apply_coo(layout, opS, opA, d, n, m, alpha, Ssub, A, lda, beta, B, ldb); return; } bool full_operator = (S.n_rows == n_rows && S.n_cols == n_cols); auto Scoo = coo_view_of_skop(S); if (full_operator) { - _lskges_compress_and_apply(layout, opS, opA, d, n, m, alpha, Scoo, A, lda, beta, B, ldb); + _lskges_compress_and_apply_coo(layout, opS, opA, d, n, m, alpha, Scoo, A, lda, beta, B, ldb); } else { // A proper submatrix of an already-sampled operator: let left_spmm handle the offsets. left_spmm(layout, opS, opA, d, n, m, alpha, Scoo, ro_s, co_s, A, lda, beta, B, ldb); @@ -697,13 +695,13 @@ inline void rskges( auto [n_rows, n_cols] = dims_before_op(n, d, opS); if (S.nnz < 0) { auto Ssub = submatrix_as_coo(S, n_rows, n_cols, ro_s, co_s); - _rskges_compress_and_apply(layout, opS, opA, m, d, n, alpha, A, lda, Ssub, beta, B, ldb); + _rskges_compress_and_apply_coo(layout, opS, opA, m, d, n, alpha, A, lda, Ssub, beta, B, ldb); return; } bool full_operator = (S.n_rows == n_rows && S.n_cols == n_cols); auto Scoo = coo_view_of_skop(S); if (full_operator) { - _rskges_compress_and_apply(layout, opS, opA, m, d, n, alpha, A, lda, Scoo, beta, B, ldb); + _rskges_compress_and_apply_coo(layout, opS, opA, m, d, n, alpha, A, lda, Scoo, beta, B, ldb); } else { // A proper submatrix of an already-sampled operator: let right_spmm handle the offsets. right_spmm(layout, opA, opS, m, d, n, alpha, A, lda, Scoo, ro_s, co_s, beta, B, ldb); diff --git a/RandBLAS/sparse_data/DevNotes.md b/RandBLAS/sparse_data/DevNotes.md index 900fbe6b..c575abf2 100644 --- a/RandBLAS/sparse_data/DevNotes.md +++ b/RandBLAS/sparse_data/DevNotes.md @@ -46,17 +46,23 @@ Sketching dense data with a sparse operator is typically handled with ``sketch_g which is defined in ``skge.hh``. If we call this function with a SparseSkOp object, ``S``, we'd immediately get routed to -either ``lskges`` or ``rskges``. Here's what would happen after we entered one of those functions: +either ``lskges`` or ``rskges``. Both are defined in ``skge.hh``. Here's what happens once +we're inside one of those functions. - 1. If necessary, we'd sample the defining data of ``S`` with ``RandBLAS::fill_sparse(S)``. + 1. We get a COO view of ``S`` (sampling its defining data first, if that hasn't happened yet). - 2. We'd obtain a lightweight view of ``S`` as a COOMatrix, and we'd pass that matrix to ``left_spmm`` - (if inside ``lskges``) or ``right_spmm`` (if inside ``rskges``). + 2. If we've been asked for a *proper submatrix* of an already-sampled operator, we hand the + COO view and the ``(ro_s, co_s)`` offsets straight to ``[left/right]_spmm`` and let that + function deal with the offsets. Otherwise, we have the full operator, and we proceed to compress it. + 3. Compression and application happens in ``_[l/r]skges_compress_and_apply_coo``. These functions + normalize-away the transposition and use a heuristic to choose the compressed format (CSR or CSC). + This heuristic can differ from that used if we had called `[left/right]_spmm` directly on + a COO matrix. ## Sketching sparse data with dense operators -If we call ``sketch_sparse`` with a DenseSkOp, ``S``, and a sparse matrix, ``A``, then we'll get routed to either +If we call ``sketch_sparse`` with a DenseSkOp, ``S``, and a sparse matrix, ``A``, then we'll get routed to either ``lsksp3`` or ``rsksp3``. From there, we'll do the following. From 94a09bfbb6455cbded42acee2d33d3089b84b9cf Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Mon, 6 Jul 2026 14:56:50 -0700 Subject: [PATCH 18/20] agents.md --- AGENTS.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0249d621..af91f65e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -244,7 +244,7 @@ GitHub Actions workflows test: All CI tests must pass before merging. -## Working with Codex on RandBLAS +## Working on RandBLAS with agents ### Preferred Workflow @@ -316,7 +316,3 @@ All CI tests must pass before merging. - GitHub Issues: https://github.com/BallisticLA/RandBLAS/issues - Documentation: https://randblas.readthedocs.io/ - Contact: Project maintainers listed in repository - ---- - -*This AGENTS.md file was created to help Codex understand the RandBLAS project structure, conventions, and workflows. Update it as the project evolves.* From e6550539606621e35c318d66d518609b36236b86 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Mon, 6 Jul 2026 14:58:12 -0700 Subject: [PATCH 19/20] revert unnecessary change --- test/datastructures/test_coo_matrix.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datastructures/test_coo_matrix.cc b/test/datastructures/test_coo_matrix.cc index d23ea142..4f5f5e23 100644 --- a/test/datastructures/test_coo_matrix.cc +++ b/test/datastructures/test_coo_matrix.cc @@ -64,7 +64,7 @@ void sparseskop_to_dense( sint_t row = S0.rows[i]; sint_t col = S0.cols[i]; T val = S0.vals[i]; - mat[idx(row, col)] += val; + mat[idx(row, col)] = val; } } From c03133b40ac8069e5f348b3b578d66c8a83055c0 Mon Sep 17 00:00:00 2001 From: Riley Murray Date: Mon, 6 Jul 2026 15:18:00 -0700 Subject: [PATCH 20/20] eliminate unnecessary scan over data --- RandBLAS/sparse_skops.hh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/RandBLAS/sparse_skops.hh b/RandBLAS/sparse_skops.hh index 390d8e4f..70e4c55f 100644 --- a/RandBLAS/sparse_skops.hh +++ b/RandBLAS/sparse_skops.hh @@ -873,9 +873,14 @@ COOMatrix submatrix_as_coo( A.cols = cols; A.nnz = nnz; // fill_sparse_unpacked emits each major-axis vector in sorted order, so the sampled - // submatrix is CSR- or CSC-sorted; label it as such (rather than None) so downstream - // conversions can take an O(nnz) fast path. - A.sort = RandBLAS::sparse_data::coo_arrays_determine_sort(A.nnz, A.rows, A.cols); + // submatrix is CSR- or CSC-sorted; label it as such. + if (D.dim_major == D.dim_minor) { + // degenerate case; recompute sort order with another scan over the data. + A.sort = RandBLAS::sparse_data::coo_arrays_determine_sort(A.nnz, A.rows, A.cols); + } else { + using RandBLAS::sparse_data::NonzeroSort; + A.sort = (D.dim_major == D.n_rows) ? NonzeroSort::CSC : NonzeroSort::CSR; + } return A; }