From 7629ee3e6a73e0553cb7f31296f0ab2888832610 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Tue, 28 Apr 2026 11:36:31 -0400 Subject: [PATCH 01/14] Adding funNystrom++ --- RandLAPACK.hh | 3 + RandLAPACK/comps/rl_hutchinson.hh | 87 ++++++++ RandLAPACK/comps/rl_lanczos_fa.hh | 253 ++++++++++++++++++++++++ RandLAPACK/comps/rl_syrf.hh | 13 +- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 189 ++++++++++++++++++ RandLAPACK/drivers/rl_revd2.hh | 176 ++++++++--------- RandLAPACK/misc/rl_util.hh | 3 +- test/CMakeLists.txt | 2 + test/comps/test_lanczos_fa.cc | 191 ++++++++++++++++++ test/comps/test_syrf.cc | 2 +- test/drivers/test_fun_nystrom_pp.cc | 147 ++++++++++++++ 11 files changed, 967 insertions(+), 99 deletions(-) create mode 100644 RandLAPACK/comps/rl_hutchinson.hh create mode 100644 RandLAPACK/comps/rl_lanczos_fa.hh create mode 100644 RandLAPACK/drivers/rl_fun_nystrom_pp.hh create mode 100644 test/comps/test_lanczos_fa.cc create mode 100644 test/drivers/test_fun_nystrom_pp.cc diff --git a/RandLAPACK.hh b/RandLAPACK.hh index c9741e3ee..91c4a31a7 100644 --- a/RandLAPACK.hh +++ b/RandLAPACK.hh @@ -27,6 +27,8 @@ #include "RandLAPACK/comps/rl_syrf.hh" #include "RandLAPACK/comps/rl_orth.hh" #include "RandLAPACK/comps/rl_rpchol.hh" +#include "RandLAPACK/comps/rl_lanczos_fa.hh" +#include "RandLAPACK/comps/rl_hutchinson.hh" // Drivers #include "RandLAPACK/drivers/rl_rsvd.hh" @@ -39,6 +41,7 @@ #include "RandLAPACK/drivers/rl_revd2.hh" #include "RandLAPACK/drivers/rl_abrik.hh" #include "RandLAPACK/drivers/rl_krill.hh" +#include "RandLAPACK/drivers/rl_fun_nystrom_pp.hh" // Cuda functions - issues with linking/visibility when present if the below is uncommented. // A temporary fix is to add the below directly in the test/benchmark files. diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh new file mode 100644 index 000000000..59bb856f1 --- /dev/null +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -0,0 +1,87 @@ +#pragma once + +#include "rl_blaspp.hh" + +#include +#include +#include +#include + +namespace RandLAPACK { + + +/// Hutchinson stochastic trace estimator. +/// +/// Estimates tr(M) using the identity E[ω^T M ω] = tr(M) for any zero-mean +/// unit-variance ω. Draws s independent Gaussian vectors, applies M to all at once, then +/// returns the averaged quadratic form (1/s) * <Ω, Z>_F where Z = M*Ω. +/// +/// Standalone and reusable: the operator M is provided as a callable rather +/// than a fixed type, so this component can be used for any trace estimation +/// problem in RandLAPACK without modification. +/// +/// @tparam T Floating-point scalar type. +/// @tparam RNG Random number generator type for RandBLAS. +template +class Hutchinson { +public: + // Internal sketch buffer — grown with calloc/free, never shrunk. + T* Omega = nullptr; int64_t Omega_sz = 0; + + ~Hutchinson() { free(Omega); } + + // ------------------------------------------------------------------ + /// Low-level estimator: given precomputed Ω (n×s) and Z = M*Ω (n×s), + /// returns <Ω, Z>_F / s = (1/s) * Σ_j ω_j^T (M ω_j). + /// Uses blas::dot on flattened n*s arrays — no allocation. + /// + /// @param[in] Omega_buf n×s sketch matrix (column-major). + /// @param[in] Z n×s result of M applied to Omega_buf (column-major). + /// @param[in] n Ambient dimension. + /// @param[in] s Number of samples. + T estimate(const T* Omega_buf, const T* Z, int64_t n, int64_t s) const { + // Frobenius inner product of two n×s matrices, treated as flat n*s vectors + return blas::dot(n * s, Omega_buf, 1, Z, 1) / static_cast(s); + } + + // ------------------------------------------------------------------ + /// High-level estimator: draws Ω internally, calls apply_M to fill Z, + /// then returns the trace estimate. + /// + /// apply_M must have signature: (const T* Omega, T* Z, int64_t n, int64_t s) + /// It receives the n×s Rademacher matrix and must overwrite Z with M*Ω. + /// + /// @param[in] apply_M Callable that applies M to an n×s matrix. + /// @param[in] n Ambient dimension. + /// @param[in] s Number of Hutchinson samples. + /// @param[in] state RandBLAS RNG state; advanced on return. + template + T call(ApplyM apply_M, int64_t n, int64_t s, + RandBLAS::RNGState& state) { + // Grow Omega buffer if needed (reuse across repeated calls) + if (n * s > Omega_sz) { + free(Omega); + Omega = (T*) calloc(n * s, sizeof(T)); + Omega_sz = n * s; + } + + // Draw Ω with iid Rademacher entries (Unif{±1}). + // RandBLAS has no ScalarDist::Rademacher, but ScalarDist::Uniform fills + // with Unif[-1,1] via r123ext::uneg11; sign-transforming gives exact ±1. + RandBLAS::DenseDist D(n, s, RandBLAS::ScalarDist::Uniform); + state = RandBLAS::fill_dense(D, Omega, state); + for (int64_t i = 0; i < n * s; ++i) + Omega[i] = (Omega[i] >= 0) ? (T)1 : (T)-1; + + // Allocate Z and apply the operator + T* Z = (T*) calloc(n * s, sizeof(T)); + apply_M(Omega, Z, n, s); + + T result = estimate(Omega, Z, n, s); + free(Z); + return result; + } +}; + + +} // end namespace RandLAPACK diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh new file mode 100644 index 000000000..b8f2f1db2 --- /dev/null +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -0,0 +1,253 @@ +#pragma once + +#include "rl_blaspp.hh" +#include "rl_lapackpp.hh" +#include "rl_linops.hh" + +#include +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +namespace RandLAPACK { + + +/// d-step block Lanczos for matrix function application f(A)B. +/// +/// Given a SymmetricLinearOperator A and a matrix B (n×s), approximates +/// f(A)B column-wise using a Krylov subspace of dimension d per column. +/// +/// Each column b_j of B generates an independent Krylov sequence. The +/// recurrence produces a d×d symmetric tridiagonal T_j per column; f(A)b_j +/// is approximated by the first column of Q_j * f(T_j) * Q_j' * b_j, where +/// Q_j = [q_1,...,q_d] is the Lanczos basis and f(T_j) is evaluated via a +/// dense d×d tridiagonal eigendecomposition (lapack::stev). +/// +/// Key formula (Tyler Chen): since q_1 = b_j/||b_j||, we have +/// f(A)b_j ≈ ||b_j|| * Q_j * S_j * diag(f(θ_j)) * S_j[0,:]^T +/// where (S_j, θ_j) = eig(T_j) from stev and S_j[0,:] is the first row +/// of the eigenvector matrix. +/// +/// f is any scalar callable T → T (e.g., sqrt, log, pow). +/// The stev calls per column are independent and run in parallel (OpenMP). +/// +/// Memory layout: K is stored as (d+1) contiguous n×s blocks. +/// K[step * n * s + col * n + row] = entry (row, col) of the step-th block. +/// This keeps each step's n×s matrix contiguous for batch matvec via SLO, +/// while allowing strided gemv (lda = n*s) for per-column reconstruction. +/// +/// @tparam T Floating-point scalar type. +/// @tparam RNG Random number generator type (unused here; kept for API uniformity). +template +class LanczosFA { +public: + /// Reorthogonalization control. + /// -1 = full (project out all previous Krylov vectors after each step). + /// 0 = none (vanilla Lanczos, per Persson's reference implementation). + /// Lanczos-FA tolerates loss of orthogonality better than eigenvalue + /// Lanczos (Paige-Greenbaum theory), so vanilla often works in practice. + /// Full reorthogonalization is the safe default for a numerical library. + int64_t reorth = -1; + + // Internal buffers — grown with calloc/free, never shrunk between calls. + // K: (d+1) × n × s — Krylov basis blocks; see layout note above. + // alpha: s × d — tridiagonal diagonals, alpha[j*d + i] = α_{i,j}. + // beta: s × (d-1) — tridiagonal subdiagonals, beta[j*(d-1) + i] = β_{i+1,j}. + // normb: s — column norms of B before normalization. + T* K = nullptr; int64_t K_sz = 0; + T* alpha = nullptr; int64_t alpha_sz = 0; + T* beta = nullptr; int64_t beta_sz = 0; + T* normb = nullptr; int64_t normb_sz = 0; + + ~LanczosFA() { free(K); free(alpha); free(beta); free(normb); } + + // ------------------------------------------------------------------ + /// Run d-step block Lanczos recurrence. + /// Fills K, alpha, beta, normb from B (n×s column-major). + /// Calls A exactly d times, each application to an n×s matrix. + /// + /// @param[in] A SymmetricLinearOperator — matvec oracle. + /// @param[in] B n×s input matrix (column-major); not modified. + /// @param[in] n Dimension of A. + /// @param[in] s Number of right-hand sides (Hutchinson samples). + /// @param[in] d Number of Lanczos steps. + template + void run(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { + // Grow buffers if needed + if ((d + 1) * n * s > K_sz) { + free(K); + K = (T*) calloc((d + 1) * n * s, sizeof(T)); + K_sz = (d + 1) * n * s; + } + // omega_needed = max(d*s, 4) to match error-est buffer sizing in REVD2 + if (d * s > alpha_sz) { + free(alpha); + alpha = (T*) calloc(d * s, sizeof(T)); + alpha_sz = d * s; + } + if ((d - 1) * s > beta_sz && d > 1) { + free(beta); + beta = (T*) calloc((d - 1) * s, sizeof(T)); + beta_sz = (d - 1) * s; + } + if (s > normb_sz) { + free(normb); + normb = (T*) calloc(s, sizeof(T)); + normb_sz = s; + } + + // Step 0: q_1 = column-normalize B; store in K[:,:,0] + T* K0 = K; + std::copy(B, B + n * s, K0); + for (int64_t j = 0; j < s; ++j) { + T nrm = blas::nrm2(n, K0 + j * n, 1); + normb[j] = nrm; + blas::scal(n, (T)1.0 / nrm, K0 + j * n, 1); + } + + // Step 0 matvec: K[:,:,1] = A * K[:,:,0] + T* K1 = K + n * s; + A(Layout::ColMajor, s, (T)1.0, K0, n, (T)0.0, K1, n); + + // α[0, j] = dot(K1[:,j], K0[:,j]) — diagonal entry for step 0 + for (int64_t j = 0; j < s; ++j) + alpha[j * d + 0] = blas::dot(n, K1 + j * n, 1, K0 + j * n, 1); + + // Main Lanczos loop: steps 1..d-1 + // At the start of iteration i: + // K[:,:,i] = A*q_i - β_i*q_{i-1} (partial three-term, β part done last iter) + // K[:,:,i+1] is free scratch + // This iteration: + // (1) subtract α_i*q_i to complete three-term → K_{i} becomes unnormalized q_{i+1} + // (2) optional reorthogonalization + // (3) β_{i+1} = ||K_{i+1}|| + // (4) normalize K_{i+1} → q_{i+1} + // (5) K_{i+2} = A*q_{i+1} - β_{i+1}*q_i (start of next three-term) + // (6) α_{i+1} = dot(K_{i+2}, q_{i+1}) + for (int64_t i = 0; i < d - 1; ++i) { + T* K_prev = K + i * n * s; // K[:,:,i] = q_i (normalized) + T* K_curr = K + (i + 1) * n * s; // K[:,:,i+1] = partial (A*q_i - β_i*q_{i-1}) + T* K_new = K + (i + 2) * n * s; // K[:,:,i+2] = scratch for A*q_{i+1} + + // (1) Complete three-term: K_curr -= α_i * K_prev, per column + for (int64_t j = 0; j < s; ++j) + blas::axpy(n, -alpha[j * d + i], K_prev + j * n, 1, K_curr + j * n, 1); + + // (2) Optional full reorthogonalization against all q_0,...,q_i + // Subtract projections onto previous Krylov vectors to recover lost + // orthogonality accumulated in floating-point arithmetic. + int64_t reorth_steps = (reorth < 0) ? (i + 1) : reorth; + for (int64_t prev = 0; prev < reorth_steps; ++prev) { + T* K_p = K + prev * n * s; + for (int64_t j = 0; j < s; ++j) { + T coeff = blas::dot(n, K_curr + j * n, 1, K_p + j * n, 1); + blas::axpy(n, -coeff, K_p + j * n, 1, K_curr + j * n, 1); + } + } + + // (3) β_{i+1} = column norms and (4) normalize to get q_{i+1} + for (int64_t j = 0; j < s; ++j) { + T nrm = blas::nrm2(n, K_curr + j * n, 1); + beta[j * (d - 1) + i] = nrm; + blas::scal(n, (T)1.0 / nrm, K_curr + j * n, 1); + } + + // (5) K_new = A*q_{i+1} - β_{i+1}*q_i (one batch matvec + axpy per column) + A(Layout::ColMajor, s, (T)1.0, K_curr, n, (T)0.0, K_new, n); + for (int64_t j = 0; j < s; ++j) + blas::axpy(n, -beta[j * (d - 1) + i], K_prev + j * n, 1, K_new + j * n, 1); + + // (6) α_{i+1} = dot(K_new[:,j], q_{i+1}[:,j]) + for (int64_t j = 0; j < s; ++j) + alpha[j * d + (i + 1)] = blas::dot(n, K_new + j * n, 1, K_curr + j * n, 1); + } + } + + // ------------------------------------------------------------------ + /// Evaluate f(A)B from precomputed Krylov data (K, alpha, beta, normb). + /// Per column j: eigendecompose T_j via lapack::stev, then compute + /// out[:,j] = normb[j] * Q_j * S_j * diag(f(θ_j)) * S_j[0,:]^T + /// where S_j[0,:] is the first row of the eigenvector matrix. + /// Per-column stev calls are independent — parallelized with OpenMP. + /// + /// @param[in] f Scalar callable T→T applied to tridiagonal eigenvalues. + /// @param[in] n Dimension of A. + /// @param[in] s Number of right-hand sides. + /// @param[in] d Number of Lanczos steps (tridiagonal size). + /// @param[out] out n×s output matrix (column-major); overwritten. + template F> + void apply(F f, int64_t n, int64_t s, int64_t d, T* out) { + // Per-thread scratch: alpha_j(d), beta_j(d-1), Z_j(d*d), c_j(d), v_j(d) + // Total per thread = d + (d-1) + d*d + d + d = d^2 + 4d - 1 + int64_t scratch_per_thread = d * d + 3 * d + std::max(d - 1, (int64_t)0); + int nthreads = 1; +#ifdef _OPENMP + nthreads = omp_get_max_threads(); +#endif + T* scratch = (T*) calloc(nthreads * scratch_per_thread, sizeof(T)); + +#pragma omp parallel for schedule(static) + for (int64_t j = 0; j < s; ++j) { + int tid = 0; +#ifdef _OPENMP + tid = omp_get_thread_num(); +#endif + T* base = scratch + tid * scratch_per_thread; + T* alpha_j = base; + T* beta_j = alpha_j + d; + T* Z_j = beta_j + std::max(d - 1, (int64_t)0); + T* c_j = Z_j + d * d; + T* v_j = c_j + d; + + // Copy tridiagonal for column j (stev destroys alpha and beta in-place) + std::copy(alpha + j * d, alpha + j * d + d, alpha_j); + if (d > 1) + std::copy(beta + j * (d-1), beta + j * (d-1) + (d-1), beta_j); + + // d×d tridiagonal eigendecomposition: T_j = Z_j * diag(θ) * Z_j^T + // alpha_j → eigenvalues θ (ascending); Z_j → eigenvectors (column-major) + lapack::stev(lapack::Job::Vec, d, alpha_j, beta_j, Z_j, d); + + // c_j[i] = f(θ_i) * S_j[0, i] + // In column-major Z_j (d×d): entry (row=0, col=i) = Z_j[i*d + 0] + for (int64_t i = 0; i < d; ++i) + c_j[i] = f(alpha_j[i]) * Z_j[i * d + 0]; + + // v_j = Z_j * c_j (d×d matrix times d-vector) + blas::gemv(Layout::ColMajor, Op::NoTrans, d, d, + (T)1.0, Z_j, d, c_j, 1, (T)0.0, v_j, 1); + + // out[:,j] = normb[j] * Q_j * v_j + // Q_j is n×d with column stride n*s (strided view into K buffer) + blas::gemv(Layout::ColMajor, Op::NoTrans, n, d, + normb[j], K + j * n, n * s, v_j, 1, (T)0.0, out + j * n, 1); + } + + free(scratch); + } + + // ------------------------------------------------------------------ + /// Combined run + apply: compute f(A)B in one call. + /// + /// @param[in] A SymmetricLinearOperator. + /// @param[in] B n×s input matrix (column-major). + /// @param[in] n Dimension of A. + /// @param[in] s Number of right-hand sides. + /// @param[in] f Scalar function T→T. + /// @param[in] d Lanczos steps. + /// @param[out] out n×s output, overwritten with f(A)B approximation. + template F> + void call(SLO& A, const T* B, int64_t n, int64_t s, F f, int64_t d, T* out) { + run(A, B, n, s, d); + apply(f, n, s, d, out); + } +}; + + +} // end namespace RandLAPACK diff --git a/RandLAPACK/comps/rl_syrf.hh b/RandLAPACK/comps/rl_syrf.hh index 66d4da01c..023acbd62 100644 --- a/RandLAPACK/comps/rl_syrf.hh +++ b/RandLAPACK/comps/rl_syrf.hh @@ -74,7 +74,7 @@ class SYRF { int64_t m, const T* A, int64_t k, - std::vector &Q, + T* Q, RandBLAS::RNGState &state, T* work_buff ) { @@ -86,7 +86,7 @@ class SYRF { int call( SLO &A, int64_t k, - std::vector &Q, + T* Q, RandBLAS::RNGState &state, T* work_buff ) { @@ -97,19 +97,18 @@ class SYRF { RandBLAS::util::safe_scal(m * k, (T) 0.0, work_buff); - T* Q_dat = util::upsize(m * k, Q); - syps.call(A, k, state, work_buff, Q_dat); + syps.call(A, k, state, work_buff, Q); // Q = orth(A * Omega) - A(Layout::ColMajor, k, (T) 1.0, work_buff, m, (T) 0.0, Q_dat, m); + A(Layout::ColMajor, k, (T) 1.0, work_buff, m, (T) 0.0, Q, m); if(this->cond_check) { util::upsize(m * k, this->cond_work_mat); util::upsize(k, this->cond_work_vec); this->cond_nums.push_back( - util::cond_num_check(m, k, Q.data(), this->verbose) + util::cond_num_check(m, k, Q, this->verbose) ); } - if(this->orth.call(m, k, Q.data())) + if(this->orth.call(m, k, Q)) throw std::runtime_error("Orthogonalization failed."); if (!callers_work_buff) diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh new file mode 100644 index 000000000..217ef472c --- /dev/null +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -0,0 +1,189 @@ +#pragma once + +#include "rl_blaspp.hh" +#include "rl_linops.hh" + +#include +#include +#include +#include +#include +#include +#include + +namespace RandLAPACK { + + +/// funNyström++ trace estimator: estimates tr(f(A)) for symmetric PSD A. +/// +/// Algorithm (3 phases): +/// Phase 1 — Nyström approximation via REVD2 (k matvecs): +/// (V, λ) = REVD2(A, k) →  = V diag(λ) V^T +/// tr(f(Â)) = Σ f(λ_i) + (n-k) * f(0) [free from the eigenvalues] +/// +/// Phase 2 — Hutchinson correction on the residual (d*s matvecs): +/// Ω = Rademacher(n×s) +/// Z1 = f(A)*Ω via LanczosFA [d batch matvecs] +/// Z2 = f(Â)*Ω = V * (f(λ) ⊙ (V^T Ω)) [two GEMMs, free] +/// correction = <Ω, Z1 - Z2>_F / s +/// +/// return tr(f(Â)) + correction +/// +/// Complexity: O(√κ/ε) matvecs total, versus O(√κ/ε²) for naive Hutchinson. +/// The variance reduction comes from subtracting the control variate f(Â)Ω: +/// because  ≈ A in the dominant eigenspace, f(A)-f(Â) has small Frobenius +/// norm and the Hutchinson estimator converges in far fewer samples. +/// +/// Requirements on A: +/// - Symmetric positive definite (λ_min > 0). The spectral interval [a,b] +/// with 0 < a < b is assumed throughout (see paper). +/// - For f = log, A must be strictly PD (log(0) = -∞). Pass f_zero = 0 only +/// for functions where f(0) is well-defined (e.g., sqrt, x^α with α > 0). +/// +/// Parameter selection (f = sqrt, κ = λ_max/λ_min): +/// d = O(√κ log(n/ε)) Lanczos steps — caller's responsibility +/// s = O(1/(√κ ε)) Hutchinson samples — caller's responsibility +/// k = O(√κ/ε) Nyström rank — caller's responsibility +/// Use error_est_power_iters=0 in REVD2 and tol=0 to get fixed-rank behavior +/// (k never changes; the adaptive loop exits after one iteration). +/// +/// @tparam REVD2_t Type of the REVD2 Nyström component. +/// @tparam LanczosFA_t Type of the LanczosFA component. +/// @tparam Hutchinson_t Type of the Hutchinson trace estimator component. +template +class FunNystromPP { +public: + using T = typename REVD2_t::T; + using RNG = typename REVD2_t::RNG; + + REVD2_t& revd2; + LanczosFA_t& lanczos_fa; + Hutchinson_t& hutchinson; + + // Persistent output/working buffers — grown with calloc/free, never shrunk. + // V, eigvals: REVD2 outputs reused across calls with same (n, k). + // F_vec: f applied elementwise to eigvals. + // tmp: k×s scratch for V^T Ω and F_vec ⊙ (V^T Ω). + // Z1: n×s LanczosFA output f(A)*Ω. + // Z2: n×s control variate f(Â)*Ω = V*(F_vec ⊙ V^T Ω). + std::vector V_buf, eigvals_buf; // std::vector for REVD2's adaptive resize + T* F_vec = nullptr; int64_t F_vec_sz = 0; + T* tmp = nullptr; int64_t tmp_sz = 0; + T* Z1 = nullptr; int64_t Z1_sz = 0; + T* Z2 = nullptr; int64_t Z2_sz = 0; + + ~FunNystromPP() { free(F_vec); free(tmp); free(Z1); free(Z2); } + + FunNystromPP(REVD2_t& r, LanczosFA_t& l, Hutchinson_t& h) + : revd2(r), lanczos_fa(l), hutchinson(h) {} + + // ------------------------------------------------------------------ + /// Estimate tr(f(A)). + /// + /// @param[in] A SymmetricLinearOperator — matvec oracle for A. + /// @param[in] f Scalar function T→T (e.g., sqrt, log, pow(x,α)). + /// @param[in] f_zero f(0): value at zero. Use 0.0 for sqrt/power. + /// Must be finite; assert fires if infinite. + /// For log, A must be strictly PD — pass any finite + /// placeholder and ensure λ_min > 0. + /// @param[in] k Nyström rank. Caller chooses based on κ, ε. + /// @param[in] s Hutchinson samples. Caller chooses based on κ, ε. + /// @param[in] d Lanczos steps per sample. Caller chooses based on κ. + /// @param[in] state RandBLAS RNG state; advanced on return. + /// @returns Estimate of tr(f(A)). + template F> + T call( + SLO& A, + F f, + T f_zero, + int64_t k, + int64_t s, + int64_t d, + RandBLAS::RNGState& state + ) { + assert(std::isfinite(f_zero) && "f_zero must be finite; for log, ensure A is strictly PD"); + + int64_t n = A.dim; + + // ------------------------------------------------------------------ + // Phase 1: Nyström approximation + // REVD2 with error_est_power_iters=0 and tol=0 runs a single pass + // with fixed rank k. V_buf (n×k) and eigvals_buf (k) are the outputs. + // ------------------------------------------------------------------ + // REVD2::call takes rank by reference and may increase it adaptively. + // We pass k_in (a copy) so the caller's k stays unchanged; we still + // need the original k below for the (n-k)*f(0) tail correction. + int64_t k_in = k; + revd2.call(A, k_in, (T)0.0, V_buf, eigvals_buf, state); + T* V = V_buf.data(); + T* eigvals = eigvals_buf.data(); + + // f(λ): apply scalar f to k eigenvalues + if (k > F_vec_sz) { + free(F_vec); + F_vec = (T*) calloc(k, sizeof(T)); + F_vec_sz = k; + } + for (int64_t i = 0; i < k; ++i) + F_vec[i] = f(eigvals[i]); + + // tr(f(Â)) = Σ f(λ_i) + (n-k)*f(0) + T tr_Ahat = (T)0.0; + for (int64_t i = 0; i < k; ++i) + tr_Ahat += F_vec[i]; + tr_Ahat += static_cast(n - k) * f_zero; + + // ------------------------------------------------------------------ + // Phase 2: Hutchinson correction on residual f(A) - f(Â) + // Ω is drawn inside hutchinson.call; Z = (f(A)-f(Â))*Ω is computed + // by the lambda below, then the inner product <Ω,Z>/s is returned. + // ------------------------------------------------------------------ + if (k * s > tmp_sz) { + free(tmp); + tmp = (T*) calloc(k * s, sizeof(T)); + tmp_sz = k * s; + } + if (n * s > Z1_sz) { + free(Z1); + Z1 = (T*) calloc(n * s, sizeof(T)); + Z1_sz = n * s; + } + if (n * s > Z2_sz) { + free(Z2); + Z2 = (T*) calloc(n * s, sizeof(T)); + Z2_sz = n * s; + } + + // Residual operator: Z = f(A)*Ω - f(Â)*Ω + // Captures V, F_vec, k, s, d, n, Z1, Z2, tmp by reference. + auto apply_residual = [&](const T* Omega, T* Z, int64_t n_, int64_t s_) { + // Z1 = f(A)*Ω via block Lanczos-FA (d batch matvecs) + lanczos_fa.call(A, Omega, n_, s_, f, d, Z1); + + // tmp = V^T * Ω (k×s) + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, + k, s_, n_, (T)1.0, V, n_, Omega, n_, (T)0.0, tmp, k); + + // tmp[:,j] *= F_vec (scale each of the s columns by F_vec elementwise) + // tmp is k×s column-major; column j is tmp + j*k + for (int64_t j = 0; j < s_; ++j) + for (int64_t i = 0; i < k; ++i) + tmp[j * k + i] *= F_vec[i]; + + // Z2 = V * tmp (n×s) = f(Â)*Ω + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + n_, s_, k, (T)1.0, V, n_, tmp, k, (T)0.0, Z2, n_); + + // Z = Z1 - Z2 (in-place: copy Z1 → Z then subtract Z2) + std::copy(Z1, Z1 + n_ * s_, Z); + blas::axpy(n_ * s_, (T)-1.0, Z2, 1, Z, 1); + }; + + T correction = hutchinson.call(apply_residual, n, s, state); + + return tr_Ahat + correction; + } +}; + + +} // end namespace RandLAPACK diff --git a/RandLAPACK/drivers/rl_revd2.hh b/RandLAPACK/drivers/rl_revd2.hh index 3667691a3..266360250 100644 --- a/RandLAPACK/drivers/rl_revd2.hh +++ b/RandLAPACK/drivers/rl_revd2.hh @@ -9,6 +9,7 @@ #include #include +#include #include namespace RandLAPACK { @@ -16,10 +17,9 @@ namespace RandLAPACK { // ----------------------------------------------------------------------------- /// Power scheme for error estimation, based on Algorithm E.1 from https://arxiv.org/pdf/2110.02820.pdf. -/// This routine is too specialized to be included into RandLAPACK::utils /// p - number of algorithm iterations -/// vector_buf - buffer for vector operations -/// Mat_buf - buffer for matrix operations +/// vector_buf - buffer for vector operations (must hold at least 4*m elements) +/// Mat_buf - buffer for matrix operations (must hold at least m*k elements) /// All other parameters come from REVD2 template T power_error_est( @@ -35,35 +35,27 @@ T power_error_est( T err = 0; for(int i = 0; i < p; ++i) { T g_norm = blas::nrm2(m, vector_buf, 1); - // Compute g = g / ||g|| - we need this because dot product does not take in an alpha blas::scal(m, 1 / g_norm, vector_buf, 1); - // Compute V' * g / ||g|| - // Using the second column of vector_buff as a buffer for matrix-vector product + // V' * g/||g|| — result in column 1 of vector_buf gemv(Layout::ColMajor, Op::Trans, m, k, 1.0, V, m, vector_buf, 1, 0.0, &vector_buf[m], 1); - // Compute V*E, eigvals diag - // Using Mat_buf as a buffer for V * diag(eigvals). + // V * diag(eigvals) into Mat_buf for (int i = 0, j = 0; i < m * k; ++i) { Mat_buf[i] = V[i] * eigvals[j]; if((i + 1) % m == 0 && i != 0) ++j; } - // Compute V * diag(eigvals) * V' * g / ||g|| - // Using the third column of vector_buf as a buffer for matrix-vector product + // V * diag(eigvals) * V' * g/||g|| — result in column 2 of vector_buf gemv(Layout::ColMajor, Op::NoTrans, m, k, 1.0, Mat_buf, m, &vector_buf[m], 1, 0.0, &vector_buf[2 * m], 1); - // Compute A * g / ||g|| - // Using the forth column of vector_buff as a buffer for matrix-vector product + // A * g/||g|| — result in column 3 of vector_buf A(Layout::ColMajor, 1, 1.0, vector_buf, m, 0.0, &vector_buf[3*m], m); - // symv(Layout::ColMajor, uplo, m, 1.0, A, m, vector_buf, 1, 0.0, &vector_buf[3 * m], 1); - // Compute w = (A * g / ||g|| - V * diag(eigvals) * V' * g / ||g||) - // Result is stored in the 4th column of vector_buf + // w = A*g/||g|| - V*diag(eigvals)*V'*g/||g|| blas::axpy(m, -1.0, &vector_buf[2 * m], 1, &vector_buf[3 * m], 1); - // Compute (g / ||g||)' * w - this is our measure for the error - err = blas::dot(m, vector_buf, 1, &vector_buf[3 * m], 1); - // v_0 <- v + // error estimate: (g/||g||)' * w + err = blas::dot(m, vector_buf, 1, &vector_buf[3 * m], 1); std::copy(&vector_buf[3 * m], &vector_buf[4 * m], vector_buf); } return err; @@ -79,13 +71,13 @@ class REVD2 { int error_est_p; bool verbose; - std::vector Y; - std::vector Omega; - std::vector R; - std::vector S; - std::vector symrf_work; + // Internal working buffers — owned by this object, grown with calloc/free + T* Y = nullptr; int64_t Y_sz = 0; + T* Omega = nullptr; int64_t Omega_sz = 0; + T* R = nullptr; int64_t R_sz = 0; + T* S = nullptr; int64_t S_sz = 0; + T* symrf_work = nullptr; int64_t symrf_work_sz = 0; - // Constructor REVD2( SYRF_t &syrf_obj, int error_est_power_iters, @@ -95,37 +87,30 @@ class REVD2 { verbose = verb; } + ~REVD2() { + free(Y); + free(Omega); + free(R); + free(S); + free(symrf_work); + } + /// Computes a rank-k approximation to an EVD of a symmetric positive semidefinite matrix: /// A_hat = V diag(eigvals) V^*, /// where V is a matrix of eigenvectors and eigvals is a vector of eigenvalues. - /// - /// This function is adaptive. If the tolerance is not met, increases the rank - /// estimation parameter by 2. Tolerance is the maximum of user-specified tol times - /// 5 and computed 'nu' times 5. This is motivated by the fact that the approximation - /// error will never be smaller than nu. - /// - /// This code is identical to Algorithm E2 from https://arxiv.org/pdf/2110.02820.pdf. - /// It uses a SymmetricRangeFinder for constructing a sketching operator. - /// It has a lot of potential in terms of storage space optimization, - /// which, however, will affect readability. - /// - /// @param[in] m - /// The number of rows in the matrix A. /// - /// @param[in] A - /// The m-by-m matrix A, stored in a column-major format. - /// Must be SPD. + /// Adaptive: if tolerance is not met, doubles k until convergence or k == m. + /// Set error_est_power_iters=0 and tol=0 for fixed-rank single-pass behavior + /// (k never changes, exits after one iteration). /// - /// @param[in] k - /// Column dimension of a sketch, k <= n. - /// - /// @param[in, out] V - /// On entry, is empty and may not have any space allocated for it. - /// On exit, stores m-by-k matrix matrix of (approximate) eigenvectors. + /// This code is identical to Algorithm E2 from https://arxiv.org/pdf/2110.02820.pdf. /// - /// @param[in, out] eigvals - /// On entry, is empty and may not have any space allocated for it. - /// On exit, stores k eigenvalues. + /// @param[in] m Number of rows/cols of A. + /// @param[in] A m-by-m matrix, column-major, must be SPD. + /// @param[in,out] k Sketch rank on entry; final rank on exit (may grow in adaptive mode). + /// @param[in] tol Convergence tolerance. Use 0.0 for fixed-rank. + /// @param[in,out] V On exit, m-by-k matrix of approximate eigenvectors. + /// @param[in,out] eigvals On exit, k approximate eigenvalues. /// int call( Uplo uplo, @@ -155,74 +140,85 @@ class REVD2 { RandBLAS::RNGState error_est_state(state.counter, state.key); error_est_state.key.incr(1); while(true) { + // Resize caller-owned output buffers (std::vector handles dynamic k) util::upsize(k, eigvals); T* V_dat = util::upsize(m * k, V); - T* Y_dat = util::upsize(m * k, this->Y); - T* Omega_dat = util::upsize(m * k, this->Omega); - T* R_dat = util::upsize(k * k, this->R); - T* S_dat = util::upsize(k * k, this->S); - T* symrf_work_dat = util::upsize(m * k, this->symrf_work); - // Construnct a sketching operator - // If CholeskyQR is used for stab/orth here, RF can fail - this->syrf.call(A, k, this->Omega, state, symrf_work_dat); + // Grow internal working buffers if needed + if (m * k > Y_sz) { + free(Y); + Y = (T*) calloc(m * k, sizeof(T)); + Y_sz = m * k; + } + // Omega must hold at least max(m*k, m*4) — the error estimator needs 4 columns + int64_t omega_needed = std::max(m * k, m * (int64_t)4); + if (omega_needed > Omega_sz) { + free(Omega); + Omega = (T*) calloc(omega_needed, sizeof(T)); + Omega_sz = omega_needed; + } + if (k * k > R_sz) { + free(R); + R = (T*) calloc(k * k, sizeof(T)); + R_sz = k * k; + } + if (k * k > S_sz) { + free(S); + S = (T*) calloc(k * k, sizeof(T)); + S_sz = k * k; + } + if (m * k > symrf_work_sz) { + free(symrf_work); + symrf_work = (T*) calloc(m * k, sizeof(T)); + symrf_work_sz = m * k; + } + + // Construct sketching operator: Omega = orth(A * sketch) + this->syrf.call(A, k, this->Omega, state, symrf_work); // Y = A * Omega - A(Layout::ColMajor, k, 1.0, Omega_dat, m, 0.0, Y_dat, m); + A(Layout::ColMajor, k, 1.0, Omega, m, 0.0, Y, m); - T nu = std::numeric_limits::epsilon() * lapack::lange(Norm::Fro, m, k, Y_dat, m); + // Stabilization parameter nu = eps * ||Y||_F + T nu = std::numeric_limits::epsilon() * lapack::lange(Norm::Fro, m, k, Y, m); - // We need Y = Y + v Omega - // We further need R = chol(Omega' Y) - // Solve this as R = chol(Omega' Y + v Omega'Omega) - // Compute v Omega' Omega; syrk only computes the lower triangular part. Need full. - blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega_dat, m, 0.0, R_dat, k); + // R = chol(Omega'*Y + nu * Omega'*Omega) — regularized to ensure PD + // syrk fills lower triangle; copy to upper for full symmetric matrix + blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega, m, 0.0, R, k); for(int i = 1; i < k; ++i) - blas::copy(k - i, &R_dat[i + ((i-1) * k)], 1, &R_dat[(i - 1) + (i * k)], k); - // Compute Omega' Y + v Omega' Omega - blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, 1.0, Omega_dat, m, Y_dat, m, 1.0, R_dat, k); + blas::copy(k - i, &R[i + ((i-1) * k)], 1, &R[(i - 1) + (i * k)], k); + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, 1.0, Omega, m, Y, m, 1.0, R, k); - // Compute R = chol(Omega' Y + v Omega' Omega) - // Looks like if POTRF gets passed a non-triangular matrix, it will also output a non-triangular one - if(lapack::potrf(Uplo::Upper, k, R_dat, k)) + if(lapack::potrf(Uplo::Upper, k, R, k)) throw std::runtime_error("Cholesky decomposition failed."); - RandLAPACK::util::get_U(k, k, R_dat, k); + RandLAPACK::util::get_U(k, k, R, k); - // B = Y(R')^-1 - need to transpose R - blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, 1.0, R_dat, k, Y_dat, m); + // B = Y * (R')^{-1} — overwrites Y + blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, 1.0, R, k, Y, m); - //[V, S, ~] = SVD(B) - // Although we don't need the right singular vectors, we need to give space for those. - // Use R as a buffer for that. - lapack::gesdd(Job::SomeVec, m, k, Y_dat, m, S_dat, V_dat, m, R_dat, k); + // [V, S, ~] = SVD(B); use R as buffer for discarded right singular vectors + lapack::gesdd(Job::SomeVec, m, k, Y, m, S, V_dat, m, R, k); - // eigvals = diag(S^2) + // eigvals = S^2 - nu (undo regularization; clamp to zero) T buf; int64_t r = 0; int i; for(i = 0; i < k; ++i) { buf = std::pow(S[i], 2); eigvals[i] = buf; - // r = number of entries in eigvals that are greater than v if(buf > nu) ++r; } - - // Undo regularlization - // Need to make sure no eigenvalue is negative for(i = 0; i < r; ++i) - (eigvals[i] - nu < 0) ? 0 : eigvals[i] -=nu; + (eigvals[i] - nu < 0) ? 0 : eigvals[i] -= nu; std::fill(&V_dat[m * r], &V_dat[m * k], 0.0); - // Error estimation - // Using the first column of Omega as a buffer for a random vector - // To perform the following safely, need to make sure Omega has at least 4 columns - Omega_dat = util::upsize(m * 4, this->Omega); - RandBLAS::DenseDist g(m, 1); - error_est_state = RandBLAS::fill_dense(g, Omega_dat, error_est_state); + // Error estimation using Omega as a scratch buffer (needs 4 columns = 4*m) + RandBLAS::DenseDist g(m, 1); + error_est_state = RandBLAS::fill_dense(g, Omega, error_est_state); - err = power_error_est(A, k, this->error_est_p, Omega_dat, V_dat, Y_dat, eigvals.data()); + err = power_error_est(A, k, this->error_est_p, Omega, V_dat, Y, eigvals.data()); if(err <= 5 * std::max(tol, nu) || k == m) { break; diff --git a/RandLAPACK/misc/rl_util.hh b/RandLAPACK/misc/rl_util.hh index 0c8012d6e..aee5072e5 100644 --- a/RandLAPACK/misc/rl_util.hh +++ b/RandLAPACK/misc/rl_util.hh @@ -186,7 +186,7 @@ void col_swap( } } -/// Checks if the given size is larger than available. +/// Checks if the given size is larger than available. /// If so, resizes the vector. template T* upsize( @@ -199,6 +199,7 @@ T* upsize( return A.data(); } + /// Uses recursion to find the rank of the matrix pointed to by A_dat. /// Does so by attempting to find the smallest k such that /// ||A[k:, k:]||_F <= tau_trunk * ||A||. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 815ae3586..1ff28611a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,6 +20,8 @@ if (GTest_FOUND) drivers/test_bqrrp.cc drivers/test_revd2.cc drivers/test_hqrrp.cc + comps/test_lanczos_fa.cc + drivers/test_fun_nystrom_pp.cc drivers/test_abrik.cc drivers/test_orth_linop.cc misc/test_util.cc diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc new file mode 100644 index 000000000..88f7ab19e --- /dev/null +++ b/test/comps/test_lanczos_fa.cc @@ -0,0 +1,191 @@ +#include "RandLAPACK.hh" +#include "rl_blaspp.hh" +#include "rl_lapackpp.hh" +#include "rl_gen.hh" + +#include +#include +#include +#include +#include + +namespace linops = RandLAPACK::linops; + +class TestLanczosFA : public ::testing::Test { +protected: + virtual void SetUp() {} + virtual void TearDown() {} + + // Build a full symmetric diagonal matrix from a vector of diagonal entries. + // Returns column-major n×n storage with all off-diagonals zero. + static std::vector make_diag_matrix(const std::vector& diag) { + int64_t n = diag.size(); + std::vector A(n * n, 0.0); + for (int64_t i = 0; i < n; ++i) + A[i + i * n] = diag[i]; + return A; + } + + // Direct reference: f(A)*B for diagonal A = diag(d). + // Returns n×s result where result[:,j] = d .* B[:,j] component-wise via f. + static std::vector diag_fa_ref( + const std::vector& d, const std::vector& B, + int64_t n, int64_t s, std::function f + ) { + std::vector out(n * s, 0.0); + for (int64_t j = 0; j < s; ++j) + for (int64_t i = 0; i < n; ++i) + out[j * n + i] = f(d[i]) * B[j * n + i]; + return out; + } +}; + + +// LanczosFA on a diagonal matrix with f=sqrt. +// A = diag(1,...,50), d=20 Lanczos steps approximate f(A)B from a 20-dimensional +// Krylov subspace; good but not machine-precision accuracy (d < n = 50). +TEST_F(TestLanczosFA, DiagonalSqrt) { + using RNG = r123::Philox4x32; + int64_t n = 50, s = 5, d = 20; + auto state = RandBLAS::RNGState(42); + + // Diagonal entries: 1, 2, ..., n + std::vector diag(n); + std::iota(diag.begin(), diag.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag); + + // Random B (n×s) + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + + // LanczosFA output + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + + using LFA = RandLAPACK::LanczosFA; + LFA lfa; + lfa.reorth = -1; // full reorthogonalization + auto f_sqrt = [](double x){ return std::sqrt(x); }; + lfa.call(A_op, B.data(), n, s, f_sqrt, d, out.data()); + + // Reference + auto ref = diag_fa_ref(diag, B, n, s, f_sqrt); + + // Relative error per column + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("LanczosFA diagonal sqrt: relative error = %e\n", rel_err); + ASSERT_LT(rel_err, 1e-4); +} + + +// LanczosFA on a diagonal matrix with f=log. +TEST_F(TestLanczosFA, DiagonalLog) { + using RNG = r123::Philox4x32; + int64_t n = 50, s = 4, d = 20; + auto state = RandBLAS::RNGState(7); + + // Diagonal entries: 1, 2, ..., n (all positive, so log is well-defined) + std::vector diag(n); + std::iota(diag.begin(), diag.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag); + + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + + using LFA = RandLAPACK::LanczosFA; + LFA lfa; + auto f_log = [](double x){ return std::log(x); }; + lfa.call(A_op, B.data(), n, s, f_log, d, out.data()); + + auto ref = diag_fa_ref(diag, B, n, s, f_log); + + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("LanczosFA diagonal log: relative error = %e\n", rel_err); + ASSERT_LT(rel_err, 1e-3); +} + + +// Hutchinson standalone: estimate tr(M) for an explicit M. +// Use M = diag(1, 2, ..., n) so tr(M) = n*(n+1)/2 is known exactly. +TEST_F(TestLanczosFA, HutchinsonStandalone) { + using RNG = r123::Philox4x32; + int64_t n = 100, s = 500; + auto state = RandBLAS::RNGState(13); + + std::vector d(n); + std::iota(d.begin(), d.end(), 1.0); + double true_trace = n * (n + 1.0) / 2.0; + + // apply_M: multiply by diagonal matrix (element-wise per column) + auto apply_M = [&](const double* Omega, double* Z, int64_t n_, int64_t s_) { + for (int64_t j = 0; j < s_; ++j) + for (int64_t i = 0; i < n_; ++i) + Z[j * n_ + i] = d[i] * Omega[j * n_ + i]; + }; + + using Hutch = RandLAPACK::Hutchinson; + Hutch hutch; + double est = hutch.call(apply_M, n, s, state); + + double rel_err = std::abs(est - true_trace) / true_trace; + printf("Hutchinson trace estimate: est=%e, true=%e, rel_err=%e\n", est, true_trace, rel_err); + // With 500 Rademacher samples, variance should be small enough for 5% tolerance + ASSERT_LT(rel_err, 0.05); +} + + +// Vanilla (no reorth) vs full reorth on a well-conditioned diagonal matrix. +// Both should give the same answer — this checks the reorth code path doesn't break things. +TEST_F(TestLanczosFA, ReorthVsVanilla) { + using RNG = r123::Philox4x32; + int64_t n = 40, s = 3, d = 15; + auto state = RandBLAS::RNGState(99); + + std::vector diag(n); + std::iota(diag.begin(), diag.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag); + + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + auto f_sqrt = [](double x){ return std::sqrt(x); }; + + using LFA = RandLAPACK::LanczosFA; + + std::vector out_full(n * s, 0.0), out_van(n * s, 0.0); + LFA lfa_full, lfa_van; + lfa_full.reorth = -1; // full + lfa_van.reorth = 0; // vanilla + + lfa_full.call(A_op, B.data(), n, s, f_sqrt, d, out_full.data()); + lfa_van.call( A_op, B.data(), n, s, f_sqrt, d, out_van.data()); + + double diff = 0.0, norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out_full[i] - out_van[i]; + diff += r * r; + norm += out_full[i] * out_full[i]; + } + double rel_diff = std::sqrt(diff / norm); + printf("Full reorth vs vanilla relative diff: %e\n", rel_diff); + ASSERT_LT(rel_diff, 1e-6); +} diff --git a/test/comps/test_syrf.cc b/test/comps/test_syrf.cc index b8afc19b7..638e640ab 100644 --- a/test/comps/test_syrf.cc +++ b/test/comps/test_syrf.cc @@ -99,7 +99,7 @@ class TestSYRF : public ::testing::Test auto m = all_data.row; auto k = all_data.rank; - all_algs.syrf.call(Uplo::Upper, m, all_data.A.data(), k, all_data.Q, state, NULL); + all_algs.syrf.call(Uplo::Upper, m, all_data.A.data(), k, all_data.Q.data(), state, NULL); // Reassing pointers because Q, B have been resized T* Q_dat = all_data.Q.data(); diff --git a/test/drivers/test_fun_nystrom_pp.cc b/test/drivers/test_fun_nystrom_pp.cc new file mode 100644 index 000000000..51f3ddf03 --- /dev/null +++ b/test/drivers/test_fun_nystrom_pp.cc @@ -0,0 +1,147 @@ +#include "RandLAPACK.hh" +#include "rl_blaspp.hh" +#include "rl_lapackpp.hh" +#include "rl_gen.hh" + +#include +#include +#include +#include +#include + +namespace linops = RandLAPACK::linops; + +class TestFunNystromPP : public ::testing::Test { +protected: + virtual void SetUp() {} + virtual void TearDown() {} + + // Compute tr(f(A)) exactly via dense eigendecomposition. + // A is n×n column-major symmetric; returns sum of f(λ_i). + static double true_trace_fa( + int64_t n, std::vector A_copy, + std::function f + ) { + std::vector eigvals(n); + // dsyevd overwrites A with eigenvectors; we only need eigenvalues + lapack::syevd(lapack::Job::NoVec, lapack::Uplo::Upper, n, + A_copy.data(), n, eigvals.data()); + double tr = 0.0; + for (int64_t i = 0; i < n; ++i) + tr += f(eigvals[i]); + return tr; + } + + // Construct algorithm stack for FunNystromPP + template + struct Algs { + using SYPS_t = RandLAPACK::SYPS; + using Orth_t = RandLAPACK::HQRQ; + using SYRF_t = RandLAPACK::SYRF; + using REVD2_t = RandLAPACK::REVD2; + using LFA_t = RandLAPACK::LanczosFA; + using Hutch_t = RandLAPACK::Hutchinson; + using Driver_t = RandLAPACK::FunNystromPP; + + SYPS_t syps; + Orth_t orth; + SYRF_t syrf; + REVD2_t revd2; + LFA_t lfa; + Hutch_t hutch; + Driver_t driver; + + Algs() : + syps(3, 1, false, false), + orth(false, false), + syrf(syps, orth), + revd2(syrf, 0), // error_est_power_iters=0 → fixed-rank single pass + driver(revd2, lfa, hutch) {} + }; +}; + + +// FunNystromPP on a small dense PSD matrix with f=sqrt. +// Compare estimate to tr(sqrt(A)) from direct eigendecomposition. +TEST_F(TestFunNystromPP, DensePSDSqrt) { + using RNG = r123::Philox4x32; + int64_t n = 30, k = 10, s = 200, d = 15; + auto state = RandBLAS::RNGState(0); + + // Generate a random PSD matrix: A = B^T B + I (ensures strict PD) + std::vector B_raw(n * n); + RandBLAS::DenseDist DB(n, n); + state = RandBLAS::fill_dense(DB, B_raw.data(), state); + + std::vector A(n * n, 0.0); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, 1.0, B_raw.data(), n, 0.0, A.data(), n); + // Add identity to ensure strict positive definiteness + for (int64_t i = 0; i < n; ++i) + A[i + i * n] += (double)n; + // Fill lower triangle from upper (ExplicitSymLinOp only reads one triangle, + // but true_trace_fa uses syevd which needs a full triangular half) + for (int64_t j = 0; j < n; ++j) + for (int64_t i = j + 1; i < n; ++i) + A[i + j * n] = A[j + i * n]; + + // True tr(sqrt(A)) via dense eigendecomp + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + double true_tr = true_trace_fa(n, A, f_sqrt); + + // FunNystromPP estimate + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + Algs algs; + double est = algs.driver.call(A_op, f_sqrt, 0.0, k, s, d, state); + + double rel_err = std::abs(est - true_tr) / true_tr; + printf("FunNystromPP sqrt: est=%e, true=%e, rel_err=%e\n", est, true_tr, rel_err); + ASSERT_LT(rel_err, 0.1); +} + + +// FunNystromPP on a diagonal matrix with f=sqrt — easier to control. +// Diagonal A = diag(1, 2, ..., n), so tr(sqrt(A)) = Σ sqrt(i) exactly. +TEST_F(TestFunNystromPP, DiagonalSqrt) { + using RNG = r123::Philox4x32; + int64_t n = 50, k = 15, s = 300, d = 20; + auto state = RandBLAS::RNGState(1); + + std::vector A(n * n, 0.0); + double true_tr = 0.0; + for (int64_t i = 0; i < n; ++i) { + A[i + i * n] = (double)(i + 1); + true_tr += std::sqrt((double)(i + 1)); + } + + auto f_sqrt = [](double x){ return std::sqrt(x); }; + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + Algs algs; + double est = algs.driver.call(A_op, f_sqrt, 0.0, k, s, d, state); + + double rel_err = std::abs(est - true_tr) / true_tr; + printf("FunNystromPP diagonal sqrt: est=%e, true=%e, rel_err=%e\n", est, true_tr, rel_err); + ASSERT_LT(rel_err, 0.05); +} + + +// Verify that REVD2 with error_est_power_iters=0 and tol=0 does not increase k. +// REVD2 takes k by reference and may grow it adaptively; the fixed-rank setting +// must leave k unchanged so FunNystromPP's (n-k)*f(0) tail correction is correct. +TEST_F(TestFunNystromPP, REVD2FixedKBypass) { + using RNG = r123::Philox4x32; + int64_t n = 40, k = 10; + auto state = RandBLAS::RNGState(5); + + // Simple diagonal PSD matrix + std::vector A(n * n, 0.0); + for (int64_t i = 0; i < n; ++i) + A[i + i * n] = (double)(i + 1); + + Algs algs; + std::vector V_out(n * k, 0.0), eigvals_out(k, 0.0); + int64_t k_before = k; + algs.revd2.call(blas::Uplo::Upper, n, A.data(), k, 0.0, V_out, eigvals_out, state); + + printf("REVD2 fixed-k: k_before=%lld, k_after=%lld\n", (long long)k_before, (long long)k); + ASSERT_EQ(k, k_before); +} From 6779aaac4d7e02194307c2a65fc57823c0e635b0 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Tue, 28 Apr 2026 11:43:51 -0400 Subject: [PATCH 02/14] C++ style --- RandLAPACK/comps/rl_hutchinson.hh | 13 ++++++------ RandLAPACK/comps/rl_lanczos_fa.hh | 27 +++++++++++-------------- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 21 +++++++++---------- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index 59bb856f1..63e0b8099 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -4,7 +4,6 @@ #include #include -#include #include namespace RandLAPACK { @@ -25,10 +24,10 @@ namespace RandLAPACK { template class Hutchinson { public: - // Internal sketch buffer — grown with calloc/free, never shrunk. + // Internal sketch buffer — grown with new/delete[], never shrunk. T* Omega = nullptr; int64_t Omega_sz = 0; - ~Hutchinson() { free(Omega); } + ~Hutchinson() { delete[] Omega; } // ------------------------------------------------------------------ /// Low-level estimator: given precomputed Ω (n×s) and Z = M*Ω (n×s), @@ -60,8 +59,8 @@ public: RandBLAS::RNGState& state) { // Grow Omega buffer if needed (reuse across repeated calls) if (n * s > Omega_sz) { - free(Omega); - Omega = (T*) calloc(n * s, sizeof(T)); + delete[] Omega; + Omega = new T[n * s]; Omega_sz = n * s; } @@ -74,11 +73,11 @@ public: Omega[i] = (Omega[i] >= 0) ? (T)1 : (T)-1; // Allocate Z and apply the operator - T* Z = (T*) calloc(n * s, sizeof(T)); + T* Z = new T[n * s]; apply_M(Omega, Z, n, s); T result = estimate(Omega, Z, n, s); - free(Z); + delete[] Z; return result; } }; diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index b8f2f1db2..da0d4dd9c 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -6,8 +6,6 @@ #include #include -#include -#include #include #include @@ -55,7 +53,7 @@ public: /// Full reorthogonalization is the safe default for a numerical library. int64_t reorth = -1; - // Internal buffers — grown with calloc/free, never shrunk between calls. + // Internal buffers — grown with new/delete[], never shrunk between calls. // K: (d+1) × n × s — Krylov basis blocks; see layout note above. // alpha: s × d — tridiagonal diagonals, alpha[j*d + i] = α_{i,j}. // beta: s × (d-1) — tridiagonal subdiagonals, beta[j*(d-1) + i] = β_{i+1,j}. @@ -65,7 +63,7 @@ public: T* beta = nullptr; int64_t beta_sz = 0; T* normb = nullptr; int64_t normb_sz = 0; - ~LanczosFA() { free(K); free(alpha); free(beta); free(normb); } + ~LanczosFA() { delete[] K; delete[] alpha; delete[] beta; delete[] normb; } // ------------------------------------------------------------------ /// Run d-step block Lanczos recurrence. @@ -81,24 +79,23 @@ public: void run(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { // Grow buffers if needed if ((d + 1) * n * s > K_sz) { - free(K); - K = (T*) calloc((d + 1) * n * s, sizeof(T)); + delete[] K; + K = new T[(d + 1) * n * s]; K_sz = (d + 1) * n * s; } - // omega_needed = max(d*s, 4) to match error-est buffer sizing in REVD2 if (d * s > alpha_sz) { - free(alpha); - alpha = (T*) calloc(d * s, sizeof(T)); + delete[] alpha; + alpha = new T[d * s]; alpha_sz = d * s; } if ((d - 1) * s > beta_sz && d > 1) { - free(beta); - beta = (T*) calloc((d - 1) * s, sizeof(T)); + delete[] beta; + beta = new T[(d - 1) * s]; beta_sz = (d - 1) * s; } if (s > normb_sz) { - free(normb); - normb = (T*) calloc(s, sizeof(T)); + delete[] normb; + normb = new T[s]; normb_sz = s; } @@ -190,7 +187,7 @@ public: #ifdef _OPENMP nthreads = omp_get_max_threads(); #endif - T* scratch = (T*) calloc(nthreads * scratch_per_thread, sizeof(T)); + T* scratch = new T[nthreads * scratch_per_thread]; #pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) { @@ -229,7 +226,7 @@ public: normb[j], K + j * n, n * s, v_j, 1, (T)0.0, out + j * n, 1); } - free(scratch); + delete[] scratch; } // ------------------------------------------------------------------ diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index 217ef472c..450479198 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -5,7 +5,6 @@ #include #include -#include #include #include #include @@ -60,7 +59,7 @@ public: LanczosFA_t& lanczos_fa; Hutchinson_t& hutchinson; - // Persistent output/working buffers — grown with calloc/free, never shrunk. + // Persistent output/working buffers — grown with new/delete[], never shrunk. // V, eigvals: REVD2 outputs reused across calls with same (n, k). // F_vec: f applied elementwise to eigvals. // tmp: k×s scratch for V^T Ω and F_vec ⊙ (V^T Ω). @@ -72,7 +71,7 @@ public: T* Z1 = nullptr; int64_t Z1_sz = 0; T* Z2 = nullptr; int64_t Z2_sz = 0; - ~FunNystromPP() { free(F_vec); free(tmp); free(Z1); free(Z2); } + ~FunNystromPP() { delete[] F_vec; delete[] tmp; delete[] Z1; delete[] Z2; } FunNystromPP(REVD2_t& r, LanczosFA_t& l, Hutchinson_t& h) : revd2(r), lanczos_fa(l), hutchinson(h) {} @@ -120,8 +119,8 @@ public: // f(λ): apply scalar f to k eigenvalues if (k > F_vec_sz) { - free(F_vec); - F_vec = (T*) calloc(k, sizeof(T)); + delete[] F_vec; + F_vec = new T[k]; F_vec_sz = k; } for (int64_t i = 0; i < k; ++i) @@ -139,18 +138,18 @@ public: // by the lambda below, then the inner product <Ω,Z>/s is returned. // ------------------------------------------------------------------ if (k * s > tmp_sz) { - free(tmp); - tmp = (T*) calloc(k * s, sizeof(T)); + delete[] tmp; + tmp = new T[k * s]; tmp_sz = k * s; } if (n * s > Z1_sz) { - free(Z1); - Z1 = (T*) calloc(n * s, sizeof(T)); + delete[] Z1; + Z1 = new T[n * s]; Z1_sz = n * s; } if (n * s > Z2_sz) { - free(Z2); - Z2 = (T*) calloc(n * s, sizeof(T)); + delete[] Z2; + Z2 = new T[n * s]; Z2_sz = n * s; } From 9847c5c168e482625c2afea3b052b229291f9847 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Tue, 28 Apr 2026 15:37:40 -0400 Subject: [PATCH 03/14] Linop fix --- RandLAPACK/comps/rl_hutchinson.hh | 33 ++++----- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 98 +++++++++++++++++-------- test/comps/test_lanczos_fa.cc | 21 ++++-- 3 files changed, 96 insertions(+), 56 deletions(-) diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index 63e0b8099..b0651937d 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -1,6 +1,7 @@ #pragma once #include "rl_blaspp.hh" +#include "rl_linops.hh" #include #include @@ -11,16 +12,14 @@ namespace RandLAPACK { /// Hutchinson stochastic trace estimator. /// -/// Estimates tr(M) using the identity E[ω^T M ω] = tr(M) for any zero-mean -/// unit-variance ω. Draws s independent Gaussian vectors, applies M to all at once, then -/// returns the averaged quadratic form (1/s) * <Ω, Z>_F where Z = M*Ω. +/// Estimates tr(M) using the identity E[ω^T M ω] = tr(M) for zero-mean +/// unit-variance ω. Draws s independent Rademacher vectors (iid Unif{±1}), +/// applies M to all at once, and returns (1/s) * <Ω, Z>_F where Z = M*Ω. /// -/// Standalone and reusable: the operator M is provided as a callable rather -/// than a fixed type, so this component can be used for any trace estimation -/// problem in RandLAPACK without modification. +/// M must satisfy linops::SymmetricLinearOperator. /// /// @tparam T Floating-point scalar type. -/// @tparam RNG Random number generator type for RandBLAS. +/// @tparam RNG Random number generator type. template class Hutchinson { public: @@ -44,19 +43,16 @@ public: } // ------------------------------------------------------------------ - /// High-level estimator: draws Ω internally, calls apply_M to fill Z, - /// then returns the trace estimate. + /// High-level estimator: draws Ω internally, applies M, returns trace estimate. + /// n is taken from apply_M.dim. /// - /// apply_M must have signature: (const T* Omega, T* Z, int64_t n, int64_t s) - /// It receives the n×s Rademacher matrix and must overwrite Z with M*Ω. - /// - /// @param[in] apply_M Callable that applies M to an n×s matrix. - /// @param[in] n Ambient dimension. + /// @param[in] apply_M Operator satisfying SymmetricLinearOperator. /// @param[in] s Number of Hutchinson samples. /// @param[in] state RandBLAS RNG state; advanced on return. - template - T call(ApplyM apply_M, int64_t n, int64_t s, - RandBLAS::RNGState& state) { + template + T call(SLO& apply_M, int64_t s, RandBLAS::RNGState& state) { + int64_t n = apply_M.dim; + // Grow Omega buffer if needed (reuse across repeated calls) if (n * s > Omega_sz) { delete[] Omega; @@ -72,9 +68,8 @@ public: for (int64_t i = 0; i < n * s; ++i) Omega[i] = (Omega[i] >= 0) ? (T)1 : (T)-1; - // Allocate Z and apply the operator T* Z = new T[n * s]; - apply_M(Omega, Z, n, s); + apply_M(Layout::ColMajor, s, (T)1.0, Omega, n, (T)0.0, Z, n); T result = estimate(Omega, Z, n, s); delete[] Z; diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index 450479198..4a1a06233 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -13,6 +13,66 @@ namespace RandLAPACK { +/// Residual operator (f(A) - f(Â)) for the funNyström++ Hutchinson correction. +/// +/// Satisfies SymmetricLinearOperator so it can be passed directly to Hutchinson. +/// Computes C = α*(f(A)*B - f(Â)*B) + β*C via: +/// Z1 = f(A)*B using LanczosFA (d batch matvecs) +/// Z2 = f(Â)*B = V*(diag(F_vec)*(V^T B)) using two GEMMs +/// tmp, Z1, Z2 are workspace owned by FunNystromPP and borrowed here by pointer. +/// +/// @tparam T Scalar type. +/// @tparam SLO_t Type of the operator A. +/// @tparam LFA_t Type of the LanczosFA component. +/// @tparam F_t Scalar function type T→T. +template +struct ResidualOp { + using scalar_t = T; + const int64_t dim; + + SLO_t& A; + LFA_t& lanczos_fa; + F_t f; + int64_t d; + int64_t k; + T* V; + T* F_vec; + T* tmp; // k×s workspace + T* Z1; // n×s workspace: f(A)*Ω + T* Z2; // n×s workspace: f(Â)*Ω + + ResidualOp(int64_t n_, SLO_t& A_, LFA_t& lfa_, F_t f_, + int64_t d_, int64_t k_, T* V_, T* Fv_, T* tmp_, T* Z1_, T* Z2_) + : dim(n_), A(A_), lanczos_fa(lfa_), f(f_), + d(d_), k(k_), V(V_), F_vec(Fv_), tmp(tmp_), Z1(Z1_), Z2(Z2_) {} + + void operator()(Layout layout, int64_t n_vecs, T alpha, + T* const B, int64_t ldb, T beta, T* C, int64_t ldc) { + // Z1 = f(A)*B via Lanczos-FA (d batch matvecs) + lanczos_fa.call(A, B, dim, n_vecs, f, d, Z1); + + // Z2 = f(Â)*B = V * (diag(F_vec) * (V^T B)) + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, + k, n_vecs, dim, (T)1.0, V, dim, B, ldb, (T)0.0, tmp, k); + for (int64_t j = 0; j < n_vecs; ++j) + for (int64_t i = 0; i < k; ++i) + tmp[j * k + i] *= F_vec[i]; + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + dim, n_vecs, k, (T)1.0, V, dim, tmp, k, (T)0.0, Z2, dim); + + // C = alpha*(Z1 - Z2) + beta*C + if (beta == (T)0) { + for (int64_t i = 0; i < dim * n_vecs; ++i) + C[i] = alpha * (Z1[i] - Z2[i]); + } else { + blas::scal(dim * n_vecs, beta, C, 1); + blas::axpy(dim * n_vecs, alpha, Z1, 1, C, 1); + blas::axpy(dim * n_vecs, -alpha, Z2, 1, C, 1); + } + } +}; + + /// funNyström++ trace estimator: estimates tr(f(A)) for symmetric PSD A. /// /// Algorithm (3 phases): @@ -62,9 +122,7 @@ public: // Persistent output/working buffers — grown with new/delete[], never shrunk. // V, eigvals: REVD2 outputs reused across calls with same (n, k). // F_vec: f applied elementwise to eigvals. - // tmp: k×s scratch for V^T Ω and F_vec ⊙ (V^T Ω). - // Z1: n×s LanczosFA output f(A)*Ω. - // Z2: n×s control variate f(Â)*Ω = V*(F_vec ⊙ V^T Ω). + // tmp, Z1, Z2: workspace owned here and lent to ResidualOp each call. std::vector V_buf, eigvals_buf; // std::vector for REVD2's adaptive resize T* F_vec = nullptr; int64_t F_vec_sz = 0; T* tmp = nullptr; int64_t tmp_sz = 0; @@ -134,8 +192,8 @@ public: // ------------------------------------------------------------------ // Phase 2: Hutchinson correction on residual f(A) - f(Â) - // Ω is drawn inside hutchinson.call; Z = (f(A)-f(Â))*Ω is computed - // by the lambda below, then the inner product <Ω,Z>/s is returned. + // ResidualOp wraps the correction computation as a SymmetricLinearOperator + // so Hutchinson::call can draw Ω and compute <Ω, (f(A)-f(Â))Ω>_F / s. // ------------------------------------------------------------------ if (k * s > tmp_sz) { delete[] tmp; @@ -153,32 +211,10 @@ public: Z2_sz = n * s; } - // Residual operator: Z = f(A)*Ω - f(Â)*Ω - // Captures V, F_vec, k, s, d, n, Z1, Z2, tmp by reference. - auto apply_residual = [&](const T* Omega, T* Z, int64_t n_, int64_t s_) { - // Z1 = f(A)*Ω via block Lanczos-FA (d batch matvecs) - lanczos_fa.call(A, Omega, n_, s_, f, d, Z1); - - // tmp = V^T * Ω (k×s) - blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, - k, s_, n_, (T)1.0, V, n_, Omega, n_, (T)0.0, tmp, k); - - // tmp[:,j] *= F_vec (scale each of the s columns by F_vec elementwise) - // tmp is k×s column-major; column j is tmp + j*k - for (int64_t j = 0; j < s_; ++j) - for (int64_t i = 0; i < k; ++i) - tmp[j * k + i] *= F_vec[i]; - - // Z2 = V * tmp (n×s) = f(Â)*Ω - blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, - n_, s_, k, (T)1.0, V, n_, tmp, k, (T)0.0, Z2, n_); - - // Z = Z1 - Z2 (in-place: copy Z1 → Z then subtract Z2) - std::copy(Z1, Z1 + n_ * s_, Z); - blas::axpy(n_ * s_, (T)-1.0, Z2, 1, Z, 1); - }; - - T correction = hutchinson.call(apply_residual, n, s, state); + ResidualOp res_op( + n, A, lanczos_fa, f, d, k, V, F_vec, tmp, Z1, Z2 + ); + T correction = hutchinson.call(res_op, s, state); return tr_Ahat + correction; } diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index 88f7ab19e..af575beec 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -133,16 +133,25 @@ TEST_F(TestLanczosFA, HutchinsonStandalone) { std::iota(d.begin(), d.end(), 1.0); double true_trace = n * (n + 1.0) / 2.0; - // apply_M: multiply by diagonal matrix (element-wise per column) - auto apply_M = [&](const double* Omega, double* Z, int64_t n_, int64_t s_) { - for (int64_t j = 0; j < s_; ++j) - for (int64_t i = 0; i < n_; ++i) - Z[j * n_ + i] = d[i] * Omega[j * n_ + i]; + // Diagonal operator satisfying SymmetricLinearOperator: C = α*diag(d)*B + β*C + struct DiagonalOp { + using scalar_t = double; + const int64_t dim; + const std::vector& diag; + void operator()(Layout layout, int64_t n_vecs, double alpha, + double* const B, int64_t ldb, double beta, double* C, int64_t ldc) { + for (int64_t j = 0; j < n_vecs; ++j) + for (int64_t i = 0; i < dim; ++i) { + double val = alpha * diag[i] * B[j * ldb + i]; + C[j * ldc + i] = (beta == 0.0) ? val : val + beta * C[j * ldc + i]; + } + } }; + DiagonalOp M{n, d}; using Hutch = RandLAPACK::Hutchinson; Hutch hutch; - double est = hutch.call(apply_M, n, s, state); + double est = hutch.call(M, s, state); double rel_err = std::abs(est - true_trace) / true_trace; printf("Hutchinson trace estimate: est=%e, true=%e, rel_err=%e\n", est, true_trace, rel_err); From cc3f9e5cf8cffb53245112354800c9a2ae36b4c5 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Tue, 28 Apr 2026 15:54:44 -0400 Subject: [PATCH 04/14] Minor fixes --- RandLAPACK/comps/rl_hutchinson.hh | 14 +++++++------- RandLAPACK/comps/rl_lanczos_fa.hh | 8 +++++--- test/comps/test_lanczos_fa.cc | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index b0651937d..1c0dfba39 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -44,14 +44,14 @@ public: // ------------------------------------------------------------------ /// High-level estimator: draws Ω internally, applies M, returns trace estimate. - /// n is taken from apply_M.dim. + /// n is taken from M.dim. /// - /// @param[in] apply_M Operator satisfying SymmetricLinearOperator. - /// @param[in] s Number of Hutchinson samples. - /// @param[in] state RandBLAS RNG state; advanced on return. + /// @param[in] M Operator satisfying SymmetricLinearOperator. + /// @param[in] s Number of Hutchinson samples. + /// @param[in] state RandBLAS RNG state; advanced on return. template - T call(SLO& apply_M, int64_t s, RandBLAS::RNGState& state) { - int64_t n = apply_M.dim; + T call(SLO& M, int64_t s, RandBLAS::RNGState& state) { + int64_t n = M.dim; // Grow Omega buffer if needed (reuse across repeated calls) if (n * s > Omega_sz) { @@ -69,7 +69,7 @@ public: Omega[i] = (Omega[i] >= 0) ? (T)1 : (T)-1; T* Z = new T[n * s]; - apply_M(Layout::ColMajor, s, (T)1.0, Omega, n, (T)0.0, Z, n); + M(Layout::ColMajor, s, (T)1.0, Omega, n, (T)0.0, Z, n); T result = estimate(Omega, Z, n, s); delete[] Z; diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index da0d4dd9c..004788326 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -46,14 +46,16 @@ template class LanczosFA { public: /// Reorthogonalization control. - /// -1 = full (project out all previous Krylov vectors after each step). + /// 1 = full (project out all previous Krylov vectors after each step). /// 0 = none (vanilla Lanczos, per Persson's reference implementation). /// Lanczos-FA tolerates loss of orthogonality better than eigenvalue /// Lanczos (Paige-Greenbaum theory), so vanilla often works in practice. /// Full reorthogonalization is the safe default for a numerical library. - int64_t reorth = -1; + int64_t reorth = 1; // Internal buffers — grown with new/delete[], never shrunk between calls. + // Dimension key: n = operator dimension, s = number of RHS vectors (columns of B), + // d = number of Lanczos steps. // K: (d+1) × n × s — Krylov basis blocks; see layout note above. // alpha: s × d — tridiagonal diagonals, alpha[j*d + i] = α_{i,j}. // beta: s × (d-1) — tridiagonal subdiagonals, beta[j*(d-1) + i] = β_{i+1,j}. @@ -139,7 +141,7 @@ public: // (2) Optional full reorthogonalization against all q_0,...,q_i // Subtract projections onto previous Krylov vectors to recover lost // orthogonality accumulated in floating-point arithmetic. - int64_t reorth_steps = (reorth < 0) ? (i + 1) : reorth; + int64_t reorth_steps = reorth ? (i + 1) : 0; for (int64_t prev = 0; prev < reorth_steps; ++prev) { T* K_p = K + prev * n * s; for (int64_t j = 0; j < s; ++j) { diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index af575beec..6d6031c67 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -65,7 +65,7 @@ TEST_F(TestLanczosFA, DiagonalSqrt) { using LFA = RandLAPACK::LanczosFA; LFA lfa; - lfa.reorth = -1; // full reorthogonalization + lfa.reorth = 1; // full reorthogonalization auto f_sqrt = [](double x){ return std::sqrt(x); }; lfa.call(A_op, B.data(), n, s, f_sqrt, d, out.data()); @@ -182,7 +182,7 @@ TEST_F(TestLanczosFA, ReorthVsVanilla) { std::vector out_full(n * s, 0.0), out_van(n * s, 0.0); LFA lfa_full, lfa_van; - lfa_full.reorth = -1; // full + lfa_full.reorth = 1; // full lfa_van.reorth = 0; // vanilla lfa_full.call(A_op, B.data(), n, s, f_sqrt, d, out_full.data()); From e633d854e707f79cfdaa5f70e053d422915b8bc5 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Tue, 28 Apr 2026 16:01:37 -0400 Subject: [PATCH 05/14] Comments update --- RandLAPACK/comps/rl_lanczos_fa.hh | 35 +++++++++---------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 004788326..2b00bd8e5 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -17,28 +17,8 @@ namespace RandLAPACK { /// d-step block Lanczos for matrix function application f(A)B. -/// -/// Given a SymmetricLinearOperator A and a matrix B (n×s), approximates -/// f(A)B column-wise using a Krylov subspace of dimension d per column. -/// -/// Each column b_j of B generates an independent Krylov sequence. The -/// recurrence produces a d×d symmetric tridiagonal T_j per column; f(A)b_j -/// is approximated by the first column of Q_j * f(T_j) * Q_j' * b_j, where -/// Q_j = [q_1,...,q_d] is the Lanczos basis and f(T_j) is evaluated via a -/// dense d×d tridiagonal eigendecomposition (lapack::stev). -/// -/// Key formula (Tyler Chen): since q_1 = b_j/||b_j||, we have -/// f(A)b_j ≈ ||b_j|| * Q_j * S_j * diag(f(θ_j)) * S_j[0,:]^T -/// where (S_j, θ_j) = eig(T_j) from stev and S_j[0,:] is the first row -/// of the eigenvector matrix. -/// -/// f is any scalar callable T → T (e.g., sqrt, log, pow). -/// The stev calls per column are independent and run in parallel (OpenMP). -/// -/// Memory layout: K is stored as (d+1) contiguous n×s blocks. -/// K[step * n * s + col * n + row] = entry (row, col) of the step-th block. -/// This keeps each step's n×s matrix contiguous for batch matvec via SLO, -/// while allowing strided gemv (lda = n*s) for per-column reconstruction. +/// Approximates f(A)B column-wise via independent Krylov subspaces of dimension d. +/// See: T. Chen, "A Lanczos-FA algorithm for matrix function approximation" (2022). /// /// @tparam T Floating-point scalar type. /// @tparam RNG Random number generator type (unused here; kept for API uniformity). @@ -56,7 +36,11 @@ public: // Internal buffers — grown with new/delete[], never shrunk between calls. // Dimension key: n = operator dimension, s = number of RHS vectors (columns of B), // d = number of Lanczos steps. - // K: (d+1) × n × s — Krylov basis blocks; see layout note above. + // + // K: (d+1) × n × s — Krylov basis blocks. + // Layout: K[step * n*s + col * n + row] = row-th entry of step-th basis vector for column col. + // Storing steps as contiguous n×s slices keeps each batch matvec contiguous, + // while the per-column stride (n*s) lets apply() use strided gemv for reconstruction. // alpha: s × d — tridiagonal diagonals, alpha[j*d + i] = α_{i,j}. // beta: s × (d-1) — tridiagonal subdiagonals, beta[j*(d-1) + i] = β_{i+1,j}. // normb: s — column norms of B before normalization. @@ -170,9 +154,10 @@ public: // ------------------------------------------------------------------ /// Evaluate f(A)B from precomputed Krylov data (K, alpha, beta, normb). - /// Per column j: eigendecompose T_j via lapack::stev, then compute + /// Per column j: eigendecompose T_j = S_j diag(θ_j) S_j^T via lapack::stev, then: /// out[:,j] = normb[j] * Q_j * S_j * diag(f(θ_j)) * S_j[0,:]^T - /// where S_j[0,:] is the first row of the eigenvector matrix. + /// where Q_j is the n×d Lanczos basis stored in K and S_j[0,:] is the first row + /// of the eigenvector matrix (Chen 2022, eq. 2.3). /// Per-column stev calls are independent — parallelized with OpenMP. /// /// @param[in] f Scalar callable T→T applied to tridiagonal eigenvalues. From f8597df8a517b8a606d6e44f24cc95f339707ab3 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Tue, 28 Apr 2026 18:11:01 -0400 Subject: [PATCH 06/14] Update --- RandLAPACK/comps/rl_hutchinson.hh | 8 +- RandLAPACK/comps/rl_lanczos_fa.hh | 111 +++++++++++++----------- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 25 ++---- RandLAPACK/misc/rl_util.hh | 11 +++ 4 files changed, 77 insertions(+), 78 deletions(-) diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index 1c0dfba39..e3e10ed50 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -2,6 +2,7 @@ #include "rl_blaspp.hh" #include "rl_linops.hh" +#include "rl_util.hh" #include #include @@ -53,12 +54,7 @@ public: T call(SLO& M, int64_t s, RandBLAS::RNGState& state) { int64_t n = M.dim; - // Grow Omega buffer if needed (reuse across repeated calls) - if (n * s > Omega_sz) { - delete[] Omega; - Omega = new T[n * s]; - Omega_sz = n * s; - } + util::regrow(Omega, Omega_sz, n * s); // Draw Ω with iid Rademacher entries (Unif{±1}). // RandBLAS has no ScalarDist::Rademacher, but ScalarDist::Uniform fills diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 2b00bd8e5..66eb9055b 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -3,6 +3,7 @@ #include "rl_blaspp.hh" #include "rl_lapackpp.hh" #include "rl_linops.hh" +#include "rl_util.hh" #include #include @@ -43,6 +44,8 @@ public: // while the per-column stride (n*s) lets apply() use strided gemv for reconstruction. // alpha: s × d — tridiagonal diagonals, alpha[j*d + i] = α_{i,j}. // beta: s × (d-1) — tridiagonal subdiagonals, beta[j*(d-1) + i] = β_{i+1,j}. + // lapack::stevd expects the diagonal and subdiagonal as separate arrays, + // so alpha and beta are stored separately rather than interleaved. // normb: s — column norms of B before normalization. T* K = nullptr; int64_t K_sz = 0; T* alpha = nullptr; int64_t alpha_sz = 0; @@ -52,7 +55,7 @@ public: ~LanczosFA() { delete[] K; delete[] alpha; delete[] beta; delete[] normb; } // ------------------------------------------------------------------ - /// Run d-step block Lanczos recurrence. + /// Run the d-step block Lanczos recurrence on B. /// Fills K, alpha, beta, normb from B (n×s column-major). /// Calls A exactly d times, each application to an n×s matrix. /// @@ -62,32 +65,17 @@ public: /// @param[in] s Number of right-hand sides (Hutchinson samples). /// @param[in] d Number of Lanczos steps. template - void run(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { + void run_lanczos(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { // Grow buffers if needed - if ((d + 1) * n * s > K_sz) { - delete[] K; - K = new T[(d + 1) * n * s]; - K_sz = (d + 1) * n * s; - } - if (d * s > alpha_sz) { - delete[] alpha; - alpha = new T[d * s]; - alpha_sz = d * s; - } - if ((d - 1) * s > beta_sz && d > 1) { - delete[] beta; - beta = new T[(d - 1) * s]; - beta_sz = (d - 1) * s; - } - if (s > normb_sz) { - delete[] normb; - normb = new T[s]; - normb_sz = s; - } + util::regrow(K, K_sz, (d + 1) * n * s); + util::regrow(alpha, alpha_sz, d * s); + if (d > 1) util::regrow(beta, beta_sz, (d - 1) * s); + util::regrow(normb, normb_sz, s); // Step 0: q_1 = column-normalize B; store in K[:,:,0] T* K0 = K; - std::copy(B, B + n * s, K0); + lapack::lacpy(lapack::MatrixType::General, n, s, B, n, K0, n); +#pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) { T nrm = blas::nrm2(n, K0 + j * n, 1); normb[j] = nrm; @@ -98,55 +86,64 @@ public: T* K1 = K + n * s; A(Layout::ColMajor, s, (T)1.0, K0, n, (T)0.0, K1, n); - // α[0, j] = dot(K1[:,j], K0[:,j]) — diagonal entry for step 0 + // α[0, j] = q_1[:,j] · (A q_1)[:,j] — s independent inner products, + // one tridiagonal diagonal entry per column. +#pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) alpha[j * d + 0] = blas::dot(n, K1 + j * n, 1, K0 + j * n, 1); // Main Lanczos loop: steps 1..d-1 // At the start of iteration i: // K[:,:,i] = A*q_i - β_i*q_{i-1} (partial three-term, β part done last iter) - // K[:,:,i+1] is free scratch + // K[:,:,i+1] is free workspace // This iteration: - // (1) subtract α_i*q_i to complete three-term → K_{i} becomes unnormalized q_{i+1} + // (1) subtract α_i*q_i to complete three-term → K_{i+1} becomes unnormalized q_{i+1} // (2) optional reorthogonalization - // (3) β_{i+1} = ||K_{i+1}|| - // (4) normalize K_{i+1} → q_{i+1} + // (3) β_{i+1} = ||K_{i+1}||, (4) normalize → q_{i+1} // (5) K_{i+2} = A*q_{i+1} - β_{i+1}*q_i (start of next three-term) - // (6) α_{i+1} = dot(K_{i+2}, q_{i+1}) + // (6) α_{i+1} = q_{i+1}[:,j] · K_{i+2}[:,j] + // All per-column loops below are independent across j and parallelized. for (int64_t i = 0; i < d - 1; ++i) { T* K_prev = K + i * n * s; // K[:,:,i] = q_i (normalized) T* K_curr = K + (i + 1) * n * s; // K[:,:,i+1] = partial (A*q_i - β_i*q_{i-1}) - T* K_new = K + (i + 2) * n * s; // K[:,:,i+2] = scratch for A*q_{i+1} + T* K_new = K + (i + 2) * n * s; // K[:,:,i+2] = workspace for A*q_{i+1} - // (1) Complete three-term: K_curr -= α_i * K_prev, per column + // (1) Complete three-term: K_curr -= α_i * K_prev + // Each column has a different scalar α_{i,j}, so GEMM would cost O(n·s²); axpy is optimal. +#pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) blas::axpy(n, -alpha[j * d + i], K_prev + j * n, 1, K_curr + j * n, 1); - // (2) Optional full reorthogonalization against all q_0,...,q_i - // Subtract projections onto previous Krylov vectors to recover lost - // orthogonality accumulated in floating-point arithmetic. + // (2) Optional full reorthogonalization: project K_curr[:,j] out of all q_0..q_i. + // Outer loop over j is parallel (columns are independent); inner prev-loop is + // sequential per column (each projection modifies K_curr[:,j] in place). int64_t reorth_steps = reorth ? (i + 1) : 0; - for (int64_t prev = 0; prev < reorth_steps; ++prev) { - T* K_p = K + prev * n * s; - for (int64_t j = 0; j < s; ++j) { +#pragma omp parallel for schedule(static) + for (int64_t j = 0; j < s; ++j) { + for (int64_t prev = 0; prev < reorth_steps; ++prev) { + T* K_p = K + prev * n * s; T coeff = blas::dot(n, K_curr + j * n, 1, K_p + j * n, 1); blas::axpy(n, -coeff, K_p + j * n, 1, K_curr + j * n, 1); } } - // (3) β_{i+1} = column norms and (4) normalize to get q_{i+1} + // (3) β_{i+1} = column norms, (4) normalize → q_{i+1} +#pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) { T nrm = blas::nrm2(n, K_curr + j * n, 1); beta[j * (d - 1) + i] = nrm; blas::scal(n, (T)1.0 / nrm, K_curr + j * n, 1); } - // (5) K_new = A*q_{i+1} - β_{i+1}*q_i (one batch matvec + axpy per column) + // (5) K_new = A*q_{i+1} - β_{i+1}*q_i + // Different β per column — same reasoning as (1), axpy is optimal. A(Layout::ColMajor, s, (T)1.0, K_curr, n, (T)0.0, K_new, n); +#pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) blas::axpy(n, -beta[j * (d - 1) + i], K_prev + j * n, 1, K_new + j * n, 1); - // (6) α_{i+1} = dot(K_new[:,j], q_{i+1}[:,j]) + // (6) α_{i+1} = q_{i+1}[:,j] · K_new[:,j] +#pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) alpha[j * d + (i + 1)] = blas::dot(n, K_new + j * n, 1, K_curr + j * n, 1); } @@ -165,16 +162,19 @@ public: /// @param[in] s Number of right-hand sides. /// @param[in] d Number of Lanczos steps (tridiagonal size). /// @param[out] out n×s output matrix (column-major); overwritten. + // F must be callable as T f(T x) — lambda, function pointer, or functor all work. + // std::invocable is a C++20 concept that enforces this at the call site, + // giving a readable error instead of a cryptic substitution failure deep inside. template F> - void apply(F f, int64_t n, int64_t s, int64_t d, T* out) { - // Per-thread scratch: alpha_j(d), beta_j(d-1), Z_j(d*d), c_j(d), v_j(d) + void apply_f(F f, int64_t n, int64_t s, int64_t d, T* out) { + // Per-thread workspace: alpha_j(d), beta_j(d-1), Z_j(d*d), c_j(d), v_j(d) // Total per thread = d + (d-1) + d*d + d + d = d^2 + 4d - 1 - int64_t scratch_per_thread = d * d + 3 * d + std::max(d - 1, (int64_t)0); + int64_t workspace_per_thread = d * d + 3 * d + std::max(d - 1, (int64_t)0); int nthreads = 1; #ifdef _OPENMP nthreads = omp_get_max_threads(); #endif - T* scratch = new T[nthreads * scratch_per_thread]; + T* workspace = new T[nthreads * workspace_per_thread]; #pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) { @@ -182,21 +182,28 @@ public: #ifdef _OPENMP tid = omp_get_thread_num(); #endif - T* base = scratch + tid * scratch_per_thread; + T* base = workspace + tid * workspace_per_thread; T* alpha_j = base; T* beta_j = alpha_j + d; T* Z_j = beta_j + std::max(d - 1, (int64_t)0); T* c_j = Z_j + d * d; T* v_j = c_j + d; - // Copy tridiagonal for column j (stev destroys alpha and beta in-place) - std::copy(alpha + j * d, alpha + j * d + d, alpha_j); + // Copy per-column tridiagonal entries into per-thread workspace. + // stevd overwrites its alpha/beta arrays in-place (they become eigenvalues + // and workspace), so we must work on copies — otherwise the member alpha/beta + // would be destroyed and a second apply_f call (with a different f) would + // produce garbage without re-running run_lanczos. + // alpha[j*d .. j*d+d-1] and beta[j*(d-1) .. j*(d-1)+(d-2)] are flat 1D + // vectors of length d and d-1, not 2D matrices — blas::copy is correct here; + // lacpy is for 2D matrices with distinct leading dimensions. + blas::copy(d, alpha + j * d, 1, alpha_j, 1); if (d > 1) - std::copy(beta + j * (d-1), beta + j * (d-1) + (d-1), beta_j); + blas::copy(d - 1, beta + j * (d - 1), 1, beta_j, 1); // d×d tridiagonal eigendecomposition: T_j = Z_j * diag(θ) * Z_j^T // alpha_j → eigenvalues θ (ascending); Z_j → eigenvectors (column-major) - lapack::stev(lapack::Job::Vec, d, alpha_j, beta_j, Z_j, d); + lapack::stevd(lapack::Job::Vec, d, alpha_j, beta_j, Z_j, d); // c_j[i] = f(θ_i) * S_j[0, i] // In column-major Z_j (d×d): entry (row=0, col=i) = Z_j[i*d + 0] @@ -213,7 +220,7 @@ public: normb[j], K + j * n, n * s, v_j, 1, (T)0.0, out + j * n, 1); } - delete[] scratch; + delete[] workspace; } // ------------------------------------------------------------------ @@ -228,8 +235,8 @@ public: /// @param[out] out n×s output, overwritten with f(A)B approximation. template F> void call(SLO& A, const T* B, int64_t n, int64_t s, F f, int64_t d, T* out) { - run(A, B, n, s, d); - apply(f, n, s, d, out); + run_lanczos(A, B, n, s, d); + apply_f(f, n, s, d, out); } }; diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index 4a1a06233..b5371808f 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -2,6 +2,7 @@ #include "rl_blaspp.hh" #include "rl_linops.hh" +#include "rl_util.hh" #include #include @@ -176,11 +177,7 @@ public: T* eigvals = eigvals_buf.data(); // f(λ): apply scalar f to k eigenvalues - if (k > F_vec_sz) { - delete[] F_vec; - F_vec = new T[k]; - F_vec_sz = k; - } + util::regrow(F_vec, F_vec_sz, k); for (int64_t i = 0; i < k; ++i) F_vec[i] = f(eigvals[i]); @@ -195,21 +192,9 @@ public: // ResidualOp wraps the correction computation as a SymmetricLinearOperator // so Hutchinson::call can draw Ω and compute <Ω, (f(A)-f(Â))Ω>_F / s. // ------------------------------------------------------------------ - if (k * s > tmp_sz) { - delete[] tmp; - tmp = new T[k * s]; - tmp_sz = k * s; - } - if (n * s > Z1_sz) { - delete[] Z1; - Z1 = new T[n * s]; - Z1_sz = n * s; - } - if (n * s > Z2_sz) { - delete[] Z2; - Z2 = new T[n * s]; - Z2_sz = n * s; - } + util::regrow(tmp, tmp_sz, k * s); + util::regrow(Z1, Z1_sz, n * s); + util::regrow(Z2, Z2_sz, n * s); ResidualOp res_op( n, A, lanczos_fa, f, d, k, V, F_vec, tmp, Z1, Z2 diff --git a/RandLAPACK/misc/rl_util.hh b/RandLAPACK/misc/rl_util.hh index aee5072e5..3179feb92 100644 --- a/RandLAPACK/misc/rl_util.hh +++ b/RandLAPACK/misc/rl_util.hh @@ -199,6 +199,17 @@ T* upsize( return A.data(); } +/// Grow a raw buffer to at least `needed` elements. +/// Replaces the allocation; existing contents are not preserved. +template +void regrow(T*& buf, int64_t& buf_sz, int64_t needed) { + if (needed > buf_sz) { + delete[] buf; + buf = new T[needed]; + buf_sz = needed; + } +} + /// Uses recursion to find the rank of the matrix pointed to by A_dat. /// Does so by attempting to find the smallest k such that From 47fe004c3ce4e5a353995d12f8fa6c5f763ce34d Mon Sep 17 00:00:00 2001 From: mmelnich Date: Wed, 29 Apr 2026 16:42:06 -0400 Subject: [PATCH 07/14] Reorganization --- RandLAPACK/comps/rl_hutchinson.hh | 2 +- RandLAPACK/comps/rl_lanczos_fa.hh | 8 +- RandLAPACK/comps/rl_syrf.hh | 4 +- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 102 ++------ RandLAPACK/drivers/rl_revd2.hh | 118 ++++++--- RandLAPACK/linops/rl_linops.hh | 1 + RandLAPACK/linops/rl_matfun_linops.hh | 77 ++++++ RandLAPACK/misc/rl_util.hh | 10 +- benchmark/bench_BQRRP/BQRRP_error_analysis.cc | 2 +- .../bench_CQRRPT/CQRRPT_error_analysis.cc | 2 +- test/comps/test_lanczos_fa.cc | 247 ++++++++++-------- test/drivers/test_bqrrp.cc | 4 +- test/drivers/test_bqrrp_gpu.cu | 4 +- test/drivers/test_cqrrpt.cc | 2 +- test/drivers/test_cqrrpt_gpu.cu | 2 +- test/drivers/test_cqrrt.cc | 2 +- test/drivers/test_fun_nystrom_pp.cc | 102 +++----- test/drivers/test_hqrrp.cc | 4 +- test/drivers/test_revd2.cc | 30 ++- 19 files changed, 414 insertions(+), 309 deletions(-) create mode 100644 RandLAPACK/linops/rl_matfun_linops.hh diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index e3e10ed50..cd70fd9ae 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -54,7 +54,7 @@ public: T call(SLO& M, int64_t s, RandBLAS::RNGState& state) { int64_t n = M.dim; - util::regrow(Omega, Omega_sz, n * s); + util::resize(Omega, Omega_sz, n * s); // Draw Ω with iid Rademacher entries (Unif{±1}). // RandBLAS has no ScalarDist::Rademacher, but ScalarDist::Uniform fills diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 66eb9055b..13a8dc547 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -67,10 +67,10 @@ public: template void run_lanczos(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { // Grow buffers if needed - util::regrow(K, K_sz, (d + 1) * n * s); - util::regrow(alpha, alpha_sz, d * s); - if (d > 1) util::regrow(beta, beta_sz, (d - 1) * s); - util::regrow(normb, normb_sz, s); + util::resize(K, K_sz, (d + 1) * n * s); + util::resize(alpha, alpha_sz, d * s); + if (d > 1) util::resize(beta, beta_sz, (d - 1) * s); + util::resize(normb, normb_sz, s); // Step 0: q_1 = column-normalize B; store in K[:,:,0] T* K0 = K; diff --git a/RandLAPACK/comps/rl_syrf.hh b/RandLAPACK/comps/rl_syrf.hh index 023acbd62..7b225a743 100644 --- a/RandLAPACK/comps/rl_syrf.hh +++ b/RandLAPACK/comps/rl_syrf.hh @@ -102,8 +102,8 @@ class SYRF { // Q = orth(A * Omega) A(Layout::ColMajor, k, (T) 1.0, work_buff, m, (T) 0.0, Q, m); if(this->cond_check) { - util::upsize(m * k, this->cond_work_mat); - util::upsize(k, this->cond_work_vec); + util::resize(m * k, this->cond_work_mat); + util::resize(k, this->cond_work_vec); this->cond_nums.push_back( util::cond_num_check(m, k, Q, this->verbose) ); diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index b5371808f..685c90fc6 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -10,70 +10,11 @@ #include #include #include +#include namespace RandLAPACK { -/// Residual operator (f(A) - f(Â)) for the funNyström++ Hutchinson correction. -/// -/// Satisfies SymmetricLinearOperator so it can be passed directly to Hutchinson. -/// Computes C = α*(f(A)*B - f(Â)*B) + β*C via: -/// Z1 = f(A)*B using LanczosFA (d batch matvecs) -/// Z2 = f(Â)*B = V*(diag(F_vec)*(V^T B)) using two GEMMs -/// tmp, Z1, Z2 are workspace owned by FunNystromPP and borrowed here by pointer. -/// -/// @tparam T Scalar type. -/// @tparam SLO_t Type of the operator A. -/// @tparam LFA_t Type of the LanczosFA component. -/// @tparam F_t Scalar function type T→T. -template -struct ResidualOp { - using scalar_t = T; - const int64_t dim; - - SLO_t& A; - LFA_t& lanczos_fa; - F_t f; - int64_t d; - int64_t k; - T* V; - T* F_vec; - T* tmp; // k×s workspace - T* Z1; // n×s workspace: f(A)*Ω - T* Z2; // n×s workspace: f(Â)*Ω - - ResidualOp(int64_t n_, SLO_t& A_, LFA_t& lfa_, F_t f_, - int64_t d_, int64_t k_, T* V_, T* Fv_, T* tmp_, T* Z1_, T* Z2_) - : dim(n_), A(A_), lanczos_fa(lfa_), f(f_), - d(d_), k(k_), V(V_), F_vec(Fv_), tmp(tmp_), Z1(Z1_), Z2(Z2_) {} - - void operator()(Layout layout, int64_t n_vecs, T alpha, - T* const B, int64_t ldb, T beta, T* C, int64_t ldc) { - // Z1 = f(A)*B via Lanczos-FA (d batch matvecs) - lanczos_fa.call(A, B, dim, n_vecs, f, d, Z1); - - // Z2 = f(Â)*B = V * (diag(F_vec) * (V^T B)) - blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, - k, n_vecs, dim, (T)1.0, V, dim, B, ldb, (T)0.0, tmp, k); - for (int64_t j = 0; j < n_vecs; ++j) - for (int64_t i = 0; i < k; ++i) - tmp[j * k + i] *= F_vec[i]; - blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, - dim, n_vecs, k, (T)1.0, V, dim, tmp, k, (T)0.0, Z2, dim); - - // C = alpha*(Z1 - Z2) + beta*C - if (beta == (T)0) { - for (int64_t i = 0; i < dim * n_vecs; ++i) - C[i] = alpha * (Z1[i] - Z2[i]); - } else { - blas::scal(dim * n_vecs, beta, C, 1); - blas::axpy(dim * n_vecs, alpha, Z1, 1, C, 1); - blas::axpy(dim * n_vecs, -alpha, Z2, 1, C, 1); - } - } -}; - - /// funNyström++ trace estimator: estimates tr(f(A)) for symmetric PSD A. /// /// Algorithm (3 phases): @@ -123,14 +64,18 @@ public: // Persistent output/working buffers — grown with new/delete[], never shrunk. // V, eigvals: REVD2 outputs reused across calls with same (n, k). // F_vec: f applied elementwise to eigvals. - // tmp, Z1, Z2: workspace owned here and lent to ResidualOp each call. - std::vector V_buf, eigvals_buf; // std::vector for REVD2's adaptive resize - T* F_vec = nullptr; int64_t F_vec_sz = 0; - T* tmp = nullptr; int64_t tmp_sz = 0; - T* Z1 = nullptr; int64_t Z1_sz = 0; - T* Z2 = nullptr; int64_t Z2_sz = 0; - - ~FunNystromPP() { delete[] F_vec; delete[] tmp; delete[] Z1; delete[] Z2; } + // tmp, Z1, Z2: workspace owned here and lent to linops::ResidualOp each call. + T* V_buf = nullptr; int64_t V_buf_sz = 0; + T* eigvals_buf = nullptr; int64_t eigvals_buf_sz = 0; + T* F_vec = nullptr; int64_t F_vec_sz = 0; + T* tmp = nullptr; int64_t tmp_sz = 0; + T* Z1 = nullptr; int64_t Z1_sz = 0; + T* Z2 = nullptr; int64_t Z2_sz = 0; + + ~FunNystromPP() { + delete[] V_buf; delete[] eigvals_buf; + delete[] F_vec; delete[] tmp; delete[] Z1; delete[] Z2; + } FunNystromPP(REVD2_t& r, LanczosFA_t& l, Hutchinson_t& h) : revd2(r), lanczos_fa(l), hutchinson(h) {} @@ -172,14 +117,11 @@ public: // We pass k_in (a copy) so the caller's k stays unchanged; we still // need the original k below for the (n-k)*f(0) tail correction. int64_t k_in = k; - revd2.call(A, k_in, (T)0.0, V_buf, eigvals_buf, state); - T* V = V_buf.data(); - T* eigvals = eigvals_buf.data(); + revd2.call(A, k_in, (T)0.0, V_buf, V_buf_sz, eigvals_buf, eigvals_buf_sz, state); // f(λ): apply scalar f to k eigenvalues - util::regrow(F_vec, F_vec_sz, k); - for (int64_t i = 0; i < k; ++i) - F_vec[i] = f(eigvals[i]); + util::resize(F_vec, F_vec_sz, k); + std::transform(eigvals_buf, eigvals_buf + k, F_vec, f); // tr(f(Â)) = Σ f(λ_i) + (n-k)*f(0) T tr_Ahat = (T)0.0; @@ -189,15 +131,15 @@ public: // ------------------------------------------------------------------ // Phase 2: Hutchinson correction on residual f(A) - f(Â) - // ResidualOp wraps the correction computation as a SymmetricLinearOperator + // linops::ResidualOp wraps the correction as a SymmetricLinearOperator // so Hutchinson::call can draw Ω and compute <Ω, (f(A)-f(Â))Ω>_F / s. // ------------------------------------------------------------------ - util::regrow(tmp, tmp_sz, k * s); - util::regrow(Z1, Z1_sz, n * s); - util::regrow(Z2, Z2_sz, n * s); + util::resize(tmp, tmp_sz, k * s); + util::resize(Z1, Z1_sz, n * s); + util::resize(Z2, Z2_sz, n * s); - ResidualOp res_op( - n, A, lanczos_fa, f, d, k, V, F_vec, tmp, Z1, Z2 + linops::ResidualOp res_op( + n, A, lanczos_fa, f, d, k, V_buf, F_vec, tmp, Z1, Z2 ); T correction = hutchinson.call(res_op, s, state); diff --git a/RandLAPACK/drivers/rl_revd2.hh b/RandLAPACK/drivers/rl_revd2.hh index 266360250..134c6e7d2 100644 --- a/RandLAPACK/drivers/rl_revd2.hh +++ b/RandLAPACK/drivers/rl_revd2.hh @@ -88,11 +88,11 @@ class REVD2 { } ~REVD2() { - free(Y); - free(Omega); - free(R); - free(S); - free(symrf_work); + delete[] Y; + delete[] Omega; + delete[] R; + delete[] S; + delete[] symrf_work; } /// Computes a rank-k approximation to an EVD of a symmetric positive semidefinite matrix: @@ -141,37 +141,16 @@ class REVD2 { error_est_state.key.incr(1); while(true) { // Resize caller-owned output buffers (std::vector handles dynamic k) - util::upsize(k, eigvals); - T* V_dat = util::upsize(m * k, V); + util::resize(k, eigvals); + T* V_dat = util::resize(m * k, V); // Grow internal working buffers if needed - if (m * k > Y_sz) { - free(Y); - Y = (T*) calloc(m * k, sizeof(T)); - Y_sz = m * k; - } + util::resize(Y, Y_sz, m * k); // Omega must hold at least max(m*k, m*4) — the error estimator needs 4 columns - int64_t omega_needed = std::max(m * k, m * (int64_t)4); - if (omega_needed > Omega_sz) { - free(Omega); - Omega = (T*) calloc(omega_needed, sizeof(T)); - Omega_sz = omega_needed; - } - if (k * k > R_sz) { - free(R); - R = (T*) calloc(k * k, sizeof(T)); - R_sz = k * k; - } - if (k * k > S_sz) { - free(S); - S = (T*) calloc(k * k, sizeof(T)); - S_sz = k * k; - } - if (m * k > symrf_work_sz) { - free(symrf_work); - symrf_work = (T*) calloc(m * k, sizeof(T)); - symrf_work_sz = m * k; - } + util::resize(Omega, Omega_sz, std::max(m * k, m * (int64_t)4)); + util::resize(R, R_sz, k * k); + util::resize(S, S_sz, k * k); + util::resize(symrf_work, symrf_work_sz, m * k); // Construct sketching operator: Omega = orth(A * sketch) this->syrf.call(A, k, this->Omega, state, symrf_work); @@ -231,6 +210,79 @@ class REVD2 { return 0; } + /// Raw-pointer overload: same algorithm but caller-owned buffers are + /// T*/int64_t pairs managed with util::resize (new/delete[]). + template + int call( + SLO &A, + int64_t &k, + T tol, + T*& V, int64_t& V_sz, + T*& eigvals, int64_t& eigvals_sz, + RandBLAS::RNGState &state + ) { + int64_t m = A.dim; + T err = 0; + RandBLAS::RNGState error_est_state(state.counter, state.key); + error_est_state.key.incr(1); + while(true) { + util::resize(eigvals, eigvals_sz, k); + util::resize(V, V_sz, m * k); + T* V_dat = V; + + util::resize(Y, Y_sz, m * k); + util::resize(Omega, Omega_sz, std::max(m * k, m * (int64_t)4)); + util::resize(R, R_sz, k * k); + util::resize(S, S_sz, k * k); + util::resize(symrf_work, symrf_work_sz, m * k); + + this->syrf.call(A, k, this->Omega, state, symrf_work); + A(Layout::ColMajor, k, 1.0, Omega, m, 0.0, Y, m); + + T nu = std::numeric_limits::epsilon() * lapack::lange(Norm::Fro, m, k, Y, m); + + blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega, m, 0.0, R, k); + for(int i = 1; i < k; ++i) + blas::copy(k - i, &R[i + ((i-1) * k)], 1, &R[(i - 1) + (i * k)], k); + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, 1.0, Omega, m, Y, m, 1.0, R, k); + + if(lapack::potrf(Uplo::Upper, k, R, k)) + throw std::runtime_error("Cholesky decomposition failed."); + RandLAPACK::util::get_U(k, k, R, k); + + blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, 1.0, R, k, Y, m); + lapack::gesdd(Job::SomeVec, m, k, Y, m, S, V_dat, m, R, k); + + T buf; + int64_t r = 0; + int i; + for(i = 0; i < k; ++i) { + buf = std::pow(S[i], 2); + eigvals[i] = buf; + if(buf > nu) + ++r; + } + for(i = 0; i < r; ++i) + (eigvals[i] - nu < 0) ? 0 : eigvals[i] -= nu; + + std::fill(&V_dat[m * r], &V_dat[m * k], 0.0); + + RandBLAS::DenseDist g(m, 1); + error_est_state = RandBLAS::fill_dense(g, Omega, error_est_state); + + err = power_error_est(A, k, this->error_est_p, Omega, V_dat, Y, eigvals); + + if(err <= 5 * std::max(tol, nu) || k == m) { + break; + } else if (2 * k > m) { + k = m; + } else { + k = 2 * k; + } + } + return 0; + } + }; diff --git a/RandLAPACK/linops/rl_linops.hh b/RandLAPACK/linops/rl_linops.hh index b981f2c35..5eaf89100 100644 --- a/RandLAPACK/linops/rl_linops.hh +++ b/RandLAPACK/linops/rl_linops.hh @@ -16,3 +16,4 @@ #include "rl_composite_linop.hh" #include "rl_sym_linops.hh" #include "rl_materialize.hh" +#include "rl_matfun_linops.hh" diff --git a/RandLAPACK/linops/rl_matfun_linops.hh b/RandLAPACK/linops/rl_matfun_linops.hh new file mode 100644 index 000000000..1f5a30e8d --- /dev/null +++ b/RandLAPACK/linops/rl_matfun_linops.hh @@ -0,0 +1,77 @@ +#pragma once + +#include "rl_blaspp.hh" + +#include + +namespace RandLAPACK::linops { + + +/// Residual operator (f(A) - f(Â)) for use as a SymmetricLinearOperator. +/// +/// Computes C = α*(f(A)*B - f(Â)*B) + β*C via: +/// Z1 = f(A)*B using a matrix-function oracle (e.g. LanczosFA) +/// Z2 = f(Â)*B = V*(diag(F_vec)*(V^T B)) using two GEMMs +/// +/// Primary use: Hutchinson correction in funNyström++ (FunNystromPP driver). +///  = V diag(λ) V^T is the Nyström approximation; V (n×k) and F_vec (k, +/// holding f(λ)) are provided by the caller. +/// +/// @tparam T Scalar type. +/// @tparam SLO_t Type of the operator A. +/// @tparam LFA_t Type of the matrix-function oracle (e.g. LanczosFA). +/// @tparam F_t Callable type T→T. +template +struct ResidualOp { + using scalar_t = T; + const int64_t dim; + + SLO_t& A; + LFA_t& matfun_oracle; // e.g. LanczosFA: computes f(A)*B + F_t f; + int64_t d; // Lanczos steps (or oracle depth) + int64_t k; // Nyström rank + T* V; // n×k Nyström eigenvectors + T* F_vec; // k-vector: f applied to Nyström eigenvalues + T* tmp; // k×s workspace (owned by caller) + T* Z1; // n×s workspace: f(A)*Ω (owned by caller) + T* Z2; // n×s workspace: f(Â)*Ω (owned by caller) + + ResidualOp(int64_t n_, SLO_t& A_, LFA_t& oracle_, F_t f_, + int64_t d_, int64_t k_, T* V_, T* Fv_, T* tmp_, T* Z1_, T* Z2_) + : dim(n_), A(A_), matfun_oracle(oracle_), f(f_), + d(d_), k(k_), V(V_), F_vec(Fv_), tmp(tmp_), Z1(Z1_), Z2(Z2_) {} + + void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, T alpha, + T* const B, int64_t ldb, T beta, T* C, [[maybe_unused]] int64_t ldc) { + // Z1 = f(A)*B via matrix-function oracle + matfun_oracle.call(A, B, dim, n_vecs, f, d, Z1); + + // Z2 = f(Â)*B = V * (diag(F_vec) * (V^T B)) + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, + k, n_vecs, dim, (T)1.0, V, dim, B, ldb, (T)0.0, tmp, k); +#pragma omp parallel for schedule(static) + for (int64_t j = 0; j < n_vecs; ++j) + for (int64_t i = 0; i < k; ++i) + tmp[j * k + i] *= F_vec[i]; + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + dim, n_vecs, k, (T)1.0, V, dim, tmp, k, (T)0.0, Z2, dim); + + // C = alpha*(Z1 - Z2) + beta*C + // beta==0 branch uses a fused loop: one pass reads Z1,Z2 and writes C, + // which is cheaper than copy+axpy+scal (three separate passes). scal(0,C) + // is avoided because it propagates NaN via 0*NaN in some BLAS implementations. + if (beta == (T)0) { +#pragma omp parallel for schedule(static) + for (int64_t i = 0; i < dim * n_vecs; ++i) + C[i] = alpha * (Z1[i] - Z2[i]); + } else { + blas::scal(dim * n_vecs, beta, C, 1); + blas::axpy(dim * n_vecs, alpha, Z1, 1, C, 1); + blas::axpy(dim * n_vecs, -alpha, Z2, 1, C, 1); + } + } +}; + + +} // end namespace RandLAPACK::linops diff --git a/RandLAPACK/misc/rl_util.hh b/RandLAPACK/misc/rl_util.hh index 3179feb92..0dc421c48 100644 --- a/RandLAPACK/misc/rl_util.hh +++ b/RandLAPACK/misc/rl_util.hh @@ -186,10 +186,10 @@ void col_swap( } } -/// Checks if the given size is larger than available. -/// If so, resizes the vector. +/// Grow a std::vector to at least target_sz elements (zero-fills new entries). +/// Returns a raw pointer to the underlying data. template -T* upsize( +T* resize( int64_t target_sz, std::vector &A ) { @@ -202,7 +202,7 @@ T* upsize( /// Grow a raw buffer to at least `needed` elements. /// Replaces the allocation; existing contents are not preserved. template -void regrow(T*& buf, int64_t& buf_sz, int64_t needed) { +void resize(T*& buf, int64_t& buf_sz, int64_t needed) { if (needed > buf_sz) { delete[] buf; buf = new T[needed]; @@ -258,7 +258,7 @@ void normc( const std::vector &A, std::vector &A_norm ) { - util::upsize(m * n, A_norm); + util::resize(m * n, A_norm); T col_nrm = 0.0; for(int i = 0; i < n; ++i) { diff --git a/benchmark/bench_BQRRP/BQRRP_error_analysis.cc b/benchmark/bench_BQRRP/BQRRP_error_analysis.cc index 3060103b0..14299bd05 100644 --- a/benchmark/bench_BQRRP/BQRRP_error_analysis.cc +++ b/benchmark/bench_BQRRP/BQRRP_error_analysis.cc @@ -70,7 +70,7 @@ error_check(QR_benchmark_data &all_data, auto n = all_data.col; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); diff --git a/benchmark/bench_CQRRPT/CQRRPT_error_analysis.cc b/benchmark/bench_CQRRPT/CQRRPT_error_analysis.cc index 2f7c816c6..a12a3877e 100644 --- a/benchmark/bench_CQRRPT/CQRRPT_error_analysis.cc +++ b/benchmark/bench_CQRRPT/CQRRPT_error_analysis.cc @@ -72,7 +72,7 @@ error_check(QR_benchmark_data &all_data, auto n = col_sz; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index 6d6031c67..77a348554 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -16,8 +16,6 @@ class TestLanczosFA : public ::testing::Test { virtual void SetUp() {} virtual void TearDown() {} - // Build a full symmetric diagonal matrix from a vector of diagonal entries. - // Returns column-major n×n storage with all off-diagonals zero. static std::vector make_diag_matrix(const std::vector& diag) { int64_t n = diag.size(); std::vector A(n * n, 0.0); @@ -26,8 +24,7 @@ class TestLanczosFA : public ::testing::Test { return A; } - // Direct reference: f(A)*B for diagonal A = diag(d). - // Returns n×s result where result[:,j] = d .* B[:,j] component-wise via f. + // Exact reference for f(A)B when A = diag(d): result[:,j] = f(d) .* B[:,j]. static std::vector diag_fa_ref( const std::vector& d, const std::vector& B, int64_t n, int64_t s, std::function f @@ -38,60 +35,127 @@ class TestLanczosFA : public ::testing::Test { out[j * n + i] = f(d[i]) * B[j * n + i]; return out; } -}; - -// LanczosFA on a diagonal matrix with f=sqrt. -// A = diag(1,...,50), d=20 Lanczos steps approximate f(A)B from a 20-dimensional -// Krylov subspace; good but not machine-precision accuracy (d < n = 50). -TEST_F(TestLanczosFA, DiagonalSqrt) { - using RNG = r123::Philox4x32; - int64_t n = 50, s = 5, d = 20; - auto state = RandBLAS::RNGState(42); + // Exact reference for f(A)B for general symmetric A via dense eigendecomposition: + // A = Q diag(λ) Q' (syevd, reads upper triangle), then f(A)B = Q diag(f(λ)) Q'B. + // A_copy passed by value because syevd overwrites it with eigenvectors. + static std::vector dense_fa_ref( + std::vector A_copy, + const std::vector& B, + int64_t n, int64_t s, + std::function f + ) { + std::vector eigvals(n); + lapack::syevd(lapack::Job::Vec, lapack::Uplo::Upper, n, + A_copy.data(), n, eigvals.data()); + std::vector tmp(n * s); + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, + n, s, n, 1.0, A_copy.data(), n, B.data(), n, 0.0, tmp.data(), n); + for (int64_t j = 0; j < s; ++j) + for (int64_t i = 0; i < n; ++i) + tmp[j * n + i] *= f(eigvals[i]); + std::vector out(n * s); + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + n, s, n, 1.0, A_copy.data(), n, tmp.data(), n, 0.0, out.data(), n); + return out; + } - // Diagonal entries: 1, 2, ..., n - std::vector diag(n); - std::iota(diag.begin(), diag.end(), 1.0); - std::vector A_mat = make_diag_matrix(diag); + // A = diag(1,...,n); reference is diag_fa_ref (free, exact). + // With d < n steps the output is approximate; tolerance encodes expected accuracy. + template + static void run_diagonal_fa_test(F f, int64_t n, int64_t s, int64_t d, + double tol, uint64_t seed) { + using RNG = r123::Philox4x32; + auto state = RandBLAS::RNGState(seed); + std::vector diag_vec(n); + std::iota(diag_vec.begin(), diag_vec.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag_vec); + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + RandLAPACK::LanczosFA lfa; + lfa.reorth = 1; + lfa.call(A_op, B.data(), n, s, f, d, out.data()); + auto ref = diag_fa_ref(diag_vec, B, n, s, f); + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("||f(A)B - ref||/||ref|| = %e\n", rel_err); + ASSERT_LT(rel_err, tol); + } - // Random B (n×s) - std::vector B(n * s); - RandBLAS::DenseDist DB(n, s); - state = RandBLAS::fill_dense(DB, B.data(), state); + // A = B'B + n*I (κ ≈ 5, λ_min ≥ n); reference is dense_fa_ref (syevd). + // Upper triangle only — ExplicitSymLinOp and syevd both read Uplo::Upper. + template + static void run_dense_fa_test(F f, int64_t n, int64_t s, int64_t d, + double tol, uint64_t seed) { + using RNG = r123::Philox4x32; + auto state = RandBLAS::RNGState(seed); + std::vector B_raw(n * n); + RandBLAS::DenseDist D1(n, n); + state = RandBLAS::fill_dense(D1, B_raw.data(), state); + std::vector A(n * n, 0.0); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, 1.0, B_raw.data(), n, 0.0, A.data(), n); + for (int64_t i = 0; i < n; ++i) + A[i + i * n] += (double)n; + std::vector B(n * s); + RandBLAS::DenseDist D2(n, s); + state = RandBLAS::fill_dense(D2, B.data(), state); + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + RandLAPACK::LanczosFA lfa; + lfa.reorth = 1; + lfa.call(A_op, B.data(), n, s, f, d, out.data()); + auto ref = dense_fa_ref(A, B, n, s, f); + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("||f(A)B - ref||/||ref|| = %e\n", rel_err); + ASSERT_LT(rel_err, tol); + } +}; - // LanczosFA output - std::vector out(n * s, 0.0); - linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); - using LFA = RandLAPACK::LanczosFA; - LFA lfa; - lfa.reorth = 1; // full reorthogonalization - auto f_sqrt = [](double x){ return std::sqrt(x); }; - lfa.call(A_op, B.data(), n, s, f_sqrt, d, out.data()); +// f=sqrt on diag(1,...,50), d=20 < n=50. Good but not exact (Krylov subspace smaller than spectrum). +TEST_F(TestLanczosFA, DiagonalSqrt) { + run_diagonal_fa_test([](double x){ return std::sqrt(x); }, 50, 5, 20, 1e-4, 42); +} - // Reference - auto ref = diag_fa_ref(diag, B, n, s, f_sqrt); +// f=log on diag(1,...,50). Entries ≥ 1 so log is well-defined. Tolerance relaxed to 1e-3 +// (log has slower polynomial convergence on [1,50] than sqrt). +TEST_F(TestLanczosFA, DiagonalLog) { + run_diagonal_fa_test([](double x){ return std::log(x); }, 50, 4, 20, 1e-3, 7); +} - // Relative error per column - double err = 0.0, ref_norm = 0.0; - for (int64_t i = 0; i < n * s; ++i) { - double r = out[i] - ref[i]; - err += r * r; - ref_norm += ref[i] * ref[i]; - } - double rel_err = std::sqrt(err / ref_norm); - printf("LanczosFA diagonal sqrt: relative error = %e\n", rel_err); - ASSERT_LT(rel_err, 1e-4); +// f=sqrt on a random dense PSD matrix. d=n/2=25 steps → near-exact convergence on κ≈5. +TEST_F(TestLanczosFA, DensePSDSqrt) { + run_dense_fa_test([](double x){ return std::sqrt(std::max(x, 0.0)); }, 50, 4, 25, 1e-6, 31); } +// f=exp on the same dense PSD setup. Eigenvalues in [50,~250] stay within double range. +// Exercises a function not covered by the diagonal tests (per the reference implementation). +TEST_F(TestLanczosFA, DensePSDExp) { + run_dense_fa_test([](double x){ return std::exp(x); }, 50, 4, 25, 1e-6, 77); +} -// LanczosFA on a diagonal matrix with f=log. -TEST_F(TestLanczosFA, DiagonalLog) { +// Both reorth modes agree on a well-conditioned diagonal matrix (κ=40). +// Tests that reorthogonalization doesn't introduce a bug, not that it helps +// (for that one would need κ >> 1 where vanilla loses orthogonality). +TEST_F(TestLanczosFA, ReorthVsVanilla) { using RNG = r123::Philox4x32; - int64_t n = 50, s = 4, d = 20; - auto state = RandBLAS::RNGState(7); + int64_t n = 40, s = 3, d = 15; + auto state = RandBLAS::RNGState(99); - // Diagonal entries: 1, 2, ..., n (all positive, so log is well-defined) std::vector diag(n); std::iota(diag.begin(), diag.end(), 1.0); std::vector A_mat = make_diag_matrix(diag); @@ -100,31 +164,45 @@ TEST_F(TestLanczosFA, DiagonalLog) { RandBLAS::DenseDist DB(n, s); state = RandBLAS::fill_dense(DB, B.data(), state); - std::vector out(n * s, 0.0); linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + auto f_sqrt = [](double x){ return std::sqrt(x); }; using LFA = RandLAPACK::LanczosFA; - LFA lfa; - auto f_log = [](double x){ return std::log(x); }; - lfa.call(A_op, B.data(), n, s, f_log, d, out.data()); + std::vector out_full(n * s, 0.0), out_van(n * s, 0.0); + LFA lfa_full, lfa_van; + lfa_full.reorth = 1; + lfa_van.reorth = 0; - auto ref = diag_fa_ref(diag, B, n, s, f_log); + lfa_full.call(A_op, B.data(), n, s, f_sqrt, d, out_full.data()); + lfa_van.call( A_op, B.data(), n, s, f_sqrt, d, out_van.data()); - double err = 0.0, ref_norm = 0.0; + double diff = 0.0, norm = 0.0; for (int64_t i = 0; i < n * s; ++i) { - double r = out[i] - ref[i]; - err += r * r; - ref_norm += ref[i] * ref[i]; + double r = out_full[i] - out_van[i]; + diff += r * r; + norm += out_full[i] * out_full[i]; } - double rel_err = std::sqrt(err / ref_norm); - printf("LanczosFA diagonal log: relative error = %e\n", rel_err); - ASSERT_LT(rel_err, 1e-3); + double rel_diff = std::sqrt(diff / norm); + printf("Full reorth vs vanilla relative diff: %e\n", rel_diff); + ASSERT_LT(rel_diff, 1e-6); } -// Hutchinson standalone: estimate tr(M) for an explicit M. -// Use M = diag(1, 2, ..., n) so tr(M) = n*(n+1)/2 is known exactly. -TEST_F(TestLanczosFA, HutchinsonStandalone) { +// --------------------------------------------------------------------------- +// Hutchinson tests live here because Hutchinson is a building block of +// FunNystromPP alongside LanczosFA, not a standalone algorithm with its own file. + +class TestHutchinson : public ::testing::Test { +protected: + virtual void SetUp() {} + virtual void TearDown() {} +}; + +// M = diag(1,...,100): tr(M) = n*(n+1)/2 = 5050 exactly. +// s=500 Rademacher samples gives std/mean ≈ 0.5%; 5% tolerance is conservative. +// Uses a custom DiagonalOp to test the SymmetricLinearOperator interface independently +// of ExplicitSymLinOp. +TEST_F(TestHutchinson, DiagonalTrace) { using RNG = r123::Philox4x32; int64_t n = 100, s = 500; auto state = RandBLAS::RNGState(13); @@ -133,12 +211,11 @@ TEST_F(TestLanczosFA, HutchinsonStandalone) { std::iota(d.begin(), d.end(), 1.0); double true_trace = n * (n + 1.0) / 2.0; - // Diagonal operator satisfying SymmetricLinearOperator: C = α*diag(d)*B + β*C struct DiagonalOp { using scalar_t = double; const int64_t dim; const std::vector& diag; - void operator()(Layout layout, int64_t n_vecs, double alpha, + void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, double alpha, double* const B, int64_t ldb, double beta, double* C, int64_t ldc) { for (int64_t j = 0; j < n_vecs; ++j) for (int64_t i = 0; i < dim; ++i) { @@ -149,52 +226,10 @@ TEST_F(TestLanczosFA, HutchinsonStandalone) { }; DiagonalOp M{n, d}; - using Hutch = RandLAPACK::Hutchinson; - Hutch hutch; + RandLAPACK::Hutchinson hutch; double est = hutch.call(M, s, state); double rel_err = std::abs(est - true_trace) / true_trace; printf("Hutchinson trace estimate: est=%e, true=%e, rel_err=%e\n", est, true_trace, rel_err); - // With 500 Rademacher samples, variance should be small enough for 5% tolerance ASSERT_LT(rel_err, 0.05); } - - -// Vanilla (no reorth) vs full reorth on a well-conditioned diagonal matrix. -// Both should give the same answer — this checks the reorth code path doesn't break things. -TEST_F(TestLanczosFA, ReorthVsVanilla) { - using RNG = r123::Philox4x32; - int64_t n = 40, s = 3, d = 15; - auto state = RandBLAS::RNGState(99); - - std::vector diag(n); - std::iota(diag.begin(), diag.end(), 1.0); - std::vector A_mat = make_diag_matrix(diag); - - std::vector B(n * s); - RandBLAS::DenseDist DB(n, s); - state = RandBLAS::fill_dense(DB, B.data(), state); - - linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); - auto f_sqrt = [](double x){ return std::sqrt(x); }; - - using LFA = RandLAPACK::LanczosFA; - - std::vector out_full(n * s, 0.0), out_van(n * s, 0.0); - LFA lfa_full, lfa_van; - lfa_full.reorth = 1; // full - lfa_van.reorth = 0; // vanilla - - lfa_full.call(A_op, B.data(), n, s, f_sqrt, d, out_full.data()); - lfa_van.call( A_op, B.data(), n, s, f_sqrt, d, out_van.data()); - - double diff = 0.0, norm = 0.0; - for (int64_t i = 0; i < n * s; ++i) { - double r = out_full[i] - out_van[i]; - diff += r * r; - norm += out_full[i] * out_full[i]; - } - double rel_diff = std::sqrt(diff / norm); - printf("Full reorth vs vanilla relative diff: %e\n", rel_diff); - ASSERT_LT(rel_diff, 1e-6); -} diff --git a/test/drivers/test_bqrrp.cc b/test/drivers/test_bqrrp.cc index f58b7ad43..db9e1d117 100644 --- a/test/drivers/test_bqrrp.cc +++ b/test/drivers/test_bqrrp.cc @@ -67,7 +67,7 @@ class TestBQRRP : public ::testing::Test auto n = all_data.col; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); @@ -131,7 +131,7 @@ class TestBQRRP : public ::testing::Test all_data.rank = BQRRP.rank; printf("RANK AS RETURNED BY BQRRP %4ld\n", all_data.rank); - RandLAPACK::util::upsize(all_data.rank * n, all_data.R); + RandLAPACK::util::resize(all_data.rank * n, all_data.R); lapack::lacpy(MatrixType::Upper, all_data.rank, n, all_data.A.data(), m, all_data.R.data(), all_data.rank); diff --git a/test/drivers/test_bqrrp_gpu.cu b/test/drivers/test_bqrrp_gpu.cu index 62f187d9b..29ed87e41 100644 --- a/test/drivers/test_bqrrp_gpu.cu +++ b/test/drivers/test_bqrrp_gpu.cu @@ -118,7 +118,7 @@ class TestBQRRP : public ::testing::TestWithParam auto n = all_data.col; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); @@ -188,7 +188,7 @@ class TestBQRRP : public ::testing::TestWithParam cudaMemcpy(all_data.J.data(), all_data.J_device, n * sizeof(int64_t), cudaMemcpyDeviceToHost); lapack::ungqr(m, n, n, all_data.Q.data(), m, all_data.tau.data()); - RandLAPACK::util::upsize(all_data.rank * n, all_data.R); + RandLAPACK::util::resize(all_data.rank * n, all_data.R); lapack::lacpy(MatrixType::Upper, all_data.rank, n, all_data.R_full.data(), m, all_data.R.data(), all_data.rank); RandLAPACK::util::col_swap(m, n, n, all_data.A_cpy1.data(), m, all_data.J); diff --git a/test/drivers/test_cqrrpt.cc b/test/drivers/test_cqrrpt.cc index 6a07ee3c5..1bc32e984 100644 --- a/test/drivers/test_cqrrpt.cc +++ b/test/drivers/test_cqrrpt.cc @@ -63,7 +63,7 @@ class TestCQRRPT : public ::testing::Test auto n = all_data.col; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); diff --git a/test/drivers/test_cqrrpt_gpu.cu b/test/drivers/test_cqrrpt_gpu.cu index fd51e6b2e..4dd541586 100644 --- a/test/drivers/test_cqrrpt_gpu.cu +++ b/test/drivers/test_cqrrpt_gpu.cu @@ -69,7 +69,7 @@ class TestCQRRPT : public ::testing::Test auto n = all_data.col; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); diff --git a/test/drivers/test_cqrrt.cc b/test/drivers/test_cqrrt.cc index fe8e65876..d0e682773 100644 --- a/test/drivers/test_cqrrt.cc +++ b/test/drivers/test_cqrrt.cc @@ -59,7 +59,7 @@ class TestCQRRT : public ::testing::Test auto n = all_data.col; auto k = n; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); diff --git a/test/drivers/test_fun_nystrom_pp.cc b/test/drivers/test_fun_nystrom_pp.cc index 51f3ddf03..126ccd914 100644 --- a/test/drivers/test_fun_nystrom_pp.cc +++ b/test/drivers/test_fun_nystrom_pp.cc @@ -16,14 +16,13 @@ class TestFunNystromPP : public ::testing::Test { virtual void SetUp() {} virtual void TearDown() {} - // Compute tr(f(A)) exactly via dense eigendecomposition. - // A is n×n column-major symmetric; returns sum of f(λ_i). + // Exact reference for tr(f(A)) via dense syevd: sum of f(λ_i). + // A_copy passed by value because syevd overwrites it with eigenvectors. static double true_trace_fa( int64_t n, std::vector A_copy, std::function f ) { std::vector eigvals(n); - // dsyevd overwrites A with eigenvectors; we only need eigenvalues lapack::syevd(lapack::Job::NoVec, lapack::Uplo::Upper, n, A_copy.data(), n, eigvals.data()); double tr = 0.0; @@ -32,75 +31,81 @@ class TestFunNystromPP : public ::testing::Test { return tr; } - // Construct algorithm stack for FunNystromPP + // Full algorithm stack. error_est_power_iters=0 → fixed-rank single pass in REVD2. template struct Algs { - using SYPS_t = RandLAPACK::SYPS; - using Orth_t = RandLAPACK::HQRQ; - using SYRF_t = RandLAPACK::SYRF; - using REVD2_t = RandLAPACK::REVD2; - using LFA_t = RandLAPACK::LanczosFA; - using Hutch_t = RandLAPACK::Hutchinson; - using Driver_t = RandLAPACK::FunNystromPP; - - SYPS_t syps; - Orth_t orth; - SYRF_t syrf; - REVD2_t revd2; - LFA_t lfa; - Hutch_t hutch; + using SYPS_t = RandLAPACK::SYPS; + using Orth_t = RandLAPACK::HQRQ; + using SYRF_t = RandLAPACK::SYRF; + using REVD2_t = RandLAPACK::REVD2; + using LFA_t = RandLAPACK::LanczosFA; + using Hutch_t = RandLAPACK::Hutchinson; + using Driver_t = RandLAPACK::FunNystromPP; + + SYPS_t syps; + Orth_t orth; + SYRF_t syrf; + REVD2_t revd2; + LFA_t lfa; + Hutch_t hutch; Driver_t driver; Algs() : syps(3, 1, false, false), orth(false, false), syrf(syps, orth), - revd2(syrf, 0), // error_est_power_iters=0 → fixed-rank single pass + revd2(syrf, 0), driver(revd2, lfa, hutch) {} }; + + // Common test body: run the full pipeline, print and assert relative error. + template + static void test_FunNystromPP_general( + Algs& algs, + linops::ExplicitSymLinOp& A_op, + F f, double f_zero, double true_tr, + int64_t k, int64_t s, int64_t d, double tol, + RandBLAS::RNGState& state + ) { + double est = algs.driver.call(A_op, f, f_zero, k, s, d, state); + double rel_err = std::abs(est - true_tr) / true_tr; + printf("FunNystromPP: est=%e, true=%e, rel_err=%e\n", est, true_tr, rel_err); + ASSERT_LT(rel_err, tol); + } }; -// FunNystromPP on a small dense PSD matrix with f=sqrt. -// Compare estimate to tr(sqrt(A)) from direct eigendecomposition. +// Random dense PSD matrix A = B'B + n*I, f=sqrt. Reference via syevd. +// k=10 << n=30: Nyström is coarse, Hutchinson correction carries real weight. Tol = 10%. TEST_F(TestFunNystromPP, DensePSDSqrt) { using RNG = r123::Philox4x32; int64_t n = 30, k = 10, s = 200, d = 15; auto state = RandBLAS::RNGState(0); - // Generate a random PSD matrix: A = B^T B + I (ensures strict PD) std::vector B_raw(n * n); RandBLAS::DenseDist DB(n, n); state = RandBLAS::fill_dense(DB, B_raw.data(), state); std::vector A(n * n, 0.0); blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, 1.0, B_raw.data(), n, 0.0, A.data(), n); - // Add identity to ensure strict positive definiteness for (int64_t i = 0; i < n; ++i) A[i + i * n] += (double)n; - // Fill lower triangle from upper (ExplicitSymLinOp only reads one triangle, - // but true_trace_fa uses syevd which needs a full triangular half) + // Symmetrize so both syevd (Uplo::Upper) and true_trace_fa see a consistent matrix. for (int64_t j = 0; j < n; ++j) for (int64_t i = j + 1; i < n; ++i) A[i + j * n] = A[j + i * n]; - // True tr(sqrt(A)) via dense eigendecomp auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; double true_tr = true_trace_fa(n, A, f_sqrt); - // FunNystromPP estimate linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); Algs algs; - double est = algs.driver.call(A_op, f_sqrt, 0.0, k, s, d, state); - - double rel_err = std::abs(est - true_tr) / true_tr; - printf("FunNystromPP sqrt: est=%e, true=%e, rel_err=%e\n", est, true_tr, rel_err); - ASSERT_LT(rel_err, 0.1); + test_FunNystromPP_general(algs, A_op, f_sqrt, 0.0, true_tr, k, s, d, 0.1, state); } -// FunNystromPP on a diagonal matrix with f=sqrt — easier to control. -// Diagonal A = diag(1, 2, ..., n), so tr(sqrt(A)) = Σ sqrt(i) exactly. +// Diagonal A = diag(1,...,n), f=sqrt. True tr(sqrt(A)) = Σ sqrt(i) requires no eigensolver. +// k=15 of n=50 captures the dominant eigenvalues; tail is (n-k)*f(0)=0. Tol = 5%. TEST_F(TestFunNystromPP, DiagonalSqrt) { using RNG = r123::Philox4x32; int64_t n = 50, k = 15, s = 300, d = 20; @@ -116,32 +121,5 @@ TEST_F(TestFunNystromPP, DiagonalSqrt) { auto f_sqrt = [](double x){ return std::sqrt(x); }; linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); Algs algs; - double est = algs.driver.call(A_op, f_sqrt, 0.0, k, s, d, state); - - double rel_err = std::abs(est - true_tr) / true_tr; - printf("FunNystromPP diagonal sqrt: est=%e, true=%e, rel_err=%e\n", est, true_tr, rel_err); - ASSERT_LT(rel_err, 0.05); -} - - -// Verify that REVD2 with error_est_power_iters=0 and tol=0 does not increase k. -// REVD2 takes k by reference and may grow it adaptively; the fixed-rank setting -// must leave k unchanged so FunNystromPP's (n-k)*f(0) tail correction is correct. -TEST_F(TestFunNystromPP, REVD2FixedKBypass) { - using RNG = r123::Philox4x32; - int64_t n = 40, k = 10; - auto state = RandBLAS::RNGState(5); - - // Simple diagonal PSD matrix - std::vector A(n * n, 0.0); - for (int64_t i = 0; i < n; ++i) - A[i + i * n] = (double)(i + 1); - - Algs algs; - std::vector V_out(n * k, 0.0), eigvals_out(k, 0.0); - int64_t k_before = k; - algs.revd2.call(blas::Uplo::Upper, n, A.data(), k, 0.0, V_out, eigvals_out, state); - - printf("REVD2 fixed-k: k_before=%lld, k_after=%lld\n", (long long)k_before, (long long)k); - ASSERT_EQ(k, k_before); + test_FunNystromPP_general(algs, A_op, f_sqrt, 0.0, true_tr, k, s, d, 0.05, state); } diff --git a/test/drivers/test_hqrrp.cc b/test/drivers/test_hqrrp.cc index b2a546bc6..419ffe288 100644 --- a/test/drivers/test_hqrrp.cc +++ b/test/drivers/test_hqrrp.cc @@ -66,7 +66,7 @@ class TestHQRRP : public ::testing::Test auto n = all_data.col; auto k = all_data.rank; - RandLAPACK::util::upsize(k * k, all_data.I_ref); + RandLAPACK::util::resize(k * k, all_data.I_ref); RandLAPACK::util::eye(k, k, all_data.I_ref); T* A_dat = all_data.A_cpy1.data(); @@ -124,7 +124,7 @@ class TestHQRRP : public ::testing::Test RandLAPACK::hqrrp(m, n, all_data.A.data(), m, all_data.J.data(), all_data.tau.data(), b_sz, (int64_t) (d_factor * b_sz), panel_pivoting, use_cholqr, state, (T**) nullptr); - RandLAPACK::util::upsize(all_data.rank * n, all_data.R); + RandLAPACK::util::resize(all_data.rank * n, all_data.R); lapack::lacpy(MatrixType::Upper, all_data.rank, n, all_data.A.data(), m, all_data.R.data(), all_data.rank); lapack::ungqr(m, n, n, all_data.A.data(), m, all_data.tau.data()); diff --git a/test/drivers/test_revd2.cc b/test/drivers/test_revd2.cc index 6cfad98bb..d34cb1a7b 100644 --- a/test/drivers/test_revd2.cc +++ b/test/drivers/test_revd2.cc @@ -154,8 +154,8 @@ class TestREVD2 : public ::testing::Test int64_t k = k_start; all_algs.revd2.call(blas::Uplo::Upper, m, all_data.A.data(), k, tol, all_data.V, all_data.eigvals, state); - T* E_dat = RandLAPACK::util::upsize(k * k, all_data.E); - T* Buf_dat = RandLAPACK::util::upsize(m * k, all_data.Buf); + T* E_dat = RandLAPACK::util::resize(k * k, all_data.E); + T* Buf_dat = RandLAPACK::util::resize(m * k, all_data.Buf); T* A_cpy_dat = all_data.A_cpy.data(); T* V_dat = all_data.V.data(); @@ -193,8 +193,8 @@ class TestREVD2 : public ::testing::Test all_algs.revd2.call(blas::Uplo::Upper, m, all_data.A_u.data(), k, tol, all_data.V_u, all_data.eigvals_u, state); all_algs.revd2.call(blas::Uplo::Lower, m, all_data.A_l.data(), k, tol, all_data.V_l, all_data.eigvals_l, state); - T* E_u_dat = RandLAPACK::util::upsize(k * k, all_data.E_u); - T* E_l_dat = RandLAPACK::util::upsize(k * k, all_data.E_l); + T* E_u_dat = RandLAPACK::util::resize(k * k, all_data.E_u); + T* E_l_dat = RandLAPACK::util::resize(k * k, all_data.E_l); T* V_u_dat = all_data.V_u.data(); T* V_l_dat = all_data.V_l.data(); T* work_u_dat = all_data.A_u.data(); @@ -411,7 +411,27 @@ TEST_F(TestREVD2, Exactness) { ); } -TEST_F(TestREVD2, Uplo) { +// Verify that error_est_power_iters=0 and tol=0 produce fixed-rank behavior: +// k must not increase, since FunNystromPP's (n-k)*f(0) tail correction depends on k staying fixed. +TEST_F(TestREVD2, FixedRank) { + using RNG = r123::Philox4x32; + int64_t m = 40, k = 10; + auto state = RandBLAS::RNGState(5); + + std::vector A(m * m, 0.0); + for (int64_t i = 0; i < m; ++i) + A[i + i * m] = (double)(i + 1); + + algorithm_objects all_algs(false, false, 3, 1, /*error_est_p=*/0); + std::vector V_out, eigvals_out; + int64_t k_before = k; + all_algs.revd2.call(blas::Uplo::Upper, m, A.data(), k, 0.0, V_out, eigvals_out, state); + + printf("REVD2 fixed-k: k_before=%lld, k_after=%lld\n", (long long)k_before, (long long)k); + ASSERT_EQ(k, k_before); +} + +TEST_F(TestREVD2, Uplo) { using RNG = r123::Philox4x32; int64_t m = 100; From a0b5a61dca5e6893180ae4932176316e8eff8a49 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Wed, 29 Apr 2026 17:01:15 -0400 Subject: [PATCH 08/14] Added a benchmark --- benchmark/CMakeLists.txt | 3 + .../FunNystromPP_benchmark.cc | 290 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 6aa7749d1..5d7969e27 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -165,6 +165,9 @@ add_benchmark(NAME ABRIK_runtime_breakdown_sparse CXX_SOURCES bench_ABRIK/ABRIK_ add_benchmark(NAME ABRIK_speed_comparisons CXX_SOURCES bench_ABRIK/ABRIK_speed_comparisons.cc LINK_LIBS ${Benchmark_libs_external}) add_benchmark(NAME ABRIK_speed_comparisons_sparse CXX_SOURCES bench_ABRIK/ABRIK_speed_comparisons_sparse.cc LINK_LIBS ${Benchmark_libs_external}) +# FunNystromPP benchmarks +add_benchmark(NAME FunNystromPP_benchmark CXX_SOURCES bench_FunNystromPP/FunNystromPP_benchmark.cc LINK_LIBS ${Benchmark_libs}) + # GPU benchmarks (if CUDA is available) if (HAVE_CUDA) message(STATUS "Building GPU benchmarks...") diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc new file mode 100644 index 000000000..018f2bcb6 --- /dev/null +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -0,0 +1,290 @@ +#if defined(__APPLE__) +int main() {return 0;} +#else +/* +FunNystromPP benchmark — compares funNyström++ against plain Hutchinson+LanczosFA +for estimating tr(sqrt(A)) on a random dense PSD matrix A = B'B + n*I. + +Two algorithms are timed for each matrix size n: + 1. FunNystromPP(k, s, d): Nyström rank-k approximation plus Hutchinson correction + with s samples each using d Lanczos steps. + 2. Hutchinson+LanczosFA(s, d): plain Hutchinson with s Rademacher samples, + each using LanczosFA with d steps to apply sqrt(A). No Nyström component. + Uses the same s and d as FunNystromPP (correction-phase budget parity). + +Both algorithms use a fixed seed so comparisons are reproducible across sizes. +For n <= 500 a syevd reference is computed to measure relative error. + +Usage: + FunNystromPP_benchmark [n_2 ...] + dir_path : output directory (use "." for current directory) + num_runs : number of timed repetitions per matrix size + k_ratio : k = n / k_ratio (Nyström rank; e.g., 10 -> k = n/10) + s_ratio : s = k * s_ratio (Hutchinson samples; e.g., 5 -> s = 5*k) + d_steps : Lanczos depth per Hutchinson sample + n_1 ... : matrix dimensions to sweep +*/ + +#include "RandLAPACK.hh" +#include "rl_blaspp.hh" +#include "rl_lapackpp.hh" +#include "rl_gen.hh" + +#include +#include +#include +#include +#include +#include +#include + +namespace linops = RandLAPACK::linops; +using RNG = r123::Philox4x32; + +// ============================================================================ +// Benchmark data struct +// ============================================================================ + +template +struct FunNystromPP_benchmark_data { + int64_t n; // current matrix dimension (may be < A's allocation) + int64_t k; // Nyström rank + int64_t s; // Hutchinson samples + int64_t d; // Lanczos steps per sample + std::vector A; // n_max × n_max full symmetric matrix + + FunNystromPP_benchmark_data(int64_t n_max, int64_t k_, int64_t s_, int64_t d_) : + n(n_max), k(k_), s(s_), d(d_), + A(n_max * n_max, 0.0) + {} +}; + +// ============================================================================ +// data_regen: build A = B'B + n*I, fully symmetrized, for the current n. +// ============================================================================ + +template +static void data_regen( + FunNystromPP_benchmark_data& data, + RandBLAS::RNGState& state +) { + int64_t n = data.n; + std::vector B(n * n); + RandBLAS::DenseDist DB(n, n); + state = RandBLAS::fill_dense(DB, B.data(), state); + + std::fill(data.A.begin(), data.A.begin() + n * n, (T)0.0); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, + (T)1.0, B.data(), n, (T)0.0, data.A.data(), n); + for (int64_t i = 0; i < n; ++i) + data.A[i + i * n] += (T)n; + for (int64_t j = 0; j < n; ++j) + for (int64_t i = j + 1; i < n; ++i) + data.A[i + j * n] = data.A[j + i * n]; +} + +// ============================================================================ +// LFAOp: wraps LanczosFA as a SymmetricLinearOperator for plain Hutchinson. +// Computes C = alpha * f(A)*B + beta*C using lfa.call internally. +// ============================================================================ + +template +struct LFAOp { + using scalar_t = double; + const int64_t dim; + SLO_t& A; + LFA_t& lfa; + F_t f; + int64_t d; + std::vector Z_buf; + + LFAOp(int64_t n, SLO_t& A_, LFA_t& lfa_, F_t f_, int64_t d_) + : dim(n), A(A_), lfa(lfa_), f(f_), d(d_), Z_buf(n) {} + + void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, double alpha, + double* const B, [[maybe_unused]] int64_t ldb, + double beta, double* C, [[maybe_unused]] int64_t ldc) + { + Z_buf.assign(dim * n_vecs, 0.0); + lfa.call(A, B, dim, n_vecs, f, d, Z_buf.data()); + if (beta == 0.0) { + for (int64_t i = 0; i < dim * n_vecs; ++i) + C[i] = alpha * Z_buf[i]; + } else { + blas::scal(dim * n_vecs, beta, C, 1); + blas::axpy(dim * n_vecs, alpha, Z_buf.data(), 1, C, 1); + } + } +}; + +// ============================================================================ +// call_all_algs: timed comparison for one matrix size. +// Creates algorithm objects fresh each call (buffers start empty, grow as needed). +// ============================================================================ + +template +static void call_all_algs( + int64_t numruns, + FunNystromPP_benchmark_data& data, + RandBLAS::RNGState& state, + const std::string& output_filename +) { + int64_t n = data.n, k = data.k, s = data.s, d = data.d; + + // Build full algorithm stack for FunNystromPP. + RandLAPACK::SYPS syps(3, 1, false, false); + RandLAPACK::HQRQ orth(false, false); + RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); + RandLAPACK::REVD2 revd2(syrf, 0); + RandLAPACK::LanczosFA lfa_pp; + RandLAPACK::Hutchinson hutch_pp; + RandLAPACK::FunNystromPP + driver(revd2, lfa_pp, hutch_pp); + + // Separate LanczosFA + Hutchinson for plain baseline. + RandLAPACK::LanczosFA lfa_base; + RandLAPACK::Hutchinson hutch_base; + + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + + auto state_gen = state; + auto state_alg = state; + + for (int i = 0; i < numruns; ++i) { + printf("ITERATION %d, N=%ld, K=%ld, S=%ld, D=%ld\n", i, n, k, s, d); + + // ---- FunNystromPP -------------------------------------------------- + state_gen = state; + data_regen(data, state_gen); + state_alg = state; + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); + + auto start_pp = steady_clock::now(); + double est_pp = driver.call(A_op, f_sqrt, 0.0, k, s, d, state_alg); + auto stop_pp = steady_clock::now(); + long dur_pp = duration_cast(stop_pp - start_pp).count(); + printf(" FunNystromPP: est=%.6e time=%ld us\n", est_pp, dur_pp); + + // ---- Hutchinson + LanczosFA (no Nyström) --------------------------- + state_gen = state; + data_regen(data, state_gen); + state_alg = state; + linops::ExplicitSymLinOp A_op2(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); + LFAOp, RandLAPACK::LanczosFA, decltype(f_sqrt)> + lfa_op(n, A_op2, lfa_base, f_sqrt, d); + + auto start_h = steady_clock::now(); + double est_h = hutch_base.call(lfa_op, s, state_alg); + auto stop_h = steady_clock::now(); + long dur_h = duration_cast(stop_h - start_h).count(); + printf(" Hutchinson+LFA: est=%.6e time=%ld us\n", est_h, dur_h); + + // ---- Syevd reference (only for small n) ---------------------------- + state_gen = state; + data_regen(data, state_gen); + + double true_tr = -1.0; + if (n <= 500) { + std::vector A_copy(data.A.begin(), data.A.begin() + n * n); + std::vector eigvals(n); + lapack::syevd(lapack::Job::NoVec, lapack::Uplo::Upper, n, + A_copy.data(), n, eigvals.data()); + true_tr = 0.0; + for (int64_t j = 0; j < n; ++j) + true_tr += std::sqrt(std::max(eigvals[j], 0.0)); + } + + double err_pp = (true_tr > 0) ? std::abs(est_pp - true_tr) / true_tr : -1.0; + double err_h = (true_tr > 0) ? std::abs(est_h - true_tr) / true_tr : -1.0; + if (true_tr > 0) + printf(" Reference: true=%.6e err_pp=%e err_h=%e\n", + true_tr, err_pp, err_h); + + std::ofstream file(output_filename, std::ios::out | std::ios::app); + file << n << ", " << k << ", " << s << ", " << d << ", " + << dur_pp << ", " << dur_h << ", " + << est_pp << ", " << est_h << ", " + << true_tr << ", " << err_pp << ", " << err_h << ",\n"; + } +} + +// ============================================================================ +// main +// ============================================================================ + +int main(int argc, char* argv[]) { + if (argc < 7) { + std::cerr << "Usage: " << argv[0] + << " " + " [n_2 ...]\n" + << " dir_path : output directory (use '.' for current dir)\n" + << " num_runs : timed repetitions per matrix size\n" + << " k_ratio : k = n / k_ratio (Nystrom rank)\n" + << " s_ratio : s = k * s_ratio (Hutchinson samples)\n" + << " d_steps : Lanczos depth per sample\n"; + return 1; + } + + int64_t numruns = std::stol(argv[2]); + double k_ratio = std::stod(argv[3]); + double s_ratio = std::stod(argv[4]); + int64_t d_steps = std::stol(argv[5]); + + std::vector n_sizes; + for (int i = 6; i < argc; ++i) + n_sizes.push_back(std::stol(argv[i])); + + std::ostringstream oss; + for (auto v : n_sizes) oss << v << ", "; + std::string sizes_str = oss.str(); + + auto state = RandBLAS::RNGState(); + auto state_constant = state; + + int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); + int64_t k_max = std::max((int64_t)1, (int64_t)(n_max / k_ratio)); + int64_t s_max = std::max((int64_t)1, (int64_t)(k_max * s_ratio)); + + FunNystromPP_benchmark_data all_data(n_max, k_max, s_max, d_steps); + + // Output file + std::string output_filename = "_FunNystromPP_benchmark_num_info_lines_7.txt"; + std::string path; + if (std::string(argv[1]) != ".") + path = std::string(argv[1]) + output_filename; + else + path = output_filename; + + std::ofstream file(path, std::ios::out | std::ios::app); + file << "Description: FunNystromPP vs Hutchinson+LanczosFA benchmark for tr(sqrt(A)), A = B'B + n*I." + "\nFile format: 11 columns: n, k, s, d, time_fun_pp (us), time_hutch (us)," + " est_fun_pp, est_hutch, true_tr (syevd; -1 for n>500), err_fun_pp, err_hutch;" + " rows = numruns repetitions per matrix size." + "\nNum OMP threads: " + std::to_string(RandLAPACK::util::get_omp_threads()) + + "\nInput sizes: " + sizes_str + + "\nParameters: k_ratio=" + std::string(argv[3]) + + " s_ratio=" + std::string(argv[4]) + + " d=" + std::to_string(d_steps) + + "\nNum runs per size: " + std::to_string(numruns) + + "\n"; + file.flush(); + + auto start_all = steady_clock::now(); + for (int64_t n : n_sizes) { + int64_t k = std::max((int64_t)1, (int64_t)(n / k_ratio)); + int64_t s = std::max((int64_t)1, (int64_t)(k * s_ratio)); + all_data.n = n; + all_data.k = k; + all_data.s = s; + all_data.d = d_steps; + + call_all_algs(numruns, all_data, state_constant, path); + } + auto stop_all = steady_clock::now(); + long dur_all = duration_cast(stop_all - start_all).count(); + file << "Total benchmark execution time: " + std::to_string(dur_all) + "\n"; + file.flush(); + + return 0; +} +#endif From b4d2b7a11dee979fe3e502a0ca7b26e6d9ad413a Mon Sep 17 00:00:00 2001 From: mmelnich Date: Thu, 30 Apr 2026 11:55:39 -0400 Subject: [PATCH 09/14] More fixes --- RandLAPACK/comps/rl_hutchinson.hh | 13 +++--- RandLAPACK/comps/rl_lanczos_fa.hh | 10 ++++- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 42 ++++++++++--------- RandLAPACK/drivers/rl_revd2.hh | 3 +- RandLAPACK/linops/rl_matfun_linops.hh | 34 +++++++++------ RandLAPACK/misc/rl_util.hh | 1 + .../FunNystromPP_benchmark.cc | 10 ++--- test/comps/test_lanczos_fa.cc | 33 +++++++++++++++ test/drivers/test_revd2.cc | 4 +- 9 files changed, 100 insertions(+), 50 deletions(-) diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index cd70fd9ae..f59f8285b 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -24,10 +24,11 @@ namespace RandLAPACK { template class Hutchinson { public: - // Internal sketch buffer — grown with new/delete[], never shrunk. - T* Omega = nullptr; int64_t Omega_sz = 0; + // Internal buffers — grown with new/delete[], never shrunk. + T* Omega = nullptr; int64_t Omega_sz = 0; + T* Z = nullptr; int64_t Z_sz = 0; - ~Hutchinson() { delete[] Omega; } + ~Hutchinson() { delete[] Omega; delete[] Z; } // ------------------------------------------------------------------ /// Low-level estimator: given precomputed Ω (n×s) and Z = M*Ω (n×s), @@ -64,12 +65,10 @@ public: for (int64_t i = 0; i < n * s; ++i) Omega[i] = (Omega[i] >= 0) ? (T)1 : (T)-1; - T* Z = new T[n * s]; + util::resize(Z, Z_sz, n * s); M(Layout::ColMajor, s, (T)1.0, Omega, n, (T)0.0, Z, n); - T result = estimate(Omega, Z, n, s); - delete[] Z; - return result; + return estimate(Omega, Z, n, s); } }; diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 13a8dc547..3a997dec3 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -79,7 +79,9 @@ public: for (int64_t j = 0; j < s; ++j) { T nrm = blas::nrm2(n, K0 + j * n, 1); normb[j] = nrm; - blas::scal(n, (T)1.0 / nrm, K0 + j * n, 1); + // Zero input column: skip normalization; normb[j]=0 so apply_f outputs zero. + if (nrm > (T)0) + blas::scal(n, (T)1.0 / nrm, K0 + j * n, 1); } // Step 0 matvec: K[:,:,1] = A * K[:,:,0] @@ -128,11 +130,15 @@ public: } // (3) β_{i+1} = column norms, (4) normalize → q_{i+1} + // Zero norm means the Krylov basis has collapsed for that column. + // Store β=0 (the tridiagonal subdiagonal entry) and skip normalization; + // stevd handles a zero subdiagonal correctly (independent 1×1 blocks). #pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) { T nrm = blas::nrm2(n, K_curr + j * n, 1); beta[j * (d - 1) + i] = nrm; - blas::scal(n, (T)1.0 / nrm, K_curr + j * n, 1); + if (nrm > (T)0) + blas::scal(n, (T)1.0 / nrm, K_curr + j * n, 1); } // (5) K_new = A*q_{i+1} - β_{i+1}*q_i diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index 685c90fc6..9ffdd3d59 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -17,8 +16,8 @@ namespace RandLAPACK { /// funNyström++ trace estimator: estimates tr(f(A)) for symmetric PSD A. /// -/// Algorithm (3 phases): -/// Phase 1 — Nyström approximation via REVD2 (k matvecs): +/// Algorithm (2 phases): +/// Phase 1 — Nyström approximation via REVD2 (O(k) matvecs): /// (V, λ) = REVD2(A, k) →  = V diag(λ) V^T /// tr(f(Â)) = Σ f(λ_i) + (n-k) * f(0) [free from the eigenvalues] /// @@ -36,10 +35,11 @@ namespace RandLAPACK { /// norm and the Hutchinson estimator converges in far fewer samples. /// /// Requirements on A: -/// - Symmetric positive definite (λ_min > 0). The spectral interval [a,b] -/// with 0 < a < b is assumed throughout (see paper). -/// - For f = log, A must be strictly PD (log(0) = -∞). Pass f_zero = 0 only -/// for functions where f(0) is well-defined (e.g., sqrt, x^α with α > 0). +/// - Symmetric positive semidefinite (λ_min ≥ 0) for most functions +/// (e.g., sqrt, x^α with α > 0 where f(0) is finite). +/// - For f = log, A must be strictly positive definite (λ_min > 0), +/// since log(0) = -∞. Pass any finite f_zero placeholder and ensure +/// λ_min > 0 at the call site. /// /// Parameter selection (f = sqrt, κ = λ_max/λ_min): /// d = O(√κ log(n/ε)) Lanczos steps — caller's responsibility @@ -86,7 +86,7 @@ public: /// @param[in] A SymmetricLinearOperator — matvec oracle for A. /// @param[in] f Scalar function T→T (e.g., sqrt, log, pow(x,α)). /// @param[in] f_zero f(0): value at zero. Use 0.0 for sqrt/power. - /// Must be finite; assert fires if infinite. + /// Must be finite; throws std::invalid_argument if not. /// For log, A must be strictly PD — pass any finite /// placeholder and ensure λ_min > 0. /// @param[in] k Nyström rank. Caller chooses based on κ, ε. @@ -104,42 +104,44 @@ public: int64_t d, RandBLAS::RNGState& state ) { - assert(std::isfinite(f_zero) && "f_zero must be finite; for log, ensure A is strictly PD"); + if (!std::isfinite(f_zero)) + throw std::invalid_argument("f_zero must be finite; for log, ensure A is strictly PD"); int64_t n = A.dim; // ------------------------------------------------------------------ // Phase 1: Nyström approximation // REVD2 with error_est_power_iters=0 and tol=0 runs a single pass - // with fixed rank k. V_buf (n×k) and eigvals_buf (k) are the outputs. + // with fixed rank k. V_buf (n×k_in) and eigvals_buf (k_in) are the outputs. // ------------------------------------------------------------------ // REVD2::call takes rank by reference and may increase it adaptively. - // We pass k_in (a copy) so the caller's k stays unchanged; we still - // need the original k below for the (n-k)*f(0) tail correction. + // We pass k_in (a copy) so the caller's k stays unchanged. + // All post-call uses reference k_in (not k) so they stay consistent + // with the actual rank used, even in adaptive mode. int64_t k_in = k; revd2.call(A, k_in, (T)0.0, V_buf, V_buf_sz, eigvals_buf, eigvals_buf_sz, state); - // f(λ): apply scalar f to k eigenvalues - util::resize(F_vec, F_vec_sz, k); - std::transform(eigvals_buf, eigvals_buf + k, F_vec, f); + // f(λ): apply scalar f to k_in eigenvalues + util::resize(F_vec, F_vec_sz, k_in); + std::transform(eigvals_buf, eigvals_buf + k_in, F_vec, f); - // tr(f(Â)) = Σ f(λ_i) + (n-k)*f(0) + // tr(f(Â)) = Σ f(λ_i) + (n-k_in)*f(0) T tr_Ahat = (T)0.0; - for (int64_t i = 0; i < k; ++i) + for (int64_t i = 0; i < k_in; ++i) tr_Ahat += F_vec[i]; - tr_Ahat += static_cast(n - k) * f_zero; + tr_Ahat += static_cast(n - k_in) * f_zero; // ------------------------------------------------------------------ // Phase 2: Hutchinson correction on residual f(A) - f(Â) // linops::ResidualOp wraps the correction as a SymmetricLinearOperator // so Hutchinson::call can draw Ω and compute <Ω, (f(A)-f(Â))Ω>_F / s. // ------------------------------------------------------------------ - util::resize(tmp, tmp_sz, k * s); + util::resize(tmp, tmp_sz, k_in * s); util::resize(Z1, Z1_sz, n * s); util::resize(Z2, Z2_sz, n * s); linops::ResidualOp res_op( - n, A, lanczos_fa, f, d, k, V_buf, F_vec, tmp, Z1, Z2 + n, A, lanczos_fa, f, d, k_in, V_buf, F_vec, tmp, Z1, Z2 ); T correction = hutchinson.call(res_op, s, state); diff --git a/RandLAPACK/drivers/rl_revd2.hh b/RandLAPACK/drivers/rl_revd2.hh index 134c6e7d2..a257c7e49 100644 --- a/RandLAPACK/drivers/rl_revd2.hh +++ b/RandLAPACK/drivers/rl_revd2.hh @@ -9,7 +9,6 @@ #include #include -#include #include namespace RandLAPACK { @@ -71,7 +70,7 @@ class REVD2 { int error_est_p; bool verbose; - // Internal working buffers — owned by this object, grown with calloc/free + // Internal working buffers — owned by this object, grown with new[]/delete[] via util::resize T* Y = nullptr; int64_t Y_sz = 0; T* Omega = nullptr; int64_t Omega_sz = 0; T* R = nullptr; int64_t R_sz = 0; diff --git a/RandLAPACK/linops/rl_matfun_linops.hh b/RandLAPACK/linops/rl_matfun_linops.hh index 1f5a30e8d..d333485f0 100644 --- a/RandLAPACK/linops/rl_matfun_linops.hh +++ b/RandLAPACK/linops/rl_matfun_linops.hh @@ -2,6 +2,10 @@ #include "rl_blaspp.hh" +#ifdef _OPENMP +#include +#endif + #include namespace RandLAPACK::linops { @@ -43,32 +47,38 @@ struct ResidualOp { d(d_), k(k_), V(V_), F_vec(Fv_), tmp(tmp_), Z1(Z1_), Z2(Z2_) {} void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, T alpha, - T* const B, int64_t ldb, T beta, T* C, [[maybe_unused]] int64_t ldc) { + T* const B, int64_t ldb, T beta, T* C, int64_t ldc) { // Z1 = f(A)*B via matrix-function oracle matfun_oracle.call(A, B, dim, n_vecs, f, d, Z1); // Z2 = f(Â)*B = V * (diag(F_vec) * (V^T B)) blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, n_vecs, dim, (T)1.0, V, dim, B, ldb, (T)0.0, tmp, k); -#pragma omp parallel for schedule(static) +#ifdef _OPENMP + #pragma omp parallel for schedule(static) +#endif for (int64_t j = 0; j < n_vecs; ++j) for (int64_t i = 0; i < k; ++i) tmp[j * k + i] *= F_vec[i]; blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, dim, n_vecs, k, (T)1.0, V, dim, tmp, k, (T)0.0, Z2, dim); - // C = alpha*(Z1 - Z2) + beta*C - // beta==0 branch uses a fused loop: one pass reads Z1,Z2 and writes C, - // which is cheaper than copy+axpy+scal (three separate passes). scal(0,C) - // is avoided because it propagates NaN via 0*NaN in some BLAS implementations. + // C = alpha*(Z1 - Z2) + beta*C column-by-column to respect ldc. + // Z1 and Z2 are always stored flat (stride dim); C may have ldc >= dim. + // beta==0 uses a fused read of Z1,Z2 to avoid scal(0,C) propagating NaN. if (beta == (T)0) { -#pragma omp parallel for schedule(static) - for (int64_t i = 0; i < dim * n_vecs; ++i) - C[i] = alpha * (Z1[i] - Z2[i]); +#ifdef _OPENMP + #pragma omp parallel for schedule(static) +#endif + for (int64_t j = 0; j < n_vecs; ++j) + for (int64_t i = 0; i < dim; ++i) + C[j * ldc + i] = alpha * (Z1[j * dim + i] - Z2[j * dim + i]); } else { - blas::scal(dim * n_vecs, beta, C, 1); - blas::axpy(dim * n_vecs, alpha, Z1, 1, C, 1); - blas::axpy(dim * n_vecs, -alpha, Z2, 1, C, 1); + for (int64_t j = 0; j < n_vecs; ++j) { + blas::scal(dim, beta, C + j * ldc, 1); + blas::axpy(dim, alpha, Z1 + j * dim, 1, C + j * ldc, 1); + blas::axpy(dim, -alpha, Z2 + j * dim, 1, C + j * ldc, 1); + } } } }; diff --git a/RandLAPACK/misc/rl_util.hh b/RandLAPACK/misc/rl_util.hh index 0dc421c48..ec535cf24 100644 --- a/RandLAPACK/misc/rl_util.hh +++ b/RandLAPACK/misc/rl_util.hh @@ -201,6 +201,7 @@ T* resize( /// Grow a raw buffer to at least `needed` elements. /// Replaces the allocation; existing contents are not preserved. +/// New storage is uninitialized — callers must write before reading. template void resize(T*& buf, int64_t& buf_sz, int64_t needed) { if (needed > buf_sz) { diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc index 018f2bcb6..eff2d5958 100644 --- a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -99,7 +99,7 @@ struct LFAOp { std::vector Z_buf; LFAOp(int64_t n, SLO_t& A_, LFA_t& lfa_, F_t f_, int64_t d_) - : dim(n), A(A_), lfa(lfa_), f(f_), d(d_), Z_buf(n) {} + : dim(n), A(A_), lfa(lfa_), f(f_), d(d_), Z_buf() {} void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, double alpha, double* const B, [[maybe_unused]] int64_t ldb, @@ -151,7 +151,7 @@ static void call_all_algs( auto state_alg = state; for (int i = 0; i < numruns; ++i) { - printf("ITERATION %d, N=%ld, K=%ld, S=%ld, D=%ld\n", i, n, k, s, d); + printf("ITERATION %d, N=%lld, K=%lld, S=%lld, D=%lld\n", i, (long long)n, (long long)k, (long long)s, (long long)d); // ---- FunNystromPP -------------------------------------------------- state_gen = state; @@ -163,7 +163,7 @@ static void call_all_algs( double est_pp = driver.call(A_op, f_sqrt, 0.0, k, s, d, state_alg); auto stop_pp = steady_clock::now(); long dur_pp = duration_cast(stop_pp - start_pp).count(); - printf(" FunNystromPP: est=%.6e time=%ld us\n", est_pp, dur_pp); + printf(" FunNystromPP: est=%.6e time=%lld us\n", est_pp, (long long)dur_pp); // ---- Hutchinson + LanczosFA (no Nyström) --------------------------- state_gen = state; @@ -177,7 +177,7 @@ static void call_all_algs( double est_h = hutch_base.call(lfa_op, s, state_alg); auto stop_h = steady_clock::now(); long dur_h = duration_cast(stop_h - start_h).count(); - printf(" Hutchinson+LFA: est=%.6e time=%ld us\n", est_h, dur_h); + printf(" Hutchinson+LFA: est=%.6e time=%lld us\n", est_h, (long long)dur_h); // ---- Syevd reference (only for small n) ---------------------------- state_gen = state; @@ -248,7 +248,7 @@ int main(int argc, char* argv[]) { FunNystromPP_benchmark_data all_data(n_max, k_max, s_max, d_steps); // Output file - std::string output_filename = "_FunNystromPP_benchmark_num_info_lines_7.txt"; + std::string output_filename = "_FunNystromPP_benchmark_num_info_lines_" + std::to_string(7) + ".txt"; std::string path; if (std::string(argv[1]) != ".") path = std::string(argv[1]) + output_filename; diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index 77a348554..356ba8bb0 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -131,6 +131,39 @@ TEST_F(TestLanczosFA, DiagonalSqrt) { run_diagonal_fa_test([](double x){ return std::sqrt(x); }, 50, 5, 20, 1e-4, 42); } +// d=1 on A = 4*I: all eigenvalues equal, so the 1-step Krylov result is exact. +// Exercises the paths where beta is never allocated and the main recurrence loop +// (0..d-2) does not execute. stevd is called on a 1×1 tridiagonal. +TEST_F(TestLanczosFA, ScalarMatrixSqrt_d1) { + using RNG = r123::Philox4x32; + int64_t n = 8, s = 3, d = 1; + auto state = RandBLAS::RNGState(55); + + std::vector A_mat(n * n, 0.0); + for (int64_t i = 0; i < n; ++i) A_mat[i + i * n] = 4.0; + + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + RandLAPACK::LanczosFA lfa; + lfa.reorth = 1; + lfa.call(A_op, B.data(), n, s, [](double x){ return std::sqrt(x); }, d, out.data()); + + // f(A)B = sqrt(4)*B = 2*B exactly for A = 4*I + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - 2.0 * B[i]; + err += r * r; + ref_norm += 4.0 * B[i] * B[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("d=1 on 4*I: ||f(A)B - ref||/||ref|| = %e\n", rel_err); + ASSERT_LT(rel_err, 1e-10); +} + // f=log on diag(1,...,50). Entries ≥ 1 so log is well-defined. Tolerance relaxed to 1e-3 // (log has slower polynomial convergence on [1,50] than sqrt). TEST_F(TestLanczosFA, DiagonalLog) { diff --git a/test/drivers/test_revd2.cc b/test/drivers/test_revd2.cc index d34cb1a7b..22309bcb0 100644 --- a/test/drivers/test_revd2.cc +++ b/test/drivers/test_revd2.cc @@ -172,7 +172,7 @@ class TestREVD2 : public ::testing::Test T norm_0 = lapack::lange(Norm::Fro, m, m, A_cpy_dat, m); printf("||A - VEV'||_F / ||A||_F: %e\n", norm_0 / norm_A); ASSERT_NEAR(norm_0 / norm_A, err_expectation, 10 * err_expectation); - ASSERT_NEAR(k, rank_expectation, std::numeric_limits::epsilon()); + ASSERT_EQ(k, rank_expectation); } /// General test for REVD: @@ -335,7 +335,7 @@ TEST_F(TestREVD2, Overestimation1) { ); } -TEST_F(TestREVD2, Oversetimation2) { +TEST_F(TestREVD2, Overestimation2) { using RNG = r123::Philox4x32; int64_t m = 1000; From 37c22a36b6c8d2bc06873812b68acf405657b691 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Fri, 1 May 2026 14:54:34 -0400 Subject: [PATCH 10/14] Various fixes and updates --- RandLAPACK.hh | 3 +- RandLAPACK/CMakeLists.txt | 2 +- RandLAPACK/comps/rl_hutchinson.hh | 1 + RandLAPACK/comps/rl_lanczos_fa.hh | 22 +- RandLAPACK/comps/rl_matfun.hh | 36 ++ RandLAPACK/comps/rl_preconditioners.hh | 14 +- RandLAPACK/comps/rl_syps.hh | 18 +- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 62 ++- .../{rl_revd2.hh => rl_nystrom_evd.hh} | 123 +---- RandLAPACK/linops/rl_matfun_linops.hh | 15 + RandLAPACK/linops/rl_sparse_linop.hh | 43 ++ RandLAPACK/testing/rl_gen.hh | 176 +++++++ .../FunNystromPP_benchmark.cc | 465 +++++++++++++----- test/CMakeLists.txt | 2 +- test/comps/test_lanczos_fa.cc | 43 ++ test/drivers/test_fun_nystrom_pp.cc | 87 +++- .../{test_revd2.cc => test_nystrom_evd.cc} | 137 +++--- 17 files changed, 922 insertions(+), 327 deletions(-) create mode 100644 RandLAPACK/comps/rl_matfun.hh rename RandLAPACK/drivers/{rl_revd2.hh => rl_nystrom_evd.hh} (57%) rename test/drivers/{test_revd2.cc => test_nystrom_evd.cc} (78%) diff --git a/RandLAPACK.hh b/RandLAPACK.hh index 91c4a31a7..40edf3755 100644 --- a/RandLAPACK.hh +++ b/RandLAPACK.hh @@ -18,6 +18,7 @@ #include "RandLAPACK/testing/rl_test_utils.hh" // Computational routines +#include "RandLAPACK/comps/rl_matfun.hh" #include "RandLAPACK/comps/rl_determiter.hh" #include "RandLAPACK/comps/rl_preconditioners.hh" #include "RandLAPACK/comps/rl_qb.hh" @@ -38,7 +39,7 @@ #include "RandLAPACK/drivers/rl_scholqr3_linops.hh" #include "RandLAPACK/drivers/rl_cqrrpt.hh" #include "RandLAPACK/drivers/rl_bqrrp.hh" -#include "RandLAPACK/drivers/rl_revd2.hh" +#include "RandLAPACK/drivers/rl_nystrom_evd.hh" #include "RandLAPACK/drivers/rl_abrik.hh" #include "RandLAPACK/drivers/rl_krill.hh" #include "RandLAPACK/drivers/rl_fun_nystrom_pp.hh" diff --git a/RandLAPACK/CMakeLists.txt b/RandLAPACK/CMakeLists.txt index 7d70a1d17..434f8edd4 100644 --- a/RandLAPACK/CMakeLists.txt +++ b/RandLAPACK/CMakeLists.txt @@ -6,7 +6,7 @@ set(RandLAPACK_cxx_sources rl_cqrrpt.hh rl_bqrrp.hh rl_rsvd.hh - rl_revd2.hh + rl_nystrom_evd.hh rl_qb.hh rl_orth.hh rl_util.hh diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index f59f8285b..47c0f9f5c 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -39,6 +39,7 @@ public: /// @param[in] Z n×s result of M applied to Omega_buf (column-major). /// @param[in] n Ambient dimension. /// @param[in] s Number of samples. + /// @returns Frobenius inner product <Ω, Z>_F / s. T estimate(const T* Omega_buf, const T* Z, int64_t n, int64_t s) const { // Frobenius inner product of two n×s matrices, treated as flat n*s vectors return blas::dot(n * s, Omega_buf, 1, Z, 1) / static_cast(s); diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 3a997dec3..345f06821 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -47,12 +47,13 @@ public: // lapack::stevd expects the diagonal and subdiagonal as separate arrays, // so alpha and beta are stored separately rather than interleaved. // normb: s — column norms of B before normalization. - T* K = nullptr; int64_t K_sz = 0; - T* alpha = nullptr; int64_t alpha_sz = 0; - T* beta = nullptr; int64_t beta_sz = 0; - T* normb = nullptr; int64_t normb_sz = 0; + T* K = nullptr; int64_t K_sz = 0; + T* alpha = nullptr; int64_t alpha_sz = 0; + T* beta = nullptr; int64_t beta_sz = 0; + T* normb = nullptr; int64_t normb_sz = 0; + T* workspace = nullptr; int64_t workspace_sz = 0; - ~LanczosFA() { delete[] K; delete[] alpha; delete[] beta; delete[] normb; } + ~LanczosFA() { delete[] K; delete[] alpha; delete[] beta; delete[] normb; delete[] workspace; } // ------------------------------------------------------------------ /// Run the d-step block Lanczos recurrence on B. @@ -163,14 +164,13 @@ public: /// of the eigenvector matrix (Chen 2022, eq. 2.3). /// Per-column stev calls are independent — parallelized with OpenMP. /// - /// @param[in] f Scalar callable T→T applied to tridiagonal eigenvalues. + /// @tparam F Callable as T(T) — lambda, function pointer, or functor. + /// std::invocable (C++20) enforces this at the call site. + /// @param[in] f Instance of F applied elementwise to tridiagonal eigenvalues θ. /// @param[in] n Dimension of A. /// @param[in] s Number of right-hand sides. /// @param[in] d Number of Lanczos steps (tridiagonal size). /// @param[out] out n×s output matrix (column-major); overwritten. - // F must be callable as T f(T x) — lambda, function pointer, or functor all work. - // std::invocable is a C++20 concept that enforces this at the call site, - // giving a readable error instead of a cryptic substitution failure deep inside. template F> void apply_f(F f, int64_t n, int64_t s, int64_t d, T* out) { // Per-thread workspace: alpha_j(d), beta_j(d-1), Z_j(d*d), c_j(d), v_j(d) @@ -180,7 +180,7 @@ public: #ifdef _OPENMP nthreads = omp_get_max_threads(); #endif - T* workspace = new T[nthreads * workspace_per_thread]; + util::resize(workspace, workspace_sz, (int64_t)nthreads * workspace_per_thread); #pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) { @@ -225,8 +225,6 @@ public: blas::gemv(Layout::ColMajor, Op::NoTrans, n, d, normb[j], K + j * n, n * s, v_j, 1, (T)0.0, out + j * n, 1); } - - delete[] workspace; } // ------------------------------------------------------------------ diff --git a/RandLAPACK/comps/rl_matfun.hh b/RandLAPACK/comps/rl_matfun.hh new file mode 100644 index 000000000..e6b142c74 --- /dev/null +++ b/RandLAPACK/comps/rl_matfun.hh @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +namespace RandLAPACK { + +/// SqrtFun: f(x) = sqrt(max(x, 0)). +/// Clamps to zero to tolerate floating-point roundoff near zero eigenvalues. +template +struct SqrtFun { + T operator()(T x) const { return std::sqrt(std::max(x, (T)0)); } +}; + +/// LogFun: f(x) = log(x). +/// Undefined for x <= 0. The caller must ensure the matrix is strictly PD +/// (all eigenvalues > 0) before using this function. +template +struct LogFun { + T operator()(T x) const { return std::log(x); } +}; + +/// PolyFun: f(x) = x*(x + lambda) = x^2 + lambda*x. +/// +/// Models tr(K(K + λI)) = tr(K² + λK) for a symmetric PSD kernel matrix K. +/// LanczosFA is exact for this function in d=2 Lanczos steps, since Gauss +/// quadrature integrates polynomials of degree ≤ 2d-1 exactly (d=2 covers +/// degree 3, which contains all degree-2 polynomials). +template +struct PolyFun { + T lam; + explicit PolyFun(T lambda) : lam(lambda) {} + T operator()(T x) const { return x * (x + lam); } +}; + +} // end namespace RandLAPACK diff --git a/RandLAPACK/comps/rl_preconditioners.hh b/RandLAPACK/comps/rl_preconditioners.hh index afbdcd090..5109ef1df 100644 --- a/RandLAPACK/comps/rl_preconditioners.hh +++ b/RandLAPACK/comps/rl_preconditioners.hh @@ -9,7 +9,7 @@ #include "rl_syps.hh" #include "rl_syrf.hh" #include "rl_rpchol.hh" -#include "rl_revd2.hh" +#include "rl_nystrom_evd.hh" #include #include @@ -305,7 +305,7 @@ RandBLAS::RNGState nystrom_pc_data( // ^ Define the symmetric rangefinder algorithm. // (*) Use power sketching followed by Householder orthogonalization. // (*) Do not check condition numbers or log to std::out. - RandLAPACK::REVD2 NystromAlg(syrf, num_steps_power_iter_error_est, false); + RandLAPACK::NystromEVD NystromAlg(syrf, num_steps_power_iter_error_est, false); // ^ Define the algorithm for low-rank approximation via Nystrom. // (*) Handle accuracy requests by estimating ||A - V diag(eigvals) V'|| // with "num_steps_power_iter_error_est" steps of power iteration. @@ -313,7 +313,15 @@ RandBLAS::RNGState nystrom_pc_data( T tol = mu_min / 5; // ^ Set tolerance to something materially smaller than the smallest // regularization parameter the user claims to need. - return NystromAlg.call(A, k, tol, V, eigvals, state); + T* V_buf = nullptr; int64_t V_sz = 0; + T* ev_buf = nullptr; int64_t ev_sz = 0; + auto out_state = NystromAlg.call(A, k, tol, V_buf, V_sz, ev_buf, ev_sz, state); + int64_t m = A.dim; + V.assign(V_buf, V_buf + m * k); + eigvals.assign(ev_buf, ev_buf + k); + delete[] V_buf; + delete[] ev_buf; + return out_state; } /** diff --git a/RandLAPACK/comps/rl_syps.hh b/RandLAPACK/comps/rl_syps.hh index 4bda9a60c..dcc3313a0 100644 --- a/RandLAPACK/comps/rl_syps.hh +++ b/RandLAPACK/comps/rl_syps.hh @@ -27,6 +27,8 @@ class SYPS { bool verbose; bool cond_check; std::vector cond_nums; + // 0 = Gaussian (DenseDist, default); 1 = SJLT (SparseDist, vec_nnz=4 per column) + int sketch_type = 0; SYPS( int64_t p, // number of passes @@ -109,8 +111,20 @@ class SYPS { bool callers_skop_buff = skop_buff != nullptr; if (!callers_skop_buff) skop_buff = new T[m * k]; - RandBLAS::DenseDist D(m, k); - state = RandBLAS::fill_dense(D, skop_buff, state); + if (sketch_type == 1) { + // SJLT: generate a sparse sketch and densify into skop_buff. + // T and sint_t (int64_t) must be explicit since SparseDist::sample + // can't deduce T from the RNGState argument alone. + auto S = RandBLAS::SparseDist(m, k, 4).sample(state); + state = S.next_state; + RandBLAS::fill_sparse(S); + auto Scoo = RandBLAS::coo_view_of_skop(S); + std::fill(skop_buff, skop_buff + m * k, (T)0.0); + RandLAPACK::util::sparse_to_dense(Scoo, Layout::ColMajor, skop_buff); + } else { + RandBLAS::DenseDist D(m, k); + state = RandBLAS::fill_dense(D, skop_buff, state); + } bool callers_work_buff = work_buff != nullptr; if (!callers_work_buff) diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index 9ffdd3d59..ebac9a48c 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -5,6 +5,7 @@ #include "rl_util.hh" #include +#include #include #include #include @@ -13,12 +14,18 @@ namespace RandLAPACK { +/// Per-call timing breakdown for FunNystromPP::call. +/// Pass a pointer to call() to collect Phase 1 and Phase 2 wall-clock times. +struct FunNystromPP_timing { + long phase1_us = 0; // NystromEVD Nyström approximation (Phase 1) + long phase2_us = 0; // Hutchinson trace correction (Phase 2) +}; /// funNyström++ trace estimator: estimates tr(f(A)) for symmetric PSD A. /// /// Algorithm (2 phases): -/// Phase 1 — Nyström approximation via REVD2 (O(k) matvecs): -/// (V, λ) = REVD2(A, k) →  = V diag(λ) V^T +/// Phase 1 — Nyström approximation via NystromEVD (O(k) matvecs): +/// (V, λ) = NystromEVD(A, k) →  = V diag(λ) V^T /// tr(f(Â)) = Σ f(λ_i) + (n-k) * f(0) [free from the eigenvalues] /// /// Phase 2 — Hutchinson correction on the residual (d*s matvecs): @@ -45,24 +52,24 @@ namespace RandLAPACK { /// d = O(√κ log(n/ε)) Lanczos steps — caller's responsibility /// s = O(1/(√κ ε)) Hutchinson samples — caller's responsibility /// k = O(√κ/ε) Nyström rank — caller's responsibility -/// Use error_est_power_iters=0 in REVD2 and tol=0 to get fixed-rank behavior +/// Use error_est_power_iters=0 in NystromEVD and tol=0 to get fixed-rank behavior /// (k never changes; the adaptive loop exits after one iteration). /// -/// @tparam REVD2_t Type of the REVD2 Nyström component. +/// @tparam NystromEVD_t Type of the NystromEVD Nyström component. /// @tparam LanczosFA_t Type of the LanczosFA component. /// @tparam Hutchinson_t Type of the Hutchinson trace estimator component. -template +template class FunNystromPP { public: - using T = typename REVD2_t::T; - using RNG = typename REVD2_t::RNG; + using T = typename NystromEVD_t::T; + using RNG = typename NystromEVD_t::RNG; - REVD2_t& revd2; - LanczosFA_t& lanczos_fa; - Hutchinson_t& hutchinson; + NystromEVD_t& nystrom_evd; // Phase 1: Nyström approximation + LanczosFA_t& lanczos_fa; // Phase 2: f(A)*Ω oracle + Hutchinson_t& hutchinson; // Phase 2: trace correction estimator // Persistent output/working buffers — grown with new/delete[], never shrunk. - // V, eigvals: REVD2 outputs reused across calls with same (n, k). + // V, eigvals: NystromEVD outputs reused across calls with same (n, k). // F_vec: f applied elementwise to eigvals. // tmp, Z1, Z2: workspace owned here and lent to linops::ResidualOp each call. T* V_buf = nullptr; int64_t V_buf_sz = 0; @@ -77,8 +84,16 @@ public: delete[] F_vec; delete[] tmp; delete[] Z1; delete[] Z2; } - FunNystromPP(REVD2_t& r, LanczosFA_t& l, Hutchinson_t& h) - : revd2(r), lanczos_fa(l), hutchinson(h) {} + // ------------------------------------------------------------------ + /// Construct a FunNystromPP estimator from pre-built components. + /// + /// @param r NystromEVD instance for Phase 1 (Nyström approximation). + /// @param l LanczosFA instance for computing f(A)*Ω in Phase 2. + /// @param h Hutchinson instance for Phase 2 trace correction. + /// + /// All three components are stored by reference and must outlive this object. + FunNystromPP(NystromEVD_t& r, LanczosFA_t& l, Hutchinson_t& h) + : nystrom_evd(r), lanczos_fa(l), hutchinson(h) {} // ------------------------------------------------------------------ /// Estimate tr(f(A)). @@ -93,6 +108,8 @@ public: /// @param[in] s Hutchinson samples. Caller chooses based on κ, ε. /// @param[in] d Lanczos steps per sample. Caller chooses based on κ. /// @param[in] state RandBLAS RNG state; advanced on return. + /// @param[out] timing Optional. If non-null, receives Phase 1 and Phase 2 + /// wall-clock times in microseconds. /// @returns Estimate of tr(f(A)). template F> T call( @@ -102,7 +119,8 @@ public: int64_t k, int64_t s, int64_t d, - RandBLAS::RNGState& state + RandBLAS::RNGState& state, + FunNystromPP_timing* timing = nullptr ) { if (!std::isfinite(f_zero)) throw std::invalid_argument("f_zero must be finite; for log, ensure A is strictly PD"); @@ -111,15 +129,17 @@ public: // ------------------------------------------------------------------ // Phase 1: Nyström approximation - // REVD2 with error_est_power_iters=0 and tol=0 runs a single pass + // NystromEVD with error_est_power_iters=0 and tol=0 runs a single pass // with fixed rank k. V_buf (n×k_in) and eigvals_buf (k_in) are the outputs. // ------------------------------------------------------------------ - // REVD2::call takes rank by reference and may increase it adaptively. + // NystromEVD::call takes rank by reference and may increase it adaptively. // We pass k_in (a copy) so the caller's k stays unchanged. // All post-call uses reference k_in (not k) so they stay consistent // with the actual rank used, even in adaptive mode. int64_t k_in = k; - revd2.call(A, k_in, (T)0.0, V_buf, V_buf_sz, eigvals_buf, eigvals_buf_sz, state); + auto t_phase1_start = std::chrono::steady_clock::now(); + nystrom_evd.call(A, k_in, (T)0.0, V_buf, V_buf_sz, eigvals_buf, eigvals_buf_sz, state); + auto t_phase1_end = std::chrono::steady_clock::now(); // f(λ): apply scalar f to k_in eigenvalues util::resize(F_vec, F_vec_sz, k_in); @@ -143,7 +163,15 @@ public: linops::ResidualOp res_op( n, A, lanczos_fa, f, d, k_in, V_buf, F_vec, tmp, Z1, Z2 ); + auto t_phase2_start = std::chrono::steady_clock::now(); T correction = hutchinson.call(res_op, s, state); + auto t_phase2_end = std::chrono::steady_clock::now(); + + if (timing) { + using us = std::chrono::microseconds; + timing->phase1_us = std::chrono::duration_cast(t_phase1_end - t_phase1_start).count(); + timing->phase2_us = std::chrono::duration_cast(t_phase2_end - t_phase2_start).count(); + } return tr_Ahat + correction; } diff --git a/RandLAPACK/drivers/rl_revd2.hh b/RandLAPACK/drivers/rl_nystrom_evd.hh similarity index 57% rename from RandLAPACK/drivers/rl_revd2.hh rename to RandLAPACK/drivers/rl_nystrom_evd.hh index a257c7e49..e4464ac69 100644 --- a/RandLAPACK/drivers/rl_revd2.hh +++ b/RandLAPACK/drivers/rl_nystrom_evd.hh @@ -9,7 +9,6 @@ #include #include -#include namespace RandLAPACK { @@ -19,7 +18,7 @@ namespace RandLAPACK { /// p - number of algorithm iterations /// vector_buf - buffer for vector operations (must hold at least 4*m elements) /// Mat_buf - buffer for matrix operations (must hold at least m*k elements) -/// All other parameters come from REVD2 +/// All other parameters come from NystromEVD template T power_error_est( SLO &A, @@ -62,7 +61,7 @@ T power_error_est( template -class REVD2 { +class NystromEVD { public: using T = typename SYRF_t::T; using RNG = typename SYRF_t::RNG; @@ -77,7 +76,7 @@ class REVD2 { T* S = nullptr; int64_t S_sz = 0; T* symrf_work = nullptr; int64_t symrf_work_sz = 0; - REVD2( + NystromEVD( SYRF_t &syrf_obj, int error_est_power_iters, bool verb = false @@ -86,7 +85,7 @@ class REVD2 { verbose = verb; } - ~REVD2() { + ~NystromEVD() { delete[] Y; delete[] Omega; delete[] R; @@ -96,20 +95,22 @@ class REVD2 { /// Computes a rank-k approximation to an EVD of a symmetric positive semidefinite matrix: /// A_hat = V diag(eigvals) V^*, - /// where V is a matrix of eigenvectors and eigvals is a vector of eigenvalues. + /// where V is a matrix of approximate eigenvectors and eigvals holds the eigenvalues. /// /// Adaptive: if tolerance is not met, doubles k until convergence or k == m. - /// Set error_est_power_iters=0 and tol=0 for fixed-rank single-pass behavior - /// (k never changes, exits after one iteration). + /// Set error_est_power_iters=0 and tol=0 for fixed-rank single-pass behavior. /// /// This code is identical to Algorithm E2 from https://arxiv.org/pdf/2110.02820.pdf. /// - /// @param[in] m Number of rows/cols of A. - /// @param[in] A m-by-m matrix, column-major, must be SPD. - /// @param[in,out] k Sketch rank on entry; final rank on exit (may grow in adaptive mode). - /// @param[in] tol Convergence tolerance. Use 0.0 for fixed-rank. - /// @param[in,out] V On exit, m-by-k matrix of approximate eigenvectors. - /// @param[in,out] eigvals On exit, k approximate eigenvalues. + /// @param[in] uplo Triangle of A that holds the data (Upper or Lower). + /// @param[in] m Number of rows/cols of A. + /// @param[in] A m×m matrix, column-major. + /// @param[in,out] k Sketch rank on entry; final rank on exit (may grow adaptively). + /// @param[in] tol Convergence tolerance. Use 0.0 for fixed-rank. + /// @param[in,out] V Caller-owned buffer; on exit holds m×k eigenvectors. + /// @param[in,out] V_sz Capacity of V in elements; grown by util::resize as needed. + /// @param[in,out] eigvals Caller-owned buffer; on exit holds k eigenvalues. + /// @param[in,out] eigvals_sz Capacity of eigvals; grown by util::resize as needed. /// int call( Uplo uplo, @@ -117,100 +118,14 @@ class REVD2 { const T* A, int64_t &k, T tol, - std::vector &V, - std::vector &eigvals, + T*& V, int64_t& V_sz, + T*& eigvals, int64_t& eigvals_sz, RandBLAS::RNGState &state ) { linops::ExplicitSymLinOp A_linop(m, uplo, A, m, Layout::ColMajor); - return this->call(A_linop, k, tol, V, eigvals, state); - } - - template - int call( - SLO &A, - int64_t &k, - T tol, - std::vector &V, - std::vector &eigvals, - RandBLAS::RNGState &state - ) { - int64_t m = A.dim; - T err = 0; - RandBLAS::RNGState error_est_state(state.counter, state.key); - error_est_state.key.incr(1); - while(true) { - // Resize caller-owned output buffers (std::vector handles dynamic k) - util::resize(k, eigvals); - T* V_dat = util::resize(m * k, V); - - // Grow internal working buffers if needed - util::resize(Y, Y_sz, m * k); - // Omega must hold at least max(m*k, m*4) — the error estimator needs 4 columns - util::resize(Omega, Omega_sz, std::max(m * k, m * (int64_t)4)); - util::resize(R, R_sz, k * k); - util::resize(S, S_sz, k * k); - util::resize(symrf_work, symrf_work_sz, m * k); - - // Construct sketching operator: Omega = orth(A * sketch) - this->syrf.call(A, k, this->Omega, state, symrf_work); - - // Y = A * Omega - A(Layout::ColMajor, k, 1.0, Omega, m, 0.0, Y, m); - - // Stabilization parameter nu = eps * ||Y||_F - T nu = std::numeric_limits::epsilon() * lapack::lange(Norm::Fro, m, k, Y, m); - - // R = chol(Omega'*Y + nu * Omega'*Omega) — regularized to ensure PD - // syrk fills lower triangle; copy to upper for full symmetric matrix - blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega, m, 0.0, R, k); - for(int i = 1; i < k; ++i) - blas::copy(k - i, &R[i + ((i-1) * k)], 1, &R[(i - 1) + (i * k)], k); - blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, 1.0, Omega, m, Y, m, 1.0, R, k); - - if(lapack::potrf(Uplo::Upper, k, R, k)) - throw std::runtime_error("Cholesky decomposition failed."); - RandLAPACK::util::get_U(k, k, R, k); - - // B = Y * (R')^{-1} — overwrites Y - blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, 1.0, R, k, Y, m); - - // [V, S, ~] = SVD(B); use R as buffer for discarded right singular vectors - lapack::gesdd(Job::SomeVec, m, k, Y, m, S, V_dat, m, R, k); - - // eigvals = S^2 - nu (undo regularization; clamp to zero) - T buf; - int64_t r = 0; - int i; - for(i = 0; i < k; ++i) { - buf = std::pow(S[i], 2); - eigvals[i] = buf; - if(buf > nu) - ++r; - } - for(i = 0; i < r; ++i) - (eigvals[i] - nu < 0) ? 0 : eigvals[i] -= nu; - - std::fill(&V_dat[m * r], &V_dat[m * k], 0.0); - - // Error estimation using Omega as a scratch buffer (needs 4 columns = 4*m) - RandBLAS::DenseDist g(m, 1); - error_est_state = RandBLAS::fill_dense(g, Omega, error_est_state); - - err = power_error_est(A, k, this->error_est_p, Omega, V_dat, Y, eigvals.data()); - - if(err <= 5 * std::max(tol, nu) || k == m) { - break; - } else if (2 * k > m) { - k = m; - } else { - k = 2 * k; - } - } - return 0; + return this->call(A_linop, k, tol, V, V_sz, eigvals, eigvals_sz, state); } - /// Raw-pointer overload: same algorithm but caller-owned buffers are - /// T*/int64_t pairs managed with util::resize (new/delete[]). template int call( SLO &A, @@ -262,7 +177,7 @@ class REVD2 { ++r; } for(i = 0; i < r; ++i) - (eigvals[i] - nu < 0) ? 0 : eigvals[i] -= nu; + if (eigvals[i] > nu) eigvals[i] -= nu; std::fill(&V_dat[m * r], &V_dat[m * k], 0.0); diff --git a/RandLAPACK/linops/rl_matfun_linops.hh b/RandLAPACK/linops/rl_matfun_linops.hh index d333485f0..0f562da01 100644 --- a/RandLAPACK/linops/rl_matfun_linops.hh +++ b/RandLAPACK/linops/rl_matfun_linops.hh @@ -1,6 +1,7 @@ #pragma once #include "rl_blaspp.hh" +#include #ifdef _OPENMP #include @@ -46,8 +47,22 @@ struct ResidualOp { : dim(n_), A(A_), matfun_oracle(oracle_), f(f_), d(d_), k(k_), V(V_), F_vec(Fv_), tmp(tmp_), Z1(Z1_), Z2(Z2_) {} + // ------------------------------------------------------------------ + /// Apply: C = α*(f(A)*B - f(Â)*B) + β*C. + /// + /// @param[in] layout Ignored; the operator always uses ColMajor internally. + /// @param[in] n_vecs Number of right-hand sides. + /// @param[in] alpha Scalar multiplier for the residual. + /// @param[in] B n×n_vecs input matrix (col-major, ldb). + /// @param[in] ldb Leading dimension of B. + /// @param[in] beta Scalar multiplier for existing C; β=0 is handled without + /// reading C, so NaN in C does not propagate. + /// @param[in,out] C n×n_vecs output matrix (col-major, ldc); overwritten. + /// @param[in] ldc Leading dimension of C; may exceed dim. void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, T alpha, T* const B, int64_t ldb, T beta, T* C, int64_t ldc) { + if (ldb != dim) + throw std::invalid_argument("ResidualOp: B must be contiguous (ldb == dim)"); // Z1 = f(A)*B via matrix-function oracle matfun_oracle.call(A, B, dim, n_vecs, f, d, Z1); diff --git a/RandLAPACK/linops/rl_sparse_linop.hh b/RandLAPACK/linops/rl_sparse_linop.hh index 108c69827..5d1b707e9 100644 --- a/RandLAPACK/linops/rl_sparse_linop.hh +++ b/RandLAPACK/linops/rl_sparse_linop.hh @@ -464,4 +464,47 @@ public: } }; + +/*********************************************************/ +/* */ +/* SparseSymLinOp */ +/* */ +/*********************************************************/ + +/// @brief Symmetric sparse linear operator satisfying the SymmetricLinearOperator concept. +/// +/// Wraps a sparse matrix assumed to be symmetric with both triangles explicitly stored. +/// Delegates to `left_spmm` with NoTrans for SYMM-like semantics: C = alpha * A * B + beta * C. +/// +/// Satisfies SymmetricLinearOperator (provides `dim` and SYMM-like operator()). +/// Also provides `n_rows` and `n_cols` (both equal to `dim`) for LinearOperator compatibility. +/// +/// @tparam SpMat Sparse matrix type (CSR, CSC, or COO format). +/// +/// @note The sparse matrix must be square (n×n) and symmetric with both triangles stored. +/// The caller's SpMat must outlive this operator. +template +struct SparseSymLinOp { + using T = typename SpMat::scalar_t; + using scalar_t = T; + const int64_t dim; + const int64_t n_rows; + const int64_t n_cols; + SpMat& A_sp; + + /// @brief Construct from an existing symmetric sparse matrix. + /// @param n Matrix dimension (n × n); A_sp must be n × n. + /// @param src Sparse matrix; must be square, symmetric, both triangles stored. + SparseSymLinOp(int64_t n, SpMat& src) + : dim(n), n_rows(n), n_cols(n), A_sp(src) {} + + /// @brief Apply: C = alpha * A_sp * B + beta * C. + void operator()(Layout layout, int64_t n_vecs, T alpha, + T* const B, int64_t ldb, + T beta, T* C, int64_t ldc) { + RandBLAS::sparse_data::left_spmm(layout, Op::NoTrans, Op::NoTrans, + dim, n_vecs, dim, alpha, A_sp, 0, 0, B, ldb, beta, C, ldc); + } +}; + } // end namespace RandLAPACK::linops diff --git a/RandLAPACK/testing/rl_gen.hh b/RandLAPACK/testing/rl_gen.hh index f70c94482..a3b94c5bc 100644 --- a/RandLAPACK/testing/rl_gen.hh +++ b/RandLAPACK/testing/rl_gen.hh @@ -3,6 +3,10 @@ #include "rl_blaspp.hh" #include "rl_lapackpp.hh" +#ifdef _OPENMP +#include +#endif + #include #include #include @@ -885,4 +889,176 @@ std::vector gen_poly_mat_psd(int64_t m, T cond_num, T exponent, uint32_t seed return G; } +/// Write k algebraically-decaying spectral values into s[0..k-1]: +/// s[i] = scale * (i+1)^{-exponent}, i = 0, 1, …, k-1. +/// Matches Persson & Kressner (SIMAX 2023) Figure 5(a): scale=100, exponent=2. +/// +/// @param[in] k Number of values to generate. +/// @param[in] scale Overall scale factor. +/// @param[in] exponent Decay exponent (positive). +/// @param[out] s Output buffer of length k; overwritten. +template +void gen_alg_decay_singvals(int64_t k, T scale, T exponent, T* s) { + #ifdef _OPENMP + #pragma omp parallel for schedule(static) + #endif + for (int64_t i = 0; i < k; ++i) + s[i] = scale * std::pow((T)(i + 1), -exponent); +} + +/// Write k exponentially-decaying spectral values into s[0..k-1]: +/// s[i] = scale * exp(-(i+1)/decay_rate), i = 0, 1, …, k-1. +/// Matches Persson & Kressner (SIMAX 2023) Figure 5(b): scale=1, decay_rate=100. +/// +/// @param[in] k Number of values to generate. +/// @param[in] scale Overall scale factor. +/// @param[in] decay_rate Rate parameter (larger = slower decay). +/// @param[out] s Output buffer of length k; overwritten. +template +void gen_exp_decay_singvals(int64_t k, T scale, T decay_rate, T* s) { + #ifdef _OPENMP + #pragma omp parallel for schedule(static) + #endif + for (int64_t i = 0; i < k; ++i) + s[i] = scale * std::exp(-(T)(i + 1) / decay_rate); +} + +/// Form the upper triangle of A = Q diag(eigvals) Q' with a Haar-random n×k matrix Q. +/// +/// Q is obtained via QR factorization of an n×k Gaussian matrix. Column j of Q is +/// scaled by sqrt(eigvals[j]) in-place — no k×k spectrum matrix is formed. The upper +/// triangle of A is then filled via syrk. The n - k trailing eigenvalues are implicitly zero. +/// +/// @param[in] n Matrix dimension (n × n output) +/// @param[in] k Rank; k ≤ n +/// @param[out] A Output buffer, upper triangle on exit (col-major, leading dim lda) +/// @param[in] lda Leading dimension of A +/// @param[in] eigvals Pointer to k non-negative eigenvalues +/// @param[in,out] state RNG state +template +void gen_sym_psd_lowrank( + int64_t n, int64_t k, + T* A, int64_t lda, + const T* eigvals, + RandBLAS::RNGState& state +) { + T* Q = new T[n * k](); + T* tau = new T[k](); + + auto dist = RandBLAS::DenseDist(n, k); + state = RandBLAS::fill_dense(dist, Q, state); + + lapack::geqrf(n, k, Q, n, tau); + lapack::orgqr(n, k, k, Q, n, tau); + + #ifdef _OPENMP + #pragma omp parallel for schedule(static) + #endif + for (int64_t j = 0; j < k; ++j) + blas::scal(n, std::sqrt(eigvals[j]), Q + j * n, 1); + + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::NoTrans, + n, k, (T)1.0, Q, n, (T)0.0, A, lda); + + delete[] Q; + delete[] tau; +} + +/// Form an n×n symmetric kernel matrix K from n random data points in R^d. +/// +/// Data points x_1,...,x_n are drawn i.i.d. from N(0, I_d). The upper +/// triangle of K is filled with K[i,j] = k(x_i, x_j); the lower triangle +/// is left unset (use Uplo::Upper with LAPACK symmetric routines). +/// +/// Pairwise dot products are computed via a single SYRK call (BLAS-3, +/// multi-threaded via MKL/OpenBLAS), so assembly is O(n²d) dominated. +/// +/// Two kernels are supported via kernel_type: +/// 0 = RBF: k(x,y) = exp(-||x-y||² / (2*bandwidth²)) +/// 1 = Polynomial: k(x,y) = (x·y + poly_c)^poly_p +/// +/// For RBF with Gaussian data, the expected squared distance is 2d, so +/// bandwidth = sqrt(d) (median heuristic) gives average kernel value ~exp(-1). +/// +/// @param[in] n Matrix dimension; also the number of data points +/// @param[in] d Data point dimension (ambient space) +/// @param[out] K Output buffer; upper triangle on exit (col-major, ldk) +/// @param[in] ldk Leading dimension of K +/// @param[in] kernel_type 0=RBF, 1=polynomial +/// @param[in] bandwidth RBF length scale σ; ignored for polynomial +/// @param[in] poly_c Polynomial additive constant c; ignored for RBF +/// @param[in] poly_p Polynomial degree p; ignored for RBF +/// @param[in,out] state RNG state; advanced on return +template +void gen_kernel_matrix( + int64_t n, int64_t d, + T* K, int64_t ldk, + int kernel_type, + T bandwidth, + T poly_c, + int poly_p, + RandBLAS::RNGState& state +) { + // X: n×d col-major; row i is data point x_i ~ N(0, I_d) + T* X = new T[n * d]; + state = RandBLAS::fill_dense(RandBLAS::DenseDist(n, d), X, state); + + // G = X*X^T (upper triangle, col-major): G[i + j*n] = x_i · x_j (i <= j) + T* G = new T[n * n](); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::NoTrans, + n, d, (T)1.0, X, n, (T)0.0, G, n); + delete[] X; + + if (kernel_type == 0) { + T inv_denom = (T)1.0 / ((T)2.0 * bandwidth * bandwidth); + #ifdef _OPENMP + #pragma omp parallel for schedule(static) + #endif + for (int64_t j = 0; j < n; ++j) { + for (int64_t i = 0; i <= j; ++i) { + T sq_dist = G[i + i*n] - (T)2.0 * G[i + j*n] + G[j + j*n]; + K[i + j*ldk] = std::exp(-sq_dist * inv_denom); + } + } + } else { + #ifdef _OPENMP + #pragma omp parallel for schedule(static) + #endif + for (int64_t j = 0; j < n; ++j) { + for (int64_t i = 0; i <= j; ++i) + K[i + j*ldk] = std::pow(G[i + j*n] + poly_c, (T)poly_p); + } + } + delete[] G; +} + +/// Form A = Q diag(eigvals) Q' + noise * I, upper triangle. +/// +/// Calls gen_sym_psd_lowrank for the low-rank component, then adds `noise` +/// to each diagonal entry. When noise > 0, every eigenvalue is shifted by +/// noise, making A strictly PD — required for f = log (λ_min > 0). +/// +/// Useful for log det(I + K) benchmarks where A = I + K: set noise = 1 and +/// use the kernel eigenvalues as `eigvals`. +/// +/// @param[in] n Matrix dimension (n × n output) +/// @param[in] k Rank of the low-rank component; k ≤ n +/// @param[out] A Output buffer, upper triangle on exit (col-major, leading dim lda) +/// @param[in] lda Leading dimension of A +/// @param[in] eigvals Pointer to k non-negative eigenvalues for the low-rank part +/// @param[in] noise Diagonal shift (≥ 0). Use 0 for no shift. +/// @param[in,out] state RNG state +template +void gen_shifted_lowrank_psd( + int64_t n, int64_t k, + T* A, int64_t lda, + const T* eigvals, + T noise, + RandBLAS::RNGState& state +) { + gen_sym_psd_lowrank(n, k, A, lda, eigvals, state); + for (int64_t i = 0; i < n; ++i) + A[i * lda + i] += noise; +} + } diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc index eff2d5958..e54bc651c 100644 --- a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -3,26 +3,72 @@ int main() {return 0;} #else /* FunNystromPP benchmark — compares funNyström++ against plain Hutchinson+LanczosFA -for estimating tr(sqrt(A)) on a random dense PSD matrix A = B'B + n*I. +for estimating tr(f(A)). -Two algorithms are timed for each matrix size n: +Two algorithms are timed and their matvec counts recorded for each matrix size: 1. FunNystromPP(k, s, d): Nyström rank-k approximation plus Hutchinson correction - with s samples each using d Lanczos steps. + with s samples, each using d Lanczos steps. 2. Hutchinson+LanczosFA(s, d): plain Hutchinson with s Rademacher samples, - each using LanczosFA with d steps to apply sqrt(A). No Nyström component. - Uses the same s and d as FunNystromPP (correction-phase budget parity). - -Both algorithms use a fixed seed so comparisons are reproducible across sizes. -For n <= 500 a syevd reference is computed to measure relative error. + each using LanczosFA with d steps to apply f(A). No Nyström component. + Uses the same s and d as FunNystromPP (equal correction-phase budget). + +Matrix input (mat_or_file): + 1 = psd_alg: A = Q diag(λ) Q', λᵢ = 100 * i^{-2} (algebraic decay, κ ≈ n²/100) + 2 = psd_exp: A = Q diag(λ) Q', λᵢ = exp(-i/100) (exponential decay) + 3 = rbf_kernel: K[i,j] = exp(-||x_i-x_j||²/(2σ²)), x_i~N(0,I_d), σ=sqrt(d) + 4 = poly_kernel: K[i,j] = (x_i·x_j + 1)², x_i~N(0,I_d) + /path.txt = load dense square symmetric matrix from space-separated text file + (row-major, full matrix stored; upper triangle used by algorithms) + TODO: add .mtx, .m, and other common formats + + For types 1/2: k_mat eigenvalues are nonzero; exact tr(f(A)) is always available. + For types 3/4: k_mat_ratio reinterpreted as data dimension d = n/k_mat_ratio; + reference requires compute_ref=1 (syevd, only feasible for small n). + +Function types (func_type): + 0 = sqrt(x) [f_zero = 0; safe for SPSD matrices] + 1 = log(x) [requires all eigenvalues > 0; for generated types, noise=1 is applied + automatically (A = lowrank + I), so all eigenvalues ≥ 1] + 2 = x*(x + poly_lambda) [degree-2 polynomial; exact in d=2 Lanczos steps; use for tr(K(K+λI))] + +Sketch types (sketch_type): + 0 = Gaussian (DenseDist, default) + 1 = SJLT (SparseDist, vec_nnz=4 per column) + +compute_ref: + 0 = skip reference (report -1 in output) + 1 = run syevd to compute exact tr(f(A)) + For types 1/2 this flag is ignored; the reference is always available from eigenvalues. + For types 3/4 and file input, syevd is run only when compute_ref=1. + +Both algorithms use a fixed seed so comparisons are reproducible. +Matvec counts reflect actual A applications inside each algorithm (not estimates). +The matrix is constructed/loaded once per n; that cost is not timed. + +Output format: 16 comma-separated columns per row (trailing comma): + n, k, k_mat, s, d, + matvec_pp, matvec_hutch, + time_pp_total (us), time_pp_phase1 (us), time_pp_phase2 (us), + time_hutch (us), + est_fun_pp, est_hutch, + true_tr (-1 if unavailable), err_fun_pp, err_hutch Usage: - FunNystromPP_benchmark [n_2 ...] - dir_path : output directory (use "." for current directory) - num_runs : number of timed repetitions per matrix size - k_ratio : k = n / k_ratio (Nyström rank; e.g., 10 -> k = n/10) - s_ratio : s = k * s_ratio (Hutchinson samples; e.g., 5 -> s = 5*k) - d_steps : Lanczos depth per Hutchinson sample - n_1 ... : matrix dimensions to sweep + FunNystromPP_benchmark + + [n_1 n_2 ...] + dir_path : output directory (use "." for current directory) + num_runs : number of timed repetitions per matrix size + k_ratio : k = n / k_ratio (Nyström rank; e.g., 10 -> k = n/10) + k_mat_ratio : k_mat = n / k_mat_ratio (matrix rank for generated types; ignored for file) + s_ratio : s = k * s_ratio (Hutchinson samples; e.g., 5 -> s = 5*k) + d_steps : Lanczos depth per Hutchinson sample + mat_or_file : 1=psd_alg, 2=psd_exp, or path to .txt file + func_type : 0=sqrt, 1=log, 2=poly (x*(x+poly_lambda)) + compute_ref : 0=skip eigendecomp reference, 1=run syevd (file input only) + sketch_type : 0=Gaussian, 1=SJLT (vec_nnz=4 per column) + poly_lambda : lambda parameter for func_type=2 (ignored for func_type=0,1) + n_1 ... : matrix dimensions (generated types only; ignored for file input) */ #include "RandLAPACK.hh" @@ -34,12 +80,17 @@ For n <= 500 a syevd reference is computed to measure relative error. #include #include #include +#include #include #include #include +#include namespace linops = RandLAPACK::linops; using RNG = r123::Philox4x32; +using std::chrono::steady_clock; +using std::chrono::duration_cast; +using std::chrono::microseconds; // ============================================================================ // Benchmark data struct @@ -47,40 +98,108 @@ using RNG = r123::Philox4x32; template struct FunNystromPP_benchmark_data { - int64_t n; // current matrix dimension (may be < A's allocation) - int64_t k; // Nyström rank - int64_t s; // Hutchinson samples - int64_t d; // Lanczos steps per sample - std::vector A; // n_max × n_max full symmetric matrix - - FunNystromPP_benchmark_data(int64_t n_max, int64_t k_, int64_t s_, int64_t d_) : - n(n_max), k(k_), s(s_), d(d_), - A(n_max * n_max, 0.0) + int64_t n; // current matrix dimension + int64_t k; // Nyström rank for algorithms + int64_t k_mat; // rank used to construct A (0 for file input) + int64_t s; // Hutchinson samples + int64_t d; // Lanczos steps per sample + bool from_file; // true when A was loaded from a file + T noise; // diagonal shift (noise=1 for func_type=1 on lowrank types; else 0) + int64_t d_dim; // data dimension for kernel types (mat_type=3/4); 0 otherwise + std::vector A; // n_max × n_max storage (upper triangle; rest zero) + std::vector eigvals; // k_mat eigenvalues of the low-rank component (pre-shift; empty for kernel/file types) + + FunNystromPP_benchmark_data(int64_t n_max, int64_t k_, int64_t k_mat_, + int64_t s_, int64_t d_, bool from_file_) : + n(n_max), k(k_), k_mat(k_mat_), s(s_), d(d_), from_file(from_file_), + noise((T)0.0), d_dim(0), A(n_max * n_max, 0.0) {} }; // ============================================================================ -// data_regen: build A = B'B + n*I, fully symmetrized, for the current n. +// CountingLinOp: wraps any SLO and counts individual matrix-vector products. +// Satisfies the SymmetricLinearOperator concept. +// ============================================================================ + +template +struct CountingLinOp { + using scalar_t = typename SLO_t::scalar_t; + const int64_t dim; + SLO_t& base; + int64_t count = 0; + + CountingLinOp(int64_t n, SLO_t& slo) : dim(n), base(slo) {} + void reset() { count = 0; } + + void operator()(Layout layout, int64_t n_vecs, scalar_t alpha, + scalar_t* const B, int64_t ldb, + scalar_t beta, scalar_t* C, int64_t ldc) { + count += n_vecs; + base(layout, n_vecs, alpha, B, ldb, beta, C, ldc); + } +}; + +// ============================================================================ +// data_regen: build or load A for the current n. +// Generated (mat_type 1/2): fills data.eigvals and the upper triangle of data.A. +// File input: reads the full matrix into data.A via process_input_mat. // ============================================================================ template static void data_regen( FunNystromPP_benchmark_data& data, - RandBLAS::RNGState& state + RandBLAS::RNGState& state, + int mat_type, + int func_type, + const std::string& file_path ) { int64_t n = data.n; - std::vector B(n * n); - RandBLAS::DenseDist DB(n, n); - state = RandBLAS::fill_dense(DB, B.data(), state); - std::fill(data.A.begin(), data.A.begin() + n * n, (T)0.0); - blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, - (T)1.0, B.data(), n, (T)0.0, data.A.data(), n); - for (int64_t i = 0; i < n; ++i) - data.A[i + i * n] += (T)n; - for (int64_t j = 0; j < n; ++j) - for (int64_t i = j + 1; i < n; ++i) - data.A[i + j * n] = data.A[j + i * n]; + + if (data.from_file) { + // Two-pass: dimensions already determined in main; just load data. + int64_t m = n, ncols = n; + int wq = 0; + RandLAPACK::gen::process_input_mat( + m, ncols, data.A.data(), + const_cast(file_path.c_str()), wq); + data.eigvals.clear(); + data.noise = (T)0.0; + data.d_dim = 0; + } else if (mat_type >= 3) { + // Kernel matrix: k_mat reinterpreted as data dimension d_dim = n / k_mat_ratio. + data.d_dim = data.k_mat; + data.noise = (T)0.0; + data.eigvals.clear(); + T bandwidth = std::sqrt((T)data.d_dim); + if (mat_type == 3) + RandLAPACK::gen::gen_kernel_matrix( + n, data.d_dim, data.A.data(), n, 0, bandwidth, (T)0.0, 0, state); + else // mat_type == 4 + RandLAPACK::gen::gen_kernel_matrix( + n, data.d_dim, data.A.data(), n, 1, (T)0.0, (T)1.0, 2, state); + } else { + data.d_dim = 0; + data.eigvals.resize(data.k_mat); + if (mat_type == 1) + RandLAPACK::gen::gen_alg_decay_singvals(data.k_mat, (T)100.0, (T)2.0, + data.eigvals.data()); + else + RandLAPACK::gen::gen_exp_decay_singvals(data.k_mat, (T)1.0, (T)100.0, + data.eigvals.data()); + if (func_type == 1) { + // log requires strict PD; shift by 1 so all eigenvalues >= 1. + // Models A = I + K (log det(I+K) use case). f(noise) = log(1) = 0, + // so f_zero=0 remains correct for the Nyström tail approximation. + data.noise = (T)1.0; + RandLAPACK::gen::gen_shifted_lowrank_psd(n, data.k_mat, data.A.data(), n, + data.eigvals.data(), data.noise, state); + } else { + data.noise = (T)0.0; + RandLAPACK::gen::gen_sym_psd_lowrank(n, data.k_mat, data.A.data(), n, + data.eigvals.data(), state); + } + } } // ============================================================================ @@ -118,8 +237,8 @@ struct LFAOp { }; // ============================================================================ -// call_all_algs: timed comparison for one matrix size. -// Creates algorithm objects fresh each call (buffers start empty, grow as needed). +// call_all_algs: timed + matvec-counted comparison for one matrix size. +// data.A and (for generated types) data.eigvals must already be filled. // ============================================================================ template @@ -127,84 +246,132 @@ static void call_all_algs( int64_t numruns, FunNystromPP_benchmark_data& data, RandBLAS::RNGState& state, + int mat_type, + int func_type, + double poly_lambda, + bool compute_ref, + int sketch_type, const std::string& output_filename ) { int64_t n = data.n, k = data.k, s = data.s, d = data.d; + std::function f; + const char* fname; + double f_zero = 0.0; + if (func_type == 0) { + f = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + fname = "sqrt"; + f_zero = 0.0; + } else if (func_type == 1) { + f = [](double x){ return std::log(x); }; + fname = "log"; + f_zero = 0.0; // placeholder; caller must ensure λ_min > 0 + } else { + double lam = poly_lambda; + f = [lam](double x){ return x * (x + lam); }; + fname = "poly"; + f_zero = 0.0; + } + + // Reference: exact formula for low-rank types; syevd for kernel/file types. + bool ref_available = false; + double true_tr = 0.0; + if (!data.from_file && mat_type < 3) { + // Low-rank types: exact reference from eigenvalues. + // Actual eigenvalues: eigvals[j] + noise (dominant k_mat) and noise (tail). + // f(noise) = 0 for all func_types: sqrt(0)=0, log(1)=0, 0*(0+lam)=0. + for (int64_t j = 0; j < data.k_mat; ++j) + true_tr += f(data.eigvals[j] + data.noise); + true_tr += (T)(n - data.k_mat) * f(data.noise); + ref_available = true; + } else if (compute_ref) { + // Kernel types and file input: compute reference via full eigendecomposition. + std::vector A_copy(data.A.begin(), data.A.begin() + n * n); + std::vector ev(n); + lapack::syevd(lapack::Job::NoVec, lapack::Uplo::Upper, n, + A_copy.data(), n, ev.data()); + for (int64_t j = 0; j < n; ++j) + true_tr += f(ev[j]); + ref_available = true; + } + // Build full algorithm stack for FunNystromPP. RandLAPACK::SYPS syps(3, 1, false, false); + syps.sketch_type = sketch_type; RandLAPACK::HQRQ orth(false, false); RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); - RandLAPACK::REVD2 revd2(syrf, 0); + RandLAPACK::NystromEVD nystrom_evd(syrf, 0); RandLAPACK::LanczosFA lfa_pp; RandLAPACK::Hutchinson hutch_pp; - RandLAPACK::FunNystromPP - driver(revd2, lfa_pp, hutch_pp); + RandLAPACK::FunNystromPP + driver(nystrom_evd, lfa_pp, hutch_pp); // Separate LanczosFA + Hutchinson for plain baseline. RandLAPACK::LanczosFA lfa_base; RandLAPACK::Hutchinson hutch_base; - auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + using ESLO = linops::ExplicitSymLinOp; + using CSLO = CountingLinOp; + using FuncT = std::function; - auto state_gen = state; - auto state_alg = state; + const char* mat_str = data.from_file ? "file" : + mat_type == 1 ? "psd_alg" : + mat_type == 2 ? "psd_exp" : + mat_type == 3 ? "rbf_kernel" : "poly_kernel"; for (int i = 0; i < numruns; ++i) { - printf("ITERATION %d, N=%lld, K=%lld, S=%lld, D=%lld\n", i, (long long)n, (long long)k, (long long)s, (long long)d); + printf("ITERATION %d, N=%lld, K=%lld, K_MAT=%lld, S=%lld, D=%lld [%s, %s]\n", + i, (long long)n, (long long)k, (long long)data.k_mat, + (long long)s, (long long)d, fname, mat_str); + + auto state_alg = state; // ---- FunNystromPP -------------------------------------------------- - state_gen = state; - data_regen(data, state_gen); - state_alg = state; - linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); + ESLO A_op(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); + CSLO A_pp(n, A_op); + RandLAPACK::FunNystromPP_timing pp_timing; auto start_pp = steady_clock::now(); - double est_pp = driver.call(A_op, f_sqrt, 0.0, k, s, d, state_alg); + double est_pp = driver.call(A_pp, f, (T)f_zero, k, s, d, state_alg, &pp_timing); auto stop_pp = steady_clock::now(); long dur_pp = duration_cast(stop_pp - start_pp).count(); - printf(" FunNystromPP: est=%.6e time=%lld us\n", est_pp, (long long)dur_pp); + int64_t mv_pp = A_pp.count; + printf(" FunNystromPP: est=%.6e time=%lld us (ph1=%lld ph2=%lld) matvecs=%lld\n", + est_pp, (long long)dur_pp, + (long long)pp_timing.phase1_us, (long long)pp_timing.phase2_us, + (long long)mv_pp); // ---- Hutchinson + LanczosFA (no Nyström) --------------------------- - state_gen = state; - data_regen(data, state_gen); state_alg = state; - linops::ExplicitSymLinOp A_op2(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); - LFAOp, RandLAPACK::LanczosFA, decltype(f_sqrt)> - lfa_op(n, A_op2, lfa_base, f_sqrt, d); + ESLO A_op2(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); + CSLO A_hutch(n, A_op2); + LFAOp, FuncT> lfa_op(n, A_hutch, lfa_base, f, d); auto start_h = steady_clock::now(); double est_h = hutch_base.call(lfa_op, s, state_alg); auto stop_h = steady_clock::now(); - long dur_h = duration_cast(stop_h - start_h).count(); - printf(" Hutchinson+LFA: est=%.6e time=%lld us\n", est_h, (long long)dur_h); - - // ---- Syevd reference (only for small n) ---------------------------- - state_gen = state; - data_regen(data, state_gen); - - double true_tr = -1.0; - if (n <= 500) { - std::vector A_copy(data.A.begin(), data.A.begin() + n * n); - std::vector eigvals(n); - lapack::syevd(lapack::Job::NoVec, lapack::Uplo::Upper, n, - A_copy.data(), n, eigvals.data()); - true_tr = 0.0; - for (int64_t j = 0; j < n; ++j) - true_tr += std::sqrt(std::max(eigvals[j], 0.0)); - } - - double err_pp = (true_tr > 0) ? std::abs(est_pp - true_tr) / true_tr : -1.0; - double err_h = (true_tr > 0) ? std::abs(est_h - true_tr) / true_tr : -1.0; - if (true_tr > 0) - printf(" Reference: true=%.6e err_pp=%e err_h=%e\n", + long dur_h = duration_cast(stop_h - start_h).count(); + int64_t mv_h = A_hutch.count; + printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", + est_h, (long long)dur_h, (long long)mv_h); + + double err_pp = ref_available ? std::abs(est_pp - true_tr) / std::abs(true_tr) : -1.0; + double err_h = ref_available ? std::abs(est_h - true_tr) / std::abs(true_tr) : -1.0; + if (ref_available) + printf(" Reference: true=%.6e err_pp=%e err_h=%e\n", true_tr, err_pp, err_h); + else + printf(" Reference: N/A\n"); + double out_tr = ref_available ? true_tr : -1.0; std::ofstream file(output_filename, std::ios::out | std::ios::app); - file << n << ", " << k << ", " << s << ", " << d << ", " - << dur_pp << ", " << dur_h << ", " - << est_pp << ", " << est_h << ", " - << true_tr << ", " << err_pp << ", " << err_h << ",\n"; + file << n << ", " << k << ", " << data.k_mat << ", " + << s << ", " << d << ", " + << mv_pp << ", " << mv_h << ", " + << dur_pp << ", " << pp_timing.phase1_us << ", " << pp_timing.phase2_us << ", " + << dur_h << ", " + << est_pp << ", " << est_h << ", " + << out_tr << ", " << err_pp << ", " << err_h << ",\n"; } } @@ -213,26 +380,74 @@ static void call_all_algs( // ============================================================================ int main(int argc, char* argv[]) { - if (argc < 7) { + if (argc < 12) { std::cerr << "Usage: " << argv[0] - << " " - " [n_2 ...]\n" - << " dir_path : output directory (use '.' for current dir)\n" - << " num_runs : timed repetitions per matrix size\n" - << " k_ratio : k = n / k_ratio (Nystrom rank)\n" - << " s_ratio : s = k * s_ratio (Hutchinson samples)\n" - << " d_steps : Lanczos depth per sample\n"; + << " " + " " + " [n_1 n_2 ...]\n" + << " dir_path : output directory (use '.' for current dir)\n" + << " num_runs : timed repetitions per matrix size\n" + << " k_ratio : k = n / k_ratio (Nystrom rank)\n" + << " k_mat_ratio : k_mat = n / k_mat_ratio (matrix rank; generated only)\n" + << " s_ratio : s = k * s_ratio (Hutchinson samples)\n" + << " d_steps : Lanczos depth per sample\n" + << " mat_or_file : 1=psd_alg, 2=psd_exp, or path to .txt file\n" + << " func_type : 0=sqrt, 1=log, 2=poly (x*(x+poly_lambda))\n" + << " compute_ref : 0=skip eigendecomp, 1=run syevd (file input only)\n" + << " sketch_type : 0=Gaussian, 1=SJLT (vec_nnz=4)\n" + << " poly_lambda : lambda for func_type=2 (ignored otherwise)\n" + << " n_1 ... : matrix sizes (generated types only)\n"; return 1; } - int64_t numruns = std::stol(argv[2]); - double k_ratio = std::stod(argv[3]); - double s_ratio = std::stod(argv[4]); - int64_t d_steps = std::stol(argv[5]); + int64_t numruns = std::stol(argv[2]); + double k_ratio = std::stod(argv[3]); + double k_mat_ratio = std::stod(argv[4]); + double s_ratio = std::stod(argv[5]); + int64_t d_steps = std::stol(argv[6]); + int func_type = std::stoi(argv[8]); + bool compute_ref = std::stoi(argv[9]) != 0; + int sketch_type = std::stoi(argv[10]); + double poly_lambda = std::stod(argv[11]); + + // argv[7]: integer mat_type or file path + bool from_file = false; + std::string mat_file_path; + int mat_type = 0; + try { + mat_type = std::stoi(argv[7]); + } catch (...) { + from_file = true; + mat_file_path = std::string(argv[7]); + } + const char* func_str = func_type == 0 ? "sqrt" : (func_type == 1 ? "log" : "poly"); + const char* mat_str = from_file ? mat_file_path.c_str() : + mat_type == 1 ? "psd_alg" : + mat_type == 2 ? "psd_exp" : + mat_type == 3 ? "rbf_kernel" : "poly_kernel"; + const char* sketch_str = sketch_type == 1 ? "SJLT" : "Gaussian"; + + // Collect n_sizes; for file input, determine n from the file. std::vector n_sizes; - for (int i = 6; i < argc; ++i) - n_sizes.push_back(std::stol(argv[i])); + if (from_file) { + int64_t fm = 0, fn = 0; + int wq = 1; + RandLAPACK::gen::process_input_mat( + fm, fn, nullptr, + const_cast(mat_file_path.c_str()), wq); + if (fm != fn) + throw std::runtime_error("File matrix must be square (got " + + std::to_string(fm) + "x" + std::to_string(fn) + ")"); + n_sizes = {fm}; + } else { + if (argc < 13) { + std::cerr << "Error: at least one matrix size (n_1) required for generated types.\n"; + return 1; + } + for (int i = 12; i < argc; ++i) + n_sizes.push_back(std::stol(argv[i])); + } std::ostringstream oss; for (auto v : n_sizes) oss << v << ", "; @@ -241,14 +456,17 @@ int main(int argc, char* argv[]) { auto state = RandBLAS::RNGState(); auto state_constant = state; - int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); - int64_t k_max = std::max((int64_t)1, (int64_t)(n_max / k_ratio)); - int64_t s_max = std::max((int64_t)1, (int64_t)(k_max * s_ratio)); + int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); + int64_t k_max = std::max((int64_t)1, (int64_t)(n_max / k_ratio)); + int64_t k_mat_max = from_file ? 0 + : std::max((int64_t)1, (int64_t)(n_max / k_mat_ratio)); + int64_t s_max = std::max((int64_t)1, (int64_t)(k_max * s_ratio)); - FunNystromPP_benchmark_data all_data(n_max, k_max, s_max, d_steps); + FunNystromPP_benchmark_data all_data( + n_max, k_max, k_mat_max, s_max, d_steps, from_file); - // Output file - std::string output_filename = "_FunNystromPP_benchmark_num_info_lines_" + std::to_string(7) + ".txt"; + std::string output_filename = + "_FunNystromPP_benchmark_num_info_lines_" + std::to_string(8) + ".txt"; std::string path; if (std::string(argv[1]) != ".") path = std::string(argv[1]) + output_filename; @@ -256,29 +474,46 @@ int main(int argc, char* argv[]) { path = output_filename; std::ofstream file(path, std::ios::out | std::ios::app); - file << "Description: FunNystromPP vs Hutchinson+LanczosFA benchmark for tr(sqrt(A)), A = B'B + n*I." - "\nFile format: 11 columns: n, k, s, d, time_fun_pp (us), time_hutch (us)," - " est_fun_pp, est_hutch, true_tr (syevd; -1 for n>500), err_fun_pp, err_hutch;" + file << "Description: FunNystromPP vs Hutchinson+LanczosFA benchmark for tr(f(A))." + "\nFile format: 16 columns: n, k, k_mat, s, d, matvec_pp, matvec_hutch," + " time_pp_total (us), time_pp_phase1 (us), time_pp_phase2 (us)," + " time_hutch (us), est_fun_pp, est_hutch," + " true_tr (-1 if unavailable), err_fun_pp, err_hutch;" " rows = numruns repetitions per matrix size." "\nNum OMP threads: " + std::to_string(RandLAPACK::util::get_omp_threads()) + "\nInput sizes: " + sizes_str + "\nParameters: k_ratio=" + std::string(argv[3]) + - " s_ratio=" + std::string(argv[4]) + + " k_mat_ratio=" + std::string(argv[4]) + + " s_ratio=" + std::string(argv[5]) + " d=" + std::to_string(d_steps) + + " mat=" + std::string(mat_str) + + " func=" + func_str + + (func_type == 2 ? (" poly_lambda=" + std::to_string(poly_lambda)) : std::string("")) + + " sketch=" + sketch_str + "\nNum runs per size: " + std::to_string(numruns) + + "\nMatrix construction: " + (from_file ? "loaded from file" : + mat_type >= 3 ? "gen_kernel_matrix (random data points x_i~N(0,I_d); cost not timed)" : + "gen_sym_psd_lowrank (Haar-random eigenvectors; cost not timed)") + "\n"; file.flush(); auto start_all = steady_clock::now(); for (int64_t n : n_sizes) { - int64_t k = std::max((int64_t)1, (int64_t)(n / k_ratio)); - int64_t s = std::max((int64_t)1, (int64_t)(k * s_ratio)); - all_data.n = n; - all_data.k = k; - all_data.s = s; - all_data.d = d_steps; - - call_all_algs(numruns, all_data, state_constant, path); + int64_t k = std::max((int64_t)1, (int64_t)(n / k_ratio)); + int64_t k_mat = from_file ? 0 + : std::max((int64_t)1, (int64_t)(n / k_mat_ratio)); + int64_t s = std::max((int64_t)1, (int64_t)(k * s_ratio)); + all_data.n = n; + all_data.k = k; + all_data.k_mat = k_mat; + all_data.s = s; + all_data.d = d_steps; + + auto state_gen = state_constant; + data_regen(all_data, state_gen, mat_type, func_type, mat_file_path); + + call_all_algs(numruns, all_data, state_constant, mat_type, + func_type, poly_lambda, compute_ref, sketch_type, path); } auto stop_all = steady_clock::now(); long dur_all = duration_cast(stop_all - start_all).count(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1ff28611a..4851b68cb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -18,7 +18,7 @@ if (GTest_FOUND) drivers/test_cqrrpt.cc drivers/test_cqrrt.cc drivers/test_bqrrp.cc - drivers/test_revd2.cc + drivers/test_nystrom_evd.cc drivers/test_hqrrp.cc comps/test_lanczos_fa.cc drivers/test_fun_nystrom_pp.cc diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index 356ba8bb0..b030f7cd9 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -170,6 +170,12 @@ TEST_F(TestLanczosFA, DiagonalLog) { run_diagonal_fa_test([](double x){ return std::log(x); }, 50, 4, 20, 1e-3, 7); } +// f=poly x*(x+2) on diag(1,...,50). Degree-2 polynomial; d=20 >> 2 steps needed for exactness. +TEST_F(TestLanczosFA, DiagonalPoly) { + double lam = 2.0; + run_diagonal_fa_test([lam](double x){ return x * (x + lam); }, 50, 5, 20, 1e-4, 17); +} + // f=sqrt on a random dense PSD matrix. d=n/2=25 steps → near-exact convergence on κ≈5. TEST_F(TestLanczosFA, DensePSDSqrt) { run_dense_fa_test([](double x){ return std::sqrt(std::max(x, 0.0)); }, 50, 4, 25, 1e-6, 31); @@ -181,6 +187,43 @@ TEST_F(TestLanczosFA, DensePSDExp) { run_dense_fa_test([](double x){ return std::exp(x); }, 50, 4, 25, 1e-6, 77); } +// f=sqrt on an RBF kernel matrix generated by gen_kernel_matrix (n=20, d_dim=5). +// All eigenvalues in (0,1] (Mercer; diagonal=1), so sqrt is well-defined. +// Reference via dense_fa_ref (syevd). Tolerance relaxed to 1e-2: RBF spectrum +// on 5-dim data does not decay fast enough for d=10 to reach tighter accuracy. +TEST_F(TestLanczosFA, KernelRBFSqrt) { + using RNG = r123::Philox4x32; + int64_t n = 20, d_dim = 5, s = 3, d = 10; + auto state = RandBLAS::RNGState(101); + + std::vector K(n * n, 0.0); + double bandwidth = std::sqrt((double)d_dim); + RandLAPACK::gen::gen_kernel_matrix( + n, d_dim, K.data(), n, 0, bandwidth, 0.0, 0, state); + + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + + std::vector out(n * s, 0.0); + auto f = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, K.data(), n, Layout::ColMajor); + RandLAPACK::LanczosFA lfa; + lfa.reorth = 1; + lfa.call(A_op, B.data(), n, s, f, d, out.data()); + + auto ref = dense_fa_ref(K, B, n, s, f); + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("KernelRBFSqrt: ||f(A)B - ref||/||ref|| = %e\n", rel_err); + ASSERT_LT(rel_err, 1e-2); +} + // Both reorth modes agree on a well-conditioned diagonal matrix (κ=40). // Tests that reorthogonalization doesn't introduce a bug, not that it helps // (for that one would need κ >> 1 where vanilla loses orthogonality). diff --git a/test/drivers/test_fun_nystrom_pp.cc b/test/drivers/test_fun_nystrom_pp.cc index 126ccd914..89dc66230 100644 --- a/test/drivers/test_fun_nystrom_pp.cc +++ b/test/drivers/test_fun_nystrom_pp.cc @@ -31,21 +31,21 @@ class TestFunNystromPP : public ::testing::Test { return tr; } - // Full algorithm stack. error_est_power_iters=0 → fixed-rank single pass in REVD2. + // Full algorithm stack. error_est_power_iters=0 → fixed-rank single pass in NystromEVD. template struct Algs { using SYPS_t = RandLAPACK::SYPS; using Orth_t = RandLAPACK::HQRQ; using SYRF_t = RandLAPACK::SYRF; - using REVD2_t = RandLAPACK::REVD2; + using NystromEVD_t = RandLAPACK::NystromEVD; using LFA_t = RandLAPACK::LanczosFA; using Hutch_t = RandLAPACK::Hutchinson; - using Driver_t = RandLAPACK::FunNystromPP; + using Driver_t = RandLAPACK::FunNystromPP; SYPS_t syps; Orth_t orth; SYRF_t syrf; - REVD2_t revd2; + NystromEVD_t nystrom_evd; LFA_t lfa; Hutch_t hutch; Driver_t driver; @@ -54,8 +54,8 @@ class TestFunNystromPP : public ::testing::Test { syps(3, 1, false, false), orth(false, false), syrf(syps, orth), - revd2(syrf, 0), - driver(revd2, lfa, hutch) {} + nystrom_evd(syrf, 0), + driver(nystrom_evd, lfa, hutch) {} }; // Common test body: run the full pipeline, print and assert relative error. @@ -123,3 +123,78 @@ TEST_F(TestFunNystromPP, DiagonalSqrt) { Algs algs; test_FunNystromPP_general(algs, A_op, f_sqrt, 0.0, true_tr, k, s, d, 0.05, state); } + + +// A = gen_shifted_lowrank_psd(noise=1): eigenvalues {lambda_j+1} ∪ {1 repeated n-k_mat times}. +// f=log; all eigenvalues ≥ 1 so log is well-defined. f(noise) = log(1) = 0 → f_zero=0. +// true_tr = Σ log(lambda_j + 1); (n-k_mat) tail terms vanish. Tol = 20%. +TEST_F(TestFunNystromPP, ShiftedLowRankLog) { + using RNG = r123::Philox4x32; + int64_t n = 40, k_mat = 20, k = 15, s = 500, d = 20; + double noise = 1.0; + auto state = RandBLAS::RNGState(2); + + std::vector eigvals(k_mat); + RandLAPACK::gen::gen_alg_decay_singvals(k_mat, 100.0, 2.0, eigvals.data()); + + std::vector A(n * n, 0.0); + RandLAPACK::gen::gen_shifted_lowrank_psd(n, k_mat, A.data(), n, eigvals.data(), noise, state); + + auto f_log = [](double x){ return std::log(x); }; + double true_tr = 0.0; + for (int64_t j = 0; j < k_mat; ++j) + true_tr += f_log(eigvals[j] + noise); + // tail: (n - k_mat) * log(1) = 0 + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + Algs algs; + test_FunNystromPP_general(algs, A_op, f_log, 0.0, true_tr, k, s, d, 0.20, state); +} + + +// A = gen_sym_psd_lowrank; f=x*(x+1), degree-2 polynomial. d=2 Lanczos steps is exact for +// degree-2 trace estimation via Gauss quadrature (exact for degree ≤ 2d-1 = 3 quadratic forms). +// true_tr = Σ lambda_j*(lambda_j+1); (n-k_mat) tail terms: 0*(0+1)=0. Tol = 10%. +TEST_F(TestFunNystromPP, LowRankPoly) { + using RNG = r123::Philox4x32; + int64_t n = 40, k_mat = 20, k = 15, s = 200, d = 2; + double lam = 1.0; + auto state = RandBLAS::RNGState(3); + + std::vector eigvals(k_mat); + RandLAPACK::gen::gen_alg_decay_singvals(k_mat, 100.0, 2.0, eigvals.data()); + + std::vector A(n * n, 0.0); + RandLAPACK::gen::gen_sym_psd_lowrank(n, k_mat, A.data(), n, eigvals.data(), state); + + auto f_poly = [lam](double x){ return x * (x + lam); }; + double true_tr = 0.0; + for (int64_t j = 0; j < k_mat; ++j) + true_tr += f_poly(eigvals[j]); + // tail: (n - k_mat) * f(0) = 0 + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + Algs algs; + test_FunNystromPP_general(algs, A_op, f_poly, 0.0, true_tr, k, s, d, 0.10, state); +} + + +// A = RBF kernel matrix via gen_kernel_matrix (n=30, d_dim=5). Eigenvalues in (0,1]. +// f=sqrt; reference via syevd. k=10, s=300, d=15. Tol = 15%. +TEST_F(TestFunNystromPP, KernelRBFSqrt) { + using RNG = r123::Philox4x32; + int64_t n = 30, d_dim = 5, k = 10, s = 300, d = 15; + auto state = RandBLAS::RNGState(4); + + std::vector K(n * n, 0.0); + double bandwidth = std::sqrt((double)d_dim); + RandLAPACK::gen::gen_kernel_matrix( + n, d_dim, K.data(), n, 0, bandwidth, 0.0, 0, state); + + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + double true_tr = true_trace_fa(n, K, f_sqrt); + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, K.data(), n, Layout::ColMajor); + Algs algs; + test_FunNystromPP_general(algs, A_op, f_sqrt, 0.0, true_tr, k, s, d, 0.15, state); +} diff --git a/test/drivers/test_revd2.cc b/test/drivers/test_nystrom_evd.cc similarity index 78% rename from test/drivers/test_revd2.cc rename to test/drivers/test_nystrom_evd.cc index 22309bcb0..81f501939 100644 --- a/test/drivers/test_revd2.cc +++ b/test/drivers/test_nystrom_evd.cc @@ -8,7 +8,7 @@ #include -class TestREVD2 : public ::testing::Test +class TestNystromEVD : public ::testing::Test { protected: @@ -17,61 +17,62 @@ class TestREVD2 : public ::testing::Test virtual void TearDown() {}; template - struct REVD2TestData { + struct NystromEVDTestData { int64_t dim; int64_t rank; std::vector A; std::vector A_cpy; - std::vector V; - std::vector eigvals; + T* V = nullptr; int64_t V_sz = 0; + T* eigvals = nullptr; int64_t eigvals_sz = 0; std::vector E; std::vector Buf; - REVD2TestData( + NystromEVDTestData( int64_t m, int64_t k - ) : - A(m * m, 0.0), - A_cpy(m * m, 0.0), - V(m * k, 0.0), - eigvals(k, 0.0), - E(k * k, 0.0), + ) : + A(m * m, 0.0), + A_cpy(m * m, 0.0), + E(k * k, 0.0), Buf(m * k, 0.0) { dim = m; rank = k; } + + ~NystromEVDTestData() { delete[] V; delete[] eigvals; } }; - template - struct REVD2UploTestData { + template + struct NystromEVDUploTestData { int64_t dim; int64_t rank; std::vector work; std::vector A_u; - std::vector V_u; - std::vector eigvals_u; + T* V_u = nullptr; int64_t V_u_sz = 0; + T* eigvals_u = nullptr; int64_t eigvals_u_sz = 0; std::vector A_l; - std::vector V_l; - std::vector eigvals_l; + T* V_l = nullptr; int64_t V_l_sz = 0; + T* eigvals_l = nullptr; int64_t eigvals_l_sz = 0; std::vector E_u; std::vector E_l; - REVD2UploTestData( + NystromEVDUploTestData( int64_t m, int64_t k - ) : + ) : work(m * m, 0.0), A_u(m * m, 0.0), - V_u(m * k, 0.0), - eigvals_u(k, 0.0), A_l(m * m, 0.0), - V_l(m * k, 0.0), - eigvals_l(k, 0.0), - E_u(k * k, 0.0), + E_u(k * k, 0.0), E_l(k * k, 0.0) { dim = m; rank = k; } + + ~NystromEVDUploTestData() { + delete[] V_u; delete[] eigvals_u; + delete[] V_l; delete[] eigvals_l; + } }; template @@ -82,7 +83,7 @@ class TestREVD2 : public ::testing::Test SYPS_t syps; Orth_t orth; SYRF_t syrf; - RandLAPACK::REVD2 revd2; + RandLAPACK::NystromEVD nystrom_evd; algorithm_objects( @@ -95,12 +96,12 @@ class TestREVD2 : public ::testing::Test syps(num_syps_passes, passes_per_syps_stabilization, verbose, cond_check), orth(cond_check, verbose), syrf(syps, orth, verbose, cond_check), - revd2(syrf, num_steps_power_iter_error_est, verbose) + nystrom_evd(syrf, num_steps_power_iter_error_est, verbose) {} }; template - static void symm_mat_and_copy_computational_helper(T &norm_A, REVD2TestData &all_data) { + static void symm_mat_and_copy_computational_helper(T &norm_A, NystromEVDTestData &all_data) { auto m = all_data.dim; // We're using Nystrom, the original must be positive semidefinite blas::syrk( @@ -116,7 +117,7 @@ class TestREVD2 : public ::testing::Test } template - static void uplo_computational_helper(REVD2UploTestData &all_data) { + static void uplo_computational_helper(NystromEVDUploTestData &all_data) { auto m = all_data.dim; T* A_u_dat = all_data.A_u.data(); T* A_l_dat = all_data.A_l.data(); @@ -138,13 +139,13 @@ class TestREVD2 : public ::testing::Test /// General test for REVD: /// Computes the decomposition factors, then checks A-U\Sigma\transpose{V}. template - static void test_REVD2_general( + static void test_NystromEVD_general( int64_t k_start, T tol, int rank_expectation, T err_expectation, T &norm_A, - REVD2TestData &all_data, + NystromEVDTestData &all_data, algorithm_objects &all_algs, RandBLAS::RNGState state ) { @@ -152,18 +153,19 @@ class TestREVD2 : public ::testing::Test auto m = all_data.dim; int64_t k = k_start; - all_algs.revd2.call(blas::Uplo::Upper, m, all_data.A.data(), k, tol, all_data.V, all_data.eigvals, state); + all_algs.nystrom_evd.call(blas::Uplo::Upper, m, all_data.A.data(), k, tol, + all_data.V, all_data.V_sz, all_data.eigvals, all_data.eigvals_sz, state); T* E_dat = RandLAPACK::util::resize(k * k, all_data.E); T* Buf_dat = RandLAPACK::util::resize(m * k, all_data.Buf); T* A_cpy_dat = all_data.A_cpy.data(); - T* V_dat = all_data.V.data(); + T* V_dat = all_data.V; - // Construnct A_hat = U1 * S1 * VT1 + // Construct A_hat = V * diag(eigvals) * V' - // Turn vector into diagonal matrix - RandLAPACK::util::diag(k, k, all_data.eigvals.data(), k, all_data.E.data()); + // Turn array into diagonal matrix + RandLAPACK::util::diag(k, k, all_data.eigvals, k, all_data.E.data()); // V * E = Buf blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, m, k, k, 1.0, V_dat, m, E_dat, k, 0.0, Buf_dat, m); // A - Buf * V' - should be close to 0 @@ -178,11 +180,11 @@ class TestREVD2 : public ::testing::Test /// General test for REVD: /// Computes the decomposition factors, then checks A-U\Sigma\transpose{V}. template - static void test_REVD2_uplo( + static void test_NystromEVD_uplo( int64_t k_start, T tol, T err_expectation, - REVD2UploTestData &all_data, + NystromEVDUploTestData &all_data, algorithm_objects &all_algs, RandBLAS::RNGState state ) { @@ -190,19 +192,21 @@ class TestREVD2 : public ::testing::Test auto m = all_data.dim; int64_t k = k_start; - all_algs.revd2.call(blas::Uplo::Upper, m, all_data.A_u.data(), k, tol, all_data.V_u, all_data.eigvals_u, state); - all_algs.revd2.call(blas::Uplo::Lower, m, all_data.A_l.data(), k, tol, all_data.V_l, all_data.eigvals_l, state); + all_algs.nystrom_evd.call(blas::Uplo::Upper, m, all_data.A_u.data(), k, tol, + all_data.V_u, all_data.V_u_sz, all_data.eigvals_u, all_data.eigvals_u_sz, state); + all_algs.nystrom_evd.call(blas::Uplo::Lower, m, all_data.A_l.data(), k, tol, + all_data.V_l, all_data.V_l_sz, all_data.eigvals_l, all_data.eigvals_l_sz, state); T* E_u_dat = RandLAPACK::util::resize(k * k, all_data.E_u); T* E_l_dat = RandLAPACK::util::resize(k * k, all_data.E_l); - T* V_u_dat = all_data.V_u.data(); - T* V_l_dat = all_data.V_l.data(); + T* V_u_dat = all_data.V_u; + T* V_l_dat = all_data.V_l; T* work_u_dat = all_data.A_u.data(); T* work_l_dat = all_data.A_l.data(); T* A_approx_dat = all_data.work.data(); - RandLAPACK::util::diag(k, k, all_data.eigvals_u.data(), k, all_data.E_u.data()); - RandLAPACK::util::diag(k, k, all_data.eigvals_l.data(), k, all_data.E_l.data()); + RandLAPACK::util::diag(k, k, all_data.eigvals_u, k, all_data.E_u.data()); + RandLAPACK::util::diag(k, k, all_data.eigvals_l, k, all_data.E_l.data()); // Reconstruct factorizations, compare the result // V_u * E_u = work_u @@ -220,7 +224,7 @@ class TestREVD2 : public ::testing::Test } }; -TEST_F(TestREVD2, Underestimation1) { +TEST_F(TestNystromEVD, Underestimation1) { using RNG = r123::Philox4x32; int64_t m = 1000; @@ -239,7 +243,7 @@ TEST_F(TestREVD2, Underestimation1) { bool verbose = false; bool cond_check = false; - REVD2TestData all_data(m, k); + NystromEVDTestData all_data(m, k); algorithm_objects all_algs( verbose, cond_check, num_syps_passes, @@ -254,12 +258,12 @@ TEST_F(TestREVD2, Underestimation1) { RandLAPACK::gen::mat_gen(m_info, all_data.A_cpy.data(), state); symm_mat_and_copy_computational_helper(norm_A, all_data); - test_REVD2_general( + test_NystromEVD_general( k_start, tol, rank_expectation, err_expectation, norm_A, all_data, all_algs, state ); } -TEST_F(TestREVD2, Underestimation2) { +TEST_F(TestNystromEVD, Underestimation2) { using RNG = r123::Philox4x32; int64_t m = 1000; @@ -277,7 +281,7 @@ TEST_F(TestREVD2, Underestimation2) { bool verbose = false; bool cond_check = false; - REVD2TestData all_data(m, k); + NystromEVDTestData all_data(m, k); algorithm_objects all_algs( verbose, cond_check, num_syps_passes, @@ -292,12 +296,12 @@ TEST_F(TestREVD2, Underestimation2) { RandLAPACK::gen::mat_gen(m_info, all_data.A_cpy.data(), state); symm_mat_and_copy_computational_helper(norm_A, all_data); - test_REVD2_general( + test_NystromEVD_general( k_start, tol, rank_expectation, err_expectation, norm_A, all_data, all_algs, state ); } -TEST_F(TestREVD2, Overestimation1) { +TEST_F(TestNystromEVD, Overestimation1) { using RNG = r123::Philox4x32; int64_t m = 1000; @@ -315,7 +319,7 @@ TEST_F(TestREVD2, Overestimation1) { bool verbose = false; bool cond_check = false; - REVD2TestData all_data(m, k); + NystromEVDTestData all_data(m, k); algorithm_objects all_algs( verbose, cond_check, num_syps_passes, @@ -330,12 +334,12 @@ TEST_F(TestREVD2, Overestimation1) { RandLAPACK::gen::mat_gen(m_info, all_data.A_cpy.data(), state); symm_mat_and_copy_computational_helper(norm_A, all_data); - test_REVD2_general( + test_NystromEVD_general( k_start, tol, rank_expectation, err_expectation, norm_A, all_data, all_algs, state ); } -TEST_F(TestREVD2, Overestimation2) { +TEST_F(TestNystromEVD, Overestimation2) { using RNG = r123::Philox4x32; int64_t m = 1000; @@ -353,7 +357,7 @@ TEST_F(TestREVD2, Overestimation2) { bool verbose = false; bool cond_check = false; - REVD2TestData all_data(m, k); + NystromEVDTestData all_data(m, k); algorithm_objects all_algs( verbose, cond_check, num_syps_passes, @@ -368,12 +372,12 @@ TEST_F(TestREVD2, Overestimation2) { RandLAPACK::gen::mat_gen(m_info, all_data.A_cpy.data(), state); symm_mat_and_copy_computational_helper(norm_A, all_data); - test_REVD2_general( + test_NystromEVD_general( k_start, tol, rank_expectation, err_expectation, norm_A, all_data, all_algs, state ); } -TEST_F(TestREVD2, Exactness) { +TEST_F(TestNystromEVD, Exactness) { using RNG = r123::Philox4x32; int64_t m = 100; @@ -391,7 +395,7 @@ TEST_F(TestREVD2, Exactness) { bool verbose = false; bool cond_check = false; - REVD2TestData all_data(m, k); + NystromEVDTestData all_data(m, k); algorithm_objects all_algs( verbose, cond_check, num_syps_passes, @@ -406,14 +410,14 @@ TEST_F(TestREVD2, Exactness) { RandLAPACK::gen::mat_gen(m_info, all_data.A_cpy.data(), state); symm_mat_and_copy_computational_helper(norm_A, all_data); - test_REVD2_general( + test_NystromEVD_general( k_start, tol, rank_expectation, err_expectation, norm_A, all_data, all_algs, state ); } // Verify that error_est_power_iters=0 and tol=0 produce fixed-rank behavior: // k must not increase, since FunNystromPP's (n-k)*f(0) tail correction depends on k staying fixed. -TEST_F(TestREVD2, FixedRank) { +TEST_F(TestNystromEVD, FixedRank) { using RNG = r123::Philox4x32; int64_t m = 40, k = 10; auto state = RandBLAS::RNGState(5); @@ -423,15 +427,18 @@ TEST_F(TestREVD2, FixedRank) { A[i + i * m] = (double)(i + 1); algorithm_objects all_algs(false, false, 3, 1, /*error_est_p=*/0); - std::vector V_out, eigvals_out; + double* V_out = nullptr; int64_t V_out_sz = 0; + double* eigvals_out = nullptr; int64_t eigvals_out_sz = 0; int64_t k_before = k; - all_algs.revd2.call(blas::Uplo::Upper, m, A.data(), k, 0.0, V_out, eigvals_out, state); + all_algs.nystrom_evd.call(blas::Uplo::Upper, m, A.data(), k, 0.0, V_out, V_out_sz, eigvals_out, eigvals_out_sz, state); + delete[] V_out; + delete[] eigvals_out; - printf("REVD2 fixed-k: k_before=%lld, k_after=%lld\n", (long long)k_before, (long long)k); + printf("NystromEVD fixed-k: k_before=%lld, k_after=%lld\n", (long long)k_before, (long long)k); ASSERT_EQ(k, k_before); } -TEST_F(TestREVD2, Uplo) { +TEST_F(TestNystromEVD, Uplo) { using RNG = r123::Philox4x32; int64_t m = 100; @@ -447,7 +454,7 @@ TEST_F(TestREVD2, Uplo) { bool verbose = false; bool cond_check = false; - REVD2UploTestData all_data(m, k); + NystromEVDUploTestData all_data(m, k); algorithm_objects all_algs( verbose, cond_check, num_syps_passes, @@ -463,5 +470,5 @@ TEST_F(TestREVD2, Uplo) { uplo_computational_helper(all_data); - test_REVD2_uplo(k_start, tol, err_expectation, all_data, all_algs, state); + test_NystromEVD_uplo(k_start, tol, err_expectation, all_data, all_algs, state); } From a5ffa9005bbee380efe2fc41f0271bb29b6bfb09 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Fri, 1 May 2026 15:33:15 -0400 Subject: [PATCH 11/14] Runtime breakdown for NystromEVD and LanczosFA; PR review fixes Timing: - NystromEVD: bool timing flag + times[11] (alloc, syrf, matvec, gram, potrf, trsm, svd, post_svd, error_est, rest, total); += accumulation across adaptive iterations; skip error_est block entirely when error_est_p==0 && tol==0 (the fixed-rank FunNystromPP path) - LanczosFA: bool timing flag + times[5] (matvec, run_lanczos, apply_f, rest, total); _t_matvec_us accumulated inside run_lanczos per A-call, reset at top of both run_lanczos and call(); workspace_per_thread formula written as d*d+4*d-1 to match comment - FunNystromPP benchmark: nystrom_evd.timing=true, lfa_pp.timing=true; per-phase printf breakdown; 32 CSV columns (up from 16) Bug fixes: - power_error_est: inner loop variable shadowed outer iter; m=1 column- scaling bug from && i!=0 guard; replaced with explicit double loop - power_error_est: 1.0/0.0 literals -> (T)1/(T)0 for float safety; same fix applied throughout NystromEVD::call() - NystromEVD::call(): int i -> int64_t i in post-SVD eigval loop - nystrom_pc_data: auto out_state captured int return of call() instead of advanced RNGState; fixed to NystromAlg.call(); return state; - Benchmark printf: nt[9] (nystrom_rest) was dropped from console output - LFAOp: scalar_t hardcoded as double; use typename SLO_t::scalar_t - call_all_algs: state param is never modified; changed to const& Safety: - Deleted copy ctor/assignment on NystromEVD, LanczosFA (+ explicit default ctor), Hutchinson (+ explicit default ctor), FunNystromPP to prevent double-free on accidental copy Test: - ASSERT_NEAR(err, target, 10*target) -> ASSERT_LT(err, 10*target) --- RandLAPACK/comps/rl_hutchinson.hh | 4 + RandLAPACK/comps/rl_lanczos_fa.hh | 37 +- RandLAPACK/comps/rl_preconditioners.hh | 4 +- RandLAPACK/comps/rl_syps.hh | 5 +- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 3 + RandLAPACK/drivers/rl_nystrom_evd.hh | 95 ++++-- .../FunNystromPP_benchmark.cc | 319 ++++++++++++++---- benchmark/bench_FunNystromPP/load_mtx.hh | 113 +++++++ test/drivers/test_nystrom_evd.cc | 2 +- 9 files changed, 482 insertions(+), 100 deletions(-) create mode 100644 benchmark/bench_FunNystromPP/load_mtx.hh diff --git a/RandLAPACK/comps/rl_hutchinson.hh b/RandLAPACK/comps/rl_hutchinson.hh index 47c0f9f5c..b0791c5d9 100644 --- a/RandLAPACK/comps/rl_hutchinson.hh +++ b/RandLAPACK/comps/rl_hutchinson.hh @@ -28,6 +28,10 @@ public: T* Omega = nullptr; int64_t Omega_sz = 0; T* Z = nullptr; int64_t Z_sz = 0; + Hutchinson() = default; + Hutchinson(const Hutchinson&) = delete; + Hutchinson& operator=(const Hutchinson&) = delete; + ~Hutchinson() { delete[] Omega; delete[] Z; } // ------------------------------------------------------------------ diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 345f06821..4ede5bce6 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -6,9 +6,11 @@ #include "rl_util.hh" #include +#include #include #include #include +#include #ifdef _OPENMP #include @@ -53,6 +55,15 @@ public: T* normb = nullptr; int64_t normb_sz = 0; T* workspace = nullptr; int64_t workspace_sz = 0; + bool timing = false; + std::vector times; // populated after call() when timing==true + // Slots: matvec, run_lanczos, apply_f, rest, total + long _t_matvec_us = 0; // accumulated in run_lanczos() when timing==true + + LanczosFA() = default; + LanczosFA(const LanczosFA&) = delete; + LanczosFA& operator=(const LanczosFA&) = delete; + ~LanczosFA() { delete[] K; delete[] alpha; delete[] beta; delete[] normb; delete[] workspace; } // ------------------------------------------------------------------ @@ -67,6 +78,10 @@ public: /// @param[in] d Number of Lanczos steps. template void run_lanczos(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { + using namespace std::chrono; + steady_clock::time_point _mv_t0, _mv_t1; + _t_matvec_us = 0; + // Grow buffers if needed util::resize(K, K_sz, (d + 1) * n * s); util::resize(alpha, alpha_sz, d * s); @@ -87,7 +102,9 @@ public: // Step 0 matvec: K[:,:,1] = A * K[:,:,0] T* K1 = K + n * s; + if (this->timing) _mv_t0 = steady_clock::now(); A(Layout::ColMajor, s, (T)1.0, K0, n, (T)0.0, K1, n); + if (this->timing) { _mv_t1 = steady_clock::now(); _t_matvec_us += duration_cast(_mv_t1 - _mv_t0).count(); } // α[0, j] = q_1[:,j] · (A q_1)[:,j] — s independent inner products, // one tridiagonal diagonal entry per column. @@ -144,7 +161,9 @@ public: // (5) K_new = A*q_{i+1} - β_{i+1}*q_i // Different β per column — same reasoning as (1), axpy is optimal. + if (this->timing) _mv_t0 = steady_clock::now(); A(Layout::ColMajor, s, (T)1.0, K_curr, n, (T)0.0, K_new, n); + if (this->timing) { _mv_t1 = steady_clock::now(); _t_matvec_us += duration_cast(_mv_t1 - _mv_t0).count(); } #pragma omp parallel for schedule(static) for (int64_t j = 0; j < s; ++j) blas::axpy(n, -beta[j * (d - 1) + i], K_prev + j * n, 1, K_new + j * n, 1); @@ -173,9 +192,8 @@ public: /// @param[out] out n×s output matrix (column-major); overwritten. template F> void apply_f(F f, int64_t n, int64_t s, int64_t d, T* out) { - // Per-thread workspace: alpha_j(d), beta_j(d-1), Z_j(d*d), c_j(d), v_j(d) - // Total per thread = d + (d-1) + d*d + d + d = d^2 + 4d - 1 - int64_t workspace_per_thread = d * d + 3 * d + std::max(d - 1, (int64_t)0); + // Per-thread workspace: alpha_j(d) + beta_j(d-1) + Z_j(d*d) + c_j(d) + v_j(d) = d^2 + 4d - 1 + int64_t workspace_per_thread = d * d + 4 * d - 1; int nthreads = 1; #ifdef _OPENMP nthreads = omp_get_max_threads(); @@ -239,8 +257,21 @@ public: /// @param[out] out n×s output, overwritten with f(A)B approximation. template F> void call(SLO& A, const T* B, int64_t n, int64_t s, F f, int64_t d, T* out) { + using namespace std::chrono; + _t_matvec_us = 0; + steady_clock::time_point t_total_start, t_lanczos_end, t_end; + if (this->timing) t_total_start = steady_clock::now(); run_lanczos(A, B, n, s, d); + if (this->timing) t_lanczos_end = steady_clock::now(); apply_f(f, n, s, d, out); + if (this->timing) { + t_end = steady_clock::now(); + long total_us = duration_cast(t_end - t_total_start).count(); + long lanczos_us = duration_cast(t_lanczos_end - t_total_start).count(); + long apply_f_us = duration_cast(t_end - t_lanczos_end).count(); + long rest_us = total_us - lanczos_us - apply_f_us; + this->times = {_t_matvec_us, lanczos_us, apply_f_us, rest_us, total_us}; + } } }; diff --git a/RandLAPACK/comps/rl_preconditioners.hh b/RandLAPACK/comps/rl_preconditioners.hh index 5109ef1df..58c08badd 100644 --- a/RandLAPACK/comps/rl_preconditioners.hh +++ b/RandLAPACK/comps/rl_preconditioners.hh @@ -315,13 +315,13 @@ RandBLAS::RNGState nystrom_pc_data( // regularization parameter the user claims to need. T* V_buf = nullptr; int64_t V_sz = 0; T* ev_buf = nullptr; int64_t ev_sz = 0; - auto out_state = NystromAlg.call(A, k, tol, V_buf, V_sz, ev_buf, ev_sz, state); + NystromAlg.call(A, k, tol, V_buf, V_sz, ev_buf, ev_sz, state); int64_t m = A.dim; V.assign(V_buf, V_buf + m * k); eigvals.assign(ev_buf, ev_buf + k); delete[] V_buf; delete[] ev_buf; - return out_state; + return state; } /** diff --git a/RandLAPACK/comps/rl_syps.hh b/RandLAPACK/comps/rl_syps.hh index dcc3313a0..fea36416f 100644 --- a/RandLAPACK/comps/rl_syps.hh +++ b/RandLAPACK/comps/rl_syps.hh @@ -27,8 +27,9 @@ class SYPS { bool verbose; bool cond_check; std::vector cond_nums; - // 0 = Gaussian (DenseDist, default); 1 = SJLT (SparseDist, vec_nnz=4 per column) + // 0 = Gaussian (DenseDist, default); 1 = SASO (SparseDist, vec_nnz per column) int sketch_type = 0; + int vec_nnz = 4; SYPS( int64_t p, // number of passes @@ -115,7 +116,7 @@ class SYPS { // SJLT: generate a sparse sketch and densify into skop_buff. // T and sint_t (int64_t) must be explicit since SparseDist::sample // can't deduce T from the RNGState argument alone. - auto S = RandBLAS::SparseDist(m, k, 4).sample(state); + auto S = RandBLAS::SparseDist(m, k, this->vec_nnz).sample(state); state = S.next_state; RandBLAS::fill_sparse(S); auto Scoo = RandBLAS::coo_view_of_skop(S); diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index ebac9a48c..b82fae615 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -79,6 +79,9 @@ public: T* Z1 = nullptr; int64_t Z1_sz = 0; T* Z2 = nullptr; int64_t Z2_sz = 0; + FunNystromPP(const FunNystromPP&) = delete; + FunNystromPP& operator=(const FunNystromPP&) = delete; + ~FunNystromPP() { delete[] V_buf; delete[] eigvals_buf; delete[] F_vec; delete[] tmp; delete[] Z1; delete[] Z2; diff --git a/RandLAPACK/drivers/rl_nystrom_evd.hh b/RandLAPACK/drivers/rl_nystrom_evd.hh index e4464ac69..f8762bf99 100644 --- a/RandLAPACK/drivers/rl_nystrom_evd.hh +++ b/RandLAPACK/drivers/rl_nystrom_evd.hh @@ -8,7 +8,9 @@ #include "rl_linops.hh" #include +#include #include +#include namespace RandLAPACK { @@ -31,27 +33,25 @@ T power_error_est( ) { int64_t m = A.dim; T err = 0; - for(int i = 0; i < p; ++i) { + for(int iter = 0; iter < p; ++iter) { T g_norm = blas::nrm2(m, vector_buf, 1); - blas::scal(m, 1 / g_norm, vector_buf, 1); + blas::scal(m, (T)1 / g_norm, vector_buf, 1); // V' * g/||g|| — result in column 1 of vector_buf - gemv(Layout::ColMajor, Op::Trans, m, k, 1.0, V, m, vector_buf, 1, 0.0, &vector_buf[m], 1); + gemv(Layout::ColMajor, Op::Trans, m, k, (T)1, V, m, vector_buf, 1, (T)0, &vector_buf[m], 1); - // V * diag(eigvals) into Mat_buf - for (int i = 0, j = 0; i < m * k; ++i) { - Mat_buf[i] = V[i] * eigvals[j]; - if((i + 1) % m == 0 && i != 0) - ++j; - } + // V * diag(eigvals) into Mat_buf — column-major: column col occupies rows [0, m) + for (int64_t col = 0; col < k; ++col) + for (int64_t row = 0; row < m; ++row) + Mat_buf[col * m + row] = V[col * m + row] * eigvals[col]; // V * diag(eigvals) * V' * g/||g|| — result in column 2 of vector_buf - gemv(Layout::ColMajor, Op::NoTrans, m, k, 1.0, Mat_buf, m, &vector_buf[m], 1, 0.0, &vector_buf[2 * m], 1); + gemv(Layout::ColMajor, Op::NoTrans, m, k, (T)1, Mat_buf, m, &vector_buf[m], 1, (T)0, &vector_buf[2 * m], 1); // A * g/||g|| — result in column 3 of vector_buf - A(Layout::ColMajor, 1, 1.0, vector_buf, m, 0.0, &vector_buf[3*m], m); + A(Layout::ColMajor, 1, (T)1, vector_buf, m, (T)0, &vector_buf[3*m], m); // w = A*g/||g|| - V*diag(eigvals)*V'*g/||g|| - blas::axpy(m, -1.0, &vector_buf[2 * m], 1, &vector_buf[3 * m], 1); + blas::axpy(m, (T)-1, &vector_buf[2 * m], 1, &vector_buf[3 * m], 1); // error estimate: (g/||g||)' * w err = blas::dot(m, vector_buf, 1, &vector_buf[3 * m], 1); std::copy(&vector_buf[3 * m], &vector_buf[4 * m], vector_buf); @@ -76,6 +76,10 @@ class NystromEVD { T* S = nullptr; int64_t S_sz = 0; T* symrf_work = nullptr; int64_t symrf_work_sz = 0; + bool timing = false; + std::vector times; // populated after call() when timing==true + // Slots: alloc, syrf, matvec, gram, potrf, trsm, svd, post_svd, error_est, rest, total + NystromEVD( SYRF_t &syrf_obj, int error_est_power_iters, @@ -85,6 +89,9 @@ class NystromEVD { verbose = verb; } + NystromEVD(const NystromEVD&) = delete; + NystromEVD& operator=(const NystromEVD&) = delete; + ~NystromEVD() { delete[] Y; delete[] Omega; @@ -135,41 +142,73 @@ class NystromEVD { T*& eigvals, int64_t& eigvals_sz, RandBLAS::RNGState &state ) { + using namespace std::chrono; int64_t m = A.dim; T err = 0; RandBLAS::RNGState error_est_state(state.counter, state.key); error_est_state.key.incr(1); + + steady_clock::time_point t0, t1, total_t_start, total_t_stop; + long alloc_t_dur = 0; + long syrf_t_dur = 0; + long matvec_t_dur = 0; + long gram_t_dur = 0; + long potrf_t_dur = 0; + long trsm_t_dur = 0; + long svd_t_dur = 0; + long post_svd_t_dur = 0; + long error_est_t_dur = 0; + long total_t_dur = 0; + + if (this->timing) total_t_start = steady_clock::now(); + while(true) { + if (this->timing) t0 = steady_clock::now(); util::resize(eigvals, eigvals_sz, k); util::resize(V, V_sz, m * k); T* V_dat = V; - util::resize(Y, Y_sz, m * k); util::resize(Omega, Omega_sz, std::max(m * k, m * (int64_t)4)); util::resize(R, R_sz, k * k); util::resize(S, S_sz, k * k); util::resize(symrf_work, symrf_work_sz, m * k); + if (this->timing) { t1 = steady_clock::now(); alloc_t_dur += duration_cast(t1 - t0).count(); } + if (this->timing) t0 = steady_clock::now(); this->syrf.call(A, k, this->Omega, state, symrf_work); - A(Layout::ColMajor, k, 1.0, Omega, m, 0.0, Y, m); + if (this->timing) { t1 = steady_clock::now(); syrf_t_dur += duration_cast(t1 - t0).count(); } + + if (this->timing) t0 = steady_clock::now(); + A(Layout::ColMajor, k, (T)1, Omega, m, (T)0, Y, m); + if (this->timing) { t1 = steady_clock::now(); matvec_t_dur += duration_cast(t1 - t0).count(); } T nu = std::numeric_limits::epsilon() * lapack::lange(Norm::Fro, m, k, Y, m); - blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega, m, 0.0, R, k); - for(int i = 1; i < k; ++i) + if (this->timing) t0 = steady_clock::now(); + blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega, m, (T)0, R, k); + for(int64_t i = 1; i < k; ++i) blas::copy(k - i, &R[i + ((i-1) * k)], 1, &R[(i - 1) + (i * k)], k); - blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, 1.0, Omega, m, Y, m, 1.0, R, k); + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, (T)1, Omega, m, Y, m, (T)1, R, k); + if (this->timing) { t1 = steady_clock::now(); gram_t_dur += duration_cast(t1 - t0).count(); } + if (this->timing) t0 = steady_clock::now(); if(lapack::potrf(Uplo::Upper, k, R, k)) throw std::runtime_error("Cholesky decomposition failed."); RandLAPACK::util::get_U(k, k, R, k); + if (this->timing) { t1 = steady_clock::now(); potrf_t_dur += duration_cast(t1 - t0).count(); } + + if (this->timing) t0 = steady_clock::now(); + blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, (T)1, R, k, Y, m); + if (this->timing) { t1 = steady_clock::now(); trsm_t_dur += duration_cast(t1 - t0).count(); } - blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, 1.0, R, k, Y, m); + if (this->timing) t0 = steady_clock::now(); lapack::gesdd(Job::SomeVec, m, k, Y, m, S, V_dat, m, R, k); + if (this->timing) { t1 = steady_clock::now(); svd_t_dur += duration_cast(t1 - t0).count(); } + if (this->timing) t0 = steady_clock::now(); T buf; int64_t r = 0; - int i; + int64_t i; for(i = 0; i < k; ++i) { buf = std::pow(S[i], 2); eigvals[i] = buf; @@ -178,13 +217,17 @@ class NystromEVD { } for(i = 0; i < r; ++i) if (eigvals[i] > nu) eigvals[i] -= nu; + std::fill(&V_dat[m * r], &V_dat[m * k], (T)0); + if (this->timing) { t1 = steady_clock::now(); post_svd_t_dur += duration_cast(t1 - t0).count(); } - std::fill(&V_dat[m * r], &V_dat[m * k], 0.0); + // Fixed-rank mode: skip error estimation entirely. + if (this->error_est_p == 0 && tol == (T)0) break; + if (this->timing) t0 = steady_clock::now(); RandBLAS::DenseDist g(m, 1); error_est_state = RandBLAS::fill_dense(g, Omega, error_est_state); - err = power_error_est(A, k, this->error_est_p, Omega, V_dat, Y, eigvals); + if (this->timing) { t1 = steady_clock::now(); error_est_t_dur += duration_cast(t1 - t0).count(); } if(err <= 5 * std::max(tol, nu) || k == m) { break; @@ -194,6 +237,16 @@ class NystromEVD { k = 2 * k; } } + + if (this->timing) { + total_t_stop = steady_clock::now(); + total_t_dur = duration_cast(total_t_stop - total_t_start).count(); + long t_rest = total_t_dur - (alloc_t_dur + syrf_t_dur + matvec_t_dur + gram_t_dur + + potrf_t_dur + trsm_t_dur + svd_t_dur + post_svd_t_dur + error_est_t_dur); + this->times = {alloc_t_dur, syrf_t_dur, matvec_t_dur, gram_t_dur, + potrf_t_dur, trsm_t_dur, svd_t_dur, post_svd_t_dur, + error_est_t_dur, t_rest, total_t_dur}; + } return 0; } diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc index e54bc651c..58c86b379 100644 --- a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -19,7 +19,8 @@ Matrix input (mat_or_file): 4 = poly_kernel: K[i,j] = (x_i·x_j + 1)², x_i~N(0,I_d) /path.txt = load dense square symmetric matrix from space-separated text file (row-major, full matrix stored; upper triangle used by algorithms) - TODO: add .mtx, .m, and other common formats + /path.mtx = load sparse symmetric matrix in Matrix Market format + (coordinate real symmetric; both triangles expanded internally) For types 1/2: k_mat eigenvalues are nonzero; exact tr(f(A)) is always available. For types 3/4: k_mat_ratio reinterpreted as data dimension d = n/k_mat_ratio; @@ -33,7 +34,7 @@ Function types (func_type): Sketch types (sketch_type): 0 = Gaussian (DenseDist, default) - 1 = SJLT (SparseDist, vec_nnz=4 per column) + 1 = SASO (SparseDist; nonzeros per column set by vec_nnz argument) compute_ref: 0 = skip reference (report -1 in output) @@ -45,28 +46,34 @@ Both algorithms use a fixed seed so comparisons are reproducible. Matvec counts reflect actual A applications inside each algorithm (not estimates). The matrix is constructed/loaded once per n; that cost is not timed. -Output format: 16 comma-separated columns per row (trailing comma): +Output format: 32 comma-separated columns per row (trailing comma): n, k, k_mat, s, d, matvec_pp, matvec_hutch, time_pp_total (us), time_pp_phase1 (us), time_pp_phase2 (us), time_hutch (us), + nystrom_alloc, nystrom_syrf, nystrom_matvec, nystrom_gram, + nystrom_potrf, nystrom_trsm, nystrom_svd, nystrom_post_svd, + nystrom_error_est, nystrom_rest, nystrom_total (all us), + lfa_matvec, lfa_run_lanczos, lfa_apply_f, lfa_rest, lfa_total (all us), est_fun_pp, est_hutch, true_tr (-1 if unavailable), err_fun_pp, err_hutch Usage: - FunNystromPP_benchmark + FunNystromPP_benchmark - [n_1 n_2 ...] + [n_1 n_2 ...] dir_path : output directory (use "." for current directory) num_runs : number of timed repetitions per matrix size - k_ratio : k = n / k_ratio (Nyström rank; e.g., 10 -> k = n/10) - k_mat_ratio : k_mat = n / k_mat_ratio (matrix rank for generated types; ignored for file) - s_ratio : s = k * s_ratio (Hutchinson samples; e.g., 5 -> s = 5*k) + k : Nyström rank (constant; must satisfy k <= n) + k_mat : matrix rank for generated types (constant; for kernel types equals data dim d; + ignored for file input) + s : Hutchinson sample count (constant) d_steps : Lanczos depth per Hutchinson sample - mat_or_file : 1=psd_alg, 2=psd_exp, or path to .txt file + mat_or_file : 1=psd_alg, 2=psd_exp, 3=rbf_kernel, 4=poly_kernel, or path to .txt/.mtx file func_type : 0=sqrt, 1=log, 2=poly (x*(x+poly_lambda)) compute_ref : 0=skip eigendecomp reference, 1=run syevd (file input only) - sketch_type : 0=Gaussian, 1=SJLT (vec_nnz=4 per column) + sketch_type : 0=Gaussian, 1=SASO + vec_nnz : nonzeros per column for SASO sketch (sketch_type=1; ignored for Gaussian) poly_lambda : lambda parameter for func_type=2 (ignored for func_type=0,1) n_1 ... : matrix dimensions (generated types only; ignored for file input) */ @@ -75,6 +82,7 @@ Output format: 16 comma-separated columns per row (trailing comma): #include "rl_blaspp.hh" #include "rl_lapackpp.hh" #include "rl_gen.hh" +#include "load_mtx.hh" #include #include @@ -209,24 +217,24 @@ static void data_regen( template struct LFAOp { - using scalar_t = double; + using scalar_t = typename SLO_t::scalar_t; const int64_t dim; SLO_t& A; LFA_t& lfa; F_t f; int64_t d; - std::vector Z_buf; + std::vector Z_buf; LFAOp(int64_t n, SLO_t& A_, LFA_t& lfa_, F_t f_, int64_t d_) : dim(n), A(A_), lfa(lfa_), f(f_), d(d_), Z_buf() {} - void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, double alpha, - double* const B, [[maybe_unused]] int64_t ldb, - double beta, double* C, [[maybe_unused]] int64_t ldc) + void operator()([[maybe_unused]] Layout layout, int64_t n_vecs, scalar_t alpha, + scalar_t* const B, [[maybe_unused]] int64_t ldb, + scalar_t beta, scalar_t* C, [[maybe_unused]] int64_t ldc) { - Z_buf.assign(dim * n_vecs, 0.0); + Z_buf.assign(dim * n_vecs, (scalar_t)0); lfa.call(A, B, dim, n_vecs, f, d, Z_buf.data()); - if (beta == 0.0) { + if (beta == (scalar_t)0) { for (int64_t i = 0; i < dim * n_vecs; ++i) C[i] = alpha * Z_buf[i]; } else { @@ -245,12 +253,13 @@ template static void call_all_algs( int64_t numruns, FunNystromPP_benchmark_data& data, - RandBLAS::RNGState& state, + const RandBLAS::RNGState& state, int mat_type, int func_type, double poly_lambda, bool compute_ref, int sketch_type, + int vec_nnz, const std::string& output_filename ) { int64_t n = data.n, k = data.k, s = data.s, d = data.d; @@ -298,10 +307,13 @@ static void call_all_algs( // Build full algorithm stack for FunNystromPP. RandLAPACK::SYPS syps(3, 1, false, false); syps.sketch_type = sketch_type; + syps.vec_nnz = vec_nnz; RandLAPACK::HQRQ orth(false, false); RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); RandLAPACK::NystromEVD nystrom_evd(syrf, 0); + nystrom_evd.timing = true; RandLAPACK::LanczosFA lfa_pp; + lfa_pp.timing = true; RandLAPACK::Hutchinson hutch_pp; RandLAPACK::FunNystromPP driver(nystrom_evd, lfa_pp, hutch_pp); @@ -336,10 +348,22 @@ static void call_all_algs( auto stop_pp = steady_clock::now(); long dur_pp = duration_cast(stop_pp - start_pp).count(); int64_t mv_pp = A_pp.count; + + // NystromEVD sub-phase times (slots: alloc, syrf, matvec, gram, potrf, trsm, svd, post_svd, error_est, rest, total) + auto& nt = nystrom_evd.times; + // LanczosFA sub-phase times (slots: matvec, run_lanczos, apply_f, rest, total) + auto& lt = lfa_pp.times; + printf(" FunNystromPP: est=%.6e time=%lld us (ph1=%lld ph2=%lld) matvecs=%lld\n", est_pp, (long long)dur_pp, (long long)pp_timing.phase1_us, (long long)pp_timing.phase2_us, (long long)mv_pp); + printf(" NystromEVD: alloc=%lld syrf=%lld matvec=%lld gram=%lld potrf=%lld trsm=%lld svd=%lld post_svd=%lld err_est=%lld rest=%lld total=%lld\n", + (long long)nt[0], (long long)nt[1], (long long)nt[2], (long long)nt[3], + (long long)nt[4], (long long)nt[5], (long long)nt[6], (long long)nt[7], + (long long)nt[8], (long long)nt[9], (long long)nt[10]); + printf(" LanczosFA: matvec=%lld lanczos=%lld apply_f=%lld total=%lld\n", + (long long)lt[0], (long long)lt[1], (long long)lt[2], (long long)lt[4]); // ---- Hutchinson + LanczosFA (no Nyström) --------------------------- state_alg = state; @@ -370,48 +394,171 @@ static void call_all_algs( << mv_pp << ", " << mv_h << ", " << dur_pp << ", " << pp_timing.phase1_us << ", " << pp_timing.phase2_us << ", " << dur_h << ", " + << nt[0] << ", " << nt[1] << ", " << nt[2] << ", " << nt[3] << ", " + << nt[4] << ", " << nt[5] << ", " << nt[6] << ", " << nt[7] << ", " + << nt[8] << ", " << nt[9] << ", " << nt[10] << ", " + << lt[0] << ", " << lt[1] << ", " << lt[2] << ", " << lt[3] << ", " << lt[4] << ", " << est_pp << ", " << est_h << ", " << out_tr << ", " << err_pp << ", " << err_h << ",\n"; } } +// ============================================================================ +// call_all_algs_sparse: same as call_all_algs but backed by a sparse CSR matrix. +// No reference computation (syevd on large sparse matrices is impractical). +// ============================================================================ + +template +static void call_all_algs_sparse( + int64_t numruns, + int64_t n, + int64_t k, + int64_t s, + int64_t d, + RandBLAS::sparse_data::CSRMatrix& csr, + const RandBLAS::RNGState& state, + int func_type, + double poly_lambda, + int sketch_type, + int vec_nnz, + const std::string& output_filename +) { + std::function f; + const char* fname; + T f_zero = (T)0; + if (func_type == 0) { + f = [](T x){ return (T)std::sqrt((double)std::max(x, (T)0)); }; + fname = "sqrt"; + } else if (func_type == 1) { + f = [](T x){ return (T)std::log((double)x); }; + fname = "log"; + } else { + T lam = (T)poly_lambda; + f = [lam](T x){ return x * (x + lam); }; + fname = "poly"; + } + + RandLAPACK::SYPS syps(3, 1, false, false); + syps.sketch_type = sketch_type; + syps.vec_nnz = vec_nnz; + RandLAPACK::HQRQ orth(false, false); + RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); + RandLAPACK::NystromEVD nystrom_evd(syrf, 0); + nystrom_evd.timing = true; + RandLAPACK::LanczosFA lfa_pp; + lfa_pp.timing = true; + RandLAPACK::Hutchinson hutch_pp; + RandLAPACK::FunNystromPP + driver(nystrom_evd, lfa_pp, hutch_pp); + + RandLAPACK::LanczosFA lfa_base; + RandLAPACK::Hutchinson hutch_base; + + using SSLO = linops::SparseSymLinOp>; + using CSLO = CountingLinOp; + using FuncT = std::function; + + for (int i = 0; i < numruns; ++i) { + printf("ITERATION %d, N=%lld, K=%lld, K_MAT=0, S=%lld, D=%lld [%s, sparse_mtx]\n", + i, (long long)n, (long long)k, (long long)s, (long long)d, fname); + + auto state_alg = state; + + // ---- FunNystromPP -------------------------------------------------- + SSLO A_op_pp(n, csr); + CSLO A_pp(n, A_op_pp); + + RandLAPACK::FunNystromPP_timing pp_timing; + auto start_pp = steady_clock::now(); + T est_pp = driver.call(A_pp, f, f_zero, k, s, d, state_alg, &pp_timing); + auto stop_pp = steady_clock::now(); + long dur_pp = duration_cast(stop_pp - start_pp).count(); + int64_t mv_pp = A_pp.count; + + auto& nt = nystrom_evd.times; + auto& lt = lfa_pp.times; + + printf(" FunNystromPP: est=%.6e time=%lld us (ph1=%lld ph2=%lld) matvecs=%lld\n", + (double)est_pp, (long long)dur_pp, + (long long)pp_timing.phase1_us, (long long)pp_timing.phase2_us, + (long long)mv_pp); + printf(" NystromEVD: alloc=%lld syrf=%lld matvec=%lld gram=%lld potrf=%lld trsm=%lld svd=%lld post_svd=%lld err_est=%lld rest=%lld total=%lld\n", + (long long)nt[0], (long long)nt[1], (long long)nt[2], (long long)nt[3], + (long long)nt[4], (long long)nt[5], (long long)nt[6], (long long)nt[7], + (long long)nt[8], (long long)nt[9], (long long)nt[10]); + printf(" LanczosFA: matvec=%lld lanczos=%lld apply_f=%lld total=%lld\n", + (long long)lt[0], (long long)lt[1], (long long)lt[2], (long long)lt[4]); + + // ---- Hutchinson + LanczosFA ---------------------------------------- + state_alg = state; + SSLO A_op_h(n, csr); + CSLO A_hutch(n, A_op_h); + LFAOp, FuncT> lfa_op(n, A_hutch, lfa_base, f, d); + + auto start_h = steady_clock::now(); + T est_h = hutch_base.call(lfa_op, s, state_alg); + auto stop_h = steady_clock::now(); + long dur_h = duration_cast(stop_h - start_h).count(); + int64_t mv_h = A_hutch.count; + printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", + (double)est_h, (long long)dur_h, (long long)mv_h); + printf(" Reference: N/A (sparse matrix)\n"); + + std::ofstream file(output_filename, std::ios::out | std::ios::app); + file << n << ", " << k << ", " << 0 << ", " + << s << ", " << d << ", " + << mv_pp << ", " << mv_h << ", " + << dur_pp << ", " << pp_timing.phase1_us << ", " << pp_timing.phase2_us << ", " + << dur_h << ", " + << nt[0] << ", " << nt[1] << ", " << nt[2] << ", " << nt[3] << ", " + << nt[4] << ", " << nt[5] << ", " << nt[6] << ", " << nt[7] << ", " + << nt[8] << ", " << nt[9] << ", " << nt[10] << ", " + << lt[0] << ", " << lt[1] << ", " << lt[2] << ", " << lt[3] << ", " << lt[4] << ", " + << (double)est_pp << ", " << (double)est_h << ", " + << -1.0 << ", " << -1.0 << ", " << -1.0 << ",\n"; + } +} + // ============================================================================ // main // ============================================================================ int main(int argc, char* argv[]) { - if (argc < 12) { + if (argc < 13) { std::cerr << "Usage: " << argv[0] - << " " + << " " " " - " [n_1 n_2 ...]\n" + " [n_1 n_2 ...]\n" << " dir_path : output directory (use '.' for current dir)\n" << " num_runs : timed repetitions per matrix size\n" - << " k_ratio : k = n / k_ratio (Nystrom rank)\n" - << " k_mat_ratio : k_mat = n / k_mat_ratio (matrix rank; generated only)\n" - << " s_ratio : s = k * s_ratio (Hutchinson samples)\n" + << " k : Nystrom rank (constant; must satisfy k <= n)\n" + << " k_mat : matrix rank for generated types (for kernel types = data dim)\n" + << " s : Hutchinson sample count (constant)\n" << " d_steps : Lanczos depth per sample\n" - << " mat_or_file : 1=psd_alg, 2=psd_exp, or path to .txt file\n" + << " mat_or_file : 1=psd_alg, 2=psd_exp, 3=rbf_kernel, 4=poly_kernel, or path\n" << " func_type : 0=sqrt, 1=log, 2=poly (x*(x+poly_lambda))\n" << " compute_ref : 0=skip eigendecomp, 1=run syevd (file input only)\n" - << " sketch_type : 0=Gaussian, 1=SJLT (vec_nnz=4)\n" + << " sketch_type : 0=Gaussian, 1=SASO\n" + << " vec_nnz : nonzeros per column for SASO (sketch_type=1; ignored otherwise)\n" << " poly_lambda : lambda for func_type=2 (ignored otherwise)\n" << " n_1 ... : matrix sizes (generated types only)\n"; return 1; } int64_t numruns = std::stol(argv[2]); - double k_ratio = std::stod(argv[3]); - double k_mat_ratio = std::stod(argv[4]); - double s_ratio = std::stod(argv[5]); + int64_t k_const = std::stol(argv[3]); + int64_t k_mat_const = std::stol(argv[4]); + int64_t s_const = std::stol(argv[5]); int64_t d_steps = std::stol(argv[6]); int func_type = std::stoi(argv[8]); bool compute_ref = std::stoi(argv[9]) != 0; int sketch_type = std::stoi(argv[10]); - double poly_lambda = std::stod(argv[11]); + int vec_nnz = std::stoi(argv[11]); + double poly_lambda = std::stod(argv[12]); - // argv[7]: integer mat_type or file path + // argv[7]: integer mat_type or file path (.txt or .mtx) bool from_file = false; + bool is_mtx = false; std::string mat_file_path; int mat_type = 0; try { @@ -419,6 +566,8 @@ int main(int argc, char* argv[]) { } catch (...) { from_file = true; mat_file_path = std::string(argv[7]); + auto& p = mat_file_path; + is_mtx = p.size() >= 4 && p.substr(p.size() - 4) == ".mtx"; } const char* func_str = func_type == 0 ? "sqrt" : (func_type == 1 ? "log" : "poly"); @@ -426,11 +575,25 @@ int main(int argc, char* argv[]) { mat_type == 1 ? "psd_alg" : mat_type == 2 ? "psd_exp" : mat_type == 3 ? "rbf_kernel" : "poly_kernel"; - const char* sketch_str = sketch_type == 1 ? "SJLT" : "Gaussian"; + const char* sketch_str = sketch_type == 1 ? "SASO" : "Gaussian"; - // Collect n_sizes; for file input, determine n from the file. + // Collect n_sizes; for file input determine n from the file. std::vector n_sizes; - if (from_file) { + if (is_mtx) { + // Size comes from the .mtx file itself; no CLI sizes needed. + int64_t mtx_n = 0; + { + // Peek at n without loading the full matrix. + std::ifstream peek(mat_file_path); + std::string ln; + while (std::getline(peek, ln)) + if (!ln.empty() && ln[0] != '%') break; + std::istringstream ss(ln); + int64_t nr, nc, nz; ss >> nr >> nc >> nz; + mtx_n = nr; + } + n_sizes = {mtx_n}; + } else if (from_file) { int64_t fm = 0, fn = 0; int wq = 1; RandLAPACK::gen::process_input_mat( @@ -441,11 +604,11 @@ int main(int argc, char* argv[]) { + std::to_string(fm) + "x" + std::to_string(fn) + ")"); n_sizes = {fm}; } else { - if (argc < 13) { + if (argc < 14) { std::cerr << "Error: at least one matrix size (n_1) required for generated types.\n"; return 1; } - for (int i = 12; i < argc; ++i) + for (int i = 13; i < argc; ++i) n_sizes.push_back(std::stol(argv[i])); } @@ -456,15 +619,6 @@ int main(int argc, char* argv[]) { auto state = RandBLAS::RNGState(); auto state_constant = state; - int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); - int64_t k_max = std::max((int64_t)1, (int64_t)(n_max / k_ratio)); - int64_t k_mat_max = from_file ? 0 - : std::max((int64_t)1, (int64_t)(n_max / k_mat_ratio)); - int64_t s_max = std::max((int64_t)1, (int64_t)(k_max * s_ratio)); - - FunNystromPP_benchmark_data all_data( - n_max, k_max, k_mat_max, s_max, d_steps, from_file); - std::string output_filename = "_FunNystromPP_benchmark_num_info_lines_" + std::to_string(8) + ".txt"; std::string path; @@ -473,48 +627,71 @@ int main(int argc, char* argv[]) { else path = output_filename; + const std::string mat_construction_str = + is_mtx ? "loaded from .mtx (sparse; both triangles expanded)" : + from_file ? "loaded from .txt (dense)" : + mat_type >= 3 ? "gen_kernel_matrix (random data points x_i~N(0,I_d); cost not timed)" : + "gen_sym_psd_lowrank (Haar-random eigenvectors; cost not timed)"; + std::ofstream file(path, std::ios::out | std::ios::app); file << "Description: FunNystromPP vs Hutchinson+LanczosFA benchmark for tr(f(A))." - "\nFile format: 16 columns: n, k, k_mat, s, d, matvec_pp, matvec_hutch," - " time_pp_total (us), time_pp_phase1 (us), time_pp_phase2 (us)," - " time_hutch (us), est_fun_pp, est_hutch," - " true_tr (-1 if unavailable), err_fun_pp, err_hutch;" + "\nFile format: 32 columns: n, k, k_mat, s, d, matvec_pp, matvec_hutch," + " time_pp_total (us), time_pp_phase1 (us), time_pp_phase2 (us), time_hutch (us)," + " nystrom_alloc, nystrom_syrf, nystrom_matvec, nystrom_gram," + " nystrom_potrf, nystrom_trsm, nystrom_svd, nystrom_post_svd," + " nystrom_error_est, nystrom_rest, nystrom_total," + " lfa_matvec, lfa_run_lanczos, lfa_apply_f, lfa_rest, lfa_total (all us)," + " est_fun_pp, est_hutch, true_tr (-1 if unavailable), err_fun_pp, err_hutch;" " rows = numruns repetitions per matrix size." "\nNum OMP threads: " + std::to_string(RandLAPACK::util::get_omp_threads()) + "\nInput sizes: " + sizes_str + - "\nParameters: k_ratio=" + std::string(argv[3]) + - " k_mat_ratio=" + std::string(argv[4]) + - " s_ratio=" + std::string(argv[5]) + + "\nParameters: k=" + std::string(argv[3]) + + " k_mat=" + std::string(argv[4]) + + " s=" + std::string(argv[5]) + " d=" + std::to_string(d_steps) + " mat=" + std::string(mat_str) + " func=" + func_str + (func_type == 2 ? (" poly_lambda=" + std::to_string(poly_lambda)) : std::string("")) + " sketch=" + sketch_str + + (sketch_type == 1 ? (" vec_nnz=" + std::to_string(vec_nnz)) : std::string("")) + "\nNum runs per size: " + std::to_string(numruns) + - "\nMatrix construction: " + (from_file ? "loaded from file" : - mat_type >= 3 ? "gen_kernel_matrix (random data points x_i~N(0,I_d); cost not timed)" : - "gen_sym_psd_lowrank (Haar-random eigenvectors; cost not timed)") + + "\nMatrix construction: " + mat_construction_str + "\n"; file.flush(); auto start_all = steady_clock::now(); - for (int64_t n : n_sizes) { - int64_t k = std::max((int64_t)1, (int64_t)(n / k_ratio)); - int64_t k_mat = from_file ? 0 - : std::max((int64_t)1, (int64_t)(n / k_mat_ratio)); - int64_t s = std::max((int64_t)1, (int64_t)(k * s_ratio)); - all_data.n = n; - all_data.k = k; - all_data.k_mat = k_mat; - all_data.s = s; - all_data.d = d_steps; - - auto state_gen = state_constant; - data_regen(all_data, state_gen, mat_type, func_type, mat_file_path); - - call_all_algs(numruns, all_data, state_constant, mat_type, - func_type, poly_lambda, compute_ref, sketch_type, path); + + if (is_mtx) { + // Sparse path: load once, run call_all_algs_sparse. + printf("Loading sparse matrix from %s ...\n", mat_file_path.c_str()); + int64_t n = n_sizes[0]; + auto csr = FunNystromPP_bench::load_mtx(mat_file_path, n); + printf(" Loaded: n=%lld nnz=%lld\n", (long long)n, (long long)csr.nnz); + call_all_algs_sparse(numruns, n, k_const, s_const, d_steps, + csr, state_constant, + func_type, poly_lambda, sketch_type, vec_nnz, path); + } else { + // Dense path: allocate data struct and iterate over n_sizes. + int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); + int64_t k_mat_max = from_file ? 0 : k_mat_const; + FunNystromPP_benchmark_data all_data( + n_max, k_const, k_mat_max, s_const, d_steps, from_file); + + for (int64_t n : n_sizes) { + all_data.n = n; + all_data.k = k_const; + all_data.k_mat = from_file ? 0 : std::min(k_mat_const, n); + all_data.s = s_const; + all_data.d = d_steps; + + auto state_gen = state_constant; + data_regen(all_data, state_gen, mat_type, func_type, mat_file_path); + + call_all_algs(numruns, all_data, state_constant, mat_type, + func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, path); + } } + auto stop_all = steady_clock::now(); long dur_all = duration_cast(stop_all - start_all).count(); file << "Total benchmark execution time: " + std::to_string(dur_all) + "\n"; diff --git a/benchmark/bench_FunNystromPP/load_mtx.hh b/benchmark/bench_FunNystromPP/load_mtx.hh new file mode 100644 index 000000000..0493c3c91 --- /dev/null +++ b/benchmark/bench_FunNystromPP/load_mtx.hh @@ -0,0 +1,113 @@ +#pragma once +// Minimal Matrix Market (.mtx) reader. +// Supports: coordinate real/integer, symmetric or general. +// Returns a CSRMatrix with both triangles stored (required by SparseSymLinOp). + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FunNystromPP_bench { + +inline RandBLAS::sparse_data::CSRMatrix +load_mtx(const std::string& filename, int64_t& out_n) { + std::ifstream f(filename); + if (!f.is_open()) + throw std::runtime_error("load_mtx: cannot open " + filename); + + // Parse banner line. + std::string banner; + std::getline(f, banner); + std::string blo = banner; + for (char& c : blo) c = (char)std::tolower((unsigned char)c); + if (blo.find("%%matrixmarket") == std::string::npos) + throw std::runtime_error("load_mtx: missing %%MatrixMarket banner"); + if (blo.find("coordinate") == std::string::npos) + throw std::runtime_error("load_mtx: only coordinate format supported"); + if (blo.find("real") == std::string::npos && + blo.find("integer") == std::string::npos && + blo.find("double") == std::string::npos) + throw std::runtime_error("load_mtx: only real/integer/double field type supported"); + bool is_symm = blo.find("symmetric") != std::string::npos; + bool is_skew = blo.find("skew") != std::string::npos; + + // Skip comment lines; find the size line. + std::string line; + while (std::getline(f, line)) { + std::string lo = line; + for (char& c : lo) c = (char)std::tolower((unsigned char)c); + if (lo.find("%%matrixmarket") != std::string::npos) continue; + if (!line.empty() && line[0] != '%') break; + } + + int64_t nrows, ncols, nnz_file; + { + std::istringstream ss(line); + ss >> nrows >> ncols >> nnz_file; + } + if (nrows != ncols) + throw std::runtime_error("load_mtx: matrix must be square (got " + + std::to_string(nrows) + "x" + std::to_string(ncols) + ")"); + out_n = nrows; + + // Read COO entries; expand symmetric/skew-symmetric off-diagonal entries. + std::vector ri, ci; + std::vector vi; + ri.reserve(is_symm ? 2*nnz_file : nnz_file); + ci.reserve(is_symm ? 2*nnz_file : nnz_file); + vi.reserve(is_symm ? 2*nnz_file : nnz_file); + + while (std::getline(f, line)) { + if (line.empty() || line[0] == '%') continue; + std::istringstream ls(line); + int64_t r, c; double v; + if (!(ls >> r >> c >> v)) continue; + --r; --c; // .mtx is 1-indexed; convert to 0-indexed + ri.push_back(r); ci.push_back(c); vi.push_back(v); + if ((is_symm || is_skew) && r != c) { + ri.push_back(c); ci.push_back(r); + vi.push_back(is_skew ? -v : v); + } + } + int64_t total_nnz = (int64_t)vi.size(); + + // Sort COO by (row, col) to build CSR. + std::vector ord(total_nnz); + std::iota(ord.begin(), ord.end(), (int64_t)0); + std::sort(ord.begin(), ord.end(), [&](int64_t a, int64_t b) { + return ri[a] != ri[b] ? ri[a] < ri[b] : ci[a] < ci[b]; + }); + + // Build owning CSRMatrix. + using CSR = RandBLAS::sparse_data::CSRMatrix; + CSR csr(nrows, ncols); + csr.reserve(total_nnz); // allocates vals[total_nnz], colidxs[total_nnz], rowptr[nrows+1]; sets nnz + + // First pass: count entries per row into rowptr[1..nrows]. + std::fill(csr.rowptr, csr.rowptr + nrows + 1, (int64_t)0); + for (int64_t k = 0; k < total_nnz; ++k) + ++csr.rowptr[ri[ord[k]] + 1]; + // Prefix sum → rowptr[0..nrows] = start of each row. + for (int64_t i = 0; i < nrows; ++i) + csr.rowptr[i+1] += csr.rowptr[i]; + + // Second pass: fill colidxs and vals; temporarily advance rowptr[row] to track position. + std::vector pos(csr.rowptr, csr.rowptr + nrows); // copy of row starts + for (int64_t k = 0; k < total_nnz; ++k) { + int64_t row = ri[ord[k]]; + int64_t p = pos[row]++; + csr.colidxs[p] = ci[ord[k]]; + csr.vals[p] = vi[ord[k]]; + } + + return csr; +} + +} // namespace FunNystromPP_bench diff --git a/test/drivers/test_nystrom_evd.cc b/test/drivers/test_nystrom_evd.cc index 81f501939..269c87459 100644 --- a/test/drivers/test_nystrom_evd.cc +++ b/test/drivers/test_nystrom_evd.cc @@ -173,7 +173,7 @@ class TestNystromEVD : public ::testing::Test T norm_0 = lapack::lange(Norm::Fro, m, m, A_cpy_dat, m); printf("||A - VEV'||_F / ||A||_F: %e\n", norm_0 / norm_A); - ASSERT_NEAR(norm_0 / norm_A, err_expectation, 10 * err_expectation); + ASSERT_LT(norm_0 / norm_A, 10 * err_expectation); ASSERT_EQ(k, rank_expectation); } From dec0d462c30590739f251cb58794b3226ff9b611 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Wed, 6 May 2026 12:53:28 -0400 Subject: [PATCH 12/14] Adding true block lanchosz & minor updates --- RandLAPACK.hh | 1 + RandLAPACK/comps/rl_lanczos_fa_block.hh | 269 ++++++++++++++++++ RandLAPACK/linops/rl_matfun_linops.hh | 8 + .../FunNystromPP_benchmark.cc | 227 ++++++++------- install.sh | 3 +- test/comps/test_lanczos_fa.cc | 118 ++++++++ test/drivers/test_fun_nystrom_pp.cc | 47 +++ 7 files changed, 566 insertions(+), 107 deletions(-) create mode 100644 RandLAPACK/comps/rl_lanczos_fa_block.hh diff --git a/RandLAPACK.hh b/RandLAPACK.hh index 40edf3755..8a0ddb33b 100644 --- a/RandLAPACK.hh +++ b/RandLAPACK.hh @@ -29,6 +29,7 @@ #include "RandLAPACK/comps/rl_orth.hh" #include "RandLAPACK/comps/rl_rpchol.hh" #include "RandLAPACK/comps/rl_lanczos_fa.hh" +#include "RandLAPACK/comps/rl_lanczos_fa_block.hh" #include "RandLAPACK/comps/rl_hutchinson.hh" // Drivers diff --git a/RandLAPACK/comps/rl_lanczos_fa_block.hh b/RandLAPACK/comps/rl_lanczos_fa_block.hh new file mode 100644 index 000000000..5524563ec --- /dev/null +++ b/RandLAPACK/comps/rl_lanczos_fa_block.hh @@ -0,0 +1,269 @@ +#pragma once + +#include "rl_blaspp.hh" +#include "rl_lapackpp.hh" +#include "rl_linops.hh" +#include "rl_util.hh" + +#include +#include +#include +#include +#include +#include +#include + +namespace RandLAPACK { + + +/// d-step block Lanczos for matrix function application f(A)B. +/// Builds a single joint block Krylov subspace using BLAS-3 throughout, +/// replacing the s independent scalar Lanczos sequences in LanczosFA. +/// +/// Algorithm reference: T. Chen, "A handbook for matrix-function-based Krylov methods", +/// arXiv:2410.11090 (2024). Algorithm 9.2 (block Lanczos recurrence), +/// Definition 9.6 (block Lanczos-FA). +/// +/// At each recurrence step all updates (alpha, beta, Z) are BLAS-3 (GEMM + GEQRF). +/// apply_f calls a single syevd on the (d*s)×(d*s) block tridiagonal instead of +/// s separate stevd calls of size d. +/// +/// Block Lanczos-FA formula: out ≈ Q_basis * f(T_k) * E₁ * R₀ +/// Q_basis = [Q₀|...|Q_{d-1}] (n×d*s), T_k = d*s×d*s block tridiagonal, +/// E₁ = first d*s×s columns of identity, B = Q₀*R₀ (initial QR). +/// +/// Known limitation (v1): deflation is not implemented. When the block Krylov +/// space fills before d steps (B_step develops near-zero singular values), accuracy +/// degrades but the algorithm does not crash. For problems where d*s approaches the +/// effective rank of A, use a smaller d or the scalar LanczosFA. +/// +/// @tparam T Floating-point scalar type. +/// @tparam RNG Random number generator type (unused; kept for API uniformity with LanczosFA). +template +class BlockLanczosFA { +public: + /// Reorthogonalization control. + /// 1 = full (project each new block Z out of all previous Krylov blocks). + /// 0 = none. + int64_t reorth = 1; + + // Internal buffers — grown with new/delete[], never shrunk between calls. + // Dimension key: n = operator dimension, s = block size, d = Lanczos steps. + // + // K_big: (d+1)*n*s — block Krylov basis + matvec scratch. + // Layout: K_big[step*n*s .. (step+1)*n*s-1] = Q_step (n×s col-major, ld=n). + // First d blocks form Q_basis (n×d*s col-major, ld=n) used by apply_f. + // Block d is scratch for the current-step matvec output. + // R0_buf: s*s — upper triangular factor from initial QR of B. + // tau_buf: n — Householder scalars; reused at each geqrf/orgqr call. + // A_blk: d*s*s — block alphas (s×s symmetric), A_blk[step*s*s] = A_step. + // B_blk: d*s*s — block betas (s×s upper triangular), B_blk[step*s*s] = B_step. + // Only d-1 entries are populated (no beta after the last step); + // size d is allocated to keep indexing uniform. + // workspace: apply_f scratch — T_dense (d*s×d*s) + eig_vals (d*s) + G (d*s×s) + C1 (d*s×s). + T* K_big = nullptr; int64_t K_big_sz = 0; + T* R0_buf = nullptr; int64_t R0_sz = 0; + T* tau_buf = nullptr; int64_t tau_buf_sz = 0; + T* A_blk = nullptr; int64_t A_blk_sz = 0; + T* B_blk = nullptr; int64_t B_blk_sz = 0; + T* workspace = nullptr; int64_t workspace_sz = 0; + + bool timing = false; + std::vector times; + long _t_matvec_us = 0; + + BlockLanczosFA() = default; + BlockLanczosFA(const BlockLanczosFA&) = delete; + BlockLanczosFA& operator=(const BlockLanczosFA&) = delete; + + ~BlockLanczosFA() { + delete[] K_big; delete[] R0_buf; delete[] tau_buf; + delete[] A_blk; delete[] B_blk; delete[] workspace; + } + + // ------------------------------------------------------------------ + /// Run the d-step block Lanczos recurrence on B (n×s col-major). + /// Fills K_big, R0_buf, A_blk, B_blk. + /// Calls A exactly d times, each applied to an n×s block. + template + void run_lanczos(SLO& A, const T* B, int64_t n, int64_t s, int64_t d) { + using namespace std::chrono; + steady_clock::time_point _mv_t0, _mv_t1; + _t_matvec_us = 0; + + util::resize(K_big, K_big_sz, (d + 1) * n * s); + util::resize(R0_buf, R0_sz, s * s); + util::resize(tau_buf, tau_buf_sz, n); + util::resize(A_blk, A_blk_sz, d * s * s); + util::resize(B_blk, B_blk_sz, d * s * s); + + // Initial QR: B = Q0 * R0. Q0 overwrites K_big[0..n*s-1]. + T* Q0 = K_big; + lapack::lacpy(lapack::MatrixType::General, n, s, B, n, Q0, n); + lapack::geqrf(n, s, Q0, n, tau_buf); + lapack::laset(lapack::MatrixType::General, s, s, (T)0, (T)0, R0_buf, s); + lapack::lacpy(lapack::MatrixType::Upper, s, s, Q0, n, R0_buf, s); + lapack::orgqr(n, s, s, Q0, n, tau_buf); + + for (int64_t step = 0; step < d; ++step) { + T* Q_step = K_big + step * n * s; + T* Q_prev = (step > 0) ? K_big + (step - 1) * n * s : nullptr; + T* Y = K_big + (step + 1) * n * s; // matvec output, then Z in-place + T* A_step = A_blk + step * s * s; + T* B_prev = (step > 0) ? B_blk + (step - 1) * s * s : nullptr; + + // Y = A * Q_step + if (this->timing) _mv_t0 = steady_clock::now(); + A(Layout::ColMajor, s, (T)1.0, Q_step, n, (T)0.0, Y, n); + if (this->timing) { _mv_t1 = steady_clock::now(); _t_matvec_us += duration_cast(_mv_t1 - _mv_t0).count(); } + + // Y -= Q_{step-1} * B_{step-1}^T + if (step > 0) + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::Trans, + n, s, s, (T)-1.0, Q_prev, n, B_prev, s, (T)1.0, Y, n); + + // A_step = Q_step^T * Y (block alpha, s×s) + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, + s, s, n, (T)1.0, Q_step, n, Y, n, (T)0.0, A_step, s); + for (int64_t j = 0; j < s; ++j) + for (int64_t i = j + 1; i < s; ++i) + A_step[j * s + i] = A_step[i * s + j] = + (T)0.5 * (A_step[j * s + i] + A_step[i * s + j]); + + // Y -= Q_step * A_step (Z = Y - Q_step*A_step, in-place) + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + n, s, s, (T)-1.0, Q_step, n, A_step, s, (T)1.0, Y, n); + + // Optional full reorthogonalization: Z -= Q_p * (Q_p^T * Z) for each prev block + if (reorth) { + std::vector proj(s * s); + for (int64_t prev = 0; prev <= step; ++prev) { + T* Q_p = K_big + prev * n * s; + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, + s, s, n, (T)1.0, Q_p, n, Y, n, (T)0.0, proj.data(), s); + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + n, s, s, (T)-1.0, Q_p, n, proj.data(), s, (T)1.0, Y, n); + } + } + + // QR of Z → Q_{step+1} (in Y) and B_step (upper R factor). + // Skipped at the last step: Q_d is never needed by apply_f. + if (step < d - 1) { + T* B_step = B_blk + step * s * s; + lapack::geqrf(n, s, Y, n, tau_buf); + lapack::laset(lapack::MatrixType::General, s, s, (T)0, (T)0, B_step, s); + lapack::lacpy(lapack::MatrixType::Upper, s, s, Y, n, B_step, s); + lapack::orgqr(n, s, s, Y, n, tau_buf); + } + } + } + + // ------------------------------------------------------------------ + /// Evaluate f(A)B from precomputed Krylov data (K_big, R0_buf, A_blk, B_blk). + /// + /// Computation: + /// 1. Assemble T_dense (d*s × d*s block tridiagonal) from A_blk and B_blk. + /// 2. syevd: T_dense → eigenvectors V (in-place), eigenvalues λ. + /// 3. W (s×m): W[i,j] = f(λⱼ)*V[i,j] for i=0..s-1 (first s rows of V, col-scaled). + /// 4. C1 (d*s × s) = V * W^T — this equals f(T_k) * E₁. + /// 5. C1 *= R₀ (TRMM: right-multiply by upper-triangular R₀). + /// 6. out (n × s) = Q_basis * C1 (GEMM). + template F> + void apply_f(F f, int64_t n, int64_t s, int64_t d, T* out) { + int64_t m = d * s; + util::resize(workspace, workspace_sz, m * m + m + 2 * m * s); + + T* T_dense = workspace; + T* eig_vals = T_dense + m * m; + T* G = eig_vals + m; + T* C1 = G + m * s; + + // 1. Assemble T_dense. + std::memset(T_dense, 0, m * m * sizeof(T)); + for (int64_t step = 0; step < d; ++step) { + int64_t b0 = step * s; + // Diagonal block + lapack::lacpy(lapack::MatrixType::General, s, s, + A_blk + step * s * s, s, T_dense + b0 * m + b0, m); + // Off-diagonal blocks (B_step in lower, B_step^T in upper) + if (step < d - 1) { + T* B_step = B_blk + step * s * s; + int64_t b1 = (step + 1) * s; + // Lower off-diagonal: T(b1:b1+s, b0:b0+s) = B_step + lapack::lacpy(lapack::MatrixType::General, s, s, + B_step, s, T_dense + b0 * m + b1, m); + // Upper off-diagonal: T(b0:b0+s, b1:b1+s) = B_step^T + for (int64_t j = 0; j < s; ++j) + for (int64_t i = 0; i < s; ++i) + T_dense[(b1 + j) * m + (b0 + i)] = B_step[i * s + j]; + } + } + // Symmetrize to eliminate any floating-point asymmetry + for (int64_t j = 0; j < m; ++j) + for (int64_t i = j + 1; i < m; ++i) { + T avg = (T)0.5 * (T_dense[j * m + i] + T_dense[i * m + j]); + T_dense[j * m + i] = T_dense[i * m + j] = avg; + } + + // 2. Eigendecomposition: T_dense → V (eigenvectors overwrite T_dense), eig_vals → λ. + lapack::syevd(lapack::Job::Vec, blas::Uplo::Lower, m, T_dense, m, eig_vals); + + // 3. W (s×m col-major, ld=s; stored in the G buffer): W[i,j] = f(λⱼ)*V[i,j] + // for i=0..s-1, j=0..m-1. Each column j of W is the first s elements of + // column j of V, scaled by f(λⱼ). + // + // Derivation: f(T_k)*E₁ = V*diag(f(λ))*V^T*E₁. + // V^T*E₁ = first s columns of V^T = {row i of V} for i=0..s-1 stacked as cols. + // diag(f(λ))*(V^T*E₁): scale row j → f(λⱼ)*(row j of V^T*E₁) = f(λⱼ)*V[:,j][0:s]. + // That intermediate is W^T (m×s), so f(T_k)*E₁ = V * W^T. + T* W = G; // reuse G buffer; W is s×m col-major (ld=s) + for (int64_t j = 0; j < m; ++j) { + T fev = f(std::max(eig_vals[j], (T)0)); + const T* V_col = T_dense + j * m; // col j of V (contiguous, length m) + T* W_col = W + j * s; // col j of W (contiguous, length s) + for (int64_t i = 0; i < s; ++i) + W_col[i] = fev * V_col[i]; // first s rows of V[:,j], scaled + } + + // 4. C1 (m × s) = V * W^T — equals f(T_k) * E₁. + // GEMM(NoTrans, Trans, m, s, m): C = V(m×m) * W^T where W is s×m (ld=s). + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::Trans, + m, s, m, (T)1.0, T_dense, m, W, s, (T)0.0, C1, m); + + // 5. C1 *= R₀ (TRMM: C1 = C1 * R₀, right upper triangular). + blas::trmm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, + m, s, (T)1.0, R0_buf, s, C1, m); + + // 6. out (n × s) = Q_basis (n × m) * C1 (m × s). + // Q_basis = K_big[0..m*n-1] (n×m col-major, ld=n). + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, + n, s, m, (T)1.0, K_big, n, C1, m, (T)0.0, out, n); + } + + // ------------------------------------------------------------------ + /// Combined run + apply. + /// + /// Drop-in replacement for LanczosFA::call — same signature, slots into + /// FunNystromPP and ResidualOp as the LanczosFA_t template parameter. + template F> + void call(SLO& A, const T* B, int64_t n, int64_t s, F f, int64_t d, T* out) { + using namespace std::chrono; + _t_matvec_us = 0; + steady_clock::time_point t_total_start, t_lanczos_end, t_end; + if (this->timing) t_total_start = steady_clock::now(); + run_lanczos(A, B, n, s, d); + if (this->timing) t_lanczos_end = steady_clock::now(); + apply_f(f, n, s, d, out); + if (this->timing) { + t_end = steady_clock::now(); + long total_us = duration_cast(t_end - t_total_start).count(); + long lanczos_us = duration_cast(t_lanczos_end - t_total_start).count(); + long apply_f_us = duration_cast(t_end - t_lanczos_end).count(); + long rest_us = total_us - lanczos_us - apply_f_us; + this->times = {_t_matvec_us, lanczos_us, apply_f_us, rest_us, total_us}; + } + } +}; + + +} // end namespace RandLAPACK diff --git a/RandLAPACK/linops/rl_matfun_linops.hh b/RandLAPACK/linops/rl_matfun_linops.hh index 0f562da01..13e70ede3 100644 --- a/RandLAPACK/linops/rl_matfun_linops.hh +++ b/RandLAPACK/linops/rl_matfun_linops.hh @@ -22,6 +22,14 @@ namespace RandLAPACK::linops { ///  = V diag(λ) V^T is the Nyström approximation; V (n×k) and F_vec (k, /// holding f(λ)) are provided by the caller. /// +/// Precision note: Z1 and Z2 follow different arithmetic paths even when f(A)=f(Â) +/// exactly (i.e. k equals the effective rank of A). Z1 uses an iterative Krylov +/// method; Z2 uses two dense GEMMs with the eigenvectors. The per-column disagreement +/// is ~1e-14, producing a systematic Hutchinson bias of roughly 1e-5 relative when +/// Phase 2's true value is zero. This is an inherent floating-point limitation of +/// combining an iterative oracle with a direct GEMM in the same residual; it is not a +/// bug. To reach machine precision at k = effective_rank, skip Phase 2 entirely. +/// /// @tparam T Scalar type. /// @tparam SLO_t Type of the operator A. /// @tparam LFA_t Type of the matrix-function oracle (e.g. LanczosFA). diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc index 58c86b379..7dd2ca139 100644 --- a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -2,16 +2,18 @@ int main() {return 0;} #else /* -FunNystromPP benchmark — compares funNyström++ against plain Hutchinson+LanczosFA +FunNystromPP benchmark -- compares funNystrom++ against plain Hutchinson+LanczosFA for estimating tr(f(A)). Two algorithms are timed and their matvec counts recorded for each matrix size: - 1. FunNystromPP(k, s, d): Nyström rank-k approximation plus Hutchinson correction + 1. FunNystromPP(k, s, d): Nystrom rank-k approximation plus Hutchinson correction with s samples, each using d Lanczos steps. 2. Hutchinson+LanczosFA(s, d): plain Hutchinson with s Rademacher samples, - each using LanczosFA with d steps to apply f(A). No Nyström component. + each using LanczosFA with d steps to apply f(A). No Nystrom component. Uses the same s and d as FunNystromPP (equal correction-phase budget). +Both algorithms use the Lanczos variant selected by lfa_type (see below). + Matrix input (mat_or_file): 1 = psd_alg: A = Q diag(λ) Q', λᵢ = 100 * i^{-2} (algebraic decay, κ ≈ n²/100) 2 = psd_exp: A = Q diag(λ) Q', λᵢ = exp(-i/100) (exponential decay) @@ -29,9 +31,13 @@ Matrix input (mat_or_file): Function types (func_type): 0 = sqrt(x) [f_zero = 0; safe for SPSD matrices] 1 = log(x) [requires all eigenvalues > 0; for generated types, noise=1 is applied - automatically (A = lowrank + I), so all eigenvalues ≥ 1] + automatically (A = lowrank + I), so all eigenvalues >= 1] 2 = x*(x + poly_lambda) [degree-2 polynomial; exact in d=2 Lanczos steps; use for tr(K(K+λI))] +Lanczos variant (lfa_type): + 0 = scalar LanczosFA (s independent Krylov sequences; BLAS-1/2 dominant) + 1 = BlockLanczosFA (single joint block Krylov subspace; BLAS-3 dominant) + Sketch types (sketch_type): 0 = Gaussian (DenseDist, default) 1 = SASO (SparseDist; nonzeros per column set by vec_nnz argument) @@ -42,6 +48,10 @@ Sketch types (sketch_type): For types 1/2 this flag is ignored; the reference is always available from eigenvalues. For types 3/4 and file input, syevd is run only when compute_ref=1. +precision: + double = run in double precision (default) + float = run in single precision (sparse path is always double regardless) + Both algorithms use a fixed seed so comparisons are reproducible. Matvec counts reflect actual A applications inside each algorithm (not estimates). The matrix is constructed/loaded once per n; that cost is not timed. @@ -58,13 +68,15 @@ Output format: 32 comma-separated columns per row (trailing comma): est_fun_pp, est_hutch, true_tr (-1 if unavailable), err_fun_pp, err_hutch + lfa_type is encoded in the output filename: _scalar_ or _block_. + Usage: FunNystromPP_benchmark - [n_1 n_2 ...] + [n_1 n_2 ...] dir_path : output directory (use "." for current directory) num_runs : number of timed repetitions per matrix size - k : Nyström rank (constant; must satisfy k <= n) + k : Nystrom rank (constant; must satisfy k <= n) k_mat : matrix rank for generated types (constant; for kernel types equals data dim d; ignored for file input) s : Hutchinson sample count (constant) @@ -75,6 +87,8 @@ Output format: 32 comma-separated columns per row (trailing comma): sketch_type : 0=Gaussian, 1=SASO vec_nnz : nonzeros per column for SASO sketch (sketch_type=1; ignored for Gaussian) poly_lambda : lambda parameter for func_type=2 (ignored for func_type=0,1) + precision : double or float (sparse .mtx path is always double) + lfa_type : 0=scalar LanczosFA, 1=BlockLanczosFA n_1 ... : matrix dimensions (generated types only; ignored for file input) */ @@ -106,16 +120,16 @@ using std::chrono::microseconds; template struct FunNystromPP_benchmark_data { - int64_t n; // current matrix dimension - int64_t k; // Nyström rank for algorithms - int64_t k_mat; // rank used to construct A (0 for file input) - int64_t s; // Hutchinson samples - int64_t d; // Lanczos steps per sample - bool from_file; // true when A was loaded from a file - T noise; // diagonal shift (noise=1 for func_type=1 on lowrank types; else 0) - int64_t d_dim; // data dimension for kernel types (mat_type=3/4); 0 otherwise - std::vector A; // n_max × n_max storage (upper triangle; rest zero) - std::vector eigvals; // k_mat eigenvalues of the low-rank component (pre-shift; empty for kernel/file types) + int64_t n; + int64_t k; + int64_t k_mat; + int64_t s; + int64_t d; + bool from_file; + T noise; + int64_t d_dim; + std::vector A; + std::vector eigvals; FunNystromPP_benchmark_data(int64_t n_max, int64_t k_, int64_t k_mat_, int64_t s_, int64_t d_, bool from_file_) : @@ -126,7 +140,6 @@ struct FunNystromPP_benchmark_data { // ============================================================================ // CountingLinOp: wraps any SLO and counts individual matrix-vector products. -// Satisfies the SymmetricLinearOperator concept. // ============================================================================ template @@ -149,8 +162,6 @@ struct CountingLinOp { // ============================================================================ // data_regen: build or load A for the current n. -// Generated (mat_type 1/2): fills data.eigvals and the upper triangle of data.A. -// File input: reads the full matrix into data.A via process_input_mat. // ============================================================================ template @@ -165,7 +176,6 @@ static void data_regen( std::fill(data.A.begin(), data.A.begin() + n * n, (T)0.0); if (data.from_file) { - // Two-pass: dimensions already determined in main; just load data. int64_t m = n, ncols = n; int wq = 0; RandLAPACK::gen::process_input_mat( @@ -175,7 +185,6 @@ static void data_regen( data.noise = (T)0.0; data.d_dim = 0; } else if (mat_type >= 3) { - // Kernel matrix: k_mat reinterpreted as data dimension d_dim = n / k_mat_ratio. data.d_dim = data.k_mat; data.noise = (T)0.0; data.eigvals.clear(); @@ -183,7 +192,7 @@ static void data_regen( if (mat_type == 3) RandLAPACK::gen::gen_kernel_matrix( n, data.d_dim, data.A.data(), n, 0, bandwidth, (T)0.0, 0, state); - else // mat_type == 4 + else RandLAPACK::gen::gen_kernel_matrix( n, data.d_dim, data.A.data(), n, 1, (T)0.0, (T)1.0, 2, state); } else { @@ -196,9 +205,6 @@ static void data_regen( RandLAPACK::gen::gen_exp_decay_singvals(data.k_mat, (T)1.0, (T)100.0, data.eigvals.data()); if (func_type == 1) { - // log requires strict PD; shift by 1 so all eigenvalues >= 1. - // Models A = I + K (log det(I+K) use case). f(noise) = log(1) = 0, - // so f_zero=0 remains correct for the Nyström tail approximation. data.noise = (T)1.0; RandLAPACK::gen::gen_shifted_lowrank_psd(n, data.k_mat, data.A.data(), n, data.eigvals.data(), data.noise, state); @@ -212,7 +218,6 @@ static void data_regen( // ============================================================================ // LFAOp: wraps LanczosFA as a SymmetricLinearOperator for plain Hutchinson. -// Computes C = alpha * f(A)*B + beta*C using lfa.call internally. // ============================================================================ template @@ -246,10 +251,9 @@ struct LFAOp { // ============================================================================ // call_all_algs: timed + matvec-counted comparison for one matrix size. -// data.A and (for generated types) data.eigvals must already be filled. // ============================================================================ -template +template static void call_all_algs( int64_t numruns, FunNystromPP_benchmark_data& data, @@ -264,37 +268,29 @@ static void call_all_algs( ) { int64_t n = data.n, k = data.k, s = data.s, d = data.d; - std::function f; + std::function f; const char* fname; - double f_zero = 0.0; + T f_zero = (T)0; if (func_type == 0) { - f = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + f = [](T x){ return (T)std::sqrt((double)std::max(x, (T)0)); }; fname = "sqrt"; - f_zero = 0.0; } else if (func_type == 1) { - f = [](double x){ return std::log(x); }; + f = [](T x){ return (T)std::log((double)x); }; fname = "log"; - f_zero = 0.0; // placeholder; caller must ensure λ_min > 0 } else { - double lam = poly_lambda; - f = [lam](double x){ return x * (x + lam); }; + T lam = (T)poly_lambda; + f = [lam](T x){ return x * (x + lam); }; fname = "poly"; - f_zero = 0.0; } - // Reference: exact formula for low-rank types; syevd for kernel/file types. bool ref_available = false; - double true_tr = 0.0; + T true_tr = (T)0; if (!data.from_file && mat_type < 3) { - // Low-rank types: exact reference from eigenvalues. - // Actual eigenvalues: eigvals[j] + noise (dominant k_mat) and noise (tail). - // f(noise) = 0 for all func_types: sqrt(0)=0, log(1)=0, 0*(0+lam)=0. for (int64_t j = 0; j < data.k_mat; ++j) true_tr += f(data.eigvals[j] + data.noise); true_tr += (T)(n - data.k_mat) * f(data.noise); ref_available = true; } else if (compute_ref) { - // Kernel types and file input: compute reference via full eigendecomposition. std::vector A_copy(data.A.begin(), data.A.begin() + n * n); std::vector ev(n); lapack::syevd(lapack::Job::NoVec, lapack::Uplo::Upper, n, @@ -304,27 +300,25 @@ static void call_all_algs( ref_available = true; } - // Build full algorithm stack for FunNystromPP. RandLAPACK::SYPS syps(3, 1, false, false); syps.sketch_type = sketch_type; syps.vec_nnz = vec_nnz; RandLAPACK::HQRQ orth(false, false); RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); - RandLAPACK::NystromEVD nystrom_evd(syrf, 0); + RandLAPACK::NystromEVD nystrom_evd(syrf, 0); nystrom_evd.timing = true; - RandLAPACK::LanczosFA lfa_pp; + LFA_t lfa_pp; lfa_pp.timing = true; RandLAPACK::Hutchinson hutch_pp; - RandLAPACK::FunNystromPP + RandLAPACK::FunNystromPP driver(nystrom_evd, lfa_pp, hutch_pp); - // Separate LanczosFA + Hutchinson for plain baseline. - RandLAPACK::LanczosFA lfa_base; + LFA_t lfa_base; RandLAPACK::Hutchinson hutch_base; using ESLO = linops::ExplicitSymLinOp; using CSLO = CountingLinOp; - using FuncT = std::function; + using FuncT = std::function; const char* mat_str = data.from_file ? "file" : mat_type == 1 ? "psd_alg" : @@ -338,24 +332,22 @@ static void call_all_algs( auto state_alg = state; - // ---- FunNystromPP -------------------------------------------------- + // FunNystromPP ESLO A_op(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); CSLO A_pp(n, A_op); RandLAPACK::FunNystromPP_timing pp_timing; auto start_pp = steady_clock::now(); - double est_pp = driver.call(A_pp, f, (T)f_zero, k, s, d, state_alg, &pp_timing); + T est_pp = driver.call(A_pp, f, f_zero, k, s, d, state_alg, &pp_timing); auto stop_pp = steady_clock::now(); long dur_pp = duration_cast(stop_pp - start_pp).count(); int64_t mv_pp = A_pp.count; - // NystromEVD sub-phase times (slots: alloc, syrf, matvec, gram, potrf, trsm, svd, post_svd, error_est, rest, total) auto& nt = nystrom_evd.times; - // LanczosFA sub-phase times (slots: matvec, run_lanczos, apply_f, rest, total) auto& lt = lfa_pp.times; printf(" FunNystromPP: est=%.6e time=%lld us (ph1=%lld ph2=%lld) matvecs=%lld\n", - est_pp, (long long)dur_pp, + (double)est_pp, (long long)dur_pp, (long long)pp_timing.phase1_us, (long long)pp_timing.phase2_us, (long long)mv_pp); printf(" NystromEVD: alloc=%lld syrf=%lld matvec=%lld gram=%lld potrf=%lld trsm=%lld svd=%lld post_svd=%lld err_est=%lld rest=%lld total=%lld\n", @@ -365,29 +357,28 @@ static void call_all_algs( printf(" LanczosFA: matvec=%lld lanczos=%lld apply_f=%lld total=%lld\n", (long long)lt[0], (long long)lt[1], (long long)lt[2], (long long)lt[4]); - // ---- Hutchinson + LanczosFA (no Nyström) --------------------------- - state_alg = state; + // Hutchinson + LanczosFA (state_alg continues from where FunNystromPP left it) ESLO A_op2(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); CSLO A_hutch(n, A_op2); - LFAOp, FuncT> lfa_op(n, A_hutch, lfa_base, f, d); + LFAOp lfa_op(n, A_hutch, lfa_base, f, d); auto start_h = steady_clock::now(); - double est_h = hutch_base.call(lfa_op, s, state_alg); + T est_h = hutch_base.call(lfa_op, s, state_alg); auto stop_h = steady_clock::now(); long dur_h = duration_cast(stop_h - start_h).count(); int64_t mv_h = A_hutch.count; printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", - est_h, (long long)dur_h, (long long)mv_h); + (double)est_h, (long long)dur_h, (long long)mv_h); - double err_pp = ref_available ? std::abs(est_pp - true_tr) / std::abs(true_tr) : -1.0; - double err_h = ref_available ? std::abs(est_h - true_tr) / std::abs(true_tr) : -1.0; + double err_pp = ref_available ? std::abs((double)est_pp - (double)true_tr) / std::abs((double)true_tr) : -1.0; + double err_h = ref_available ? std::abs((double)est_h - (double)true_tr) / std::abs((double)true_tr) : -1.0; if (ref_available) printf(" Reference: true=%.6e err_pp=%e err_h=%e\n", - true_tr, err_pp, err_h); + (double)true_tr, err_pp, err_h); else printf(" Reference: N/A\n"); - double out_tr = ref_available ? true_tr : -1.0; + double out_tr = ref_available ? (double)true_tr : -1.0; std::ofstream file(output_filename, std::ios::out | std::ios::app); file << n << ", " << k << ", " << data.k_mat << ", " << s << ", " << d << ", " @@ -398,17 +389,16 @@ static void call_all_algs( << nt[4] << ", " << nt[5] << ", " << nt[6] << ", " << nt[7] << ", " << nt[8] << ", " << nt[9] << ", " << nt[10] << ", " << lt[0] << ", " << lt[1] << ", " << lt[2] << ", " << lt[3] << ", " << lt[4] << ", " - << est_pp << ", " << est_h << ", " + << (double)est_pp << ", " << (double)est_h << ", " << out_tr << ", " << err_pp << ", " << err_h << ",\n"; } } // ============================================================================ -// call_all_algs_sparse: same as call_all_algs but backed by a sparse CSR matrix. -// No reference computation (syevd on large sparse matrices is impractical). +// call_all_algs_sparse: sparse path (always double). // ============================================================================ -template +template static void call_all_algs_sparse( int64_t numruns, int64_t n, @@ -445,13 +435,13 @@ static void call_all_algs_sparse( RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); RandLAPACK::NystromEVD nystrom_evd(syrf, 0); nystrom_evd.timing = true; - RandLAPACK::LanczosFA lfa_pp; + LFA_t lfa_pp; lfa_pp.timing = true; RandLAPACK::Hutchinson hutch_pp; - RandLAPACK::FunNystromPP + RandLAPACK::FunNystromPP driver(nystrom_evd, lfa_pp, hutch_pp); - RandLAPACK::LanczosFA lfa_base; + LFA_t lfa_base; RandLAPACK::Hutchinson hutch_base; using SSLO = linops::SparseSymLinOp>; @@ -464,7 +454,6 @@ static void call_all_algs_sparse( auto state_alg = state; - // ---- FunNystromPP -------------------------------------------------- SSLO A_op_pp(n, csr); CSLO A_pp(n, A_op_pp); @@ -489,11 +478,10 @@ static void call_all_algs_sparse( printf(" LanczosFA: matvec=%lld lanczos=%lld apply_f=%lld total=%lld\n", (long long)lt[0], (long long)lt[1], (long long)lt[2], (long long)lt[4]); - // ---- Hutchinson + LanczosFA ---------------------------------------- state_alg = state; SSLO A_op_h(n, csr); CSLO A_hutch(n, A_op_h); - LFAOp, FuncT> lfa_op(n, A_hutch, lfa_base, f, d); + LFAOp lfa_op(n, A_hutch, lfa_base, f, d); auto start_h = steady_clock::now(); T est_h = hutch_base.call(lfa_op, s, state_alg); @@ -524,11 +512,11 @@ static void call_all_algs_sparse( // ============================================================================ int main(int argc, char* argv[]) { - if (argc < 13) { + if (argc < 15) { std::cerr << "Usage: " << argv[0] << " " " " - " [n_1 n_2 ...]\n" + " [n_1 n_2 ...]\n" << " dir_path : output directory (use '.' for current dir)\n" << " num_runs : timed repetitions per matrix size\n" << " k : Nystrom rank (constant; must satisfy k <= n)\n" @@ -541,6 +529,8 @@ int main(int argc, char* argv[]) { << " sketch_type : 0=Gaussian, 1=SASO\n" << " vec_nnz : nonzeros per column for SASO (sketch_type=1; ignored otherwise)\n" << " poly_lambda : lambda for func_type=2 (ignored otherwise)\n" + << " precision : double or float (sparse .mtx path is always double)\n" + << " lfa_type : 0=scalar LanczosFA, 1=BlockLanczosFA\n" << " n_1 ... : matrix sizes (generated types only)\n"; return 1; } @@ -555,8 +545,11 @@ int main(int argc, char* argv[]) { int sketch_type = std::stoi(argv[10]); int vec_nnz = std::stoi(argv[11]); double poly_lambda = std::stod(argv[12]); + std::string precision_str = argv[13]; + bool use_float = (precision_str == "float"); + int lfa_type = std::stoi(argv[14]); + const char* lfa_str = lfa_type == 0 ? "scalar" : "block"; - // argv[7]: integer mat_type or file path (.txt or .mtx) bool from_file = false; bool is_mtx = false; std::string mat_file_path; @@ -577,13 +570,10 @@ int main(int argc, char* argv[]) { mat_type == 3 ? "rbf_kernel" : "poly_kernel"; const char* sketch_str = sketch_type == 1 ? "SASO" : "Gaussian"; - // Collect n_sizes; for file input determine n from the file. std::vector n_sizes; if (is_mtx) { - // Size comes from the .mtx file itself; no CLI sizes needed. int64_t mtx_n = 0; { - // Peek at n without loading the full matrix. std::ifstream peek(mat_file_path); std::string ln; while (std::getline(peek, ln)) @@ -604,11 +594,11 @@ int main(int argc, char* argv[]) { + std::to_string(fm) + "x" + std::to_string(fn) + ")"); n_sizes = {fm}; } else { - if (argc < 14) { + if (argc < 16) { std::cerr << "Error: at least one matrix size (n_1) required for generated types.\n"; return 1; } - for (int i = 13; i < argc; ++i) + for (int i = 15; i < argc; ++i) n_sizes.push_back(std::stol(argv[i])); } @@ -620,7 +610,7 @@ int main(int argc, char* argv[]) { auto state_constant = state; std::string output_filename = - "_FunNystromPP_benchmark_num_info_lines_" + std::to_string(8) + ".txt"; + "_FunNystromPP_benchmark_" + precision_str + "_" + std::string(lfa_str) + "_num_info_lines_8.txt"; std::string path; if (std::string(argv[1]) != ".") path = std::string(argv[1]) + output_filename; @@ -654,6 +644,8 @@ int main(int argc, char* argv[]) { (func_type == 2 ? (" poly_lambda=" + std::to_string(poly_lambda)) : std::string("")) + " sketch=" + sketch_str + (sketch_type == 1 ? (" vec_nnz=" + std::to_string(vec_nnz)) : std::string("")) + + " precision=" + precision_str + + " lfa_type=" + std::string(lfa_str) + "\nNum runs per size: " + std::to_string(numruns) + "\nMatrix construction: " + mat_construction_str + "\n"; @@ -661,35 +653,58 @@ int main(int argc, char* argv[]) { auto start_all = steady_clock::now(); + using ScalarLFA_d = RandLAPACK::LanczosFA; + using BlockLFA_d = RandLAPACK::BlockLanczosFA; + using ScalarLFA_f = RandLAPACK::LanczosFA; + using BlockLFA_f = RandLAPACK::BlockLanczosFA; + if (is_mtx) { - // Sparse path: load once, run call_all_algs_sparse. + // Sparse path: always double. + if (use_float) + printf("Note: sparse .mtx path runs in double regardless of precision flag.\n"); printf("Loading sparse matrix from %s ...\n", mat_file_path.c_str()); int64_t n = n_sizes[0]; auto csr = FunNystromPP_bench::load_mtx(mat_file_path, n); printf(" Loaded: n=%lld nnz=%lld\n", (long long)n, (long long)csr.nnz); - call_all_algs_sparse(numruns, n, k_const, s_const, d_steps, - csr, state_constant, - func_type, poly_lambda, sketch_type, vec_nnz, path); + if (lfa_type == 0) + call_all_algs_sparse(numruns, n, k_const, s_const, d_steps, + csr, state_constant, + func_type, poly_lambda, sketch_type, vec_nnz, path); + else + call_all_algs_sparse(numruns, n, k_const, s_const, d_steps, + csr, state_constant, + func_type, poly_lambda, sketch_type, vec_nnz, path); } else { - // Dense path: allocate data struct and iterate over n_sizes. - int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); - int64_t k_mat_max = from_file ? 0 : k_mat_const; - FunNystromPP_benchmark_data all_data( - n_max, k_const, k_mat_max, s_const, d_steps, from_file); - - for (int64_t n : n_sizes) { - all_data.n = n; - all_data.k = k_const; - all_data.k_mat = from_file ? 0 : std::min(k_mat_const, n); - all_data.s = s_const; - all_data.d = d_steps; - - auto state_gen = state_constant; - data_regen(all_data, state_gen, mat_type, func_type, mat_file_path); - - call_all_algs(numruns, all_data, state_constant, mat_type, - func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, path); - } + // Dense path: dispatch on precision and lfa_type. + auto run_dense = [&]() { + int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); + int64_t k_mat_max = from_file ? 0 : k_mat_const; + FunNystromPP_benchmark_data all_data( + n_max, k_const, k_mat_max, s_const, d_steps, from_file); + + for (int64_t n : n_sizes) { + all_data.n = n; + all_data.k = k_const; + all_data.k_mat = from_file ? 0 : std::min(k_mat_const, n); + all_data.s = s_const; + all_data.d = d_steps; + + auto state_gen = state_constant; + data_regen(all_data, state_gen, mat_type, func_type, mat_file_path); + + call_all_algs(numruns, all_data, state_constant, mat_type, + func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, path); + } + }; + + if (!use_float && lfa_type == 0) + run_dense.template operator()(); + else if (!use_float && lfa_type == 1) + run_dense.template operator()(); + else if (use_float && lfa_type == 0) + run_dense.template operator()(); + else + run_dense.template operator()(); } auto stop_all = steady_clock::now(); diff --git a/install.sh b/install.sh index 0c0525173..3ad9a2a26 100755 --- a/install.sh +++ b/install.sh @@ -247,6 +247,7 @@ fi cmake -S $RANDNLA_PROJECT_DIR/lib/blaspp/ -B $RANDNLA_PROJECT_DIR/build/blaspp-build/ \ -Dgpu_backend=$RANDNLA_PROJECT_GPU_AVAIL \ -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ -Dblas_int=$BLAS_INT \ -DCMAKE_INSTALL_PREFIX=$RANDNLA_PROJECT_DIR/install/blaspp-install/ \ $MACOS_BLAS_FLAGS $MACOS_OPENMP_FLAGS @@ -262,7 +263,7 @@ fi # Configure, build, and install LAPACK++ # Add "-DBLAS_LIBRARIES='-lflame -lblis'" if using AMD AOCL -cmake -S $RANDNLA_PROJECT_DIR/lib/lapackpp/ -B $RANDNLA_PROJECT_DIR/build/lapackpp-build/ -Dgpu_backend=$RANDNLA_PROJECT_GPU_AVAIL -DCMAKE_BUILD_TYPE=Release -Dblaspp_DIR=$RANDNLA_PROJECT_DIR/install/blaspp-install/$LIB_VAR/cmake/blaspp/ -DCMAKE_INSTALL_PREFIX=$RANDNLA_PROJECT_DIR/install/lapackpp-install -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON $MACOS_LAPACK_FLAGS $MACOS_OPENMP_FLAGS +cmake -S $RANDNLA_PROJECT_DIR/lib/lapackpp/ -B $RANDNLA_PROJECT_DIR/build/lapackpp-build/ -Dgpu_backend=$RANDNLA_PROJECT_GPU_AVAIL -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -Dblaspp_DIR=$RANDNLA_PROJECT_DIR/install/blaspp-install/$LIB_VAR/cmake/blaspp/ -DCMAKE_INSTALL_PREFIX=$RANDNLA_PROJECT_DIR/install/lapackpp-install -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON $MACOS_LAPACK_FLAGS $MACOS_OPENMP_FLAGS make -C $RANDNLA_PROJECT_DIR/build/lapackpp-build/ -j20 install # Configure, build, and install RandLAPACK echo "==========================================" diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index b030f7cd9..908e8b85f 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -264,6 +264,124 @@ TEST_F(TestLanczosFA, ReorthVsVanilla) { } +// --------------------------------------------------------------------------- +// BlockLanczosFA tests: verify that the block algorithm matches the dense +// reference at sufficient depth and agrees with scalar LanczosFA. + +class TestBlockLanczosFA : public TestLanczosFA { +protected: + template + static void run_block_diagonal_fa_test(F f, int64_t n, int64_t s, int64_t d, + double tol, uint64_t seed) { + using RNG = r123::Philox4x32; + auto state = RandBLAS::RNGState(seed); + std::vector diag_vec(n); + std::iota(diag_vec.begin(), diag_vec.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag_vec); + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + RandLAPACK::BlockLanczosFA blfa; + blfa.reorth = 1; + blfa.call(A_op, B.data(), n, s, f, d, out.data()); + auto ref = diag_fa_ref(diag_vec, B, n, s, f); + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("Block ||f(A)B - ref||/||ref|| = %e\n", rel_err); + ASSERT_LT(rel_err, tol); + } + + template + static void run_block_dense_fa_test(F f, int64_t n, int64_t s, int64_t d, + double tol, uint64_t seed) { + using RNG = r123::Philox4x32; + auto state = RandBLAS::RNGState(seed); + std::vector B_raw(n * n); + RandBLAS::DenseDist D1(n, n); + state = RandBLAS::fill_dense(D1, B_raw.data(), state); + std::vector A(n * n, 0.0); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, 1.0, B_raw.data(), n, 0.0, A.data(), n); + for (int64_t i = 0; i < n; ++i) + A[i + i * n] += (double)n; + std::vector B(n * s); + RandBLAS::DenseDist D2(n, s); + state = RandBLAS::fill_dense(D2, B.data(), state); + std::vector out(n * s, 0.0); + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + RandLAPACK::BlockLanczosFA blfa; + blfa.reorth = 1; + blfa.call(A_op, B.data(), n, s, f, d, out.data()); + auto ref = dense_fa_ref(A, B, n, s, f); + double err = 0.0, ref_norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out[i] - ref[i]; + err += r * r; + ref_norm += ref[i] * ref[i]; + } + double rel_err = std::sqrt(err / ref_norm); + printf("Block dense ||f(A)B - ref||/||ref|| = %e\n", rel_err); + ASSERT_LT(rel_err, tol); + } +}; + +// f=sqrt on diag(1,...,50), d=20 < n. +TEST_F(TestBlockLanczosFA, DiagonalSqrt) { + run_block_diagonal_fa_test([](double x){ return std::sqrt(x); }, 50, 5, 20, 1e-4, 42); +} + +// f=sqrt on a random dense PSD matrix, d=n/2 steps. +TEST_F(TestBlockLanczosFA, DensePSDSqrt) { + run_block_dense_fa_test([](double x){ return std::sqrt(std::max(x, 0.0)); }, 50, 4, 25, 1e-6, 31); +} + +// BlockLanczosFA vs LanczosFA on the same problem — outputs should agree at sufficient depth. +TEST_F(TestBlockLanczosFA, AgreeWithScalar) { + using RNG = r123::Philox4x32; + int64_t n = 50, s = 4, d = 25; + auto state = RandBLAS::RNGState(31); + + std::vector B_raw(n * n); + RandBLAS::DenseDist D1(n, n); + state = RandBLAS::fill_dense(D1, B_raw.data(), state); + std::vector A(n * n, 0.0); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, 1.0, B_raw.data(), n, 0.0, A.data(), n); + for (int64_t i = 0; i < n; ++i) + A[i + i * n] += (double)n; + + std::vector B(n * s); + RandBLAS::DenseDist D2(n, s); + state = RandBLAS::fill_dense(D2, B.data(), state); + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + + std::vector out_scalar(n * s, 0.0), out_block(n * s, 0.0); + RandLAPACK::LanczosFA lfa; + lfa.reorth = 1; + lfa.call(A_op, B.data(), n, s, f_sqrt, d, out_scalar.data()); + + RandLAPACK::BlockLanczosFA blfa; + blfa.reorth = 1; + blfa.call(A_op, B.data(), n, s, f_sqrt, d, out_block.data()); + + double diff = 0.0, norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out_block[i] - out_scalar[i]; + diff += r * r; + norm += out_scalar[i] * out_scalar[i]; + } + double rel_diff = std::sqrt(diff / norm); + printf("Block vs scalar relative diff: %e\n", rel_diff); + ASSERT_LT(rel_diff, 1e-4); +} + // --------------------------------------------------------------------------- // Hutchinson tests live here because Hutchinson is a building block of // FunNystromPP alongside LanczosFA, not a standalone algorithm with its own file. diff --git a/test/drivers/test_fun_nystrom_pp.cc b/test/drivers/test_fun_nystrom_pp.cc index 89dc66230..a5b6dd3b0 100644 --- a/test/drivers/test_fun_nystrom_pp.cc +++ b/test/drivers/test_fun_nystrom_pp.cc @@ -198,3 +198,50 @@ TEST_F(TestFunNystromPP, KernelRBFSqrt) { Algs algs; test_FunNystromPP_general(algs, A_op, f_sqrt, 0.0, true_tr, k, s, d, 0.15, state); } + + +// At k=k_mat on a rank-k_mat matrix, NystromEVD captures the entire spectrum, so Phase 1 +// computes tr(f(A)) to machine precision and Phase 2's true correction is zero. +// However, Phase 2 (Hutchinson+LanczosFA) produces a small nonzero t2 because LanczosFA +// and the GEMM-based f(Â)*Omega in ResidualOp follow different floating-point arithmetic +// paths, disagreeing at ~1e-14 per column. Hutchinson accumulates this into a systematic +// bias of roughly 1e-5 relative. This test documents that expected behavior: +// - Phase 1 (t1) is machine-precision accurate: err < 1e-10 +// - Total error is bounded by the Phase 2 floor: err < 1e-3 +// - t2 is nonzero but small (the two-path arithmetic residual) +// To hit machine precision on the total, skip Phase 2 when k >= effective rank. +TEST_F(TestFunNystromPP, Phase1MachinePrecisionAtKmat) { + using RNG = r123::Philox4x32; + int64_t n = 200, k_mat = 10, k = 10, s = 200, d = 20; + auto state = RandBLAS::RNGState(7); + + std::vector eigvals(k_mat); + RandLAPACK::gen::gen_alg_decay_singvals(k_mat, 100.0, 2.0, eigvals.data()); + + std::vector A(n * n, 0.0); + RandLAPACK::gen::gen_sym_psd_lowrank(n, k_mat, A.data(), n, eigvals.data(), state); + + auto f = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + double f_zero = 0.0; + double true_tr = 0.0; + for (int64_t j = 0; j < k_mat; ++j) + true_tr += f(eigvals[j]); + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + Algs algs; + double est = algs.driver.call(A_op, f, f_zero, k, s, d, state); + + double t1 = 0.0; + for (int64_t i = 0; i < k; ++i) + t1 += algs.driver.F_vec[i]; + double t2 = est - t1; + + double err_t1 = std::abs(t1 - true_tr) / true_tr; + double err_tot = std::abs(est - true_tr) / true_tr; + + printf("Phase1MachinePrecision: t1=%e t2=%e err_t1=%e err_tot=%e\n", + t1, t2, err_t1, err_tot); + + ASSERT_LT(err_t1, 1e-10); // Phase 1 machine precision at k=k_mat + ASSERT_LT(err_tot, 1e-3); // total bounded by Phase 2 LFA floor +} From ef47108bc4c7aaaacc9e1ed5446b39a1c1245468 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Wed, 6 May 2026 17:45:01 -0400 Subject: [PATCH 13/14] More fixes --- RandLAPACK/comps/rl_lanczos_fa.hh | 3 +- RandLAPACK/comps/rl_lanczos_fa_block.hh | 24 +-- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 67 +++++--- RandLAPACK/linops/rl_matfun_linops.hh | 2 +- .../FunNystromPP_benchmark.cc | 89 ++++++----- test/comps/test_lanczos_fa.cc | 143 ++++++++++++++++-- test/drivers/test_fun_nystrom_pp.cc | 2 +- 7 files changed, 249 insertions(+), 81 deletions(-) diff --git a/RandLAPACK/comps/rl_lanczos_fa.hh b/RandLAPACK/comps/rl_lanczos_fa.hh index 4ede5bce6..9a8128e16 100644 --- a/RandLAPACK/comps/rl_lanczos_fa.hh +++ b/RandLAPACK/comps/rl_lanczos_fa.hh @@ -24,8 +24,7 @@ namespace RandLAPACK { /// See: T. Chen, "A Lanczos-FA algorithm for matrix function approximation" (2022). /// /// @tparam T Floating-point scalar type. -/// @tparam RNG Random number generator type (unused here; kept for API uniformity). -template +template class LanczosFA { public: /// Reorthogonalization control. diff --git a/RandLAPACK/comps/rl_lanczos_fa_block.hh b/RandLAPACK/comps/rl_lanczos_fa_block.hh index 5524563ec..3488f578f 100644 --- a/RandLAPACK/comps/rl_lanczos_fa_block.hh +++ b/RandLAPACK/comps/rl_lanczos_fa_block.hh @@ -38,8 +38,7 @@ namespace RandLAPACK { /// effective rank of A, use a smaller d or the scalar LanczosFA. /// /// @tparam T Floating-point scalar type. -/// @tparam RNG Random number generator type (unused; kept for API uniformity with LanczosFA). -template +template class BlockLanczosFA { public: /// Reorthogonalization control. @@ -61,12 +60,14 @@ public: // Only d-1 entries are populated (no beta after the last step); // size d is allocated to keep indexing uniform. // workspace: apply_f scratch — T_dense (d*s×d*s) + eig_vals (d*s) + G (d*s×s) + C1 (d*s×s). + // proj_buf: s*s — reorthogonalization scratch (Q_p^T * Y projection); reused across steps. T* K_big = nullptr; int64_t K_big_sz = 0; T* R0_buf = nullptr; int64_t R0_sz = 0; T* tau_buf = nullptr; int64_t tau_buf_sz = 0; T* A_blk = nullptr; int64_t A_blk_sz = 0; T* B_blk = nullptr; int64_t B_blk_sz = 0; T* workspace = nullptr; int64_t workspace_sz = 0; + T* proj_buf = nullptr; int64_t proj_buf_sz = 0; bool timing = false; std::vector times; @@ -78,7 +79,7 @@ public: ~BlockLanczosFA() { delete[] K_big; delete[] R0_buf; delete[] tau_buf; - delete[] A_blk; delete[] B_blk; delete[] workspace; + delete[] A_blk; delete[] B_blk; delete[] workspace; delete[] proj_buf; } // ------------------------------------------------------------------ @@ -96,6 +97,7 @@ public: util::resize(tau_buf, tau_buf_sz, n); util::resize(A_blk, A_blk_sz, d * s * s); util::resize(B_blk, B_blk_sz, d * s * s); + if (reorth) util::resize(proj_buf, proj_buf_sz, s * s); // Initial QR: B = Q0 * R0. Q0 overwrites K_big[0..n*s-1]. T* Q0 = K_big; @@ -125,10 +127,13 @@ public: // A_step = Q_step^T * Y (block alpha, s×s) blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, s, s, n, (T)1.0, Q_step, n, Y, n, (T)0.0, A_step, s); - for (int64_t j = 0; j < s; ++j) - for (int64_t i = j + 1; i < s; ++i) - A_step[j * s + i] = A_step[i * s + j] = - (T)0.5 * (A_step[j * s + i] + A_step[i * s + j]); + for (int64_t j = 0; j < s; ++j) { + for (int64_t i = j + 1; i < s; ++i) { + T avg = (T)0.5 * (A_step[j * s + i] + A_step[i * s + j]); + A_step[j * s + i] = avg; + A_step[i * s + j] = avg; + } + } // Y -= Q_step * A_step (Z = Y - Q_step*A_step, in-place) blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, @@ -136,13 +141,12 @@ public: // Optional full reorthogonalization: Z -= Q_p * (Q_p^T * Z) for each prev block if (reorth) { - std::vector proj(s * s); for (int64_t prev = 0; prev <= step; ++prev) { T* Q_p = K_big + prev * n * s; blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, - s, s, n, (T)1.0, Q_p, n, Y, n, (T)0.0, proj.data(), s); + s, s, n, (T)1.0, Q_p, n, Y, n, (T)0.0, proj_buf, s); blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, - n, s, s, (T)-1.0, Q_p, n, proj.data(), s, (T)1.0, Y, n); + n, s, s, (T)-1.0, Q_p, n, proj_buf, s, (T)1.0, Y, n); } } diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index b82fae615..287a54484 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -11,6 +11,7 @@ #include #include #include +#include namespace RandLAPACK { @@ -103,10 +104,13 @@ public: /// /// @param[in] A SymmetricLinearOperator — matvec oracle for A. /// @param[in] f Scalar function T→T (e.g., sqrt, log, pow(x,α)). - /// @param[in] f_zero f(0): value at zero. Use 0.0 for sqrt/power. - /// Must be finite; throws std::invalid_argument if not. - /// For log, A must be strictly PD — pass any finite - /// placeholder and ensure λ_min > 0. + /// @param[in] f_zero Optional f(0). When std::nullopt (the default), and + /// k < n, the value is auto-derived as f((T)0); for + /// functions like log where f(0) is non-finite, the + /// caller must pass an explicit value or guarantee + /// k == n. Must be finite if provided. When k == n + /// (Phase 1 captures the full spectrum), f_zero is + /// unused and may be omitted. /// @param[in] k Nyström rank. Caller chooses based on κ, ε. /// @param[in] s Hutchinson samples. Caller chooses based on κ, ε. /// @param[in] d Lanczos steps per sample. Caller chooses based on κ. @@ -118,15 +122,15 @@ public: T call( SLO& A, F f, - T f_zero, + std::optional f_zero, int64_t k, int64_t s, int64_t d, RandBLAS::RNGState& state, FunNystromPP_timing* timing = nullptr ) { - if (!std::isfinite(f_zero)) - throw std::invalid_argument("f_zero must be finite; for log, ensure A is strictly PD"); + if (f_zero.has_value() && !std::isfinite(*f_zero)) + throw std::invalid_argument("f_zero must be finite when provided"); int64_t n = A.dim; @@ -148,27 +152,56 @@ public: util::resize(F_vec, F_vec_sz, k_in); std::transform(eigvals_buf, eigvals_buf + k_in, F_vec, f); + // Resolve f_zero. The (n-k_in)*f(0) term is only used when k_in < n; + // when k_in == n the full spectrum is captured by Phase 1 and f_zero is + // unused (so we don't require the caller to provide one). + T fz; + if (f_zero.has_value()) { + fz = *f_zero; + } else if (k_in < n) { + fz = f((T)0); + if (!std::isfinite(fz)) + throw std::invalid_argument( + "f(0) is non-finite; pass explicit f_zero or ensure k captures full rank"); + } else { + fz = (T)0; // unused: (n - k_in) == 0 + } + // tr(f(Â)) = Σ f(λ_i) + (n-k_in)*f(0) T tr_Ahat = (T)0.0; for (int64_t i = 0; i < k_in; ++i) tr_Ahat += F_vec[i]; - tr_Ahat += static_cast(n - k_in) * f_zero; + tr_Ahat += static_cast(n - k_in) * fz; // ------------------------------------------------------------------ // Phase 2: Hutchinson correction on residual f(A) - f(Â) // linops::ResidualOp wraps the correction as a SymmetricLinearOperator // so Hutchinson::call can draw Ω and compute <Ω, (f(A)-f(Â))Ω>_F / s. + // + // Skip Phase 2 when k_in == n: NystromEVD has captured the full spectrum + // exactly ( = A), so f(A) - f(Â) is analytically zero. Running Phase 2 + // anyway would compute <Ω, (Z1 - Z2)>/s where Z1 (LFA approximation) and + // Z2 (V·diag(f(λ))·Vᵀ·Ω) follow different floating-point paths, leaving + // a misleading O(ε_mach × s × LFA_residual) "noise floor" on the trace. + // The structural bound k + s ≤ n implies that when k = n, s must be 0 + // for a sensible matvec budget — this skip is its natural consequence. // ------------------------------------------------------------------ - util::resize(tmp, tmp_sz, k_in * s); - util::resize(Z1, Z1_sz, n * s); - util::resize(Z2, Z2_sz, n * s); - - linops::ResidualOp res_op( - n, A, lanczos_fa, f, d, k_in, V_buf, F_vec, tmp, Z1, Z2 - ); + T correction = (T)0; auto t_phase2_start = std::chrono::steady_clock::now(); - T correction = hutchinson.call(res_op, s, state); - auto t_phase2_end = std::chrono::steady_clock::now(); + auto t_phase2_end = t_phase2_start; + + if (k_in < n) { + util::resize(tmp, tmp_sz, k_in * s); + util::resize(Z1, Z1_sz, n * s); + util::resize(Z2, Z2_sz, n * s); + + linops::ResidualOp res_op( + n, A, lanczos_fa, f, d, k_in, V_buf, F_vec, tmp, Z1, Z2 + ); + t_phase2_start = std::chrono::steady_clock::now(); + correction = hutchinson.call(res_op, s, state); + t_phase2_end = std::chrono::steady_clock::now(); + } if (timing) { using us = std::chrono::microseconds; diff --git a/RandLAPACK/linops/rl_matfun_linops.hh b/RandLAPACK/linops/rl_matfun_linops.hh index 13e70ede3..705ac651d 100644 --- a/RandLAPACK/linops/rl_matfun_linops.hh +++ b/RandLAPACK/linops/rl_matfun_linops.hh @@ -32,7 +32,7 @@ namespace RandLAPACK::linops { /// /// @tparam T Scalar type. /// @tparam SLO_t Type of the operator A. -/// @tparam LFA_t Type of the matrix-function oracle (e.g. LanczosFA). +/// @tparam LFA_t Type of the matrix-function oracle (e.g. LanczosFA). /// @tparam F_t Callable type T→T. template struct ResidualOp { diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc index 7dd2ca139..7b02f8013 100644 --- a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -38,6 +38,12 @@ Lanczos variant (lfa_type): 0 = scalar LanczosFA (s independent Krylov sequences; BLAS-1/2 dominant) 1 = BlockLanczosFA (single joint block Krylov subspace; BLAS-3 dominant) +SYRF orthogonalization (orth_type): + 0 = HouseholderQR (HQRQ) — backward-stable (geqrf+ungqr); ~2·n·k² flops + 1 = CholeskyQR (CholQRQ) — ~2× faster (syrk+potrf+trsm); fails on + ill-conditioned input (the SYRF output after + power iteration is generally well-conditioned). + Sketch types (sketch_type): 0 = Gaussian (DenseDist, default) 1 = SASO (SparseDist; nonzeros per column set by vec_nnz argument) @@ -253,7 +259,7 @@ struct LFAOp { // call_all_algs: timed + matvec-counted comparison for one matrix size. // ============================================================================ -template +template static void call_all_algs( int64_t numruns, FunNystromPP_benchmark_data& data, @@ -303,8 +309,8 @@ static void call_all_algs( RandLAPACK::SYPS syps(3, 1, false, false); syps.sketch_type = sketch_type; syps.vec_nnz = vec_nnz; - RandLAPACK::HQRQ orth(false, false); - RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); + ORTH_t orth(false, false); + RandLAPACK::SYRF, ORTH_t> syrf(syps, orth); RandLAPACK::NystromEVD nystrom_evd(syrf, 0); nystrom_evd.timing = true; LFA_t lfa_pp; @@ -398,7 +404,7 @@ static void call_all_algs( // call_all_algs_sparse: sparse path (always double). // ============================================================================ -template +template static void call_all_algs_sparse( int64_t numruns, int64_t n, @@ -431,8 +437,8 @@ static void call_all_algs_sparse( RandLAPACK::SYPS syps(3, 1, false, false); syps.sketch_type = sketch_type; syps.vec_nnz = vec_nnz; - RandLAPACK::HQRQ orth(false, false); - RandLAPACK::SYRF, RandLAPACK::HQRQ> syrf(syps, orth); + ORTH_t orth(false, false); + RandLAPACK::SYRF, ORTH_t> syrf(syps, orth); RandLAPACK::NystromEVD nystrom_evd(syrf, 0); nystrom_evd.timing = true; LFA_t lfa_pp; @@ -512,11 +518,12 @@ static void call_all_algs_sparse( // ============================================================================ int main(int argc, char* argv[]) { - if (argc < 15) { + if (argc < 16) { std::cerr << "Usage: " << argv[0] << " " " " - " [n_1 n_2 ...]\n" + " " + " [n_1 n_2 ...]\n" << " dir_path : output directory (use '.' for current dir)\n" << " num_runs : timed repetitions per matrix size\n" << " k : Nystrom rank (constant; must satisfy k <= n)\n" @@ -531,6 +538,7 @@ int main(int argc, char* argv[]) { << " poly_lambda : lambda for func_type=2 (ignored otherwise)\n" << " precision : double or float (sparse .mtx path is always double)\n" << " lfa_type : 0=scalar LanczosFA, 1=BlockLanczosFA\n" + << " orth_type : 0=HouseholderQR (HQRQ), 1=Cholesky QR (CholQRQ)\n" << " n_1 ... : matrix sizes (generated types only)\n"; return 1; } @@ -549,6 +557,8 @@ int main(int argc, char* argv[]) { bool use_float = (precision_str == "float"); int lfa_type = std::stoi(argv[14]); const char* lfa_str = lfa_type == 0 ? "scalar" : "block"; + int orth_type = std::stoi(argv[15]); + const char* orth_str = orth_type == 0 ? "hqrq" : "cholqr"; bool from_file = false; bool is_mtx = false; @@ -594,11 +604,11 @@ int main(int argc, char* argv[]) { + std::to_string(fm) + "x" + std::to_string(fn) + ")"); n_sizes = {fm}; } else { - if (argc < 16) { + if (argc < 17) { std::cerr << "Error: at least one matrix size (n_1) required for generated types.\n"; return 1; } - for (int i = 15; i < argc; ++i) + for (int i = 16; i < argc; ++i) n_sizes.push_back(std::stol(argv[i])); } @@ -610,7 +620,8 @@ int main(int argc, char* argv[]) { auto state_constant = state; std::string output_filename = - "_FunNystromPP_benchmark_" + precision_str + "_" + std::string(lfa_str) + "_num_info_lines_8.txt"; + "_FunNystromPP_benchmark_" + precision_str + "_" + std::string(lfa_str) + + "_" + std::string(orth_str) + "_num_info_lines_8.txt"; std::string path; if (std::string(argv[1]) != ".") path = std::string(argv[1]) + output_filename; @@ -646,6 +657,7 @@ int main(int argc, char* argv[]) { (sketch_type == 1 ? (" vec_nnz=" + std::to_string(vec_nnz)) : std::string("")) + " precision=" + precision_str + " lfa_type=" + std::string(lfa_str) + + " orth_type=" + std::string(orth_str) + "\nNum runs per size: " + std::to_string(numruns) + "\nMatrix construction: " + mat_construction_str + "\n"; @@ -653,10 +665,14 @@ int main(int argc, char* argv[]) { auto start_all = steady_clock::now(); - using ScalarLFA_d = RandLAPACK::LanczosFA; - using BlockLFA_d = RandLAPACK::BlockLanczosFA; - using ScalarLFA_f = RandLAPACK::LanczosFA; - using BlockLFA_f = RandLAPACK::BlockLanczosFA; + using ScalarLFA_d = RandLAPACK::LanczosFA; + using BlockLFA_d = RandLAPACK::BlockLanczosFA; + using ScalarLFA_f = RandLAPACK::LanczosFA; + using BlockLFA_f = RandLAPACK::BlockLanczosFA; + using HQRQ_d = RandLAPACK::HQRQ; + using CholQR_d = RandLAPACK::CholQRQ; + using HQRQ_f = RandLAPACK::HQRQ; + using CholQR_f = RandLAPACK::CholQRQ; if (is_mtx) { // Sparse path: always double. @@ -666,17 +682,19 @@ int main(int argc, char* argv[]) { int64_t n = n_sizes[0]; auto csr = FunNystromPP_bench::load_mtx(mat_file_path, n); printf(" Loaded: n=%lld nnz=%lld\n", (long long)n, (long long)csr.nnz); - if (lfa_type == 0) - call_all_algs_sparse(numruns, n, k_const, s_const, d_steps, - csr, state_constant, - func_type, poly_lambda, sketch_type, vec_nnz, path); - else - call_all_algs_sparse(numruns, n, k_const, s_const, d_steps, - csr, state_constant, - func_type, poly_lambda, sketch_type, vec_nnz, path); + auto run_sparse = [&]() { + call_all_algs_sparse( + numruns, n, k_const, s_const, d_steps, + csr, state_constant, + func_type, poly_lambda, sketch_type, vec_nnz, path); + }; + if (lfa_type == 0 && orth_type == 0) run_sparse.template operator()(); + else if (lfa_type == 0 && orth_type == 1) run_sparse.template operator()(); + else if (lfa_type == 1 && orth_type == 0) run_sparse.template operator()(); + else run_sparse.template operator()(); } else { - // Dense path: dispatch on precision and lfa_type. - auto run_dense = [&]() { + // Dense path: dispatch on precision, lfa_type, orth_type (8-way). + auto run_dense = [&]() { int64_t n_max = *std::max_element(n_sizes.begin(), n_sizes.end()); int64_t k_mat_max = from_file ? 0 : k_mat_const; FunNystromPP_benchmark_data all_data( @@ -692,19 +710,20 @@ int main(int argc, char* argv[]) { auto state_gen = state_constant; data_regen(all_data, state_gen, mat_type, func_type, mat_file_path); - call_all_algs(numruns, all_data, state_constant, mat_type, - func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, path); + call_all_algs( + numruns, all_data, state_constant, mat_type, + func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, path); } }; - if (!use_float && lfa_type == 0) - run_dense.template operator()(); - else if (!use_float && lfa_type == 1) - run_dense.template operator()(); - else if (use_float && lfa_type == 0) - run_dense.template operator()(); - else - run_dense.template operator()(); + if (!use_float && lfa_type == 0 && orth_type == 0) run_dense.template operator()(); + else if (!use_float && lfa_type == 0 && orth_type == 1) run_dense.template operator()(); + else if (!use_float && lfa_type == 1 && orth_type == 0) run_dense.template operator()(); + else if (!use_float && lfa_type == 1 && orth_type == 1) run_dense.template operator()(); + else if ( use_float && lfa_type == 0 && orth_type == 0) run_dense.template operator()(); + else if ( use_float && lfa_type == 0 && orth_type == 1) run_dense.template operator()(); + else if ( use_float && lfa_type == 1 && orth_type == 0) run_dense.template operator()(); + else run_dense.template operator()(); } auto stop_all = steady_clock::now(); diff --git a/test/comps/test_lanczos_fa.cc b/test/comps/test_lanczos_fa.cc index 908e8b85f..90ea176dd 100644 --- a/test/comps/test_lanczos_fa.cc +++ b/test/comps/test_lanczos_fa.cc @@ -65,7 +65,6 @@ class TestLanczosFA : public ::testing::Test { template static void run_diagonal_fa_test(F f, int64_t n, int64_t s, int64_t d, double tol, uint64_t seed) { - using RNG = r123::Philox4x32; auto state = RandBLAS::RNGState(seed); std::vector diag_vec(n); std::iota(diag_vec.begin(), diag_vec.end(), 1.0); @@ -75,7 +74,7 @@ class TestLanczosFA : public ::testing::Test { state = RandBLAS::fill_dense(DB, B.data(), state); std::vector out(n * s, 0.0); linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); - RandLAPACK::LanczosFA lfa; + RandLAPACK::LanczosFA lfa; lfa.reorth = 1; lfa.call(A_op, B.data(), n, s, f, d, out.data()); auto ref = diag_fa_ref(diag_vec, B, n, s, f); @@ -95,7 +94,6 @@ class TestLanczosFA : public ::testing::Test { template static void run_dense_fa_test(F f, int64_t n, int64_t s, int64_t d, double tol, uint64_t seed) { - using RNG = r123::Philox4x32; auto state = RandBLAS::RNGState(seed); std::vector B_raw(n * n); RandBLAS::DenseDist D1(n, n); @@ -109,7 +107,7 @@ class TestLanczosFA : public ::testing::Test { state = RandBLAS::fill_dense(D2, B.data(), state); std::vector out(n * s, 0.0); linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); - RandLAPACK::LanczosFA lfa; + RandLAPACK::LanczosFA lfa; lfa.reorth = 1; lfa.call(A_op, B.data(), n, s, f, d, out.data()); auto ref = dense_fa_ref(A, B, n, s, f); @@ -135,7 +133,6 @@ TEST_F(TestLanczosFA, DiagonalSqrt) { // Exercises the paths where beta is never allocated and the main recurrence loop // (0..d-2) does not execute. stevd is called on a 1×1 tridiagonal. TEST_F(TestLanczosFA, ScalarMatrixSqrt_d1) { - using RNG = r123::Philox4x32; int64_t n = 8, s = 3, d = 1; auto state = RandBLAS::RNGState(55); @@ -148,7 +145,7 @@ TEST_F(TestLanczosFA, ScalarMatrixSqrt_d1) { std::vector out(n * s, 0.0); linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); - RandLAPACK::LanczosFA lfa; + RandLAPACK::LanczosFA lfa; lfa.reorth = 1; lfa.call(A_op, B.data(), n, s, [](double x){ return std::sqrt(x); }, d, out.data()); @@ -208,7 +205,7 @@ TEST_F(TestLanczosFA, KernelRBFSqrt) { std::vector out(n * s, 0.0); auto f = [](double x){ return std::sqrt(std::max(x, 0.0)); }; linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, K.data(), n, Layout::ColMajor); - RandLAPACK::LanczosFA lfa; + RandLAPACK::LanczosFA lfa; lfa.reorth = 1; lfa.call(A_op, B.data(), n, s, f, d, out.data()); @@ -228,7 +225,6 @@ TEST_F(TestLanczosFA, KernelRBFSqrt) { // Tests that reorthogonalization doesn't introduce a bug, not that it helps // (for that one would need κ >> 1 where vanilla loses orthogonality). TEST_F(TestLanczosFA, ReorthVsVanilla) { - using RNG = r123::Philox4x32; int64_t n = 40, s = 3, d = 15; auto state = RandBLAS::RNGState(99); @@ -243,7 +239,7 @@ TEST_F(TestLanczosFA, ReorthVsVanilla) { linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); auto f_sqrt = [](double x){ return std::sqrt(x); }; - using LFA = RandLAPACK::LanczosFA; + using LFA = RandLAPACK::LanczosFA; std::vector out_full(n * s, 0.0), out_van(n * s, 0.0); LFA lfa_full, lfa_van; lfa_full.reorth = 1; @@ -273,7 +269,6 @@ class TestBlockLanczosFA : public TestLanczosFA { template static void run_block_diagonal_fa_test(F f, int64_t n, int64_t s, int64_t d, double tol, uint64_t seed) { - using RNG = r123::Philox4x32; auto state = RandBLAS::RNGState(seed); std::vector diag_vec(n); std::iota(diag_vec.begin(), diag_vec.end(), 1.0); @@ -283,7 +278,7 @@ class TestBlockLanczosFA : public TestLanczosFA { state = RandBLAS::fill_dense(DB, B.data(), state); std::vector out(n * s, 0.0); linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); - RandLAPACK::BlockLanczosFA blfa; + RandLAPACK::BlockLanczosFA blfa; blfa.reorth = 1; blfa.call(A_op, B.data(), n, s, f, d, out.data()); auto ref = diag_fa_ref(diag_vec, B, n, s, f); @@ -301,7 +296,6 @@ class TestBlockLanczosFA : public TestLanczosFA { template static void run_block_dense_fa_test(F f, int64_t n, int64_t s, int64_t d, double tol, uint64_t seed) { - using RNG = r123::Philox4x32; auto state = RandBLAS::RNGState(seed); std::vector B_raw(n * n); RandBLAS::DenseDist D1(n, n); @@ -315,7 +309,7 @@ class TestBlockLanczosFA : public TestLanczosFA { state = RandBLAS::fill_dense(D2, B.data(), state); std::vector out(n * s, 0.0); linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); - RandLAPACK::BlockLanczosFA blfa; + RandLAPACK::BlockLanczosFA blfa; blfa.reorth = 1; blfa.call(A_op, B.data(), n, s, f, d, out.data()); auto ref = dense_fa_ref(A, B, n, s, f); @@ -341,6 +335,125 @@ TEST_F(TestBlockLanczosFA, DensePSDSqrt) { run_block_dense_fa_test([](double x){ return std::sqrt(std::max(x, 0.0)); }, 50, 4, 25, 1e-6, 31); } +// d=1 edge case: only the initial QR of B is performed; the recurrence loop runs one +// step (computes A_step) but no further blocks are appended. The block Krylov subspace +// is span(Q_0) = span(B), so out = Q_0 * f(Q_0^T A Q_0) * R_0 — a Galerkin approximation +// of f(A)B in the original input subspace. Quality is poor for general A but the path +// must run without crashing and produce a finite, meaningful result. +TEST_F(TestBlockLanczosFA, DStepEqualsOne) { + int64_t n = 30, s = 4, d = 1; + auto state = RandBLAS::RNGState(7); + + // A = diag(1..n); B random. + std::vector diag_vec(n); + std::iota(diag_vec.begin(), diag_vec.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag_vec); + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + + std::vector out(n * s, 0.0); + RandLAPACK::BlockLanczosFA blfa; + blfa.reorth = 1; + blfa.call(A_op, B.data(), n, s, f_sqrt, d, out.data()); + + // Sanity: output is finite, nonzero, and not catastrophically wrong. + double out_norm = 0.0; + bool all_finite = true; + for (int64_t i = 0; i < n * s; ++i) { + out_norm += out[i] * out[i]; + all_finite = all_finite && std::isfinite(out[i]); + } + ASSERT_TRUE(all_finite); + ASSERT_GT(std::sqrt(out_norm), 1e-3); + printf("d=1 block ||out|| = %e (sanity check, accuracy not asserted)\n", std::sqrt(out_norm)); +} + +// reorth=0 (vanilla block recurrence) vs reorth=1 (full reorthogonalization) on the +// same problem. With moderate d on a well-conditioned matrix, both should converge +// to a similar answer; this exercises the no-reorth code path and guards against +// regressions in it. +TEST_F(TestBlockLanczosFA, NoReorthAgreesWithFull) { + int64_t n = 50, s = 4, d = 25; + auto state = RandBLAS::RNGState(31); + + // PSD A = G^T G + n*I (well-conditioned). + std::vector B_raw(n * n); + RandBLAS::DenseDist D1(n, n); + state = RandBLAS::fill_dense(D1, B_raw.data(), state); + std::vector A(n * n, 0.0); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, n, n, 1.0, B_raw.data(), n, 0.0, A.data(), n); + for (int64_t i = 0; i < n; ++i) A[i + i * n] += (double)n; + + std::vector B(n * s); + RandBLAS::DenseDist D2(n, s); + state = RandBLAS::fill_dense(D2, B.data(), state); + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A.data(), n, Layout::ColMajor); + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + + std::vector out_full(n * s, 0.0), out_van(n * s, 0.0); + RandLAPACK::BlockLanczosFA blfa_full, blfa_van; + blfa_full.reorth = 1; + blfa_van.reorth = 0; + + blfa_full.call(A_op, B.data(), n, s, f_sqrt, d, out_full.data()); + blfa_van.call (A_op, B.data(), n, s, f_sqrt, d, out_van.data()); + + double diff = 0.0, norm = 0.0; + for (int64_t i = 0; i < n * s; ++i) { + double r = out_full[i] - out_van[i]; + diff += r * r; + norm += out_full[i] * out_full[i]; + } + double rel_diff = std::sqrt(diff / norm); + printf("Block reorth=1 vs reorth=0 relative diff: %e\n", rel_diff); + ASSERT_LT(rel_diff, 1e-6); +} + +// Zero-column input: if B has a column of zeros, the corresponding output column +// must be zero. The initial QR produces R_0 with a zero column at that index, which +// propagates to a zero output column via the formula out = Q_basis * f(T_k) * E_1 * R_0. +TEST_F(TestBlockLanczosFA, ZeroColumn) { + int64_t n = 40, s = 3, d = 10; + auto state = RandBLAS::RNGState(11); + + std::vector diag_vec(n); + std::iota(diag_vec.begin(), diag_vec.end(), 1.0); + std::vector A_mat = make_diag_matrix(diag_vec); + std::vector B(n * s); + RandBLAS::DenseDist DB(n, s); + state = RandBLAS::fill_dense(DB, B.data(), state); + // Zero out the middle column. + for (int64_t i = 0; i < n; ++i) B[i + 1 * n] = 0.0; + + linops::ExplicitSymLinOp A_op(n, blas::Uplo::Upper, A_mat.data(), n, Layout::ColMajor); + auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; + + std::vector out(n * s, 0.0); + RandLAPACK::BlockLanczosFA blfa; + blfa.reorth = 1; + blfa.call(A_op, B.data(), n, s, f_sqrt, d, out.data()); + + double zero_col_norm = 0.0, other_cols_norm = 0.0; + for (int64_t i = 0; i < n; ++i) zero_col_norm += out[i + 1 * n] * out[i + 1 * n]; + for (int64_t j = 0; j < s; ++j) { + if (j == 1) continue; + for (int64_t i = 0; i < n; ++i) + other_cols_norm += out[i + j * n] * out[i + j * n]; + } + zero_col_norm = std::sqrt(zero_col_norm); + other_cols_norm = std::sqrt(other_cols_norm); + printf("Zero-column out norm: %e (others: %e)\n", zero_col_norm, other_cols_norm); + // Output column for zero input column should be at roundoff level. + ASSERT_LT(zero_col_norm, 1e-10); + // Other columns should be substantially nonzero (sanity). + ASSERT_GT(other_cols_norm, 1e-3); +} + // BlockLanczosFA vs LanczosFA on the same problem — outputs should agree at sufficient depth. TEST_F(TestBlockLanczosFA, AgreeWithScalar) { using RNG = r123::Philox4x32; @@ -363,11 +476,11 @@ TEST_F(TestBlockLanczosFA, AgreeWithScalar) { auto f_sqrt = [](double x){ return std::sqrt(std::max(x, 0.0)); }; std::vector out_scalar(n * s, 0.0), out_block(n * s, 0.0); - RandLAPACK::LanczosFA lfa; + RandLAPACK::LanczosFA lfa; lfa.reorth = 1; lfa.call(A_op, B.data(), n, s, f_sqrt, d, out_scalar.data()); - RandLAPACK::BlockLanczosFA blfa; + RandLAPACK::BlockLanczosFA blfa; blfa.reorth = 1; blfa.call(A_op, B.data(), n, s, f_sqrt, d, out_block.data()); diff --git a/test/drivers/test_fun_nystrom_pp.cc b/test/drivers/test_fun_nystrom_pp.cc index a5b6dd3b0..15f3df6ce 100644 --- a/test/drivers/test_fun_nystrom_pp.cc +++ b/test/drivers/test_fun_nystrom_pp.cc @@ -38,7 +38,7 @@ class TestFunNystromPP : public ::testing::Test { using Orth_t = RandLAPACK::HQRQ; using SYRF_t = RandLAPACK::SYRF; using NystromEVD_t = RandLAPACK::NystromEVD; - using LFA_t = RandLAPACK::LanczosFA; + using LFA_t = RandLAPACK::LanczosFA; using Hutch_t = RandLAPACK::Hutchinson; using Driver_t = RandLAPACK::FunNystromPP; From e8066b4467c672a2b97fa145702c6256b34e85d7 Mon Sep 17 00:00:00 2001 From: mmelnich Date: Thu, 7 May 2026 18:51:45 -0400 Subject: [PATCH 14/14] Update + rebase --- RandLAPACK/comps/rl_lanczos_fa_block.hh | 32 ++-- RandLAPACK/comps/rl_syps.hh | 83 +++++--- RandLAPACK/drivers/rl_fun_nystrom_pp.hh | 6 + RandLAPACK/drivers/rl_nystrom_evd.hh | 165 +++++++++++++--- RandLAPACK/linops/rl_sym_linops.hh | 45 +++++ RandLAPACK/misc/rl_util.hh | 16 ++ .../FunNystromPP_benchmark.cc | 178 +++++++++++++----- 7 files changed, 415 insertions(+), 110 deletions(-) diff --git a/RandLAPACK/comps/rl_lanczos_fa_block.hh b/RandLAPACK/comps/rl_lanczos_fa_block.hh index 3488f578f..ac6ab3602 100644 --- a/RandLAPACK/comps/rl_lanczos_fa_block.hh +++ b/RandLAPACK/comps/rl_lanczos_fa_block.hh @@ -124,16 +124,12 @@ public: blas::gemm(Layout::ColMajor, Op::NoTrans, Op::Trans, n, s, s, (T)-1.0, Q_prev, n, B_prev, s, (T)1.0, Y, n); - // A_step = Q_step^T * Y (block alpha, s×s) + // A_step = Q_step^T * Y (block alpha, s×s). Symmetric in + // exact arithmetic since Q_stepᵀ·A·Q_step is — symmetrize + // away the small finite-arithmetic asymmetry. blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, s, s, n, (T)1.0, Q_step, n, Y, n, (T)0.0, A_step, s); - for (int64_t j = 0; j < s; ++j) { - for (int64_t i = j + 1; i < s; ++i) { - T avg = (T)0.5 * (A_step[j * s + i] + A_step[i * s + j]); - A_step[j * s + i] = avg; - A_step[i * s + j] = avg; - } - } + util::symmetrize(s, A_step, s); // Y -= Q_step * A_step (Z = Y - Q_step*A_step, in-place) blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, @@ -182,7 +178,16 @@ public: T* G = eig_vals + m; T* C1 = G + m * s; - // 1. Assemble T_dense. + // 1. Assemble T_dense (m×m, m = d·s) as block tridiagonal: + // + // ┌ A₀ B₀ᵀ ┐ + // │ B₀ A₁ B₁ᵀ │ + // T_d = │ B₁ A₂ … │ + // │ ⋱ ⋱ B_{d-2}ᵀ│ + // └ B_{d-2} A_{d-1}┘ + // + // where Aᵢ is the s×s diagonal block in A_blk[i] and Bᵢ is the s×s + // upper-triangular factor produced by the QR of Z at step i+1. std::memset(T_dense, 0, m * m * sizeof(T)); for (int64_t step = 0; step < d; ++step) { int64_t b0 = step * s; @@ -202,12 +207,9 @@ public: T_dense[(b1 + j) * m + (b0 + i)] = B_step[i * s + j]; } } - // Symmetrize to eliminate any floating-point asymmetry - for (int64_t j = 0; j < m; ++j) - for (int64_t i = j + 1; i < m; ++i) { - T avg = (T)0.5 * (T_dense[j * m + i] + T_dense[i * m + j]); - T_dense[j * m + i] = T_dense[i * m + j] = avg; - } + // Symmetrize to eliminate any floating-point asymmetry between the + // explicit B_step lower block and its hand-transposed upper twin. + util::symmetrize(m, T_dense, m); // 2. Eigendecomposition: T_dense → V (eigenvectors overwrite T_dense), eig_vals → λ. lapack::syevd(lapack::Job::Vec, blas::Uplo::Lower, m, T_dense, m, eig_vals); diff --git a/RandLAPACK/comps/rl_syps.hh b/RandLAPACK/comps/rl_syps.hh index fea36416f..29e244136 100644 --- a/RandLAPACK/comps/rl_syps.hh +++ b/RandLAPACK/comps/rl_syps.hh @@ -30,6 +30,13 @@ class SYPS { // 0 = Gaussian (DenseDist, default); 1 = SASO (SparseDist, vec_nnz per column) int sketch_type = 0; int vec_nnz = 4; + // Internal stabilization (applied between power iterations). + // nullptr → use the historical hardcoded geqrf+ungqr path (HQRQ-equivalent). + // Non-null → invoke the supplied Stabilization (e.g. CholQRQ) and fall back to + // the hardcoded geqrf+ungqr if it returns nonzero (e.g. CholQR Cholesky failure). + // Power iteration produces increasingly column-correlated matrices; CholQR can + // hit potrf failure on ill-conditioned inputs, so the fallback is intentional. + Stabilization* internal_stab = nullptr; SYPS( int64_t p, // number of passes @@ -112,20 +119,6 @@ class SYPS { bool callers_skop_buff = skop_buff != nullptr; if (!callers_skop_buff) skop_buff = new T[m * k]; - if (sketch_type == 1) { - // SJLT: generate a sparse sketch and densify into skop_buff. - // T and sint_t (int64_t) must be explicit since SparseDist::sample - // can't deduce T from the RNGState argument alone. - auto S = RandBLAS::SparseDist(m, k, this->vec_nnz).sample(state); - state = S.next_state; - RandBLAS::fill_sparse(S); - auto Scoo = RandBLAS::coo_view_of_skop(S); - std::fill(skop_buff, skop_buff + m * k, (T)0.0); - RandLAPACK::util::sparse_to_dense(Scoo, Layout::ColMajor, skop_buff); - } else { - RandBLAS::DenseDist D(m, k); - state = RandBLAS::fill_dense(D, skop_buff, state); - } bool callers_work_buff = work_buff != nullptr; if (!callers_work_buff) @@ -135,18 +128,64 @@ class SYPS { T *symm_out = work_buff; T *symm_in = skop_buff; T *tau = new T[k]{}; - while (p - p_done > 0) { - A(Layout::ColMajor, k, 1.0, symm_in, m, 0.0, symm_out, m); - ++p_done; - if (p_done % q == 0) { - if(lapack::geqrf(m, k, symm_out, m, tau)) { - delete [] tau; + + // Inline lambda for the per-iter stabilization (deduplicates the path used + // for both the sparse-first-iter shortcut and the regular dense loop). + auto stabilize = [&](T* buf) { + bool fallback = (this->internal_stab == nullptr); + if (!fallback) { + fallback = (this->internal_stab->call(m, k, buf) != 0); + } + if (fallback) { + if (lapack::geqrf(m, k, buf, m, tau)) { + delete[] tau; throw std::runtime_error("GEQRF failed."); } - lapack::ungqr(m, k, k, symm_out, m, tau); + lapack::ungqr(m, k, k, buf, m, tau); + } + }; + + if (sketch_type == 1) { + // Sparse sketch path — apply A to the sparse Ω directly via the + // SkOp overload of A (dense·sparse spmm), skipping the n×k + // densification of Ω. T and sint_t (int64_t) must be explicit + // since SparseDist::sample can't deduce T from the RNGState. + auto S = RandBLAS::SparseDist(m, k, this->vec_nnz).sample(state); + state = S.next_state; + RandBLAS::fill_sparse(S); + + // SLOs that implement the SkOp-taking operator() get the fast + // sparse-aware first matvec. Other SLOs (RegExplicitSymLinOp, + // SparseSymLinOp without the SkOp overload, etc.) fall back + // to the legacy densification. + if constexpr (requires (SLO& a, decltype(S)& s) { + a(Layout::ColMajor, k, (T)1.0, s, (T)0.0, symm_out, m); + }) { + A(Layout::ColMajor, k, (T)1.0, S, (T)0.0, symm_out, m); + } else { + auto Scoo = RandBLAS::coo_view_of_skop(S); + std::fill(skop_buff, skop_buff + m * k, (T)0.0); + RandLAPACK::util::sparse_to_dense(Scoo, Layout::ColMajor, skop_buff); + A(Layout::ColMajor, k, (T)1.0, skop_buff, m, (T)0.0, symm_out, m); } + ++p_done; + if (p_done % q == 0) stabilize(symm_out); + symm_out = (p_done % 2 == 1) ? skop_buff : work_buff; + symm_in = (p_done % 2 == 1) ? work_buff : skop_buff; + } else { + // Dense Gaussian sketch — fill skop_buff in place; first matvec + // happens uniformly in the loop below. + RandBLAS::DenseDist D(m, k); + state = RandBLAS::fill_dense(D, skop_buff, state); + } + + // Remaining iterations are pure dense matvecs through the SLO interface. + while (p - p_done > 0) { + A(Layout::ColMajor, k, 1.0, symm_in, m, 0.0, symm_out, m); + ++p_done; + if (p_done % q == 0) stabilize(symm_out); symm_out = (p_done % 2 == 1) ? skop_buff : work_buff; - symm_in = (p_done % 2 == 1) ? work_buff : skop_buff; + symm_in = (p_done % 2 == 1) ? work_buff : skop_buff; } delete[] tau; if (p % 2 == 1) diff --git a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh index 287a54484..a13434957 100644 --- a/RandLAPACK/drivers/rl_fun_nystrom_pp.hh +++ b/RandLAPACK/drivers/rl_fun_nystrom_pp.hh @@ -201,6 +201,12 @@ public: t_phase2_start = std::chrono::steady_clock::now(); correction = hutchinson.call(res_op, s, state); t_phase2_end = std::chrono::steady_clock::now(); + } else { + // Phase 2 skipped: lanczos_fa.call() never executed, so its + // .times vector stays empty. Populate with zeros (5 slots: matvec, + // lanczos, apply_f, rest, total) so callers indexing into it for + // timing breakdowns get well-defined zeros instead of UB. + lanczos_fa.times.assign(5, 0L); } if (timing) { diff --git a/RandLAPACK/drivers/rl_nystrom_evd.hh b/RandLAPACK/drivers/rl_nystrom_evd.hh index f8762bf99..5389fa11e 100644 --- a/RandLAPACK/drivers/rl_nystrom_evd.hh +++ b/RandLAPACK/drivers/rl_nystrom_evd.hh @@ -182,42 +182,144 @@ class NystromEVD { A(Layout::ColMajor, k, (T)1, Omega, m, (T)0, Y, m); if (this->timing) { t1 = steady_clock::now(); matvec_t_dur += duration_cast(t1 - t0).count(); } - T nu = std::numeric_limits::epsilon() * lapack::lange(Norm::Fro, m, k, Y, m); - + // ----- Phase 1 spectral recovery ----- + // + // Two paths sharing the same Gram-matrix construction: + // + // FAST PATH (well-conditioned Gram — the common case): + // G = Ωᵀ · Y (k×k sym PSD) + // G = Lᵀ · L (Cholesky) + // Y := Y · L⁻¹ (TRSM) + // G_y = Yᵀ · Y (small Gram) + // G_y = V · Σ² · Vᵀ (syevd) + // U = Y · V · diag(1/σ) (gemm + scal) + // eigvals = Σ² + // + // The eig-of-Yᵀ·Y trick (Halko-Martinsson-Tropp §5.1) avoids + // the more expensive gesdd of n×k while keeping eigenvector + // accuracy fine because Y is well-conditioned post-TRSM. + // + // FALL-BACK (Cholesky failure on near-singular Gram): + // syevd(G) → V_G, D (pseudoinverse) + // B = Y · V_G · diag(1/√D) · V_Gᵀ (3 GEMMs) + // gesdd(B) → U, Σ + // eigvals = Σ² + // + // This is the safe path. It handles rank-deficient Gram via + // thresholded pseudoinverse. Mirrors the Persson-Kressner + // reference nystrom.m. Slightly more expensive than the + // fast path but always succeeds. + // + // No regularization shift in either path — both paths preserve + // the ε_mach accuracy of the SVD-pseudoinverse rewrite. + + T eps_mach = std::numeric_limits::epsilon(); + + // Form Gram G := Ωᵀ · Y into R (k×k). G = Ωᵀ·A·Ω in exact + // arithmetic and is symmetric, but the two non-trivial GEMMs + // (sketch + power-iter Y, then Ωᵀ Y) introduce small + // asymmetry; symmetrize so potrf and the later syevd see a + // numerically symmetric matrix. if (this->timing) t0 = steady_clock::now(); - blas::syrk(Layout::ColMajor, Uplo::Lower, Op::Trans, k, m, nu, Omega, m, (T)0, R, k); - for(int64_t i = 1; i < k; ++i) - blas::copy(k - i, &R[i + ((i-1) * k)], 1, &R[(i - 1) + (i * k)], k); - blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, (T)1, Omega, m, Y, m, (T)1, R, k); + blas::gemm(Layout::ColMajor, Op::Trans, Op::NoTrans, k, k, m, + (T)1, Omega, m, Y, m, (T)0, R, k); + RandLAPACK::util::symmetrize(k, R, k); + // Backup G into symrf_work so we can restore it if Cholesky + // fails (potrf is destructive). symrf_work is m×k = enough for + // the k×k Gram. + lapack::lacpy(lapack::MatrixType::General, k, k, R, k, symrf_work, k); if (this->timing) { t1 = steady_clock::now(); gram_t_dur += duration_cast(t1 - t0).count(); } - if (this->timing) t0 = steady_clock::now(); - if(lapack::potrf(Uplo::Upper, k, R, k)) - throw std::runtime_error("Cholesky decomposition failed."); - RandLAPACK::util::get_U(k, k, R, k); - if (this->timing) { t1 = steady_clock::now(); potrf_t_dur += duration_cast(t1 - t0).count(); } + int64_t r = 0; + // Try Cholesky. if (this->timing) t0 = steady_clock::now(); - blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, Op::NoTrans, Diag::NonUnit, m, k, (T)1, R, k, Y, m); - if (this->timing) { t1 = steady_clock::now(); trsm_t_dur += duration_cast(t1 - t0).count(); } + int chol_status = lapack::potrf(Uplo::Upper, k, R, k); + if (this->timing) { t1 = steady_clock::now(); potrf_t_dur += duration_cast(t1 - t0).count(); } - if (this->timing) t0 = steady_clock::now(); - lapack::gesdd(Job::SomeVec, m, k, Y, m, S, V_dat, m, R, k); - if (this->timing) { t1 = steady_clock::now(); svd_t_dur += duration_cast(t1 - t0).count(); } + if (chol_status == 0) { + // -------- FAST PATH -------- + RandLAPACK::util::get_U(k, k, R, k); // zero strict lower + + // Y := Y · R⁻¹ (R is upper-tri Cholesky factor) + if (this->timing) t0 = steady_clock::now(); + blas::trsm(Layout::ColMajor, Side::Right, Uplo::Upper, + Op::NoTrans, Diag::NonUnit, m, k, + (T)1, R, k, Y, m); + if (this->timing) { t1 = steady_clock::now(); trsm_t_dur += duration_cast(t1 - t0).count(); } + + // G_y = Yᵀ · Y (k×k symmetric); reuse R buffer. syrk + // writes only the upper triangle; the strict lower stays + // unspecified, which is fine — the following syevd call + // is invoked with Uplo::Upper and reads only the upper + // triangle (then overwrites all of R with eigenvectors). + if (this->timing) t0 = steady_clock::now(); + blas::syrk(Layout::ColMajor, Uplo::Upper, Op::Trans, + k, m, (T)1, Y, m, (T)0, R, k); + // syevd(G_y) → R (eigvecs V), S (Σ², ASCENDING order). + lapack::syevd(lapack::Job::Vec, Uplo::Upper, k, R, k, S); + // Reverse to descending order to match the rest of the + // pipeline (post-SVD threshold logic expects S[0]=largest). + for (int64_t ii = 0; ii < k / 2; ++ii) std::swap(S[ii], S[k - 1 - ii]); + for (int64_t jj = 0; jj < k / 2; ++jj) { + blas::swap(k, R + jj * k, 1, R + (k - 1 - jj) * k, 1); + } + // U = Y · V (left singular vectors, unscaled). + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, m, k, k, + (T)1, Y, m, R, k, (T)0, V_dat, m); + // Scale columns by 1/σ (= 1/sqrt(σ²)). Threshold tiny σ² as 0. + T sig_sq_max = (k > 0) ? std::max(S[0], (T)0) : (T)0; + T sig_sq_thresh = (T)4 * eps_mach * sig_sq_max; + for (int64_t jj = 0; jj < k; ++jj) { + if (S[jj] > sig_sq_thresh) { + T inv_sig = (T)1 / std::sqrt(S[jj]); + blas::scal(m, inv_sig, V_dat + jj * m, 1); + ++r; + } else { + std::fill(V_dat + jj * m, V_dat + (jj + 1) * m, (T)0); + } + } + // eigvals = σ² (already descending). + for (int64_t i = 0; i < k; ++i) + eigvals[i] = std::max(S[i], (T)0); + if (this->timing) { t1 = steady_clock::now(); svd_t_dur += duration_cast(t1 - t0).count(); } + } else { + // -------- FALL-BACK: syevd-pseudoinverse -------- + // Restore Gram into R from backup. + lapack::lacpy(lapack::MatrixType::General, k, k, symrf_work, k, R, k); + + if (this->timing) t0 = steady_clock::now(); + lapack::syevd(lapack::Job::Vec, Uplo::Lower, k, R, k, S); + if (this->timing) { t1 = steady_clock::now(); potrf_t_dur += duration_cast(t1 - t0).count(); } + + if (this->timing) t0 = steady_clock::now(); + T D_max = (k > 0) ? std::max(S[k - 1], (T)0) : (T)0; + T D_thresh = (T)2 * eps_mach * D_max; + // symrf_work was the Gram backup; safe to overwrite now. + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::NoTrans, m, k, k, + (T)1, Y, m, R, k, (T)0, symrf_work, m); + for (int64_t jj = 0; jj < k; ++jj) { + T scale = (S[jj] > D_thresh) ? (T)1 / std::sqrt(S[jj]) : (T)0; + blas::scal(m, scale, symrf_work + jj * m, 1); + } + blas::gemm(Layout::ColMajor, Op::NoTrans, Op::Trans, m, k, k, + (T)1, symrf_work, m, R, k, (T)0, Y, m); + if (this->timing) { t1 = steady_clock::now(); trsm_t_dur += duration_cast(t1 - t0).count(); } + + if (this->timing) t0 = steady_clock::now(); + lapack::gesdd(Job::SomeVec, m, k, Y, m, S, V_dat, m, R, k); + if (this->timing) { t1 = steady_clock::now(); svd_t_dur += duration_cast(t1 - t0).count(); } + + T S_max = (k > 0) ? S[0] : (T)0; + T S_thresh = (T)2 * eps_mach * S_max; + for (int64_t i = 0; i < k; ++i) { + eigvals[i] = std::pow(S[i], 2); + if (S[i] > S_thresh) ++r; + } + std::fill(&V_dat[m * r], &V_dat[m * k], (T)0); + } if (this->timing) t0 = steady_clock::now(); - T buf; - int64_t r = 0; - int64_t i; - for(i = 0; i < k; ++i) { - buf = std::pow(S[i], 2); - eigvals[i] = buf; - if(buf > nu) - ++r; - } - for(i = 0; i < r; ++i) - if (eigvals[i] > nu) eigvals[i] -= nu; - std::fill(&V_dat[m * r], &V_dat[m * k], (T)0); if (this->timing) { t1 = steady_clock::now(); post_svd_t_dur += duration_cast(t1 - t0).count(); } // Fixed-rank mode: skip error estimation entirely. @@ -229,7 +331,12 @@ class NystromEVD { err = power_error_est(A, k, this->error_est_p, Omega, V_dat, Y, eigvals); if (this->timing) { t1 = steady_clock::now(); error_est_t_dur += duration_cast(t1 - t0).count(); } - if(err <= 5 * std::max(tol, nu) || k == m) { + // Convergence: stop when the residual estimate hits the + // numerical noise floor (~ ε_mach · λ_max ≈ ε_mach · eigvals[0] + // since gesdd returned descending Σ → eigvals[0] is largest) + // or when the rank has saturated. + T noise_floor = eps_mach * (k > 0 ? eigvals[0] : (T)0); + if (err <= 5 * std::max(tol, noise_floor) || k == m) { break; } else if (2 * k > m) { k = m; diff --git a/RandLAPACK/linops/rl_sym_linops.hh b/RandLAPACK/linops/rl_sym_linops.hh index 524774ab2..db31e3ebd 100644 --- a/RandLAPACK/linops/rl_sym_linops.hh +++ b/RandLAPACK/linops/rl_sym_linops.hh @@ -104,6 +104,51 @@ struct ExplicitSymLinOp { return A_buff[i + j*lda]; } } + + /// SkOp overload: apply A to a SketchingOperator S (sparse or dense). + /// Computes C = alpha * A * S + beta * C. + /// + /// Mirrors the pattern in `SparseLinOp::operator()(SkOp&)`: + /// - Dense SkOp → extract S.buff and dispatch to the existing SYMM path. + /// - Sparse SkOp → use RandBLAS::sparse_data::right_spmm directly, + /// bypassing the densification of S into a dense buffer. + /// + /// PRECONDITION (sparse path only): A_buff must have BOTH triangles populated. + /// `right_spmm` reads A as a generic dense matrix and does not exploit + /// symmetry — so it will read whatever is in the lower triangle. For + /// matrices generated via `gen_sym_psd_lowrank` (which only fills upper), + /// the caller must symmetrize before constructing this operator. + /// `SparseSymLinOp` imposes the same constraint by convention. + template + void operator()( + Layout layout, + int64_t n_vecs, + T alpha, + SkOp& S, + T beta, + T* C, + int64_t ldc + ) { + if constexpr (requires { S.buff; S.layout; S.dist; }) { + // Dense sketch — extract buffer, dispatch to existing dense matvec. + // The existing path uses blas::symm and works fine with one-triangle storage. + if (S.buff == nullptr) RandBLAS::fill_dense(S); + int64_t ldS = S.dist.dim_major; + randblas_require(S.layout == layout); // simplification — caller ensures match + (*this)(layout, n_vecs, alpha, S.buff, ldS, beta, C, ldc); + } else { + // Sparse sketch — dense·sparse via right_spmm. Requires both triangles of A. + if (S.nnz < 0) RandBLAS::fill_sparse(S); + auto S_coo = RandBLAS::coo_view_of_skop(S); + RandBLAS::sparse_data::right_spmm( + layout, Op::NoTrans, Op::NoTrans, + dim, n_vecs, dim, + alpha, this->A_buff, this->lda, + S_coo, 0, 0, + beta, C, ldc + ); + } + } }; /*********************************************************/ diff --git a/RandLAPACK/misc/rl_util.hh b/RandLAPACK/misc/rl_util.hh index ec535cf24..6e57676f2 100644 --- a/RandLAPACK/misc/rl_util.hh +++ b/RandLAPACK/misc/rl_util.hh @@ -124,6 +124,22 @@ void get_U( } } +/// In-place symmetrize: replace A[i,j] and A[j,i] (i ≠ j) with their +/// average. Used to clean up small finite-arithmetic asymmetry in matrices +/// that should be symmetric in exact arithmetic (e.g. Ωᵀ·A·Ω built via two +/// non-trivial GEMMs, or block-Lanczos diagonal blocks Qᵢᵀ·A·Qᵢ). +/// Column-major; touches the strict upper and strict lower triangles. +template +void symmetrize(int64_t n, T* A, int64_t lda) { + for (int64_t j = 0; j < n; ++j) { + for (int64_t i = j + 1; i < n; ++i) { + T avg = (T)0.5 * (A[i + j * lda] + A[j + i * lda]); + A[i + j * lda] = avg; + A[j + i * lda] = avg; + } + } +} + /// Returns true if all diagonal entries of an upper-triangular matrix are /// nonzero (safe to use in TRSM). Returns false if any diagonal is exactly /// zero, which would cause division by zero during triangular solve. diff --git a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc index 7b02f8013..ae9f3f239 100644 --- a/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc +++ b/benchmark/bench_FunNystromPP/FunNystromPP_benchmark.cc @@ -168,6 +168,16 @@ struct CountingLinOp { // ============================================================================ // data_regen: build or load A for the current n. +// +// Optional disk cache: generated matrices (the synthetic-decay and kernel +// paths) are saved to /tmp/funny_A_mat_n_kmat_func_.bin +// after first generation. Subsequent invocations with the same parameters +// load from the cache instead of regenerating, saving the Haar-random QR + +// scale + syrk cost on every benchmark invocation. +// +// The cache file is just a flat binary dump of A (n*n entries, col-major). +// Eigenvalues, noise, and d_dim are recomputed deterministically each time — +// they're cheap to redo and avoid format/version concerns in the cache file. // ============================================================================ template @@ -190,17 +200,15 @@ static void data_regen( data.eigvals.clear(); data.noise = (T)0.0; data.d_dim = 0; - } else if (mat_type >= 3) { + return; + } + + // ------ Set up small bookkeeping (eigvals / noise / d_dim) — deterministic + // and cheap, recomputed every call regardless of cache hit. ------ + if (mat_type >= 3) { data.d_dim = data.k_mat; - data.noise = (T)0.0; + data.noise = (func_type == 1) ? (T)1.0 : (T)0.0; data.eigvals.clear(); - T bandwidth = std::sqrt((T)data.d_dim); - if (mat_type == 3) - RandLAPACK::gen::gen_kernel_matrix( - n, data.d_dim, data.A.data(), n, 0, bandwidth, (T)0.0, 0, state); - else - RandLAPACK::gen::gen_kernel_matrix( - n, data.d_dim, data.A.data(), n, 1, (T)0.0, (T)1.0, 2, state); } else { data.d_dim = 0; data.eigvals.resize(data.k_mat); @@ -210,16 +218,64 @@ static void data_regen( else RandLAPACK::gen::gen_exp_decay_singvals(data.k_mat, (T)1.0, (T)100.0, data.eigvals.data()); + data.noise = (func_type == 1) ? (T)1.0 : (T)0.0; + } + + // ------ Try the matrix file cache. ------ + char cache_path[256]; + std::snprintf(cache_path, sizeof(cache_path), + "/tmp/funny_A_mat%d_n%lld_kmat%lld_func%d_%c.bin", + mat_type, (long long)n, (long long)data.k_mat, func_type, + (sizeof(T) == 8) ? 'd' : 'f'); + { + std::ifstream cache_in(cache_path, std::ios::binary); + if (cache_in.good()) { + cache_in.read(reinterpret_cast(data.A.data()), + static_cast(n) * n * sizeof(T)); + if (cache_in.gcount() == + static_cast(n) * n * sizeof(T)) { + return; // cache hit + } + } + } + + // ------ Cache miss: generate. ------ + if (mat_type >= 3) { + T bandwidth = std::sqrt((T)data.d_dim); + if (mat_type == 3) + RandLAPACK::gen::gen_kernel_matrix( + n, data.d_dim, data.A.data(), n, 0, bandwidth, (T)0.0, 0, state); + else + RandLAPACK::gen::gen_kernel_matrix( + n, data.d_dim, data.A.data(), n, 1, (T)0.0, (T)1.0, 2, state); + if (data.noise > (T)0) { + for (int64_t i = 0; i < n; ++i) + data.A[i + i * n] += data.noise; + } + } else { if (func_type == 1) { - data.noise = (T)1.0; RandLAPACK::gen::gen_shifted_lowrank_psd(n, data.k_mat, data.A.data(), n, data.eigvals.data(), data.noise, state); } else { - data.noise = (T)0.0; RandLAPACK::gen::gen_sym_psd_lowrank(n, data.k_mat, data.A.data(), n, data.eigvals.data(), state); } } + + // Mirror upper into lower for the SkOp sparse-spmm path + // (right_spmm reads A as a generic dense matrix; symmetry isn't exploited). + if (mat_type < 3) { + for (int64_t j = 0; j < n; ++j) + for (int64_t i = j + 1; i < n; ++i) + data.A[i + j * n] = data.A[j + i * n]; + } + + // ------ Save the freshly generated matrix to cache for next time. ------ + std::ofstream cache_out(cache_path, std::ios::binary | std::ios::trunc); + if (cache_out.good()) { + cache_out.write(reinterpret_cast(data.A.data()), + static_cast(n) * n * sizeof(T)); + } } // ============================================================================ @@ -270,6 +326,7 @@ static void call_all_algs( bool compute_ref, int sketch_type, int vec_nnz, + bool skip_hutchinson, const std::string& output_filename ) { int64_t n = data.n, k = data.k, s = data.s, d = data.d; @@ -306,10 +363,14 @@ static void call_all_algs( ref_available = true; } - RandLAPACK::SYPS syps(3, 1, false, false); + RandLAPACK::SYPS syps(2, 1, false, false); syps.sketch_type = sketch_type; syps.vec_nnz = vec_nnz; ORTH_t orth(false, false); + // Wire ORTH_t into SYPS's internal power-iter stabilization too — same + // user choice applies. SYPS auto-falls-back to HQRQ on CholQR failure. + ORTH_t orth_inner(false, false); + syps.internal_stab = &orth_inner; RandLAPACK::SYRF, ORTH_t> syrf(syps, orth); RandLAPACK::NystromEVD nystrom_evd(syrf, 0); nystrom_evd.timing = true; @@ -363,21 +424,32 @@ static void call_all_algs( printf(" LanczosFA: matvec=%lld lanczos=%lld apply_f=%lld total=%lld\n", (long long)lt[0], (long long)lt[1], (long long)lt[2], (long long)lt[4]); - // Hutchinson + LanczosFA (state_alg continues from where FunNystromPP left it) - ESLO A_op2(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); - CSLO A_hutch(n, A_op2); - LFAOp lfa_op(n, A_hutch, lfa_base, f, d); - - auto start_h = steady_clock::now(); - T est_h = hutch_base.call(lfa_op, s, state_alg); - auto stop_h = steady_clock::now(); - long dur_h = duration_cast(stop_h - start_h).count(); - int64_t mv_h = A_hutch.count; - printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", - (double)est_h, (long long)dur_h, (long long)mv_h); + // Hutchinson + LanczosFA (skipped when --skip-hutchinson is set; + // useful for s-sweeps that only need the Hutchinson reference at one + // (lfa, d) variant — e.g. scalar d=200 — and not at every sweep + // point. CSV output preserves the column shape with zeros. + T est_h = (T)0; + long dur_h = 0; + int64_t mv_h = 0; + if (!skip_hutchinson) { + ESLO A_op2(n, blas::Uplo::Upper, data.A.data(), n, Layout::ColMajor); + CSLO A_hutch(n, A_op2); + LFAOp lfa_op(n, A_hutch, lfa_base, f, d); + + auto start_h = steady_clock::now(); + est_h = hutch_base.call(lfa_op, s, state_alg); + auto stop_h = steady_clock::now(); + dur_h = duration_cast(stop_h - start_h).count(); + mv_h = A_hutch.count; + printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", + (double)est_h, (long long)dur_h, (long long)mv_h); + } else { + printf(" Hutchinson+LFA: SKIPPED\n"); + } double err_pp = ref_available ? std::abs((double)est_pp - (double)true_tr) / std::abs((double)true_tr) : -1.0; - double err_h = ref_available ? std::abs((double)est_h - (double)true_tr) / std::abs((double)true_tr) : -1.0; + double err_h = (!skip_hutchinson && ref_available) + ? std::abs((double)est_h - (double)true_tr) / std::abs((double)true_tr) : -1.0; if (ref_available) printf(" Reference: true=%.6e err_pp=%e err_h=%e\n", (double)true_tr, err_pp, err_h); @@ -417,6 +489,7 @@ static void call_all_algs_sparse( double poly_lambda, int sketch_type, int vec_nnz, + bool skip_hutchinson, const std::string& output_filename ) { std::function f; @@ -434,10 +507,12 @@ static void call_all_algs_sparse( fname = "poly"; } - RandLAPACK::SYPS syps(3, 1, false, false); + RandLAPACK::SYPS syps(2, 1, false, false); syps.sketch_type = sketch_type; syps.vec_nnz = vec_nnz; ORTH_t orth(false, false); + ORTH_t orth_inner(false, false); + syps.internal_stab = &orth_inner; RandLAPACK::SYRF, ORTH_t> syrf(syps, orth); RandLAPACK::NystromEVD nystrom_evd(syrf, 0); nystrom_evd.timing = true; @@ -484,18 +559,25 @@ static void call_all_algs_sparse( printf(" LanczosFA: matvec=%lld lanczos=%lld apply_f=%lld total=%lld\n", (long long)lt[0], (long long)lt[1], (long long)lt[2], (long long)lt[4]); - state_alg = state; - SSLO A_op_h(n, csr); - CSLO A_hutch(n, A_op_h); - LFAOp lfa_op(n, A_hutch, lfa_base, f, d); - - auto start_h = steady_clock::now(); - T est_h = hutch_base.call(lfa_op, s, state_alg); - auto stop_h = steady_clock::now(); - long dur_h = duration_cast(stop_h - start_h).count(); - int64_t mv_h = A_hutch.count; - printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", - (double)est_h, (long long)dur_h, (long long)mv_h); + T est_h = (T)0; + long dur_h = 0; + int64_t mv_h = 0; + if (!skip_hutchinson) { + state_alg = state; + SSLO A_op_h(n, csr); + CSLO A_hutch(n, A_op_h); + LFAOp lfa_op(n, A_hutch, lfa_base, f, d); + + auto start_h = steady_clock::now(); + est_h = hutch_base.call(lfa_op, s, state_alg); + auto stop_h = steady_clock::now(); + dur_h = duration_cast(stop_h - start_h).count(); + mv_h = A_hutch.count; + printf(" Hutchinson+LFA: est=%.6e time=%lld us matvecs=%lld\n", + (double)est_h, (long long)dur_h, (long long)mv_h); + } else { + printf(" Hutchinson+LFA: SKIPPED\n"); + } printf(" Reference: N/A (sparse matrix)\n"); std::ofstream file(output_filename, std::ios::out | std::ios::app); @@ -518,12 +600,12 @@ static void call_all_algs_sparse( // ============================================================================ int main(int argc, char* argv[]) { - if (argc < 16) { + if (argc < 17) { std::cerr << "Usage: " << argv[0] << " " " " " " - " [n_1 n_2 ...]\n" + " [n_1 n_2 ...]\n" << " dir_path : output directory (use '.' for current dir)\n" << " num_runs : timed repetitions per matrix size\n" << " k : Nystrom rank (constant; must satisfy k <= n)\n" @@ -539,6 +621,10 @@ int main(int argc, char* argv[]) { << " precision : double or float (sparse .mtx path is always double)\n" << " lfa_type : 0=scalar LanczosFA, 1=BlockLanczosFA\n" << " orth_type : 0=HouseholderQR (HQRQ), 1=Cholesky QR (CholQRQ)\n" + << " skip_hutchinson : 0=run plain Hutchinson+LFA baseline,\n" + << " 1=skip it (CSV columns padded with zeros).\n" + << " Useful for s/k sweeps that only need the\n" + << " Hutchinson reference at one (lfa, d) variant.\n" << " n_1 ... : matrix sizes (generated types only)\n"; return 1; } @@ -559,6 +645,7 @@ int main(int argc, char* argv[]) { const char* lfa_str = lfa_type == 0 ? "scalar" : "block"; int orth_type = std::stoi(argv[15]); const char* orth_str = orth_type == 0 ? "hqrq" : "cholqr"; + bool skip_hutchinson = std::stoi(argv[16]) != 0; bool from_file = false; bool is_mtx = false; @@ -604,11 +691,11 @@ int main(int argc, char* argv[]) { + std::to_string(fm) + "x" + std::to_string(fn) + ")"); n_sizes = {fm}; } else { - if (argc < 17) { + if (argc < 18) { std::cerr << "Error: at least one matrix size (n_1) required for generated types.\n"; return 1; } - for (int i = 16; i < argc; ++i) + for (int i = 17; i < argc; ++i) n_sizes.push_back(std::stol(argv[i])); } @@ -658,6 +745,7 @@ int main(int argc, char* argv[]) { " precision=" + precision_str + " lfa_type=" + std::string(lfa_str) + " orth_type=" + std::string(orth_str) + + " skip_hutch=" + std::to_string((int)skip_hutchinson) + "\nNum runs per size: " + std::to_string(numruns) + "\nMatrix construction: " + mat_construction_str + "\n"; @@ -686,7 +774,8 @@ int main(int argc, char* argv[]) { call_all_algs_sparse( numruns, n, k_const, s_const, d_steps, csr, state_constant, - func_type, poly_lambda, sketch_type, vec_nnz, path); + func_type, poly_lambda, sketch_type, vec_nnz, + skip_hutchinson, path); }; if (lfa_type == 0 && orth_type == 0) run_sparse.template operator()(); else if (lfa_type == 0 && orth_type == 1) run_sparse.template operator()(); @@ -712,7 +801,8 @@ int main(int argc, char* argv[]) { call_all_algs( numruns, all_data, state_constant, mat_type, - func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, path); + func_type, poly_lambda, compute_ref, sketch_type, vec_nnz, + skip_hutchinson, path); } };