From 00538eefd793245b8189cfd57a9cf314533eaae2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Nov 2025 05:09:59 +0000 Subject: [PATCH 01/10] Implement positive policy decay Add positive policy decay based on Naphthalin's work in PRs #1173, #1288, and #2093. Formula: scaling = FastInvSqrt(1.0f + N_child / policy_decay_scale) odds = 1.0f / P - 1.0f P_eff = 1.0f / (1.0f + odds * scaling) Changes: - Add FastInvSqrt function to utils/fastmath.h for fast 1/sqrt(x) calculation - Add policy_decay_scale parameter (float, default 500.0, range 0-1000000) - When scale=0, policy decay is disabled (P_eff = P) - Lower scale = faster convergence to 100% policy - Modify GetU() in both classic and dag_classic node.h to use P_eff - Update search.cc files to pass policy_decay_scale to GetU() Behavior: - As N_child increases, all policies gradually approach 100% - Low-policy moves grow faster than high-policy moves - Preserves PUCT invariants: U decreases after visiting move, increases after visiting sibling - P_eff depends only on N_child, not N_parent --- src/search/classic/node.h | 17 ++++++++++++++--- src/search/classic/params.cc | 8 +++++++- src/search/classic/params.h | 3 +++ src/search/classic/search.cc | 10 ++++++---- src/search/dag_classic/node.h | 17 ++++++++++++++--- src/search/dag_classic/search.cc | 7 ++++--- src/utils/fastmath.h | 13 +++++++++++++ 7 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index 8a4e598fdb..858e5408b0 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -39,6 +39,7 @@ #include "chess/position.h" #include "neural/encoder.h" #include "proto/net.pb.h" +#include "utils/fastmath.h" #include "utils/mutex.h" namespace lczero { @@ -403,10 +404,20 @@ class EdgeAndNode { return edge_ ? edge_->GetMove(flip) : Move(); } - // Returns U = numerator * p / N. + // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - float GetU(float numerator) const { - return numerator * GetP() / (1 + GetNStarted()); + // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). + float GetU(float numerator, float policy_decay_scale = 0.0f) const { + float p = GetP(); + if (policy_decay_scale > 0.0f) { + // Apply positive policy decay: p_eff = 1 / (1 + odds * scaling) + // where odds = 1/p - 1 and scaling = FastInvSqrt(1 + N/scale) + float n_child = static_cast(GetN()); + float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); + float odds = 1.0f / p - 1.0f; + p = 1.0f / (1.0f + odds * scaling); + } + return numerator * p / (1 + GetNStarted()); } std::string DebugString() const; diff --git a/src/search/classic/params.cc b/src/search/classic/params.cc index e61b0f9c88..7ddf74e230 100644 --- a/src/search/classic/params.cc +++ b/src/search/classic/params.cc @@ -529,6 +529,10 @@ const OptionId BaseSearchParams::kGarbageCollectionDelayId{ "garbage-collection-delay", "GarbageCollectionDelay", "The percentage of expected move time until garbage collection start. " "Delay lets search find transpositions to freed search tree branches."}; +const OptionId BaseSearchParams::kPolicyDecayScaleId{ + "policy-decay-scale", "PolicyDecayScale", + "Scale parameter for positive policy decay. Higher values slow down " + "convergence. Set to 0 to disable policy decay (P_eff = P)."}; const OptionId SearchParams::kMaxPrefetchBatchId{ "max-prefetch", "MaxPrefetch", @@ -631,6 +635,7 @@ void BaseSearchParams::Populate(OptionsParser* options) { options->Add(kUCIRatingAdvId, -10000.0f, 10000.0f) = 0.0f; options->Add(kSearchSpinBackoffId) = false; options->Add(kGarbageCollectionDelayId, 0.0f, 100.0f) = 10.0f; + options->Add(kPolicyDecayScaleId, 0.0f, 1000000.0f) = 500.0f; } void SearchParams::Populate(OptionsParser* options) { @@ -725,7 +730,8 @@ BaseSearchParams::BaseSearchParams(const OptionsDict& options) kMaxCollisionVisitsScalingPower( options.Get(kMaxCollisionVisitsScalingPowerId)), kSearchSpinBackoff(options_.Get(kSearchSpinBackoffId)), - kGarbageCollectionDelay(options_.Get(kGarbageCollectionDelayId)) {} + kGarbageCollectionDelay(options_.Get(kGarbageCollectionDelayId)), + kPolicyDecayScale(options_.Get(kPolicyDecayScaleId)) {} SearchParams::SearchParams(const OptionsDict& options) : BaseSearchParams(options), diff --git a/src/search/classic/params.h b/src/search/classic/params.h index d84dbad5d8..70e224948b 100644 --- a/src/search/classic/params.h +++ b/src/search/classic/params.h @@ -130,6 +130,7 @@ class BaseSearchParams { return kMaxOutOfOrderEvalsFactor; } float GetNpsLimit() const { return kNpsLimit; } + float GetPolicyDecayScale() const { return kPolicyDecayScale; } int GetTaskWorkersPerSearchWorker() const { return kTaskWorkersPerSearchWorker; @@ -231,6 +232,7 @@ class BaseSearchParams { static const OptionId kUCIRatingAdvId; static const OptionId kSearchSpinBackoffId; static const OptionId kGarbageCollectionDelayId; + static const OptionId kPolicyDecayScaleId; protected: const OptionsDict& options_; @@ -290,6 +292,7 @@ class BaseSearchParams { const float kMaxCollisionVisitsScalingPower; const bool kSearchSpinBackoff; const float kGarbageCollectionDelay; + const float kPolicyDecayScale; }; class SearchParams : public BaseSearchParams { diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index 101d62e941..5772b83422 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -465,11 +465,12 @@ std::vector Search::GetVerboseStats(const Node* node) const { const float cpuct = ComputeCpuct(params_, node->GetN(), is_root); const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); + const float policy_decay_scale = params_.GetPolicyDecayScale(); std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale), edge); } std::sort(edges.begin(), edges.end()); @@ -555,8 +556,8 @@ std::vector Search::GetVerboseStats(const Node* node) const { MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale) + M, ") ", 8, 5); print_tail(&oss, edge.node()); infos.emplace_back(oss.str()); } @@ -2074,11 +2075,12 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float fpu = GetFpu(params_, node, node == search_->root_node_, draw_score); + const float policy_decay_scale = params_.GetPolicyDecayScale(); for (auto& edge : node->Edges()) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult) - edge.GetQ(fpu, draw_score), + scores.emplace_back(-edge.GetU(puct_mult, policy_decay_scale) - edge.GetQ(fpu, draw_score), edge); } diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index b74a64a9d4..e400c6d35c 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -46,6 +46,7 @@ #include "chess/gamestate.h" #include "chess/position.h" #include "neural/backend.h" +#include "utils/fastmath.h" #include "utils/mutex.h" namespace lczero { @@ -724,10 +725,20 @@ class EdgeAndNode { return edge_ ? edge_->GetMove(flip) : Move(); } - // Returns U = numerator * p / N. + // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - float GetU(float numerator) const { - return numerator * GetP() / (1 + GetNStarted()); + // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). + float GetU(float numerator, float policy_decay_scale = 0.0f) const { + float p = GetP(); + if (policy_decay_scale > 0.0f) { + // Apply positive policy decay: p_eff = 1 / (1 + odds * scaling) + // where odds = 1/p - 1 and scaling = FastInvSqrt(1 + N/scale) + float n_child = static_cast(GetN()); + float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); + float odds = 1.0f / p - 1.0f; + p = 1.0f / (1.0f + odds * scaling); + } + return numerator * p / (1 + GetNStarted()); } std::string DebugString() const; diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index e65977c38c..eee109997a 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -521,11 +521,12 @@ std::vector Search::GetVerboseStats( const float cpuct = ComputeCpuct(params_, node->GetTotalVisits(), is_root); const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); + const float policy_decay_scale = params_.GetPolicyDecayScale(); std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale), edge); } std::sort(edges.begin(), edges.end()); @@ -617,8 +618,8 @@ std::vector Search::GetVerboseStats( MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale) + M, ") ", 8, 5); print_tail(&oss, edge.node(), true); infos.emplace_back(oss.str()); } diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 6aa49412aa..f32691d7e6 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -104,4 +104,17 @@ inline float FastSign(const float a) { #endif } +// Fast approximate 1/sqrt(x) using bit manipulation. +// Based on the classic Quake III algorithm. +inline float FastInvSqrt(const float a) { + float halfx = 0.5f * a; + uint32_t i; + std::memcpy(&i, &a, sizeof(float)); + i = 0x5f3759df - (i >> 1); // Magic constant + float y; + std::memcpy(&y, &i, sizeof(float)); + y = y * (1.5f - halfx * y * y); // Newton iteration + return y; +} + } // namespace lczero From ddb175397e7317f8be8e595465d0015387d8d251 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Nov 2025 05:22:02 +0000 Subject: [PATCH 02/10] Fix policy decay in PUCT selection loops Apply positive policy decay to the actual move selection in both classic and dag_classic search implementations. The previous commit only applied decay to verbose stats and edge sorting, not the critical PUCT scoring loop that determines which moves to explore. Changes: - Apply policy decay formula in PickNodesToExtendTask() for both classic and dag_classic - Policy decay now affects current_score[idx] calculation in the hot path - This makes the policy_decay_scale parameter (default 500.0) functional Without this fix, the policy decay parameter had no effect on move selection, only on display and prefetch calculations. Fixes the issue identified by codex review bot. --- src/search/classic/search.cc | 12 ++++++++++-- src/search/dag_classic/search.cc | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index 5772b83422..b9bffd35da 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -1711,6 +1711,7 @@ void SearchWorker::PickNodesToExtendTask( const float cpuct = ComputeCpuct(params_, node->GetN(), is_root_node); const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); + const float policy_decay_scale = params_.GetPolicyDecayScale(); int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1733,8 +1734,15 @@ void SearchWorker::PickNodesToExtendTask( int nstarted = current_nstarted[idx]; const float util = current_util[idx]; if (idx > cache_filled_idx) { - current_score[idx] = - current_pol[idx] * puct_mult / (1 + nstarted) + util; + float p = current_pol[idx]; + if (policy_decay_scale > 0.0f) { + // Apply positive policy decay + float n_child = static_cast(cur_iters[idx].GetN()); + float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); + float odds = 1.0f / p - 1.0f; + p = 1.0f / (1.0f + odds * scaling); + } + current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; } if (is_root_node) { diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index eee109997a..3bab1b8f46 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -1809,6 +1809,7 @@ void SearchWorker::PickNodesToExtendTask( ComputeCpuct(params_, node->GetTotalVisits(), is_root_node); const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); + const float policy_decay_scale = params_.GetPolicyDecayScale(); int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1831,8 +1832,15 @@ void SearchWorker::PickNodesToExtendTask( int nstarted = current_nstarted[idx]; const float util = current_util[idx]; if (idx > cache_filled_idx) { - current_score[idx] = - cur_iters[idx].GetP() * puct_mult / (1 + nstarted) + util; + float p = cur_iters[idx].GetP(); + if (policy_decay_scale > 0.0f) { + // Apply positive policy decay + float n_child = static_cast(cur_iters[idx].GetN()); + float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); + float odds = 1.0f / p - 1.0f; + p = 1.0f / (1.0f + odds * scaling); + } + current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; } if (is_root_node) { From 04cfd2317a596f4d6b8bc171f50788e87e96ad1c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Nov 2025 05:43:58 +0000 Subject: [PATCH 03/10] Fix division by zero and add documentation Address critical issues identified in code review: 1. Add zero-check guards to prevent division by zero when p==0.0f - Policy values can be zero, causing division by zero in the decay formula - Added p > 0.0f check before applying decay in all locations 2. Document FastInvSqrt input requirements - Added comment noting it expects positive values - Documents undefined behavior for zero, negative, NaN, infinity 3. Extract policy decay logic to helper function - Created ApplyPolicyDecay() in utils/fastmath.h - Reduces code duplication across 4 locations - Centralizes zero-checking logic 4. Simplify GetU() implementations - Both classic and dag_classic node.h now use ApplyPolicyDecay() - Search loops also use the helper function This prevents crashes when encountering zero-policy edges and improves code maintainability by eliminating duplication. --- src/search/classic/node.h | 9 ++------- src/search/classic/search.cc | 9 +++------ src/search/dag_classic/node.h | 9 ++------- src/search/dag_classic/search.cc | 9 +++------ src/utils/fastmath.h | 14 ++++++++++++++ 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index 858e5408b0..c4a859f5e5 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -409,13 +409,8 @@ class EdgeAndNode { // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). float GetU(float numerator, float policy_decay_scale = 0.0f) const { float p = GetP(); - if (policy_decay_scale > 0.0f) { - // Apply positive policy decay: p_eff = 1 / (1 + odds * scaling) - // where odds = 1/p - 1 and scaling = FastInvSqrt(1 + N/scale) - float n_child = static_cast(GetN()); - float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); - float odds = 1.0f / p - 1.0f; - p = 1.0f / (1.0f + odds * scaling); + if (policy_decay_scale > 0.0f && p > 0.0f) { + p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale); } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index b9bffd35da..2540d0559b 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -1735,12 +1735,9 @@ void SearchWorker::PickNodesToExtendTask( const float util = current_util[idx]; if (idx > cache_filled_idx) { float p = current_pol[idx]; - if (policy_decay_scale > 0.0f) { - // Apply positive policy decay - float n_child = static_cast(cur_iters[idx].GetN()); - float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); - float odds = 1.0f / p - 1.0f; - p = 1.0f / (1.0f + odds * scaling); + if (policy_decay_scale > 0.0f && p > 0.0f) { + p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), + policy_decay_scale); } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index e400c6d35c..d377394125 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -730,13 +730,8 @@ class EdgeAndNode { // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). float GetU(float numerator, float policy_decay_scale = 0.0f) const { float p = GetP(); - if (policy_decay_scale > 0.0f) { - // Apply positive policy decay: p_eff = 1 / (1 + odds * scaling) - // where odds = 1/p - 1 and scaling = FastInvSqrt(1 + N/scale) - float n_child = static_cast(GetN()); - float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); - float odds = 1.0f / p - 1.0f; - p = 1.0f / (1.0f + odds * scaling); + if (policy_decay_scale > 0.0f && p > 0.0f) { + p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale); } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index 3bab1b8f46..8c88c919cb 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -1833,12 +1833,9 @@ void SearchWorker::PickNodesToExtendTask( const float util = current_util[idx]; if (idx > cache_filled_idx) { float p = cur_iters[idx].GetP(); - if (policy_decay_scale > 0.0f) { - // Apply positive policy decay - float n_child = static_cast(cur_iters[idx].GetN()); - float scaling = FastInvSqrt(1.0f + n_child / policy_decay_scale); - float odds = 1.0f / p - 1.0f; - p = 1.0f / (1.0f + odds * scaling); + if (policy_decay_scale > 0.0f && p > 0.0f) { + p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), + policy_decay_scale); } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index f32691d7e6..31581fd616 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -106,6 +106,8 @@ inline float FastSign(const float a) { // Fast approximate 1/sqrt(x) using bit manipulation. // Based on the classic Quake III algorithm. +// Expects positive input values. Does no range checking; produces undefined +// behavior for zero, negative, or special values (NaN, infinity). inline float FastInvSqrt(const float a) { float halfx = 0.5f * a; uint32_t i; @@ -117,4 +119,16 @@ inline float FastInvSqrt(const float a) { return y; } +// Apply positive policy decay transformation. +// Returns P_eff = 1 / (1 + odds * scaling) where: +// odds = 1/P - 1 +// scaling = 1/sqrt(1 + N/scale) +// When P=0 or scale=0, returns P unchanged. +inline float ApplyPolicyDecay(float p, float n_child, float scale) { + if (p == 0.0f || scale == 0.0f) return p; + float scaling = FastInvSqrt(1.0f + n_child / scale); + float odds = 1.0f / p - 1.0f; + return 1.0f / (1.0f + odds * scaling); +} + } // namespace lczero From fd5217526447f3d1b3ef369565d70b3e0bb140dc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Nov 2025 04:48:35 +0000 Subject: [PATCH 04/10] Make policy decay exponent tunable Add policy_decay_exponent parameter to control the strength of policy decay, making the feature more flexible and experimentally tunable. New formula: power_term = (1 + N_child/scale)^(-exponent) P_eff = 1 / (1 + odds * power_term) where odds = 1/P - 1 Changes: - Add PolicyDecayExponent parameter (range 0.01-10.0, default 1.0) - exponent=1.0: linear decay (new default, replaces sqrt) - exponent=0.5: sqrt decay (original behavior) - exponent<1.0: gentler decay - exponent>1.0: stronger decay - Update ApplyPolicyDecay() to accept exponent parameter - Optimize for exponent=1.0: use 1/(1+N/scale) directly - Optimize for exponent=0.5: use FastInvSqrt - General case: use std::pow - Update all call sites to pass exponent: - GetU() in both classic and dag_classic node.h - PUCT selection loops in both search implementations - Verbose stats output - Prefetch calculations Benefits: - More flexible tuning for different use cases - Default exponent=1.0 gives simpler, faster computation - Maintains fast path for sqrt case (exponent=0.5) - Allows exploring stronger or gentler decay behaviors --- src/search/classic/node.h | 7 +++++-- src/search/classic/params.cc | 9 ++++++++- src/search/classic/params.h | 3 +++ src/search/classic/search.cc | 13 ++++++++----- src/search/dag_classic/node.h | 7 +++++-- src/search/dag_classic/search.cc | 10 ++++++---- src/utils/fastmath.h | 27 ++++++++++++++++++++++----- 7 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index c4a859f5e5..e7b252d92c 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -407,10 +407,13 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). - float GetU(float numerator, float policy_decay_scale = 0.0f) const { + // policy_decay_exponent controls the strength of decay (default 1.0). + float GetU(float numerator, float policy_decay_scale = 0.0f, + float policy_decay_exponent = 1.0f) const { float p = GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { - p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale); + p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale, + policy_decay_exponent); } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/classic/params.cc b/src/search/classic/params.cc index 7ddf74e230..07e722aeb6 100644 --- a/src/search/classic/params.cc +++ b/src/search/classic/params.cc @@ -533,6 +533,11 @@ const OptionId BaseSearchParams::kPolicyDecayScaleId{ "policy-decay-scale", "PolicyDecayScale", "Scale parameter for positive policy decay. Higher values slow down " "convergence. Set to 0 to disable policy decay (P_eff = P)."}; +const OptionId BaseSearchParams::kPolicyDecayExponentId{ + "policy-decay-exponent", "PolicyDecayExponent", + "Exponent for policy decay formula: power_term = (1 + N/scale)^(-exponent). " + "Lower values give gentler decay, higher values give stronger decay. " + "Default 1.0 gives linear decay, 0.5 gives sqrt decay."}; const OptionId SearchParams::kMaxPrefetchBatchId{ "max-prefetch", "MaxPrefetch", @@ -636,6 +641,7 @@ void BaseSearchParams::Populate(OptionsParser* options) { options->Add(kSearchSpinBackoffId) = false; options->Add(kGarbageCollectionDelayId, 0.0f, 100.0f) = 10.0f; options->Add(kPolicyDecayScaleId, 0.0f, 1000000.0f) = 500.0f; + options->Add(kPolicyDecayExponentId, 0.01f, 10.0f) = 1.0f; } void SearchParams::Populate(OptionsParser* options) { @@ -731,7 +737,8 @@ BaseSearchParams::BaseSearchParams(const OptionsDict& options) options.Get(kMaxCollisionVisitsScalingPowerId)), kSearchSpinBackoff(options_.Get(kSearchSpinBackoffId)), kGarbageCollectionDelay(options_.Get(kGarbageCollectionDelayId)), - kPolicyDecayScale(options_.Get(kPolicyDecayScaleId)) {} + kPolicyDecayScale(options_.Get(kPolicyDecayScaleId)), + kPolicyDecayExponent(options_.Get(kPolicyDecayExponentId)) {} SearchParams::SearchParams(const OptionsDict& options) : BaseSearchParams(options), diff --git a/src/search/classic/params.h b/src/search/classic/params.h index 70e224948b..9824320057 100644 --- a/src/search/classic/params.h +++ b/src/search/classic/params.h @@ -131,6 +131,7 @@ class BaseSearchParams { } float GetNpsLimit() const { return kNpsLimit; } float GetPolicyDecayScale() const { return kPolicyDecayScale; } + float GetPolicyDecayExponent() const { return kPolicyDecayExponent; } int GetTaskWorkersPerSearchWorker() const { return kTaskWorkersPerSearchWorker; @@ -233,6 +234,7 @@ class BaseSearchParams { static const OptionId kSearchSpinBackoffId; static const OptionId kGarbageCollectionDelayId; static const OptionId kPolicyDecayScaleId; + static const OptionId kPolicyDecayExponentId; protected: const OptionsDict& options_; @@ -293,6 +295,7 @@ class BaseSearchParams { const bool kSearchSpinBackoff; const float kGarbageCollectionDelay; const float kPolicyDecayScale; + const float kPolicyDecayExponent; }; class SearchParams : public BaseSearchParams { diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index 2540d0559b..a9a16c3163 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -466,11 +466,12 @@ std::vector Search::GetVerboseStats(const Node* node) const { const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); + const float policy_decay_exponent = params_.GetPolicyDecayExponent(); std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), edge); } std::sort(edges.begin(), edges.end()); @@ -556,8 +557,8 @@ std::vector Search::GetVerboseStats(const Node* node) const { MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); print_tail(&oss, edge.node()); infos.emplace_back(oss.str()); } @@ -1712,6 +1713,7 @@ void SearchWorker::PickNodesToExtendTask( const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); + const float policy_decay_exponent = params_.GetPolicyDecayExponent(); int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1737,7 +1739,7 @@ void SearchWorker::PickNodesToExtendTask( float p = current_pol[idx]; if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale); + policy_decay_scale, policy_decay_exponent); } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; @@ -2081,11 +2083,12 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { const float fpu = GetFpu(params_, node, node == search_->root_node_, draw_score); const float policy_decay_scale = params_.GetPolicyDecayScale(); + const float policy_decay_exponent = params_.GetPolicyDecayExponent(); for (auto& edge : node->Edges()) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult, policy_decay_scale) - edge.GetQ(fpu, draw_score), + scores.emplace_back(-edge.GetU(puct_mult, policy_decay_scale, policy_decay_exponent) - edge.GetQ(fpu, draw_score), edge); } diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index d377394125..aa12f8a547 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -728,10 +728,13 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). - float GetU(float numerator, float policy_decay_scale = 0.0f) const { + // policy_decay_exponent controls the strength of decay (default 1.0). + float GetU(float numerator, float policy_decay_scale = 0.0f, + float policy_decay_exponent = 1.0f) const { float p = GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { - p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale); + p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale, + policy_decay_exponent); } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index 8c88c919cb..46d5d863b9 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -522,11 +522,12 @@ std::vector Search::GetVerboseStats( const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); + const float policy_decay_exponent = params_.GetPolicyDecayExponent(); std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), edge); } std::sort(edges.begin(), edges.end()); @@ -618,8 +619,8 @@ std::vector Search::GetVerboseStats( MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); print_tail(&oss, edge.node(), true); infos.emplace_back(oss.str()); } @@ -1810,6 +1811,7 @@ void SearchWorker::PickNodesToExtendTask( const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); + const float policy_decay_exponent = params_.GetPolicyDecayExponent(); int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1835,7 +1837,7 @@ void SearchWorker::PickNodesToExtendTask( float p = cur_iters[idx].GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale); + policy_decay_scale, policy_decay_exponent); } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 31581fd616..42133137ba 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -120,15 +120,32 @@ inline float FastInvSqrt(const float a) { } // Apply positive policy decay transformation. -// Returns P_eff = 1 / (1 + odds * scaling) where: +// Returns P_eff = 1 / (1 + odds * power_term) where: // odds = 1/P - 1 -// scaling = 1/sqrt(1 + N/scale) +// power_term = (1 + N/scale)^(-exponent) // When P=0 or scale=0, returns P unchanged. -inline float ApplyPolicyDecay(float p, float n_child, float scale) { +// Optimized for common exponent values (1.0 and 0.5). +inline float ApplyPolicyDecay(float p, float n_child, float scale, + float exponent) { if (p == 0.0f || scale == 0.0f) return p; - float scaling = FastInvSqrt(1.0f + n_child / scale); + + float base = 1.0f + n_child / scale; + float power_term; + + // Optimize common cases + if (exponent == 1.0f) { + // Linear decay: (1 + N/scale)^(-1) = 1/(1 + N/scale) + power_term = 1.0f / base; + } else if (exponent == 0.5f) { + // Sqrt decay: (1 + N/scale)^(-0.5) = 1/sqrt(1 + N/scale) + power_term = FastInvSqrt(base); + } else { + // General case: use std::pow + power_term = std::pow(base, -exponent); + } + float odds = 1.0f / p - 1.0f; - return 1.0f / (1.0f + odds * scaling); + return 1.0f / (1.0f + odds * power_term); } } // namespace lczero From fcec006fcfd686958feb11da377c1014f50e6f1b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Nov 2025 04:55:16 +0000 Subject: [PATCH 05/10] Add FastPow and FastPrecisePow implementations Integrate Martin Ankerl's widely-used fast power approximation functions to replace std::pow in policy decay calculations for significant performance improvement. New functions: - FastPow(a, b): Basic fast power approximation - ~4x faster than std::pow for fractional exponents - Accuracy within 5%, rare cases up to 12% error - Uses bit manipulation similar to FastInvSqrt - FastPrecisePow(a, b): Accurate fast power approximation - ~3x faster than std::pow - Separates integer and fractional exponent parts - Integer part handled via exponentiation by squaring - Significantly more accurate when exponent > 1 Changes to ApplyPolicyDecay: - General case now uses FastPrecisePow instead of std::pow - Provides ~3x speedup for non-optimized exponent values - Maintains fast paths for exponent=1.0 and exponent=0.5 Source: https://martin.ankerl.com/2012/01/25/optimized-approximative-pow-in-c-and-cpp/ Performance impact: - Policy decay calculations run ~3x faster in general case - Particularly beneficial when exponent != 1.0 and != 0.5 - Critical for PUCT selection loop (hot path) --- src/utils/fastmath.h | 51 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 42133137ba..2cdfa84e37 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -119,6 +119,53 @@ inline float FastInvSqrt(const float a) { return y; } +// Fast approximate pow(a, b) using bit manipulation. +// Based on Martin Ankerl's implementation (https://martin.ankerl.com/2012/01/25/optimized-approximative-pow-in-c-and-cpp/) +// Approximately 4x faster than std::pow for fractional exponents. +// Accuracy typically within 5%, with rare cases up to 12% error. +// For better accuracy with integer or near-integer exponents, use FastPrecisePow. +// Expects positive base. Does no range checking. +inline float FastPow(float a, float b) { + union { + float f; + int32_t i; + } u = {a}; + u.i = static_cast(b * (u.i - 1064866805) + 1064866805); + return u.f; +} + +// More accurate version of FastPow that handles integer exponent part separately. +// Approximately 3x faster than std::pow. +// Significantly more accurate than FastPow when exponent > 1. +// Expects positive base. Does no range checking. +inline float FastPrecisePow(float a, float b) { + // Separate integer and fractional parts + int e = static_cast(b); + union { + float f; + int32_t i; + } u = {a}; + u.i = static_cast((b - e) * (u.i - 1064866805) + 1064866805); + + // Handle integer part using exponentiation by squaring + float r = 1.0f; + float base = a; + int exp = e; + if (exp < 0) { + base = 1.0f / base; + exp = -exp; + } + while (exp) { + if (exp & 1) { + r *= base; + } + base *= base; + exp >>= 1; + } + + return r * u.f; +} + // Apply positive policy decay transformation. // Returns P_eff = 1 / (1 + odds * power_term) where: // odds = 1/P - 1 @@ -140,8 +187,8 @@ inline float ApplyPolicyDecay(float p, float n_child, float scale, // Sqrt decay: (1 + N/scale)^(-0.5) = 1/sqrt(1 + N/scale) power_term = FastInvSqrt(base); } else { - // General case: use std::pow - power_term = std::pow(base, -exponent); + // General case: use FastPrecisePow for ~3x speedup over std::pow + power_term = FastPrecisePow(base, -exponent); } float odds = 1.0f / p - 1.0f; From 4d604f7b0ef674f58ed4356cc268113b2aad676f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Nov 2025 05:42:56 +0000 Subject: [PATCH 06/10] Scale policy decay by number of legal moves Update policy decay implementation to multiply the scale parameter by the number of legal moves in each position. This makes the decay proportional to the position complexity, so positions with more legal moves require proportionally more visits before decay takes effect. Changes: - Updated ApplyPolicyDecay() to accept num_legal_moves parameter - Modified decay formula: base = 1 + N/(scale * num_legal_moves) - Updated GetU() signature in both classic and dag_classic node.h - Updated all call sites to pass node->GetNumEdges() --- src/search/classic/node.h | 5 +++-- src/search/classic/search.cc | 11 ++++++----- src/search/dag_classic/node.h | 5 +++-- src/search/dag_classic/search.cc | 9 +++++---- src/utils/fastmath.h | 11 ++++++----- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index e7b252d92c..f5d9157206 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -408,12 +408,13 @@ class EdgeAndNode { // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). // policy_decay_exponent controls the strength of decay (default 1.0). - float GetU(float numerator, float policy_decay_scale = 0.0f, + // num_edges is the number of legal moves (used to scale the decay parameter). + float GetU(float numerator, int num_edges, float policy_decay_scale = 0.0f, float policy_decay_exponent = 1.0f) const { float p = GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale, - policy_decay_exponent); + policy_decay_exponent, num_edges); } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index a9a16c3163..633cdff15c 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -471,7 +471,7 @@ std::vector Search::GetVerboseStats(const Node* node) const { edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), edge); } std::sort(edges.begin(), edges.end()); @@ -557,8 +557,8 @@ std::vector Search::GetVerboseStats(const Node* node) const { MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); print_tail(&oss, edge.node()); infos.emplace_back(oss.str()); } @@ -1739,7 +1739,8 @@ void SearchWorker::PickNodesToExtendTask( float p = current_pol[idx]; if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale, policy_decay_exponent); + policy_decay_scale, policy_decay_exponent, + node->GetNumEdges()); } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; @@ -2088,7 +2089,7 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult, policy_decay_scale, policy_decay_exponent) - edge.GetQ(fpu, draw_score), + scores.emplace_back(-edge.GetU(puct_mult, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent) - edge.GetQ(fpu, draw_score), edge); } diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index aa12f8a547..f91fbf243b 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -729,12 +729,13 @@ class EdgeAndNode { // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). // policy_decay_exponent controls the strength of decay (default 1.0). - float GetU(float numerator, float policy_decay_scale = 0.0f, + // num_edges is the number of legal moves (used to scale the decay parameter). + float GetU(float numerator, int num_edges, float policy_decay_scale = 0.0f, float policy_decay_exponent = 1.0f) const { float p = GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale, - policy_decay_exponent); + policy_decay_exponent, num_edges); } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index 46d5d863b9..e0f4971204 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -527,7 +527,7 @@ std::vector Search::GetVerboseStats( edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), edge); } std::sort(edges.begin(), edges.end()); @@ -619,8 +619,8 @@ std::vector Search::GetVerboseStats( MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); print_tail(&oss, edge.node(), true); infos.emplace_back(oss.str()); } @@ -1837,7 +1837,8 @@ void SearchWorker::PickNodesToExtendTask( float p = cur_iters[idx].GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale, policy_decay_exponent); + policy_decay_scale, policy_decay_exponent, + node->GetNumEdges()); } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 2cdfa84e37..6527ccf988 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -169,14 +169,15 @@ inline float FastPrecisePow(float a, float b) { // Apply positive policy decay transformation. // Returns P_eff = 1 / (1 + odds * power_term) where: // odds = 1/P - 1 -// power_term = (1 + N/scale)^(-exponent) -// When P=0 or scale=0, returns P unchanged. +// power_term = (1 + N/(scale * num_legal_moves))^(-exponent) +// When P=0, scale=0, or num_legal_moves<=0, returns P unchanged. // Optimized for common exponent values (1.0 and 0.5). inline float ApplyPolicyDecay(float p, float n_child, float scale, - float exponent) { - if (p == 0.0f || scale == 0.0f) return p; + float exponent, int num_legal_moves) { + if (p == 0.0f || scale == 0.0f || num_legal_moves <= 0) return p; - float base = 1.0f + n_child / scale; + float effective_scale = scale * num_legal_moves; + float base = 1.0f + n_child / effective_scale; float power_term; // Optimize common cases From 0800e09cf9eed55420b56603e07dacf07a352531 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Nov 2025 17:04:00 +0000 Subject: [PATCH 07/10] Implement sum-normalized policy decay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace simple per-move policy decay with sum-normalized approach to ensure policies always sum to 1.0 throughout search. PARAMETER CHANGES: - Rename policy_decay_scale to policy_decay_scale_per_move - Default changed from 500.0 to 20.0 - Range changed from 0-1000000 to 0-1000 - Default exponent changed from 1.0 to 0.5 (sqrt decay) - Parameter interpretation: "visits per available move before significant decay" NORMALIZATION METHOD: - Calculate sum of raw P_eff across all edges at each node - Normalize each P_eff by dividing by the sum - Ensures sum(P_eff) ā‰ˆ 1.0 at N=0 (equals original policies) - As Nā†’āˆž, P_eff ā†’ 1/num_legal (uniform distribution) IMPLEMENTATION: - ApplyPolicyDecay now returns raw (unnormalized) P_eff - Added policy_decay_sum parameter to GetU() - Calculate sum once per node evaluation before PUCT selection - Apply normalization in GetU() and inline PUCT loops - Updated verbose stats and prefetch to use normalized values FORMULA: effective_scale = scale_per_move * num_legal_moves power_term = (1 + N_child / effective_scale)^(-exponent) odds = 1/P - 1 raw_P_eff = 1 / (1 + odds * power_term) P_eff = raw_P_eff / sum(raw_P_eff for all edges) Files modified: - src/utils/fastmath.h: Updated ApplyPolicyDecay documentation - src/search/classic/params.cc: Renamed parameter and updated defaults - src/search/classic/node.h: Added policy_decay_sum to GetU() - src/search/dag_classic/node.h: Added policy_decay_sum to GetU() - src/search/classic/search.cc: Calculate sum and normalize in 3 locations - src/search/dag_classic/search.cc: Calculate sum and normalize in 2 locations --- src/search/classic/node.h | 21 ++++++++---- src/search/classic/params.cc | 14 ++++---- src/search/classic/search.cc | 59 +++++++++++++++++++++++++++++--- src/search/dag_classic/node.h | 21 ++++++++---- src/search/dag_classic/search.cc | 41 ++++++++++++++++++++-- src/utils/fastmath.h | 14 ++++---- 6 files changed, 137 insertions(+), 33 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index f5d9157206..df209352d6 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -406,15 +406,22 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). - // policy_decay_exponent controls the strength of decay (default 1.0). + // policy_decay_scale_per_move controls positive policy decay. When scale=0, no decay (p_eff = p). + // policy_decay_exponent controls the strength of decay (default 0.5). // num_edges is the number of legal moves (used to scale the decay parameter). - float GetU(float numerator, int num_edges, float policy_decay_scale = 0.0f, - float policy_decay_exponent = 1.0f) const { + // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). + float GetU(float numerator, int num_edges, + float policy_decay_scale_per_move = 0.0f, + float policy_decay_exponent = 0.5f, + float policy_decay_sum = 1.0f) const { float p = GetP(); - if (policy_decay_scale > 0.0f && p > 0.0f) { - p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale, - policy_decay_exponent, num_edges); + if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { + // Calculate raw (unnormalized) P_eff + p = ApplyPolicyDecay(p, static_cast(GetN()), + policy_decay_scale_per_move, policy_decay_exponent, + num_edges); + // Normalize by sum to ensure sum(P_eff) = 1.0 + p /= policy_decay_sum; } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/classic/params.cc b/src/search/classic/params.cc index 07e722aeb6..136346bb29 100644 --- a/src/search/classic/params.cc +++ b/src/search/classic/params.cc @@ -530,14 +530,16 @@ const OptionId BaseSearchParams::kGarbageCollectionDelayId{ "The percentage of expected move time until garbage collection start. " "Delay lets search find transpositions to freed search tree branches."}; const OptionId BaseSearchParams::kPolicyDecayScaleId{ - "policy-decay-scale", "PolicyDecayScale", - "Scale parameter for positive policy decay. Higher values slow down " - "convergence. Set to 0 to disable policy decay (P_eff = P)."}; + "policy-decay-scale-per-move", "PolicyDecayScalePerMove", + "Scale parameter for positive policy decay, specified per legal move. " + "Effective scale = scale_per_move * num_legal_moves. Controls how many " + "visits per available move before decay becomes significant. " + "Set to 0 to disable policy decay (P_eff = P)."}; const OptionId BaseSearchParams::kPolicyDecayExponentId{ "policy-decay-exponent", "PolicyDecayExponent", "Exponent for policy decay formula: power_term = (1 + N/scale)^(-exponent). " "Lower values give gentler decay, higher values give stronger decay. " - "Default 1.0 gives linear decay, 0.5 gives sqrt decay."}; + "Default 0.5 gives sqrt decay, 1.0 gives linear decay."}; const OptionId SearchParams::kMaxPrefetchBatchId{ "max-prefetch", "MaxPrefetch", @@ -640,8 +642,8 @@ void BaseSearchParams::Populate(OptionsParser* options) { options->Add(kUCIRatingAdvId, -10000.0f, 10000.0f) = 0.0f; options->Add(kSearchSpinBackoffId) = false; options->Add(kGarbageCollectionDelayId, 0.0f, 100.0f) = 10.0f; - options->Add(kPolicyDecayScaleId, 0.0f, 1000000.0f) = 500.0f; - options->Add(kPolicyDecayExponentId, 0.01f, 10.0f) = 1.0f; + options->Add(kPolicyDecayScaleId, 0.0f, 1000.0f) = 20.0f; + options->Add(kPolicyDecayExponentId, 0.01f, 10.0f) = 0.5f; } void SearchParams::Populate(OptionsParser* options) { diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index 633cdff15c..bb3a4a5434 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -467,11 +467,27 @@ std::vector Search::GetVerboseStats(const Node* node) const { cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); const float policy_decay_exponent = params_.GetPolicyDecayExponent(); + + // Calculate sum of raw P_eff for normalization + float policy_decay_sum = 1.0f; + if (policy_decay_scale > 0.0f) { + policy_decay_sum = 0.0f; + for (const auto& edge : node->Edges()) { + float p = edge.GetP(); + if (p > 0.0f) { + policy_decay_sum += ApplyPolicyDecay( + p, static_cast(edge.GetN()), policy_decay_scale, + policy_decay_exponent, node->GetNumEdges()); + } + } + if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; + } + std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), edge); } std::sort(edges.begin(), edges.end()); @@ -557,8 +573,8 @@ std::vector Search::GetVerboseStats(const Node* node) const { MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) + M, ") ", 8, 5); print_tail(&oss, edge.node()); infos.emplace_back(oss.str()); } @@ -1714,6 +1730,23 @@ void SearchWorker::PickNodesToExtendTask( cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); const float policy_decay_exponent = params_.GetPolicyDecayExponent(); + + // Calculate sum of raw P_eff for normalization + float policy_decay_sum = 1.0f; + if (policy_decay_scale > 0.0f) { + policy_decay_sum = 0.0f; + auto edge_iter = node->Edges(); + for (int i = 0; i < max_needed; ++i, ++edge_iter) { + float p = current_pol[i]; + if (p > 0.0f) { + policy_decay_sum += ApplyPolicyDecay( + p, static_cast(edge_iter.GetN()), policy_decay_scale, + policy_decay_exponent, node->GetNumEdges()); + } + } + if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; + } + int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1741,6 +1774,8 @@ void SearchWorker::PickNodesToExtendTask( p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), policy_decay_scale, policy_decay_exponent, node->GetNumEdges()); + // Normalize by sum to ensure sum(P_eff) = 1.0 + p /= policy_decay_sum; } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; @@ -2085,11 +2120,27 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { GetFpu(params_, node, node == search_->root_node_, draw_score); const float policy_decay_scale = params_.GetPolicyDecayScale(); const float policy_decay_exponent = params_.GetPolicyDecayExponent(); + + // Calculate sum of raw P_eff for normalization + float policy_decay_sum = 1.0f; + if (policy_decay_scale > 0.0f) { + policy_decay_sum = 0.0f; + for (const auto& edge : node->Edges()) { + float p = edge.GetP(); + if (p > 0.0f) { + policy_decay_sum += ApplyPolicyDecay( + p, static_cast(edge.GetN()), policy_decay_scale, + policy_decay_exponent, node->GetNumEdges()); + } + } + if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; + } + for (auto& edge : node->Edges()) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent) - edge.GetQ(fpu, draw_score), + scores.emplace_back(-edge.GetU(puct_mult, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) - edge.GetQ(fpu, draw_score), edge); } diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index f91fbf243b..567cbb407d 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -727,15 +727,22 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - // policy_decay_scale controls positive policy decay. When scale=0, no decay (p_eff = p). - // policy_decay_exponent controls the strength of decay (default 1.0). + // policy_decay_scale_per_move controls positive policy decay. When scale=0, no decay (p_eff = p). + // policy_decay_exponent controls the strength of decay (default 0.5). // num_edges is the number of legal moves (used to scale the decay parameter). - float GetU(float numerator, int num_edges, float policy_decay_scale = 0.0f, - float policy_decay_exponent = 1.0f) const { + // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). + float GetU(float numerator, int num_edges, + float policy_decay_scale_per_move = 0.0f, + float policy_decay_exponent = 0.5f, + float policy_decay_sum = 1.0f) const { float p = GetP(); - if (policy_decay_scale > 0.0f && p > 0.0f) { - p = ApplyPolicyDecay(p, static_cast(GetN()), policy_decay_scale, - policy_decay_exponent, num_edges); + if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { + // Calculate raw (unnormalized) P_eff + p = ApplyPolicyDecay(p, static_cast(GetN()), + policy_decay_scale_per_move, policy_decay_exponent, + num_edges); + // Normalize by sum to ensure sum(P_eff) = 1.0 + p /= policy_decay_sum; } return numerator * p / (1 + GetNStarted()); } diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index e0f4971204..6c682a6ec3 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -523,11 +523,27 @@ std::vector Search::GetVerboseStats( cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); const float policy_decay_exponent = params_.GetPolicyDecayExponent(); + + // Calculate sum of raw P_eff for normalization + float policy_decay_sum = 1.0f; + if (policy_decay_scale > 0.0f) { + policy_decay_sum = 0.0f; + for (const auto& edge : node->Edges()) { + float p = edge.GetP(); + if (p > 0.0f) { + policy_decay_sum += ApplyPolicyDecay( + p, static_cast(edge.GetN()), policy_decay_scale, + policy_decay_exponent, node->GetNumEdges()); + } + } + if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; + } + std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), edge); } std::sort(edges.begin(), edges.end()); @@ -619,8 +635,8 @@ std::vector Search::GetVerboseStats( MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) + M, ") ", 8, 5); print_tail(&oss, edge.node(), true); infos.emplace_back(oss.str()); } @@ -1812,6 +1828,23 @@ void SearchWorker::PickNodesToExtendTask( cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); const float policy_decay_exponent = params_.GetPolicyDecayExponent(); + + // Calculate sum of raw P_eff for normalization + float policy_decay_sum = 1.0f; + if (policy_decay_scale > 0.0f) { + policy_decay_sum = 0.0f; + auto edge_iter = node->Edges(); + for (int i = 0; i < max_needed; ++i, ++edge_iter) { + float p = edge_iter.GetP(); + if (p > 0.0f) { + policy_decay_sum += ApplyPolicyDecay( + p, static_cast(edge_iter.GetN()), policy_decay_scale, + policy_decay_exponent, node->GetNumEdges()); + } + } + if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; + } + int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1839,6 +1872,8 @@ void SearchWorker::PickNodesToExtendTask( p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), policy_decay_scale, policy_decay_exponent, node->GetNumEdges()); + // Normalize by sum to ensure sum(P_eff) = 1.0 + p /= policy_decay_sum; } current_score[idx] = p * puct_mult / (1 + nstarted) + util; cache_filled_idx++; diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 6527ccf988..88507677ba 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -167,16 +167,18 @@ inline float FastPrecisePow(float a, float b) { } // Apply positive policy decay transformation. -// Returns P_eff = 1 / (1 + odds * power_term) where: +// Returns raw (unnormalized) P_eff = 1 / (1 + odds * power_term) where: // odds = 1/P - 1 -// power_term = (1 + N/(scale * num_legal_moves))^(-exponent) +// effective_scale = scale_per_move * num_legal_moves +// power_term = (1 + N/effective_scale)^(-exponent) // When P=0, scale=0, or num_legal_moves<=0, returns P unchanged. -// Optimized for common exponent values (1.0 and 0.5). -inline float ApplyPolicyDecay(float p, float n_child, float scale, +// NOTE: Caller must normalize by sum of all raw_P_eff values to ensure sum(P_eff) = 1. +// Optimized for common exponent values (0.5 and 1.0). +inline float ApplyPolicyDecay(float p, float n_child, float scale_per_move, float exponent, int num_legal_moves) { - if (p == 0.0f || scale == 0.0f || num_legal_moves <= 0) return p; + if (p == 0.0f || scale_per_move == 0.0f || num_legal_moves <= 0) return p; - float effective_scale = scale * num_legal_moves; + float effective_scale = scale_per_move * num_legal_moves; float base = 1.0f + n_child / effective_scale; float power_term; From 363eab507f51e690e3a0a7ce0f9f3b6c0ec5366e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Nov 2025 17:15:10 +0000 Subject: [PATCH 08/10] Change default exponent to 1.0 and display P_eff in verbose stats 1. Changed default policy_decay_exponent from 0.5 to 1.0 (linear decay) - Updated default in params.cc - Updated documentation to list 1.0 first as default - Updated GetU() default parameter values in both node.h files 2. Display P_eff in verbose move statistics instead of raw P - Label remains "P" for compatibility with tools like Nibbler - Calculates normalized P_eff for each edge in verbose stats - Users can now watch P_eff evolve during search - Updated both classic and dag_classic search.cc --- src/search/classic/node.h | 4 ++-- src/search/classic/params.cc | 4 ++-- src/search/classic/search.cc | 10 +++++++++- src/search/dag_classic/node.h | 4 ++-- src/search/dag_classic/search.cc | 10 +++++++++- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index df209352d6..2c3d2dbdbc 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -407,12 +407,12 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). // policy_decay_scale_per_move controls positive policy decay. When scale=0, no decay (p_eff = p). - // policy_decay_exponent controls the strength of decay (default 0.5). + // policy_decay_exponent controls the strength of decay (default 1.0). // num_edges is the number of legal moves (used to scale the decay parameter). // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). float GetU(float numerator, int num_edges, float policy_decay_scale_per_move = 0.0f, - float policy_decay_exponent = 0.5f, + float policy_decay_exponent = 1.0f, float policy_decay_sum = 1.0f) const { float p = GetP(); if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { diff --git a/src/search/classic/params.cc b/src/search/classic/params.cc index 136346bb29..c77ce1a891 100644 --- a/src/search/classic/params.cc +++ b/src/search/classic/params.cc @@ -539,7 +539,7 @@ const OptionId BaseSearchParams::kPolicyDecayExponentId{ "policy-decay-exponent", "PolicyDecayExponent", "Exponent for policy decay formula: power_term = (1 + N/scale)^(-exponent). " "Lower values give gentler decay, higher values give stronger decay. " - "Default 0.5 gives sqrt decay, 1.0 gives linear decay."}; + "Default 1.0 gives linear decay, 0.5 gives sqrt decay."}; const OptionId SearchParams::kMaxPrefetchBatchId{ "max-prefetch", "MaxPrefetch", @@ -643,7 +643,7 @@ void BaseSearchParams::Populate(OptionsParser* options) { options->Add(kSearchSpinBackoffId) = false; options->Add(kGarbageCollectionDelayId, 0.0f, 100.0f) = 10.0f; options->Add(kPolicyDecayScaleId, 0.0f, 1000.0f) = 20.0f; - options->Add(kPolicyDecayExponentId, 0.01f, 10.0f) = 0.5f; + options->Add(kPolicyDecayExponentId, 0.01f, 10.0f) = 1.0f; } void SearchParams::Populate(OptionsParser* options) { diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index bb3a4a5434..2b1b432f1b 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -566,12 +566,20 @@ std::vector Search::GetVerboseStats(const Node* node) const { const auto& edge = std::get<2>(edge_tuple); float Q = edge.GetQ(fpu, draw_score); float M = m_evaluator.GetMUtility(edge, Q); + // Calculate P_eff for display (keeping label as "P" for compatibility) + float p_eff = edge.GetP(); + if (policy_decay_scale > 0.0f && p_eff > 0.0f) { + p_eff = ApplyPolicyDecay(p_eff, static_cast(edge.GetN()), + policy_decay_scale, policy_decay_exponent, + node->GetNumEdges()); + p_eff /= policy_decay_sum; + } std::ostringstream oss; oss << std::left; // TODO: should this be displaying transformed index? print_head(&oss, edge.GetMove(is_black_to_move).ToString(true), MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), - edge.GetNInFlight(), edge.GetP()); + edge.GetNInFlight(), p_eff); print_stats(&oss, edge.node()); print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), ") ", 6, 5); print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) + M, ") ", 8, 5); diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index 567cbb407d..e83fb7fe08 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -728,12 +728,12 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). // policy_decay_scale_per_move controls positive policy decay. When scale=0, no decay (p_eff = p). - // policy_decay_exponent controls the strength of decay (default 0.5). + // policy_decay_exponent controls the strength of decay (default 1.0). // num_edges is the number of legal moves (used to scale the decay parameter). // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). float GetU(float numerator, int num_edges, float policy_decay_scale_per_move = 0.0f, - float policy_decay_exponent = 0.5f, + float policy_decay_exponent = 1.0f, float policy_decay_sum = 1.0f) const { float p = GetP(); if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index 6c682a6ec3..2fecd23e3e 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -628,12 +628,20 @@ std::vector Search::GetVerboseStats( const auto& edge = std::get<2>(edge_tuple); float Q = edge.GetQ(fpu, draw_score); float M = m_evaluator.GetMUtility(edge, Q); + // Calculate P_eff for display (keeping label as "P" for compatibility) + float p_eff = edge.GetP(); + if (policy_decay_scale > 0.0f && p_eff > 0.0f) { + p_eff = ApplyPolicyDecay(p_eff, static_cast(edge.GetN()), + policy_decay_scale, policy_decay_exponent, + node->GetNumEdges()); + p_eff /= policy_decay_sum; + } std::ostringstream oss; oss << std::left; // TODO: should this be displaying transformed index? print_head(&oss, edge.GetMove(is_black_to_move).ToString(true), MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), - edge.GetNInFlight(), edge.GetP()); + edge.GetNInFlight(), p_eff); print_stats(&oss, edge.node()); print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), ") ", 6, 5); print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) + M, ") ", 8, 5); From 0b0f15de04c1b42c904273f13d50ceb1aa99d9b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Nov 2025 23:59:42 +0000 Subject: [PATCH 09/10] Use fixed sqrt decay (exponent=0.5) instead of tunable exponent Simplifies implementation by removing the policy_decay_exponent parameter and hardcoding sqrt decay (exponent=0.5), which always uses FastInvSqrt for optimal performance. CHANGES: 1. Updated ApplyPolicyDecay() in fastmath.h: - Removed exponent parameter - Fixed to use sqrt decay: power_term = FastInvSqrt(base) - Simplified from 3 code paths (1.0, 0.5, general) to single path - Updated documentation 2. Removed policy_decay_exponent parameter: - Deleted kPolicyDecayExponentId from params.cc - Removed GetPolicyDecayExponent() getter from params.h - Removed kPolicyDecayExponent member variable from params.h - Removed option registration in Populate() - Removed from constructor initialization 3. Updated GetU() signature in both node.h files: - Removed policy_decay_exponent parameter - Updated documentation to reflect fixed sqrt decay 4. Updated all call sites in classic/search.cc: - Removed exponent parameter from 3 locations - Verbose stats (line ~571) - PUCT selection loop (line ~1747, ~1779) - Prefetch (line ~2134) 5. Updated all call sites in dag_classic/search.cc: - Removed exponent parameter from 2 locations - Verbose stats (line ~633) - PUCT selection loop (line ~1845, ~1877) This simplification reduces code complexity and ensures consistent performance by always using the optimized FastInvSqrt path. --- src/search/classic/node.h | 10 ++++------ src/search/classic/params.cc | 13 +++---------- src/search/classic/params.h | 3 --- src/search/classic/search.cc | 23 +++++++++-------------- src/search/dag_classic/node.h | 10 ++++------ src/search/dag_classic/search.cc | 18 +++++++----------- src/utils/fastmath.h | 24 +++++++----------------- 7 files changed, 34 insertions(+), 67 deletions(-) diff --git a/src/search/classic/node.h b/src/search/classic/node.h index 2c3d2dbdbc..e0edf38a73 100644 --- a/src/search/classic/node.h +++ b/src/search/classic/node.h @@ -406,20 +406,18 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - // policy_decay_scale_per_move controls positive policy decay. When scale=0, no decay (p_eff = p). - // policy_decay_exponent controls the strength of decay (default 1.0). + // policy_decay_scale_per_move controls positive policy decay with fixed sqrt decay. + // When scale=0, no decay (p_eff = p). // num_edges is the number of legal moves (used to scale the decay parameter). // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). float GetU(float numerator, int num_edges, float policy_decay_scale_per_move = 0.0f, - float policy_decay_exponent = 1.0f, float policy_decay_sum = 1.0f) const { float p = GetP(); if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { - // Calculate raw (unnormalized) P_eff + // Calculate raw (unnormalized) P_eff with fixed sqrt decay p = ApplyPolicyDecay(p, static_cast(GetN()), - policy_decay_scale_per_move, policy_decay_exponent, - num_edges); + policy_decay_scale_per_move, num_edges); // Normalize by sum to ensure sum(P_eff) = 1.0 p /= policy_decay_sum; } diff --git a/src/search/classic/params.cc b/src/search/classic/params.cc index c77ce1a891..dee9d5603b 100644 --- a/src/search/classic/params.cc +++ b/src/search/classic/params.cc @@ -533,13 +533,8 @@ const OptionId BaseSearchParams::kPolicyDecayScaleId{ "policy-decay-scale-per-move", "PolicyDecayScalePerMove", "Scale parameter for positive policy decay, specified per legal move. " "Effective scale = scale_per_move * num_legal_moves. Controls how many " - "visits per available move before decay becomes significant. " - "Set to 0 to disable policy decay (P_eff = P)."}; -const OptionId BaseSearchParams::kPolicyDecayExponentId{ - "policy-decay-exponent", "PolicyDecayExponent", - "Exponent for policy decay formula: power_term = (1 + N/scale)^(-exponent). " - "Lower values give gentler decay, higher values give stronger decay. " - "Default 1.0 gives linear decay, 0.5 gives sqrt decay."}; + "visits per available move before decay becomes significant. Uses fixed " + "sqrt decay (exponent=0.5). Set to 0 to disable policy decay (P_eff = P)."}; const OptionId SearchParams::kMaxPrefetchBatchId{ "max-prefetch", "MaxPrefetch", @@ -643,7 +638,6 @@ void BaseSearchParams::Populate(OptionsParser* options) { options->Add(kSearchSpinBackoffId) = false; options->Add(kGarbageCollectionDelayId, 0.0f, 100.0f) = 10.0f; options->Add(kPolicyDecayScaleId, 0.0f, 1000.0f) = 20.0f; - options->Add(kPolicyDecayExponentId, 0.01f, 10.0f) = 1.0f; } void SearchParams::Populate(OptionsParser* options) { @@ -739,8 +733,7 @@ BaseSearchParams::BaseSearchParams(const OptionsDict& options) options.Get(kMaxCollisionVisitsScalingPowerId)), kSearchSpinBackoff(options_.Get(kSearchSpinBackoffId)), kGarbageCollectionDelay(options_.Get(kGarbageCollectionDelayId)), - kPolicyDecayScale(options_.Get(kPolicyDecayScaleId)), - kPolicyDecayExponent(options_.Get(kPolicyDecayExponentId)) {} + kPolicyDecayScale(options_.Get(kPolicyDecayScaleId)) {} SearchParams::SearchParams(const OptionsDict& options) : BaseSearchParams(options), diff --git a/src/search/classic/params.h b/src/search/classic/params.h index 9824320057..70e224948b 100644 --- a/src/search/classic/params.h +++ b/src/search/classic/params.h @@ -131,7 +131,6 @@ class BaseSearchParams { } float GetNpsLimit() const { return kNpsLimit; } float GetPolicyDecayScale() const { return kPolicyDecayScale; } - float GetPolicyDecayExponent() const { return kPolicyDecayExponent; } int GetTaskWorkersPerSearchWorker() const { return kTaskWorkersPerSearchWorker; @@ -234,7 +233,6 @@ class BaseSearchParams { static const OptionId kSearchSpinBackoffId; static const OptionId kGarbageCollectionDelayId; static const OptionId kPolicyDecayScaleId; - static const OptionId kPolicyDecayExponentId; protected: const OptionsDict& options_; @@ -295,7 +293,6 @@ class BaseSearchParams { const bool kSearchSpinBackoff; const float kGarbageCollectionDelay; const float kPolicyDecayScale; - const float kPolicyDecayExponent; }; class SearchParams : public BaseSearchParams { diff --git a/src/search/classic/search.cc b/src/search/classic/search.cc index 2b1b432f1b..9637fd85e8 100644 --- a/src/search/classic/search.cc +++ b/src/search/classic/search.cc @@ -466,7 +466,6 @@ std::vector Search::GetVerboseStats(const Node* node) const { const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); - const float policy_decay_exponent = params_.GetPolicyDecayExponent(); // Calculate sum of raw P_eff for normalization float policy_decay_sum = 1.0f; @@ -477,7 +476,7 @@ std::vector Search::GetVerboseStats(const Node* node) const { if (p > 0.0f) { policy_decay_sum += ApplyPolicyDecay( p, static_cast(edge.GetN()), policy_decay_scale, - policy_decay_exponent, node->GetNumEdges()); + node->GetNumEdges()); } } if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; @@ -487,7 +486,7 @@ std::vector Search::GetVerboseStats(const Node* node) const { edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum), edge); } std::sort(edges.begin(), edges.end()); @@ -570,8 +569,7 @@ std::vector Search::GetVerboseStats(const Node* node) const { float p_eff = edge.GetP(); if (policy_decay_scale > 0.0f && p_eff > 0.0f) { p_eff = ApplyPolicyDecay(p_eff, static_cast(edge.GetN()), - policy_decay_scale, policy_decay_exponent, - node->GetNumEdges()); + policy_decay_scale, node->GetNumEdges()); p_eff /= policy_decay_sum; } std::ostringstream oss; @@ -581,8 +579,8 @@ std::vector Search::GetVerboseStats(const Node* node) const { MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), p_eff); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum) + M, ") ", 8, 5); print_tail(&oss, edge.node()); infos.emplace_back(oss.str()); } @@ -1737,7 +1735,6 @@ void SearchWorker::PickNodesToExtendTask( const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); - const float policy_decay_exponent = params_.GetPolicyDecayExponent(); // Calculate sum of raw P_eff for normalization float policy_decay_sum = 1.0f; @@ -1749,7 +1746,7 @@ void SearchWorker::PickNodesToExtendTask( if (p > 0.0f) { policy_decay_sum += ApplyPolicyDecay( p, static_cast(edge_iter.GetN()), policy_decay_scale, - policy_decay_exponent, node->GetNumEdges()); + node->GetNumEdges()); } } if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; @@ -1780,8 +1777,7 @@ void SearchWorker::PickNodesToExtendTask( float p = current_pol[idx]; if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale, policy_decay_exponent, - node->GetNumEdges()); + policy_decay_scale, node->GetNumEdges()); // Normalize by sum to ensure sum(P_eff) = 1.0 p /= policy_decay_sum; } @@ -2127,7 +2123,6 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { const float fpu = GetFpu(params_, node, node == search_->root_node_, draw_score); const float policy_decay_scale = params_.GetPolicyDecayScale(); - const float policy_decay_exponent = params_.GetPolicyDecayExponent(); // Calculate sum of raw P_eff for normalization float policy_decay_sum = 1.0f; @@ -2138,7 +2133,7 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { if (p > 0.0f) { policy_decay_sum += ApplyPolicyDecay( p, static_cast(edge.GetN()), policy_decay_scale, - policy_decay_exponent, node->GetNumEdges()); + node->GetNumEdges()); } } if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; @@ -2148,7 +2143,7 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) - edge.GetQ(fpu, draw_score), + scores.emplace_back(-edge.GetU(puct_mult, node->GetNumEdges(), policy_decay_scale, policy_decay_sum) - edge.GetQ(fpu, draw_score), edge); } diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index e83fb7fe08..fb90781aaf 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -727,20 +727,18 @@ class EdgeAndNode { // Returns U = numerator * p_eff / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - // policy_decay_scale_per_move controls positive policy decay. When scale=0, no decay (p_eff = p). - // policy_decay_exponent controls the strength of decay (default 1.0). + // policy_decay_scale_per_move controls positive policy decay with fixed sqrt decay. + // When scale=0, no decay (p_eff = p). // num_edges is the number of legal moves (used to scale the decay parameter). // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). float GetU(float numerator, int num_edges, float policy_decay_scale_per_move = 0.0f, - float policy_decay_exponent = 1.0f, float policy_decay_sum = 1.0f) const { float p = GetP(); if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { - // Calculate raw (unnormalized) P_eff + // Calculate raw (unnormalized) P_eff with fixed sqrt decay p = ApplyPolicyDecay(p, static_cast(GetN()), - policy_decay_scale_per_move, policy_decay_exponent, - num_edges); + policy_decay_scale_per_move, num_edges); // Normalize by sum to ensure sum(P_eff) = 1.0 p /= policy_decay_sum; } diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index 2fecd23e3e..107cae1461 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -522,7 +522,6 @@ std::vector Search::GetVerboseStats( const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); - const float policy_decay_exponent = params_.GetPolicyDecayExponent(); // Calculate sum of raw P_eff for normalization float policy_decay_sum = 1.0f; @@ -533,7 +532,7 @@ std::vector Search::GetVerboseStats( if (p > 0.0f) { policy_decay_sum += ApplyPolicyDecay( p, static_cast(edge.GetN()), policy_decay_scale, - policy_decay_exponent, node->GetNumEdges()); + node->GetNumEdges()); } } if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; @@ -543,7 +542,7 @@ std::vector Search::GetVerboseStats( edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum), edge); } std::sort(edges.begin(), edges.end()); @@ -632,8 +631,7 @@ std::vector Search::GetVerboseStats( float p_eff = edge.GetP(); if (policy_decay_scale > 0.0f && p_eff > 0.0f) { p_eff = ApplyPolicyDecay(p_eff, static_cast(edge.GetN()), - policy_decay_scale, policy_decay_exponent, - node->GetNumEdges()); + policy_decay_scale, node->GetNumEdges()); p_eff /= policy_decay_sum; } std::ostringstream oss; @@ -643,8 +641,8 @@ std::vector Search::GetVerboseStats( MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), edge.GetNInFlight(), p_eff); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_exponent, policy_decay_sum) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum) + M, ") ", 8, 5); print_tail(&oss, edge.node(), true); infos.emplace_back(oss.str()); } @@ -1835,7 +1833,6 @@ void SearchWorker::PickNodesToExtendTask( const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float policy_decay_scale = params_.GetPolicyDecayScale(); - const float policy_decay_exponent = params_.GetPolicyDecayExponent(); // Calculate sum of raw P_eff for normalization float policy_decay_sum = 1.0f; @@ -1847,7 +1844,7 @@ void SearchWorker::PickNodesToExtendTask( if (p > 0.0f) { policy_decay_sum += ApplyPolicyDecay( p, static_cast(edge_iter.GetN()), policy_decay_scale, - policy_decay_exponent, node->GetNumEdges()); + node->GetNumEdges()); } } if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; @@ -1878,8 +1875,7 @@ void SearchWorker::PickNodesToExtendTask( float p = cur_iters[idx].GetP(); if (policy_decay_scale > 0.0f && p > 0.0f) { p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale, policy_decay_exponent, - node->GetNumEdges()); + policy_decay_scale, node->GetNumEdges()); // Normalize by sum to ensure sum(P_eff) = 1.0 p /= policy_decay_sum; } diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 88507677ba..400d0f8633 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -166,33 +166,23 @@ inline float FastPrecisePow(float a, float b) { return r * u.f; } -// Apply positive policy decay transformation. +// Apply positive policy decay transformation with fixed sqrt decay (exponent=0.5). // Returns raw (unnormalized) P_eff = 1 / (1 + odds * power_term) where: // odds = 1/P - 1 // effective_scale = scale_per_move * num_legal_moves -// power_term = (1 + N/effective_scale)^(-exponent) +// power_term = 1/sqrt(1 + N/effective_scale) // When P=0, scale=0, or num_legal_moves<=0, returns P unchanged. // NOTE: Caller must normalize by sum of all raw_P_eff values to ensure sum(P_eff) = 1. -// Optimized for common exponent values (0.5 and 1.0). +// Uses FastInvSqrt for optimal performance. inline float ApplyPolicyDecay(float p, float n_child, float scale_per_move, - float exponent, int num_legal_moves) { + int num_legal_moves) { if (p == 0.0f || scale_per_move == 0.0f || num_legal_moves <= 0) return p; float effective_scale = scale_per_move * num_legal_moves; float base = 1.0f + n_child / effective_scale; - float power_term; - - // Optimize common cases - if (exponent == 1.0f) { - // Linear decay: (1 + N/scale)^(-1) = 1/(1 + N/scale) - power_term = 1.0f / base; - } else if (exponent == 0.5f) { - // Sqrt decay: (1 + N/scale)^(-0.5) = 1/sqrt(1 + N/scale) - power_term = FastInvSqrt(base); - } else { - // General case: use FastPrecisePow for ~3x speedup over std::pow - power_term = FastPrecisePow(base, -exponent); - } + + // Sqrt decay: (1 + N/scale)^(-0.5) = 1/sqrt(1 + N/scale) + float power_term = FastInvSqrt(base); float odds = 1.0f / p - 1.0f; return 1.0f / (1.0f + odds * power_term); From 756999f8598f9948fb1ad27174ba7195740c1409 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Nov 2025 00:23:36 +0000 Subject: [PATCH 10/10] Revert dag_classic changes, clean up fastmath, update parameter range 1. REVERT DAG_CLASSIC: - Reverted all policy decay changes to dag_classic/node.h - Reverted all policy decay changes to dag_classic/search.cc - Policy decay is now only implemented for classic search 2. CLEAN UP FASTMATH: - Removed FastPow() function (no longer needed) - Removed FastPrecisePow() function (no longer needed) - Only FastInvSqrt remains for sqrt decay implementation - Simplified ApplyPolicyDecay() guard condition 3. UPDATE PARAMETER: - Changed range from 0-1000 to 1-1000000 - Changed default from 20 to 100 - Updated documentation to reflect new range - Removed "set to 0 to disable" from description This focuses the PR on classic search only and simplifies the code by removing unused fast math functions. --- src/search/classic/params.cc | 4 +-- src/search/dag_classic/node.h | 21 ++--------- src/search/dag_classic/search.cc | 60 ++++---------------------------- src/utils/fastmath.h | 51 ++------------------------- 4 files changed, 13 insertions(+), 123 deletions(-) diff --git a/src/search/classic/params.cc b/src/search/classic/params.cc index dee9d5603b..5c7637ffd6 100644 --- a/src/search/classic/params.cc +++ b/src/search/classic/params.cc @@ -534,7 +534,7 @@ const OptionId BaseSearchParams::kPolicyDecayScaleId{ "Scale parameter for positive policy decay, specified per legal move. " "Effective scale = scale_per_move * num_legal_moves. Controls how many " "visits per available move before decay becomes significant. Uses fixed " - "sqrt decay (exponent=0.5). Set to 0 to disable policy decay (P_eff = P)."}; + "sqrt decay (exponent=0.5). Range: 1-1000000, default 100."}; const OptionId SearchParams::kMaxPrefetchBatchId{ "max-prefetch", "MaxPrefetch", @@ -637,7 +637,7 @@ void BaseSearchParams::Populate(OptionsParser* options) { options->Add(kUCIRatingAdvId, -10000.0f, 10000.0f) = 0.0f; options->Add(kSearchSpinBackoffId) = false; options->Add(kGarbageCollectionDelayId, 0.0f, 100.0f) = 10.0f; - options->Add(kPolicyDecayScaleId, 0.0f, 1000.0f) = 20.0f; + options->Add(kPolicyDecayScaleId, 1.0f, 1000000.0f) = 100.0f; } void SearchParams::Populate(OptionsParser* options) { diff --git a/src/search/dag_classic/node.h b/src/search/dag_classic/node.h index fb90781aaf..b74a64a9d4 100644 --- a/src/search/dag_classic/node.h +++ b/src/search/dag_classic/node.h @@ -46,7 +46,6 @@ #include "chess/gamestate.h" #include "chess/position.h" #include "neural/backend.h" -#include "utils/fastmath.h" #include "utils/mutex.h" namespace lczero { @@ -725,24 +724,10 @@ class EdgeAndNode { return edge_ ? edge_->GetMove(flip) : Move(); } - // Returns U = numerator * p_eff / N. + // Returns U = numerator * p / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - // policy_decay_scale_per_move controls positive policy decay with fixed sqrt decay. - // When scale=0, no decay (p_eff = p). - // num_edges is the number of legal moves (used to scale the decay parameter). - // policy_decay_sum is the sum of raw P_eff for all edges (for sum normalization). - float GetU(float numerator, int num_edges, - float policy_decay_scale_per_move = 0.0f, - float policy_decay_sum = 1.0f) const { - float p = GetP(); - if (policy_decay_scale_per_move > 0.0f && p > 0.0f) { - // Calculate raw (unnormalized) P_eff with fixed sqrt decay - p = ApplyPolicyDecay(p, static_cast(GetN()), - policy_decay_scale_per_move, num_edges); - // Normalize by sum to ensure sum(P_eff) = 1.0 - p /= policy_decay_sum; - } - return numerator * p / (1 + GetNStarted()); + float GetU(float numerator) const { + return numerator * GetP() / (1 + GetNStarted()); } std::string DebugString() const; diff --git a/src/search/dag_classic/search.cc b/src/search/dag_classic/search.cc index 107cae1461..e65977c38c 100644 --- a/src/search/dag_classic/search.cc +++ b/src/search/dag_classic/search.cc @@ -521,28 +521,11 @@ std::vector Search::GetVerboseStats( const float cpuct = ComputeCpuct(params_, node->GetTotalVisits(), is_root); const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); - const float policy_decay_scale = params_.GetPolicyDecayScale(); - - // Calculate sum of raw P_eff for normalization - float policy_decay_sum = 1.0f; - if (policy_decay_scale > 0.0f) { - policy_decay_sum = 0.0f; - for (const auto& edge : node->Edges()) { - float p = edge.GetP(); - if (p > 0.0f) { - policy_decay_sum += ApplyPolicyDecay( - p, static_cast(edge.GetN()), policy_decay_scale, - node->GetNumEdges()); - } - } - if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; - } - std::vector> edges; edges.reserve(node->GetNumEdges()); for (const auto& edge : node->Edges()) { edges.emplace_back(edge.GetN(), - edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum), + edge.GetQ(fpu, draw_score) + edge.GetU(U_coeff), edge); } std::sort(edges.begin(), edges.end()); @@ -627,22 +610,15 @@ std::vector Search::GetVerboseStats( const auto& edge = std::get<2>(edge_tuple); float Q = edge.GetQ(fpu, draw_score); float M = m_evaluator.GetMUtility(edge, Q); - // Calculate P_eff for display (keeping label as "P" for compatibility) - float p_eff = edge.GetP(); - if (policy_decay_scale > 0.0f && p_eff > 0.0f) { - p_eff = ApplyPolicyDecay(p_eff, static_cast(edge.GetN()), - policy_decay_scale, node->GetNumEdges()); - p_eff /= policy_decay_sum; - } std::ostringstream oss; oss << std::left; // TODO: should this be displaying transformed index? print_head(&oss, edge.GetMove(is_black_to_move).ToString(true), MoveToNNIndex(edge.GetMove(), 0), edge.GetN(), - edge.GetNInFlight(), p_eff); + edge.GetNInFlight(), edge.GetP()); print_stats(&oss, edge.node()); - print(&oss, "(U: ", edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum), ") ", 6, 5); - print(&oss, "(S: ", Q + edge.GetU(U_coeff, node->GetNumEdges(), policy_decay_scale, policy_decay_sum) + M, ") ", 8, 5); + print(&oss, "(U: ", edge.GetU(U_coeff), ") ", 6, 5); + print(&oss, "(S: ", Q + edge.GetU(U_coeff) + M, ") ", 8, 5); print_tail(&oss, edge.node(), true); infos.emplace_back(oss.str()); } @@ -1832,24 +1808,6 @@ void SearchWorker::PickNodesToExtendTask( ComputeCpuct(params_, node->GetTotalVisits(), is_root_node); const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); - const float policy_decay_scale = params_.GetPolicyDecayScale(); - - // Calculate sum of raw P_eff for normalization - float policy_decay_sum = 1.0f; - if (policy_decay_scale > 0.0f) { - policy_decay_sum = 0.0f; - auto edge_iter = node->Edges(); - for (int i = 0; i < max_needed; ++i, ++edge_iter) { - float p = edge_iter.GetP(); - if (p > 0.0f) { - policy_decay_sum += ApplyPolicyDecay( - p, static_cast(edge_iter.GetN()), policy_decay_scale, - node->GetNumEdges()); - } - } - if (policy_decay_sum == 0.0f) policy_decay_sum = 1.0f; - } - int cache_filled_idx = -1; while (cur_limit > 0) { // Perform UCT for current node. @@ -1872,14 +1830,8 @@ void SearchWorker::PickNodesToExtendTask( int nstarted = current_nstarted[idx]; const float util = current_util[idx]; if (idx > cache_filled_idx) { - float p = cur_iters[idx].GetP(); - if (policy_decay_scale > 0.0f && p > 0.0f) { - p = ApplyPolicyDecay(p, static_cast(cur_iters[idx].GetN()), - policy_decay_scale, node->GetNumEdges()); - // Normalize by sum to ensure sum(P_eff) = 1.0 - p /= policy_decay_sum; - } - current_score[idx] = p * puct_mult / (1 + nstarted) + util; + current_score[idx] = + cur_iters[idx].GetP() * puct_mult / (1 + nstarted) + util; cache_filled_idx++; } if (is_root_node) { diff --git a/src/utils/fastmath.h b/src/utils/fastmath.h index 400d0f8633..7846ff5aef 100644 --- a/src/utils/fastmath.h +++ b/src/utils/fastmath.h @@ -119,64 +119,17 @@ inline float FastInvSqrt(const float a) { return y; } -// Fast approximate pow(a, b) using bit manipulation. -// Based on Martin Ankerl's implementation (https://martin.ankerl.com/2012/01/25/optimized-approximative-pow-in-c-and-cpp/) -// Approximately 4x faster than std::pow for fractional exponents. -// Accuracy typically within 5%, with rare cases up to 12% error. -// For better accuracy with integer or near-integer exponents, use FastPrecisePow. -// Expects positive base. Does no range checking. -inline float FastPow(float a, float b) { - union { - float f; - int32_t i; - } u = {a}; - u.i = static_cast(b * (u.i - 1064866805) + 1064866805); - return u.f; -} - -// More accurate version of FastPow that handles integer exponent part separately. -// Approximately 3x faster than std::pow. -// Significantly more accurate than FastPow when exponent > 1. -// Expects positive base. Does no range checking. -inline float FastPrecisePow(float a, float b) { - // Separate integer and fractional parts - int e = static_cast(b); - union { - float f; - int32_t i; - } u = {a}; - u.i = static_cast((b - e) * (u.i - 1064866805) + 1064866805); - - // Handle integer part using exponentiation by squaring - float r = 1.0f; - float base = a; - int exp = e; - if (exp < 0) { - base = 1.0f / base; - exp = -exp; - } - while (exp) { - if (exp & 1) { - r *= base; - } - base *= base; - exp >>= 1; - } - - return r * u.f; -} - // Apply positive policy decay transformation with fixed sqrt decay (exponent=0.5). // Returns raw (unnormalized) P_eff = 1 / (1 + odds * power_term) where: // odds = 1/P - 1 // effective_scale = scale_per_move * num_legal_moves // power_term = 1/sqrt(1 + N/effective_scale) -// When P=0, scale=0, or num_legal_moves<=0, returns P unchanged. +// When P=0 or num_legal_moves<=0, returns P unchanged. // NOTE: Caller must normalize by sum of all raw_P_eff values to ensure sum(P_eff) = 1. // Uses FastInvSqrt for optimal performance. inline float ApplyPolicyDecay(float p, float n_child, float scale_per_move, int num_legal_moves) { - if (p == 0.0f || scale_per_move == 0.0f || num_legal_moves <= 0) return p; + if (p == 0.0f || num_legal_moves <= 0) return p; float effective_scale = scale_per_move * num_legal_moves; float base = 1.0f + n_child / effective_scale;