From adffa6975ddc74213c3c626d1415924dd831d375 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 01:23:43 -0500 Subject: [PATCH 01/12] Introduce p_decay Ignore GetVisitsToReachU for now --- src/mcts/node.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mcts/node.h b/src/mcts/node.h index 217a5ceec2..9cd1d84ffc 100644 --- a/src/mcts/node.h +++ b/src/mcts/node.h @@ -396,8 +396,10 @@ class EdgeAndNode { // Returns U = numerator * p / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - float GetU(float numerator) const { - return numerator * GetP() / (1 + GetNStarted()); + float GetU(float numerator, float childVisits) const { + const float p = GetP(); + const float p_dcay = p + (1 - p) * (1 - 1 / std::sqrt(1 + p * childVisits); + return numerator * p_dcay / (1 + GetNStarted()); } int GetVisitsToReachU(float target_score, float numerator, From 4eabc40d7e8e02eb92ca22c92749880161c40dfd Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 01:33:44 -0500 Subject: [PATCH 02/12] Pass childVisits to GetU --- src/mcts/search.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index b54ca5a0c5..546251c0e7 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -253,6 +253,7 @@ std::vector Search::GetVerboseStats(Node* node) const { const float cpuct = ComputeCpuct(params_, node->GetN(), is_root); const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); + const float childVisits = node->GetChildrenVisits(); const bool logit_q = params_.GetLogitQ(); std::vector edges; @@ -263,10 +264,10 @@ std::vector Search::GetVerboseStats(Node* node) const { [&fpu, &U_coeff, &logit_q, &draw_score](EdgeAndNode a, EdgeAndNode b) { return std::forward_as_tuple( a.GetN(), - a.GetQ(fpu, draw_score, logit_q) + a.GetU(U_coeff)) < + a.GetQ(fpu, draw_score, logit_q) + a.GetU(U_coeff, childVisits) < std::forward_as_tuple( b.GetN(), - b.GetQ(fpu, draw_score, logit_q) + b.GetU(U_coeff)); + b.GetQ(fpu, draw_score, logit_q) + b.GetU(U_coeff, childVisits)); }); std::vector infos; @@ -301,11 +302,11 @@ std::vector Search::GetVerboseStats(Node* node) const { oss << "(Q: " << std::setw(8) << std::setprecision(5) << edge.GetQ(fpu, draw_score, /* logit_q= */ false) << ") "; - oss << "(U: " << std::setw(6) << std::setprecision(5) << edge.GetU(U_coeff) + oss << "(U: " << std::setw(6) << std::setprecision(5) << edge.GetU(U_coeff, childVisits) << ") "; oss << "(Q+U: " << std::setw(8) << std::setprecision(5) - << edge.GetQ(fpu, draw_score, logit_q) + edge.GetU(U_coeff) << ") "; + << edge.GetQ(fpu, draw_score, logit_q) + edge.GetU(U_coeff, childVisits) << ") "; oss << "(V: "; std::optional v; @@ -1076,7 +1077,7 @@ SearchWorker::NodeToProcess SearchWorker::PickNodeToExtend( M *= a + b * std::abs(Q) + c * Q * Q; } - const float score = child.GetU(puct_mult) + Q + M; + const float score = child.GetU(puct_mult, node->GetChildrenVisits()) + Q + M; if (score > best) { second_best = best; second_best_edge = best_edge; @@ -1313,7 +1314,7 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult) - + scores.emplace_back(-edge.GetU(puct_mult, node->GetChildrenVisits()) - edge.GetQ(fpu, draw_score, /* logit_q= */ false), edge); } From 88a9d767918b3ca33da69b29619a54a0658fcce1 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 01:35:38 -0500 Subject: [PATCH 03/12] Missing ")" --- src/mcts/search.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index 546251c0e7..6b8ff72cc6 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -264,7 +264,7 @@ std::vector Search::GetVerboseStats(Node* node) const { [&fpu, &U_coeff, &logit_q, &draw_score](EdgeAndNode a, EdgeAndNode b) { return std::forward_as_tuple( a.GetN(), - a.GetQ(fpu, draw_score, logit_q) + a.GetU(U_coeff, childVisits) < + a.GetQ(fpu, draw_score, logit_q) + a.GetU(U_coeff, childVisits)) < std::forward_as_tuple( b.GetN(), b.GetQ(fpu, draw_score, logit_q) + b.GetU(U_coeff, childVisits)); From 937424275ca54db4141a31b385b36cbe96d66df6 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 02:02:46 -0500 Subject: [PATCH 04/12] Off by 1 If childVisits = 1, then p_decay = p --- src/mcts/node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/node.h b/src/mcts/node.h index 9cd1d84ffc..b5caf328c8 100644 --- a/src/mcts/node.h +++ b/src/mcts/node.h @@ -398,7 +398,7 @@ class EdgeAndNode { // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). float GetU(float numerator, float childVisits) const { const float p = GetP(); - const float p_dcay = p + (1 - p) * (1 - 1 / std::sqrt(1 + p * childVisits); + const float p_dcay = p + (1 - p) * (1 - 1 / std::sqrt(1 + p * (childVisits - 1)); return numerator * p_dcay / (1 + GetNStarted()); } From 875457875a269e7588066a751267d4edd8e3b318 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 02:06:56 -0500 Subject: [PATCH 05/12] Missing ")" --- src/mcts/node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/node.h b/src/mcts/node.h index b5caf328c8..59745c1b5c 100644 --- a/src/mcts/node.h +++ b/src/mcts/node.h @@ -398,7 +398,7 @@ class EdgeAndNode { // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). float GetU(float numerator, float childVisits) const { const float p = GetP(); - const float p_dcay = p + (1 - p) * (1 - 1 / std::sqrt(1 + p * (childVisits - 1)); + const float p_dcay = p + (1 - p) * (1 - 1 / std::sqrt(1 + p * (childVisits - 1))); return numerator * p_dcay / (1 + GetNStarted()); } From 15c91151c988420d20ce2cc8b32f737b214a1714 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 08:56:05 -0500 Subject: [PATCH 06/12] Capture childVisits by reference --- src/mcts/search.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index 6b8ff72cc6..a7c5e26d7f 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -261,7 +261,7 @@ std::vector Search::GetVerboseStats(Node* node) const { std::sort( edges.begin(), edges.end(), - [&fpu, &U_coeff, &logit_q, &draw_score](EdgeAndNode a, EdgeAndNode b) { + [&fpu, &U_coeff, &childVisits, &logit_q, &draw_score](EdgeAndNode a, EdgeAndNode b) { return std::forward_as_tuple( a.GetN(), a.GetQ(fpu, draw_score, logit_q) + a.GetU(U_coeff, childVisits)) < From 39ba0c0823267dc2bb7846fdcdc529ff06c477bc Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 10:15:38 -0500 Subject: [PATCH 07/12] Add policy decay shift and slope parameters --- src/mcts/params.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/mcts/params.cc b/src/mcts/params.cc index f9b6d64d0f..4cd9c2637b 100644 --- a/src/mcts/params.cc +++ b/src/mcts/params.cc @@ -57,6 +57,12 @@ const OptionId SearchParams::kLogitQId{ "logit-q", "LogitQ", "Apply logit to Q when determining Q+U best child. This makes the U term " "less dominant when Q is near -1 or +1."}; +const OptionId SearchParams::kPolicyDecayShiftId{ + "policy-decay-shift", "PolicyDecayShift", + "Adjusts how soon policy effect starts to decay."}; +const OptionId SearchParams::kPolicyDecaySlopeId{ + "policy-decay-slope", "PolicyDecaySlope", + "Adjusts how fast policy effect decays to 1."}; const OptionId SearchParams::kCpuctId{ "cpuct", "CPuct", "cpuct_init constant from \"UCT search\" algorithm. Higher values promote " @@ -259,12 +265,14 @@ void SearchParams::Populate(OptionsParser* options) { options->Add(kMiniBatchSizeId, 1, 1024) = 256; options->Add(kMaxPrefetchBatchId, 0, 1024) = 32; options->Add(kLogitQId) = false; + options->Add(kPolicyDecayShiftId, 0.0f, 100.0f) = 0.1f; + options->Add(kPolicyDecaySlopeId, 0.0f, 100.0f) = 0.5f; options->Add(kCpuctId, 0.0f, 100.0f) = 2.147f; options->Add(kCpuctAtRootId, 0.0f, 100.0f) = 2.147f; options->Add(kCpuctBaseId, 1.0f, 1000000000.0f) = 18368.0f; options->Add(kCpuctBaseAtRootId, 1.0f, 1000000000.0f) = 18368.0f; - options->Add(kCpuctFactorId, 0.0f, 1000.0f) = 2.815f; - options->Add(kCpuctFactorAtRootId, 0.0f, 1000.0f) = 2.815f; + options->Add(kCpuctFactorId, 0.0f, 1000.0f) = 0.0f; + options->Add(kCpuctFactorAtRootId, 0.0f, 1000.0f) = 0.0f; options->Add(kRootHasOwnCpuctParamsId) = true; options->Add(kTemperatureId, 0.0f, 100.0f) = 0.0f; options->Add(kTempDecayMovesId, 0, 100) = 0; @@ -330,6 +338,8 @@ void SearchParams::Populate(OptionsParser* options) { SearchParams::SearchParams(const OptionsDict& options) : options_(options), kLogitQ(options.Get(kLogitQId)), + kPolicyDecayShift(options.Get(kPolicyDecayShiftId)), + kPolicyDecaySlope(options.Get(kPolicyDecaySlopeId)), kCpuct(options.Get(kCpuctId)), kCpuctAtRoot(options.Get( options.Get(kRootHasOwnCpuctParamsId) ? kCpuctAtRootId From acd7acca667f7a725989ebe82aa086d509c8c2b0 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 10:20:09 -0500 Subject: [PATCH 08/12] Include policy decay parameters --- src/mcts/params.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mcts/params.h b/src/mcts/params.h index 4ed15a71f8..4d33e9fec5 100644 --- a/src/mcts/params.h +++ b/src/mcts/params.h @@ -47,6 +47,8 @@ class SearchParams { return options_.Get(kMaxPrefetchBatchId); } bool GetLogitQ() const { return kLogitQ; } + float GetPolicyDecayShift() const { return kPolicyDecayShift; } + float GetPolicyDecaySlope() const { return kPolicyDecaySlope; } float GetCpuct(bool at_root) const { return at_root ? kCpuctAtRoot : kCpuct; } float GetCpuctBase(bool at_root) const { return at_root ? kCpuctBaseAtRoot : kCpuctBase; @@ -114,6 +116,8 @@ class SearchParams { static const OptionId kMiniBatchSizeId; static const OptionId kMaxPrefetchBatchId; static const OptionId kLogitQId; + static const OptionId kPolicyDecayShiftId; + static const OptionId kPolicyDecaySlopeId; static const OptionId kCpuctId; static const OptionId kCpuctAtRootId; static const OptionId kCpuctBaseId; @@ -170,6 +174,8 @@ class SearchParams { // TODO(crem) Some of those parameters can be converted to be dynamic after // trivial search optimizations. const bool kLogitQ; + const float kPolicyDecayShift; + const float kPolicyDecaySlope; const float kCpuct; const float kCpuctAtRoot; const float kCpuctBase; From f95f2e7c84c8c027498dff75ebc944bea9e50f8f Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 10:28:10 -0500 Subject: [PATCH 09/12] Add decay parameters to GetU --- src/mcts/node.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mcts/node.h b/src/mcts/node.h index 59745c1b5c..752b233c33 100644 --- a/src/mcts/node.h +++ b/src/mcts/node.h @@ -396,9 +396,10 @@ class EdgeAndNode { // Returns U = numerator * p / N. // Passed numerator is expected to be equal to (cpuct * sqrt(N[parent])). - float GetU(float numerator, float childVisits) const { + float GetU(float numerator, float childVisits, float shift, float slope) const { const float p = GetP(); - const float p_dcay = p + (1 - p) * (1 - 1 / std::sqrt(1 + p * (childVisits - 1))); + const float p_dcay = p + (1 - p) * + (1 - 1 / std::pow(1 + shift * p * (childVisits - 1), slope)); return numerator * p_dcay / (1 + GetNStarted()); } From 8d958867b3436f95727fd8f711ba993b5019fce9 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 10:39:04 -0500 Subject: [PATCH 10/12] Pass new parameters to GetU --- src/mcts/search.cc | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index a7c5e26d7f..20e59099a9 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -254,6 +254,8 @@ std::vector Search::GetVerboseStats(Node* node) const { const float U_coeff = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float childVisits = node->GetChildrenVisits(); + const float shift = params_.GetPolicyDecayShift(); + const float slope = params_.GetPolicyDecaySlope(); const bool logit_q = params_.GetLogitQ(); std::vector edges; @@ -264,10 +266,12 @@ std::vector Search::GetVerboseStats(Node* node) const { [&fpu, &U_coeff, &childVisits, &logit_q, &draw_score](EdgeAndNode a, EdgeAndNode b) { return std::forward_as_tuple( a.GetN(), - a.GetQ(fpu, draw_score, logit_q) + a.GetU(U_coeff, childVisits)) < + a.GetQ(fpu, draw_score, logit_q) + + a.GetU(U_coeff, childVisits, shift, slope)) < std::forward_as_tuple( b.GetN(), - b.GetQ(fpu, draw_score, logit_q) + b.GetU(U_coeff, childVisits)); + b.GetQ(fpu, draw_score, logit_q) + + b.GetU(U_coeff, childVisits, shift, slope)); }); std::vector infos; @@ -302,11 +306,12 @@ std::vector Search::GetVerboseStats(Node* node) const { oss << "(Q: " << std::setw(8) << std::setprecision(5) << edge.GetQ(fpu, draw_score, /* logit_q= */ false) << ") "; - oss << "(U: " << std::setw(6) << std::setprecision(5) << edge.GetU(U_coeff, childVisits) - << ") "; + oss << "(U: " << std::setw(6) << std::setprecision(5) + << edge.GetU(U_coeff, childVisits, shift, slope) << ") "; oss << "(Q+U: " << std::setw(8) << std::setprecision(5) - << edge.GetQ(fpu, draw_score, logit_q) + edge.GetU(U_coeff, childVisits) << ") "; + << edge.GetQ(fpu, draw_score, logit_q) + + edge.GetU(U_coeff, childVisits, shift slope) << ") "; oss << "(V: "; std::optional v; @@ -1028,6 +1033,9 @@ SearchWorker::NodeToProcess SearchWorker::PickNodeToExtend( const float cpuct = ComputeCpuct(params_, node->GetN(), is_root_node); const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); + const float childVisits = node->GetChildrenVisits(); + const float shift = params_.GetPolicyDecayShift(); + const float slope = params_.GetPolicyDecaySlope(); float best = std::numeric_limits::lowest(); float best_without_u = std::numeric_limits::lowest(); float second_best = std::numeric_limits::lowest(); @@ -1077,7 +1085,7 @@ SearchWorker::NodeToProcess SearchWorker::PickNodeToExtend( M *= a + b * std::abs(Q) + c * Q * Q; } - const float score = child.GetU(puct_mult, node->GetChildrenVisits()) + Q + M; + const float score = child.GetU(puct_mult, childVisits, shift, slope) + Q + M; if (score > best) { second_best = best; second_best_edge = best_edge; @@ -1306,6 +1314,9 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { std::vector scores; const float cpuct = ComputeCpuct(params_, node->GetN(), node == search_->root_node_); + const float childVisits = node->GetChildrenVisits(); + const float shift = params_.GetPolicyDecayShift(); + const float slope = params_.GetPolicyDecaySlope(); const float puct_mult = cpuct * std::sqrt(std::max(node->GetChildrenVisits(), 1u)); const float fpu = @@ -1314,7 +1325,7 @@ int SearchWorker::PrefetchIntoCache(Node* node, int budget, bool is_odd_depth) { if (edge.GetP() == 0.0f) continue; // Flip the sign of a score to be able to easily sort. // TODO: should this use logit_q if set?? - scores.emplace_back(-edge.GetU(puct_mult, node->GetChildrenVisits()) - + scores.emplace_back(-edge.GetU(puct_mult, childVisits, shift, slope) - edge.GetQ(fpu, draw_score, /* logit_q= */ false), edge); } From 2a703a3008d2931203cfc0b18295450548c949e8 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Fri, 24 Apr 2020 12:34:10 -0500 Subject: [PATCH 11/12] Fix syntax errors --- src/mcts/search.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index 20e59099a9..44056322c3 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -263,7 +263,8 @@ std::vector Search::GetVerboseStats(Node* node) const { std::sort( edges.begin(), edges.end(), - [&fpu, &U_coeff, &childVisits, &logit_q, &draw_score](EdgeAndNode a, EdgeAndNode b) { + [&fpu, &U_coeff, &childVisits, &shift, &slope, &logit_q, + &draw_score](EdgeAndNode a, EdgeAndNode b) { return std::forward_as_tuple( a.GetN(), a.GetQ(fpu, draw_score, logit_q) + @@ -311,7 +312,7 @@ std::vector Search::GetVerboseStats(Node* node) const { oss << "(Q+U: " << std::setw(8) << std::setprecision(5) << edge.GetQ(fpu, draw_score, logit_q) + - edge.GetU(U_coeff, childVisits, shift slope) << ") "; + edge.GetU(U_coeff, childVisits, shift, slope) << ") "; oss << "(V: "; std::optional v; From 5d68ecd288d81110e97c2d1d7af52c1208e2e675 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 25 Apr 2020 15:38:39 -0500 Subject: [PATCH 12/12] Update search.cc --- src/mcts/search.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index b406cbf0e7..d961c2cdaa 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -331,8 +331,8 @@ std::vector Search::GetVerboseStats(Node* node) const { oss << "(U: " << std::setw(6) << std::setprecision(5) << edge.GetU(U_coeff, childVisits, shift, slope) << ") "; - oss << "(Q+U: " << std::setw(8) << std::setprecision(5) - << Q + edge.GetU(U_coeff, childVisits, shift, slope) + M_effect<< ") "; + oss << "(S: " << std::setw(8) << std::setprecision(5) + << Q + edge.GetU(U_coeff, childVisits, shift, slope) + M_effect << ") "; oss << "(V: "; std::optional v;