From 763fdff31d84bff7465e867cdf18b307e6691d8a Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 14:06:33 -0500 Subject: [PATCH 01/13] Weight recent nodes more heavily --- src/mcts/node.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mcts/node.cc b/src/mcts/node.cc index ea97edd368..c142a926e3 100644 --- a/src/mcts/node.cc +++ b/src/mcts/node.cc @@ -39,6 +39,7 @@ #include "neural/network.h" #include "utils/exception.h" #include "utils/hashcat.h" +#include "utils/fastmath.h" namespace lczero { @@ -278,9 +279,10 @@ void Node::CancelScoreUpdate(int multivisit) { void Node::FinalizeScoreUpdate(float v, float d, float m, int multivisit) { // Recompute Q. - wl_ += multivisit * (v - wl_) / (n_ + multivisit); - d_ += multivisit * (d - d_) / (n_ + multivisit); - m_ += multivisit * (m - m_) / (n_ + multivisit); + float delta = FastLog2( 1.0f * n_ ) * multivisit; + wl_ += delta * (v - wl_) / (n_ + delta); + d_ += delta * (d - d_) / (n_ + delta); + m_ += delta * (m - m_) / (n_ + delta); // If first visit, update parent's sum of policies visited at least once. if (n_ == 0 && parent_ != nullptr) { From 3ca0e55fd53c1df8470ef0a27c05ad2b19c669d0 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 14:14:30 -0500 Subject: [PATCH 02/13] log(n) -> log(1+n) --- src/mcts/node.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/node.cc b/src/mcts/node.cc index c142a926e3..30d08aa66b 100644 --- a/src/mcts/node.cc +++ b/src/mcts/node.cc @@ -279,7 +279,7 @@ void Node::CancelScoreUpdate(int multivisit) { void Node::FinalizeScoreUpdate(float v, float d, float m, int multivisit) { // Recompute Q. - float delta = FastLog2( 1.0f * n_ ) * multivisit; + float delta = FastLog2( 1.0f + n_ ) * multivisit; wl_ += delta * (v - wl_) / (n_ + delta); d_ += delta * (d - d_) / (n_ + delta); m_ += delta * (m - m_) / (n_ + delta); From b0fff475e32166ecd2aa6b966c9f2298e59d82f1 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 14:48:44 -0500 Subject: [PATCH 03/13] Change constants --- src/mcts/node.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/node.cc b/src/mcts/node.cc index 30d08aa66b..2aba2e63aa 100644 --- a/src/mcts/node.cc +++ b/src/mcts/node.cc @@ -279,7 +279,7 @@ void Node::CancelScoreUpdate(int multivisit) { void Node::FinalizeScoreUpdate(float v, float d, float m, int multivisit) { // Recompute Q. - float delta = FastLog2( 1.0f + n_ ) * multivisit; + float delta = FastLog2( 2.0f + n_/400.0f ) * multivisit; wl_ += delta * (v - wl_) / (n_ + delta); d_ += delta * (d - d_) / (n_ + delta); m_ += delta * (m - m_) / (n_ + delta); From 412d9ba5c60226af1d366c9d7430be2b2414c481 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 16:54:23 -0500 Subject: [PATCH 04/13] New parameter: BackupFactor --- src/mcts/params.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mcts/params.cc b/src/mcts/params.cc index 29f247c53e..d1b597f3f5 100644 --- a/src/mcts/params.cc +++ b/src/mcts/params.cc @@ -57,6 +57,9 @@ const OptionId SearchParams::kLogitQId{ "logit-q", "LogitQ", "Apply logit to Q when determining Q+U best child. This makes the U term " "less dominant when Q is near -1 or +1."}; +const OptionId SearchParams::kBackupFactorId{ + "backup-factor", "BackupFactor", + "Parameter for weighted backup. Set 0 for standard search."}; const OptionId SearchParams::kCpuctId{ "cpuct", "CPuct", "cpuct_init constant from \"UCT search\" algorithm. Higher values promote " @@ -259,6 +262,7 @@ void SearchParams::Populate(OptionsParser* options) { options->Add(kMiniBatchSizeId, 1, 1024) = 256; options->Add(kMaxPrefetchBatchId, 0, 1024) = 32; options->Add(kLogitQId) = false; + options->Add(kBackupFactorId, 0.0f, 1.0f) = 0.0f; options->Add(kCpuctId, 0.0f, 100.0f) = 2.147f; options->Add(kCpuctAtRootId, 0.0f, 100.0f) = 2.147f; options->Add(kCpuctBaseId, 1.0f, 1000000000.0f) = 18368.0f; @@ -329,6 +333,7 @@ void SearchParams::Populate(OptionsParser* options) { SearchParams::SearchParams(const OptionsDict& options) : options_(options), kLogitQ(options.Get(kLogitQId)), + kCpuct(options.Get(kBackupFactorId)), kCpuct(options.Get(kCpuctId)), kCpuctAtRoot(options.Get( options.Get(kRootHasOwnCpuctParamsId) ? kCpuctAtRootId From 0c3d34e69c4f26430c3f83ce8651699c526732f0 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 16:55:56 -0500 Subject: [PATCH 05/13] Add BackupFactor parameter --- src/mcts/params.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mcts/params.h b/src/mcts/params.h index 4ed15a71f8..ae1175ff50 100644 --- a/src/mcts/params.h +++ b/src/mcts/params.h @@ -114,6 +114,7 @@ class SearchParams { static const OptionId kMiniBatchSizeId; static const OptionId kMaxPrefetchBatchId; static const OptionId kLogitQId; + static const OptionId kBackupFactorId; static const OptionId kCpuctId; static const OptionId kCpuctAtRootId; static const OptionId kCpuctBaseId; From 32111c5bcec836739837d0e7f1f24989c552f560 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 16:57:12 -0500 Subject: [PATCH 06/13] Add argument to FinalizeScoreUpdate --- src/mcts/node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/node.h b/src/mcts/node.h index 217a5ceec2..7e94fd9f52 100644 --- a/src/mcts/node.h +++ b/src/mcts/node.h @@ -194,7 +194,7 @@ class Node { // * Q (weighted average of all V in a subtree) // * N (+=1) // * N-in-flight (-=1) - void FinalizeScoreUpdate(float v, float d, float m, int multivisit); + void FinalizeScoreUpdate(float v, float d, float m, int multivisit, float factor); // When search decides to treat one visit as several (in case of collisions // or visiting terminal nodes several times), it amplifies the visit by // incrementing n_in_flight. From 043341a40192962044de66f89e10f23cd9b7e455 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 16:59:28 -0500 Subject: [PATCH 07/13] Introduce factor argument --- src/mcts/node.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mcts/node.cc b/src/mcts/node.cc index 2aba2e63aa..f98ccfac1a 100644 --- a/src/mcts/node.cc +++ b/src/mcts/node.cc @@ -277,9 +277,10 @@ void Node::CancelScoreUpdate(int multivisit) { best_child_cached_ = nullptr; } -void Node::FinalizeScoreUpdate(float v, float d, float m, int multivisit) { +void Node::FinalizeScoreUpdate(float v, float d, float m, int multivisit, + float factor) { // Recompute Q. - float delta = FastLog2( 2.0f + n_/400.0f ) * multivisit; + float delta = FastLog2( 2.0f + factor * n_ ) * multivisit; wl_ += delta * (v - wl_) / (n_ + delta); d_ += delta * (d - d_) / (n_ + delta); m_ += delta * (m - m_) / (n_ + delta); From b0b9b293699fc690879fe6aaf8b805b8f7257a48 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 17:01:32 -0500 Subject: [PATCH 08/13] Include BackupFactor parameter --- src/mcts/search.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/search.cc b/src/mcts/search.cc index 517090fb29..e4926cd0ba 100644 --- a/src/mcts/search.cc +++ b/src/mcts/search.cc @@ -1477,7 +1477,7 @@ void SearchWorker::DoBackupUpdateSingleNode( m = n->GetM(); } n->FinalizeScoreUpdate(v / (1.0f + params_.GetShortSightedness() * depth), - d, m, node_to_process.multivisit); + d, m, node_to_process.multivisit, params_.GetBackupFactor()); // Nothing left to do without ancestors to update. if (!p) break; From ad592c54afa44117d6a6ffb7cbbac4a0cb72b56a Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 17:02:38 -0500 Subject: [PATCH 09/13] Add GetBackupFactor() --- src/mcts/params.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mcts/params.h b/src/mcts/params.h index ae1175ff50..75bb21a2ec 100644 --- a/src/mcts/params.h +++ b/src/mcts/params.h @@ -47,6 +47,7 @@ class SearchParams { return options_.Get(kMaxPrefetchBatchId); } bool GetLogitQ() const { return kLogitQ; } + float GetBackupFactor() const { return kBackupFactor; } float GetCpuct(bool at_root) const { return at_root ? kCpuctAtRoot : kCpuct; } float GetCpuctBase(bool at_root) const { return at_root ? kCpuctBaseAtRoot : kCpuctBase; From c96790a4cac0ae3fc9c6f361d53bcabc44473822 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 17:04:32 -0500 Subject: [PATCH 10/13] Fix incomplete change after C&P --- src/mcts/params.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/params.cc b/src/mcts/params.cc index d1b597f3f5..19fdfd2cad 100644 --- a/src/mcts/params.cc +++ b/src/mcts/params.cc @@ -333,7 +333,7 @@ void SearchParams::Populate(OptionsParser* options) { SearchParams::SearchParams(const OptionsDict& options) : options_(options), kLogitQ(options.Get(kLogitQId)), - kCpuct(options.Get(kBackupFactorId)), + kBackupFactor(options.Get(kBackupFactorId)), kCpuct(options.Get(kCpuctId)), kCpuctAtRoot(options.Get( options.Get(kRootHasOwnCpuctParamsId) ? kCpuctAtRootId From f43c382c9a4d808db2be71fe6e52c9a629bf376c Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 17:13:48 -0500 Subject: [PATCH 11/13] Optimize for default 0.0f case --- src/mcts/node.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mcts/node.cc b/src/mcts/node.cc index f98ccfac1a..9235bbcda9 100644 --- a/src/mcts/node.cc +++ b/src/mcts/node.cc @@ -280,7 +280,9 @@ void Node::CancelScoreUpdate(int multivisit) { void Node::FinalizeScoreUpdate(float v, float d, float m, int multivisit, float factor) { // Recompute Q. - float delta = FastLog2( 2.0f + factor * n_ ) * multivisit; + float delta = (factor > 0.0f + ? FastLog2( 2.0f + factor * n_ ) * multivisit + : multivisit); wl_ += delta * (v - wl_) / (n_ + delta); d_ += delta * (d - d_) / (n_ + delta); m_ += delta * (m - m_) / (n_ + delta); From 54fb623cde4cd8e5aadd21310f6aff722dd39e90 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sat, 18 Apr 2020 20:07:11 -0500 Subject: [PATCH 12/13] Fix missing identifier --- src/mcts/params.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mcts/params.h b/src/mcts/params.h index 75bb21a2ec..6d28f94b5f 100644 --- a/src/mcts/params.h +++ b/src/mcts/params.h @@ -172,6 +172,7 @@ class SearchParams { // TODO(crem) Some of those parameters can be converted to be dynamic after // trivial search optimizations. const bool kLogitQ; + const float kBackupFactor; const float kCpuct; const float kCpuctAtRoot; const float kCpuctBase; From 6c71d650b9f67d67affec6daf14666b3138bca13 Mon Sep 17 00:00:00 2001 From: Alexis Olson Date: Sun, 19 Apr 2020 00:51:28 -0500 Subject: [PATCH 13/13] Expand option range for testing purposes --- src/mcts/params.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcts/params.cc b/src/mcts/params.cc index 19fdfd2cad..9aed58ace6 100644 --- a/src/mcts/params.cc +++ b/src/mcts/params.cc @@ -262,7 +262,7 @@ void SearchParams::Populate(OptionsParser* options) { options->Add(kMiniBatchSizeId, 1, 1024) = 256; options->Add(kMaxPrefetchBatchId, 0, 1024) = 32; options->Add(kLogitQId) = false; - options->Add(kBackupFactorId, 0.0f, 1.0f) = 0.0f; + options->Add(kBackupFactorId, 0.0f, 100.0f) = 0.0f; options->Add(kCpuctId, 0.0f, 100.0f) = 2.147f; options->Add(kCpuctAtRootId, 0.0f, 100.0f) = 2.147f; options->Add(kCpuctBaseId, 1.0f, 1000000000.0f) = 18368.0f;