nethack arch

config/nethack.ini

@@ -7,21 +7,16 @@ num_buffers = 4
 num_threads = 32
 
 [env]
-gold_coef = 0.00267383433
-exp_coef = 0.112984791
-descent_coef = 0.762886882
-# pays once per new unique (dnum, dlevel) floor entered (branches count)
-floor_coef = 0.0
-xp_coef = 0.2399306
-scout_coef = 0.0104040429
-ac_coef = 0.0
-death_penalty = -0.324047506
+gold_coef = 0.096887961
+score_coef = 0
+exp_coef = 0.041234985
+descent_coef = 1.0
+floor_coef = 0.38962388
+xp_coef = 0.741106272
+scout_coef = 0.000813971157
+death_penalty = -0.286484599
 
 # flags
-# scout reward scales with xp level relative to depth; <= 0 disables
-scout_ready = 0.0
-# fraction of protection-spell AC excluded from the ac reward; 1 = armor only
-ac_nospell = 1.0
 # 1 removes the verb from the action space
 mask_search20 = 0
 mask_run = 0
@@ -35,20 +30,20 @@ num_layers = 3
 [train]
 total_timesteps = 1_000_000_000
 minibatch_size = 4096
-horizon = 128
-learning_rate = 0.000765701407
-momentum = 0.987343371
+horizon = 256
+learning_rate = 0.00154131022
+momentum = 0.956484258
 verb_eps = 0.005
 verb_eps_anneal_start = 0.4
 verb_eps_anneal_end = 1.0
-gamma = 0.998305142
-gae_lambda = 0.265489638
-replay_ratio = 1.68762803
-clip_coef = 0.100000001
-vf_coef = 4
-vf_clip_coef = 0.00999999978
-max_grad_norm = 0.961730838
-ent_coef = 0.0229146201
+gamma = 0.999329984
+gae_lambda = 0.199999988
+replay_ratio = 1.20105004
+clip_coef = 0.36599344
+vf_coef = 0.882273436
+vf_clip_coef = 1.30713189
+max_grad_norm = 0.335048616
+ent_coef = 0.00555205066
 anneal_lr = 1
 anneal_ent_coef = 0
 min_lr_ratio = 0.0
@@ -67,8 +62,14 @@ scale = time
 
 [sweep.env.exp_coef]
 distribution = log_normal
-min = 0.01
-max = 0.2
+min = 0.001
+max = 0.1
+scale = auto
+
+[sweep.env.score_coef]
+distribution = log_normal
+min = 0.001
+max = 0.1
 scale = auto
 
 [sweep.env.gold_coef]
@@ -91,7 +92,7 @@ scale = auto
 
 [sweep.env.scout_coef]
 distribution = log_normal
-min = 0.001
+min = 0.0001
 max = 0.1
 scale = auto
 
@@ -113,12 +114,6 @@ min = 0.2
 max = 0.8
 scale = auto
 
-[sweep.env.ac_coef]
-distribution = log_normal
-min = 0.01
-max = 2.0
-scale = auto
-
 [sweep.env.floor_coef]
 distribution = uniform
 min = 0.0

ocean/nethack/depth.ini

@@ -1,7 +1,8 @@
-# async depth config: promoted winner of the max_depth sweep (run 0306,
-# tail depth 11.0, durable AC best-in-cohort). Rewards are descent-dominated
-# with a per-floor bonus; SEARCH20 and RUN are masked.
-# To use: copy to config/depth.ini, then ./puffer train depth
+# Depth recipe: sweep-0306 winner (tail depth 11.0, async, RNN carry) with
+# challenge multi-role. ~366M steps. SEARCH20 and RUN are masked.
+# Use: copy over config/nethack.ini, then ./build.sh nethack && ./puffer train
+# NOTE horizon*num_buffers > total_agents: runs are single-draw (the async
+# multi-stream cuBLAS nondeterminism); use num_buffers=1 for bit-repro.
 [base]
 env_name = nethack
 reset_every_horizon = 0
@@ -11,29 +12,18 @@ async = 1
 total_agents = 512
 num_buffers = 4
 num_threads = 32
-action_mask_size = 734
 
 [env]
 gold_coef = 0.00100000005
 exp_coef = 0.00999999978
 descent_coef = 1.0
-# pays once per new unique (dnum, dlevel) floor entered (branches count)
 floor_coef = 0.727649987
 xp_coef = 0.33030045
 scout_coef = 0.0165148061
-ac_coef = 0.0
 death_penalty = -0.0327779055
-
-# flags
-# scout reward scales with xp level relative to depth; <= 0 disables
-scout_ready = 0.0
-# fraction of protection-spell AC excluded from the ac reward; 1 = armor only
-ac_nospell = 1.0
-# 1 removes the verb from the action space
 mask_search20 = 1
 mask_run = 1
-# 1 = random role/race/gender/align per reset (challenge protocol)
-multi_role = 0
+multi_role = 1
 
 [policy]
 hidden_size = 512
@@ -62,60 +52,3 @@ min_lr_ratio = 0.0
 min_ent_coef_ratio = 0.1
 vtrace_rho_clip = 1.0
 vtrace_c_clip = 1.0
-
-[sweep]
-metric = max_depth
-
-[sweep.train.total_timesteps]
-distribution = log_normal
-min = 2e8
-max = 1e9
-scale = time
-
-[sweep.env.exp_coef]
-distribution = log_normal
-min = 0.01
-max = 1.0
-scale = auto
-
-[sweep.env.gold_coef]
-distribution = log_normal
-min = 0.001
-max = 1.0
-scale = auto
-
-[sweep.env.descent_coef]
-distribution = uniform
-min = 0.0
-max = 1.0
-scale = auto
-
-[sweep.env.floor_coef]
-distribution = uniform
-min = 0.0
-max = 1.0
-scale = auto
-
-[sweep.env.xp_coef]
-distribution = uniform
-min = 0.0
-max = 1.0
-scale = auto
-
-[sweep.env.scout_coef]
-distribution = log_normal
-min = 0.001
-max = 1.0
-scale = auto
-
-[sweep.env.death_penalty]
-distribution = uniform
-min = -1.0
-max = 0.0
-scale = auto
-
-[sweep.env.ac_coef]
-distribution = log_normal
-min = 0.01
-max = 2.0
-scale = auto