tmp

Author: tAnGjIa520

lzero/policy/unizero.py

@@ -57,25 +57,25 @@ def configure_optimizer_unizero(model, learning_rate, weight_decay, device_type,
     # 3. 为每组设置不同的优化器参数（特别是学习率）
     #    这里我们仍然使用AdamW，但学习率设置更合理
     optim_groups = [
+        {
+            'params': list(tokenizer_params.values()),
+            'lr': learning_rate,  # Tokenizer使用基础学习率，例如 1e-4
+            # 'lr': learning_rate * 0.1,  # 为encoder设置一个较小的学习率，例如 1e-5
+            'weight_decay': weight_decay * 5.0  # <-- 为Encoder设置5倍的权重衰减！这是一个强力正则化
+            # 'weight_decay': weight_decay  # <-- 为Encoder设置5倍的权重衰减！这是一个强力正则化
+        },
         {
             'params': list(transformer_params.values()),
             'lr': learning_rate,  # 1e-4
             # 'lr': learning_rate * 0.2,  # 为Transformer主干设置一个较小的学习率，例如 1e-5
             'weight_decay': weight_decay
             # 'weight_decay': weight_decay * 5.0 
         },
-        {
-            'params': list(tokenizer_params.values()),
-            'lr': learning_rate,  # Tokenizer使用基础学习率，例如 1e-4
-            # 'lr': learning_rate * 0.1,  # 为encoder设置一个较小的学习率，例如 1e-5
-            # 'weight_decay': weight_decay * 5.0  # <-- 为Encoder设置5倍的权重衰减！这是一个强力正则化
-            'weight_decay': weight_decay  # <-- 为Encoder设置5倍的权重衰减！这是一个强力正则化
-        },
         {
             'params': list(head_params.values()),
             'lr': learning_rate,  # Heads也使用基础学习率率，例如 1e-4
-            # 'weight_decay': 0.0  # 通常Heads的权重不做衰减
-            'weight_decay': weight_decay
+            'weight_decay': 0.0  # 通常Heads的权重不做衰减
+            # 'weight_decay': weight_decay
 
         }
     ]

zoo/atari/config/atari_unizero_segment_config.py

@@ -11,8 +11,8 @@ def main(env_id, seed):
     collector_env_num = 8
     num_segments = 8
 
-    # game_segment_length = 20
-    game_segment_length = 400 # TODO
+    game_segment_length = 20
+    # game_segment_length = 400 # TODO
 
     evaluator_env_num = 3
     num_simulations = 50
@@ -76,10 +76,10 @@ def main(env_id, seed):
                 reward_support_range=(-300., 301., 1.),
                 value_support_range=(-300., 301., 1.),
                 norm_type=norm_type,
-                # num_res_blocks=1,
-                # num_channels=64,
-                num_res_blocks=2,
-                num_channels=128,
+                num_res_blocks=1,
+                num_channels=64,
+                # num_res_blocks=2,
+                # num_channels=128,
                 world_model_cfg=dict(
                     norm_type=norm_type,
                     final_norm_option_in_obs_head='LayerNorm',
@@ -161,8 +161,8 @@ def main(env_id, seed):
             # (float) 退火的结束 clip 值 (训练后期，较严格)。
             encoder_clip_end_value=10.0,
             # (int) 完成从起始值到结束值的退火所需的训练迭代步数。
-            encoder_clip_anneal_steps=400000,  # 例如，在400k次迭代后达到最终值
-            # encoder_clip_anneal_steps=100000,  # 例如，在100k次迭代后达到最终值
+            # encoder_clip_anneal_steps=400000,  # 例如，在400k次迭代后达到最终值
+            encoder_clip_anneal_steps=100000,  # 例如，在100k次迭代后达到最终值
 
             # ==================== START: label smooth ====================
             policy_ls_eps_start=0.05, #TODO============= good start in Pong and MsPacman
@@ -225,7 +225,9 @@ def main(env_id, seed):
 
     # ============ use muzero_segment_collector instead of muzero_collector =============
     from lzero.entry import train_unizero_segment
-    main_config.exp_name = f'data_unizero_st_refactor1010/{env_id[:-14]}/{env_id[:-14]}_uz_ch128-res2_targetentropy-alpha-100k-098-07-encoder-clip30-10-400k_label-smooth_resnet-encoder_priority_adamw-wd1e-2-encoder1-trans1-head1_ln-inner-ln_brf{buffer_reanalyze_freq}-rbs{reanalyze_batch_size}-rp{reanalyze_partition}_nlayer{num_layers}_numsegments-{num_segments}_gsl{game_segment_length}_rr{replay_ratio}_Htrain{num_unroll_steps}-Hinfer{infer_context_length}_bs{batch_size}_seed{seed}'
+    main_config.exp_name = f'data_unizero_st_refactor1010/{env_id[:-14]}/{env_id[:-14]}_uz_ch64-res1_targetentropy-alpha-100k-098-07-encoder-clip30-10-100k_label-smooth_resnet-encoder_priority_adamw-wd1e-2-encoder5-trans1-head0-true_ln-inner-ln_brf{buffer_reanalyze_freq}-rbs{reanalyze_batch_size}-rp{reanalyze_partition}_nlayer{num_layers}_numsegments-{num_segments}_gsl{game_segment_length}_rr{replay_ratio}_Htrain{num_unroll_steps}-Hinfer{infer_context_length}_bs{batch_size}_seed{seed}'
+
+    # main_config.exp_name = f'data_unizero_st_refactor1010/{env_id[:-14]}/{env_id[:-14]}_uz_ch128-res2_targetentropy-alpha-100k-098-07-encoder-clip30-10-400k_label-smooth_resnet-encoder_priority_adamw-wd1e-2-encoder1-trans1-head1_ln-inner-ln_brf{buffer_reanalyze_freq}-rbs{reanalyze_batch_size}-rp{reanalyze_partition}_nlayer{num_layers}_numsegments-{num_segments}_gsl{game_segment_length}_rr{replay_ratio}_Htrain{num_unroll_steps}-Hinfer{infer_context_length}_bs{batch_size}_seed{seed}'
     train_unizero_segment([main_config, create_config], seed=seed, model_path=main_config.policy.model_path, max_env_step=max_env_step)
 
 
@@ -262,7 +264,7 @@ def main(env_id, seed):
     tmux new -s uz-st-refactor-boxing
 
     conda activate /mnt/nfs/zhangjinouwen/puyuan/conda_envs/lz
-    export CUDA_VISIBLE_DEVICES=4
+    export CUDA_VISIBLE_DEVICES=6
     cd /mnt/nfs/zhangjinouwen/puyuan/LightZero
     python /mnt/nfs/zhangjinouwen/puyuan/LightZero/zoo/atari/config/atari_unizero_segment_config.py 2>&1 | tee /mnt/nfs/zhangjinouwen/puyuan/LightZero/log/20251010_uz_st_ch128-res2_fix-encoder-clip_qbert.log
     """