mdp-playground/experiments/dqn_space_invaders_image_transforms.py at master · automl/mdp-playground · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
import itertools
from ray import tune
from collections import OrderedDict

num_seeds = 5
timesteps_total = 10_000_000

var_env_configs = OrderedDict(
    {
        "image_transforms": [
            "none",
            "shift",
            # "scale",
            # "flip",
            # "rotate",
            # "shift,scale,rotate,flip",
        ],  # image_transforms,
        "dummy_seed": [i for i in range(num_seeds)],
    }
)

var_configs = OrderedDict({"env": var_env_configs})

env_config = {
    "env": "GymEnvWrapper-Atari",
    "env_config": {
        "AtariEnv": {
            "game": "space_invaders",
            "obs_type": "image",
            "frameskip": 1,
        },
        # "GymEnvWrapper": {
        "atari_preprocessing": True,
        "frame_skip": 4,
        "grayscale_obs": False,  # grayscale_obs gives a 2-D observation tensor.
        "state_space_type": "discrete",
        "action_space_type": "discrete",
        "seed": 0,
        # },
        # 'seed': 0, #seed
    },
}

algorithm = "DQN"
agent_config = {  # Taken from Ray tuned_examples
    "adam_epsilon": 0.00015,
    "buffer_size": 200000,
    "double_q": False,
    "dueling": False,
    "exploration_config": {"epsilon_timesteps": 200000, "final_epsilon": 0.01},
    "final_prioritized_replay_beta": 1.0,
    "hiddens": [512],
    "learning_starts": 20000,
    "lr": 6.25e-05,
    "n_step": 1,
    "noisy": False,
    "num_atoms": 1,
    "num_gpus": 0,
    "num_workers": 3,
    "prioritized_replay": False,
    "prioritized_replay_alpha": 0.5,
    "prioritized_replay_beta_annealing_timesteps": 2000000,
    "rollout_fragment_length": 4,
    "target_network_update_freq": 8000,
    "timesteps_per_iteration": 10000,
    "train_batch_size": 32,
    "tf_session_args": {
        # note: overriden by `local_tf_session_args`
        "intra_op_parallelism_threads": 4,
        "inter_op_parallelism_threads": 4,
        # "gpu_options": {
        #     "allow_growth": True,
        # },
        # "log_device_placement": False,
        "device_count": {"CPU": 2},
        # "allow_soft_placement": True,  # required by PPO multi-gpu
    },
    # Override the following tf session args on the local worker
    "local_tf_session_args": {
        "intra_op_parallelism_threads": 4,
        "inter_op_parallelism_threads": 4,
    },
}

# formula [(W−K+2P)/S]+1; for padding=same: P = ((S-1)*W - S + K)/2
filters_124x124 = [
    [
        16,
        [8, 8],
        4,
    ],  # changes from 84x84x1 with padding 4 to 22x22x16 (or 32x32x16 for 124x124x1)
    [32, [4, 4], 2],  # changes to 11x11x32 with padding 2 (or 16x16x32 for 124x124x1)
    [
        256,
        [16, 16],
        1,
    ],  # changes to 1x1x256 with padding 0 (for 124x124x1??); this is the only layer with "valid" padding in Ray!
]

filters_100x100 = [
    [
        16,
        [8, 8],
        4,
    ],  # changes from 84x84x1 with padding 4 to 22x22x16 (or 26x26x16 for 100x100x1)
    [32, [4, 4], 2],  # changes to 11x11x32 with padding 2 (or 13x13x32 for 100x100x1)
    [
        64,
        [13, 13],
        1,
    ],  # changes to 1x1x64 with padding 0 (or 3x3x64 for 100x100x1); this is the only layer with valid padding in Ray!
]
# [num_outputs(=8 in this case), [1, 1], 1] conv2d appended by Ray always followed by a Dense layer with 1 output

# filters_99x99 = [
#     [16, [8, 8], 4], # 51x51x16
#     [32, [4, 4], 2],
#     [64, [13, 13], 1],
# ]

model_config = {
    "model": {
        "conv_filters": filters_124x124,
        # "fcnet_hiddens": [256, 256],
        # "fcnet_activation": "tanh",
        "use_lstm": False,
        "max_seq_len": 20,
        "lstm_cell_size": 256,
        "lstm_use_prev_action_reward": False,
    },
}


eval_config = {
    "evaluation_interval": None,  # I think this means every x training_iterations
    "evaluation_config": {
        "explore": False,
        "exploration_fraction": 0,
        "exploration_final_eps": 0,
        "evaluation_num_episodes": 10,
        "horizon": 100,
        "env_config": {
            "dummy_eval": True,  # hack Used to check if we are in evaluation mode or training mode inside Ray callback on_episode_end() to be able to write eval stats
            "transition_noise": 0
            if "state_space_type" in env_config["env_config"]
            and env_config["env_config"]["state_space_type"] == "discrete"
            else tune.function(lambda a: a.normal(0, 0)),
            "reward_noise": tune.function(lambda a: a.normal(0, 0)),
            "action_loss_weight": 0.0,
        },
    },
}