Breakout game is flickering when the episode_frame_number reaches 65536 while max_num_frames_per_episode is 108000 in the env spec, is this a known issue?

[RongpingZhou]

Breakout game is flickering when the episode_frame_number reaches 65536 while max_num_frames_per_episode is 108000 in the env spec?

The below is env spec:

EnvSpec(id='BreakoutNoFrameskip-v4', entry_point='ale_py.env:AtariEnv', reward_threshold=None, nondeterministic=False, max_episode_steps=108000, order_enforce=True, disable_env_checker=False, kwargs={'game': 'breakout', 'obs_type': 'rgb', 'repeat_action_probability': 0.0, 'full_action_space': False, 'max_num_frames_per_episode': 108000, 'frameskip': 1, 'render_mode': 'rgb_array'}, namespace=None, name='BreakoutNoFrameskip', version=4, additional_wrappers=(WrapperSpec(name='MaxAndSkipEnv', entry_point='stable_baselines3.common.atari_wrappers:MaxAndSkipEnv', kwargs=None), WrapperSpec(name='EpisodicLifeEnv', entry_point='stable_baselines3.common.atari_wrappers:EpisodicLifeEnv', kwargs=None), WrapperSpec(name='FireResetEnv', entry_point='stable_baselines3.common.atari_wrappers:FireResetEnv', kwargs=None)), vector_entry_point=None)

Here attched the captured video:

breakout_flickering.mp4

Just wondering whether there is any fix for that?

[pseudo-rnd-thoughts]

Is this an issue only everytime the agent gets to frame 65536? I would guess this is more likely a problem with Breakout itself rather than the implementation

[RongpingZhou]

Is this an issue only everytime the agent gets to frame 65536? I would guess this is more likely a problem with Breakout itself rather than the implementation

This issue occurs every time that one episode runs for over 65536 frames, here is the code to duplicate it:

`import time
import numpy as np
import random
from typing import List

import pygame
from pygame import Surface

import gymnasium as gym
from gymnasium import Env

import ale_py
gym.register_envs(ale_py)

def display_arr(
screen: Surface, arr: np.ndarray, video_size: tuple[int, int], transpose: bool
):
"""Displays a numpy array on screen.

Args:
    screen: The screen to show the array on
    arr: The array to show
    video_size: The video size of the screen
    transpose: If to transpose the array on the screen
"""
assert isinstance(arr, np.ndarray) and arr.dtype == np.uint8
pyg_img = pygame.surfarray.make_surface(arr.swapaxes(0, 1) if transpose else arr)
pyg_img = pygame.transform.scale(pyg_img, video_size)
# We might have to add black bars if surface_size is larger than video_size
surface_size = screen.get_size()
width_offset = (surface_size[0] - video_size[0]) / 2
height_offset = (surface_size[1] - video_size[1]) / 2
screen.fill((0, 0, 0))
screen.blit(pyg_img, (width_offset, height_offset))

def main():

fps = 300
seed = 1
zoom = 2.0
env = gym.make("BreakoutNoFrameskip-v4", render_mode="rgb_array")
env.reset(seed=seed)
env.action_space.seed(seed)
env.observation_space.seed(seed)
print(f"env spec: {env.spec}")
print(f"env metadata: {env.metadata}")

if env.render_mode not in {"rgb_array", "rgb_array_list"}:
    raise ValueError(
        "The program works only with rgb_array and rgb_array_list render modes, "
        f"but your environment render_mode = {env.render_mode}."
    )
    
rendered = env.render()
if isinstance(rendered, List):
    rendered = rendered[-1]
assert rendered is not None and isinstance(rendered, np.ndarray)
video_size = (rendered.shape[1], rendered.shape[0])

if zoom is not None:
    video_size = (int(video_size[0] * zoom), int(video_size[1] * zoom))

# self.video_size is the size of the video that is being displayed.
# The window size may be larger, in that case we will add black bars
screen = pygame.display.set_mode(video_size, pygame.RESIZABLE)

clock = pygame.time.Clock()    
clock.tick(fps)

obs = None
act = 0
done = True
terminated = True
truncated = False
done = terminated or truncated

inner_loop_break = False
    
while True:

    # Start of getting the state from the environment
    if done == True:
        obs, info = env.reset()
        terminated = False
        truncated = False
        done = False

    if obs is not None:
        rendered = env.render()
        # print(f"rendered shape: {rendered.shape}")                    
        if isinstance(rendered, List):
            rendered = rendered[-1]
        assert rendered is not None and isinstance(rendered, np.ndarray)
        display_arr(
            screen, rendered, transpose=True, video_size=video_size
        )

        pygame.display.flip()

    # For testing, we will use a random action
    # 0: No Operation
    # 1: Fire
    # 2: Right
    # 3: Left
    a_t = random.randint(0,3)
    a_t = random.randint(2,3)
            
    obs, _, terminated, truncated, info = env.step(a_t)
    done = terminated or truncated
    print(f"a_t: {a_t} info: {info}")
    if info["episode_frame_number"] == 60000:
        print("episode_frame_number is 60000, sleep 15 seconds, so that the video can be recorded")
        time.sleep(15)

    for event in pygame.event.get():
        if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE): 
            inner_loop_break = True
            break
    
    if inner_loop_break:
        break

print(f"info: {info}")
pygame.quit()

return

if name == "main":
main()
`
The below is the screen recording with showing frame number:

issue_65536_frames.mp4

[RongpingZhou]

By the way, I didn't let the agent fire, if it fires, it would end quickly, the episode_frame_number can't reach 65536