config.yaml

# Model
model_name: "Qwen/Qwen2.5-1.5B-Instruct"

# Data - Update these to your own paths!
dataset_path: "/home/ubuntu/trl/data/train_mini_sudoku.jsonl"
eval_dataset_path: "/home/ubuntu/trl/data/val_mini_sudoku.jsonl"

# NeMo Gym server configs (relative to Gym repo)
gym_configs:
  - resources_servers/reasoning_gym/configs/reasoning_gym.yaml
  - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml

# Logging
output_dir: "outputs/nemo_gym"
run_name: "sudoku"
report_to: "none"
project_name: "trl-nemo-gym"
log_completions: true
num_completions_to_print: 2

# Training hyperparameters
learning_rate: 1.0e-5
max_steps: 1000
num_generations: 16
per_device_train_batch_size: 1
per_device_eval_batch_size: 1
gradient_accumulation_steps: 256
max_completion_length: 10000
warmup_steps: 3
lr_scheduler_type: "linear"
optim: "adamw_torch_fused"
weight_decay: 0.01
vllm_importance_sampling_correction: true
temperature: 1.0
top_p: 0.999

# Checkpointing and eval
save_steps: 10
eval_strategy: "steps"
eval_steps: 10