struct.txt

rlaif_project/
├── data/
│   ├── sft.json                   # Instruction-response training data
│   ├── ai_feedback_pairs.json     # AI-generated preference pairs
│   └── eval.json                  # Evaluation prompts
│
├── models/
│   ├── base/                      # Pretrained model weights
│   ├── sft_weights.pt             # Fine-tuned SFT model
│   ├── reward_weights.pt          # Reward model
│   └── ppo_weights.pt             # Final PPO-aligned model
│
├── scripts/
│   ├── sft_train.py               # Supervised fine-tuning script
│   ├── generate_feedback.py       # AI feedback generation script
│   ├── train_reward.py            # Reward model training script
│   ├── train_ppo.py               # PPO training script
│   └── evaluate.py                # Evaluation script
│
├── src/                           # Core code logic
│   ├── models.py                  # Model classes (LM, reward, value)
│   ├── data_utils.py              # Dataset loading & preprocessing
│   ├── generation.py              # Generation / sampling logic
│   ├── ppo.py                     # PPO objective & optimizer
│   └── loss.py                    # Loss computations (CE, preference)
│
├── configs/
│   ├── sft.yaml                   # SFT hyperparams
│   ├── reward.yaml                # Reward model hyperparams
│   └── ppo.yaml                   # PPO hyperparams
│
├── requirements.txt
└── README.md