[BUG]: AttributeError: 'MoeFusedLinear' object has no attribute 'lora_dropout'

[chapter544]

Error message: As the title described.

[woct0rdho]

Are you using multiple GPUs and HF Accelerate? You can try to remove patch_Qwen3MoeFusedSparseMoeBlock_forward, or just copy-paste the content of that fast forward function to that original forward function if you need a bit of speedup. See unslothai/unsloth#2582 (comment)

Next time if you see errors, please show the whole error log.

[chapter544]

Oh, thank you. So the patch_Qwen3MoeFusedSparseMoeBlock_forward is only useful when I finetune using Lora. If I finetune without using Lora then I don't need patch_Qwen3MoeFusedSparseMoeBlock_forward.

Just want to clarify that patch_Qwen3MoeFusedSparseMoeBlock_forward only provides a small speed up when using lora. However, most speed up is done using the fused model Qwen3MoeFusedForCausalLM?

[woct0rdho]

Yes, most speed up should be done by the fused MoE kernel

[chapter544]

Hi, have you tried doing DPO using the fused model? SFT training looks fine, but when I tried DPO the gradients a very high and the loss is very unstable and does not converge.

Thanks,

[woct0rdho]

I haven't tried DPO. Can you show a minimal training script and the loss graph? Are you using multiple GPUs and/or Unsloth now?

[chapter544]

I am using 2 GPUs (not using Unsoth) + deepseed zero-3.

``
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model

from qwen3_moe_fused.fast_lora import patch_Qwen3MoeFusedSparseMoeBlock_forward
from qwen3_moe_fused.lora import patch_lora_config
from qwen3_moe_fused.modular_qwen3_moe_fused import Qwen3MoeFusedForCausalLM
from qwen3_moe_fused.quantize.quantizer import patch_bnb_quantizer
from peft import PeftModel, LoraConfig

#patch_Qwen3MoeFusedSparseMoeBlock_forward()

import os
import json
import datasets
import random
import torch
from datasets import load_dataset
from trl import SFTConfig, SFTTrainer

os.environ["TRITON_PRINT_AUTOTUNING"] = "1"

from trl import DPOTrainer
from trl import DPOConfig, DPOTrainer

def main():
patch_lora_config()

lora_config = LoraConfig(
    target_modules=[
        "q_proj",
        "k_proj",
        "v_proj",
        "o_proj",
        #"gate",
        "gate_proj",
        "up_proj",
        "down_proj",
    ],
    # We can set a smaller rank for MoE layers
    # With rslora, we don't need to set a different alpha for them
    rank_pattern={
        "q_proj": 64,
        "k_proj": 64,
        "v_proj": 64,
        "o_proj": 64,
        #"gate": 64,
        "gate_proj": 16,
        "up_proj": 16,
        "down_proj": 16,
    },
    lora_alpha=1,
    use_rslora=True,
    task_type="CAUSAL_LM",
    modules_to_save=[
        "lm_head",
        "embed_tokens",
    ],
)

ref_model_id = "Qwen3-30BA3-Instruct-fused"
sft_adapter_path = './trainer_output'
model_id = "Qwen3-30BA3-Instruct-fused"
output_dir_path = "haylm-30BA3-dpo-08-03-2025-lora"

max_prompt_length = 4096
max_completion_length = 4096
max_seq_length = 8192


tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = 'left' # to prevent errors with FA
tokenizer.truncation_side = 'left' # to prevent cutting off last generation


# Load model and tokenizer
ref_model = Qwen3MoeFusedForCausalLM.from_pretrained(
    ref_model_id,
    #device_map="auto",
    use_cache=False,
    attn_implementation="flash_attention_2",
    torch_dtype=torch.bfloat16,
    #quantization_config=bnb_config
)

#Load model and tokenizer
base_model = Qwen3MoeFusedForCausalLM.from_pretrained(
    model_id,
    #device_map="auto",
    use_cache=False,
    attn_implementation="flash_attention_2",
    torch_dtype=torch.bfloat16,
    #quantization_config=bnb_config
)

print(f"Loading LoRA adapters from {sft_adapter_path}")
model = PeftModel.from_pretrained(base_model, sft_adapter_path, is_trainable=True)
print("LoRA adapters loaded successfully.")


train_vi_json_fname = "train.json"
with open(train_vi_json_fname) as f:
    train_vi_json_data = json.load(f)
train_dataset = datasets.Dataset.from_list(train_vi_json_data)

dev_json_fname = "dev.json"
with open(dev_json_fname) as f:
    dev_json_data = json.load(f)
eval_dataset = datasets.Dataset.from_list(dev_json_data)


dpo_args = {
    "beta": 0.1,                            # The beta factor in DPO loss. Higher beta means less divergence
    "loss_type": "sigmoid"                  # The loss type for DPO.
}

args = DPOConfig(
    output_dir=output_dir_path,               # directory to save and repository id
    num_train_epochs=2,                     # number of training epochs
    per_device_train_batch_size=3,         # batch size per device during training
    per_device_eval_batch_size=6,           # batch size for evaluation
    gradient_accumulation_steps=4,          # number of steps before performing a backward/update pass
    gradient_checkpointing=True,            # use gradient checkpointing to save memory
    optim="adamw_torch_fused",              # use fused adamw optimizer
    learning_rate=3e-7,                     # 10x higher LR than QLoRA paper
    max_grad_norm=0.1,                      # max gradient norm based on QLoRA paper
    warmup_ratio=0.1,                       # warmup ratio based on QLoRA paper
    lr_scheduler_type="cosine",             # use cosine learning rate scheduler
    logging_steps=10,                       # log every 25 steps
    save_steps=200,                         # when to save checkpoint
    save_total_limit=5,                     # limit the total amount of checkpoints
    eval_strategy="steps",            # evaluate every 1000 steps
    eval_steps=100,                         # when to evaluate
    bf16=True,                              # use bfloat16 precision
    tf32=True,                              # use tf32 precision
    push_to_hub=False,                      # push model to hub
    report_to="wandb",                # report metrics to tensorboard
    max_length=max_seq_length,
    max_prompt_length=max_prompt_length,
    max_completion_length=max_completion_length,
    beta=dpo_args["beta"],
    loss_type=dpo_args["loss_type"],
)


trainer = DPOTrainer(
    model,
    ref_model=ref_model,
    args=args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    processing_class=tokenizer,
)

# start training, the model will be automatically saved to the hub and the output directory
checkpoint_path = None
if checkpoint_path:
    trainer_stats = trainer.train(resume_from_checkpoint=checkpoint_path)
else:
    trainer_stats = trainer.train()

# save model at the end of training
trainer.save_model()


# free the memory again
del model
del trainer
torch.cuda.empty_cache()

if name == "main":
main()
``

[woct0rdho]

Hi @chapter544 , we just found a bug that we did not restore the values when running an inplace kernel multiple times in autotune, see 668dfd5 . This may affect the correctness.