qlora.py

# -*- coding: utf-8 -*-
"""QLoRA.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/github/akatakan/QLoRa/blob/main/QLoRA.ipynb
"""

!pip install -q -U bitsandbytes
!pip install -q -U transformers
!pip install -q -U peft
!pip install -q -U accelerate
!pip install -q datasets
!pip install -q trl

from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig,DataCollatorForLanguageModeling
from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model
from huggingface_hub import notebook_login
from datasets import load_dataset, DatasetDict
import torch
from trl import SFTTrainer

from huggingface_hub import login
from google.colab import userdata
login(userdata.get('hf'))

base_model = "meta-llama/Meta-Llama-3-8B-Instruct"

# NF4 = normalize float 4 | pure FP4 , quantization çeşitleri
# Quantizationda ağıtlıklar 4 bit olarak saklanırken, hesaplamaları 16 veya 32 bit veri tipleri olarak ayarlanabilir
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True, #ikinci bir quantization kullanmamızı sağlar, bu da parametre başına 0.4 bitlik ek tasarrufluk sağlar
    bnb_4bit_quant_type='nf4',
    bnb_4bit_compute_dtype=torch.bfloat16 #daha esnek, daha fazla fine tuning
)

tokenizer = AutoTokenizer.from_pretrained(base_model)
model = AutoModelForCausalLM.from_pretrained(
    base_model,
    quantization_config = bnb_config,
    device_map="auto"
)

trained_model_id = "Llama-3-8B-turkishQA"
output_dir = './results/' + trained_model_id

model.gradient_checkpointing_enable()
model = prepare_model_for_kbit_training(model)

import bitsandbytes as bnb
def find_all_linear_names(bits, model):
    cls = bnb.nn.Linear4bit if bits == 4 else (bnb.nn.Linear8bitLt if bits == 8 else torch.nn.Linear)
    lora_module_names = set()
    for name, module in model.named_modules():
        if isinstance(module, cls):
            names = name.split('.')
            lora_module_names.add(names[0] if len(names) == 1 else names[-1])


    if 'lm_head' in lora_module_names: # needed for 16-bit
        lora_module_names.remove('lm_head')
    return list(lora_module_names)

modules = find_all_linear_names(4,model)
modules

def print_trainable_parameters(model):
    """
    Prints the number of trainable parameters in the model.
    """
    trainable_params = 0
    all_param = 0
    for _, param in model.named_parameters():
        all_param += param.numel()
        if param.requires_grad:
            trainable_params += param.numel()
    print(
        f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}"
    )

from peft import LoraConfig

peft_config = LoraConfig(
    r=8,
    lora_alpha=32,
    target_modules=modules,
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)

model = get_peft_model(model, peft_config)
print_trainable_parameters(model)

dataset = "Yudum/turkish-instruct-dataset"

train_dataset = load_dataset(dataset,split="train")

def generate_prompt(data_point):
    if data_point["input"]:
        return f"""Aşağıda, daha geniş bir bağlam sağlayan girdiyle birlikte bir görevi açıklayan talimat bulunmaktadır. Talimatı yeterince sağlayan bir çıktı yaz.

                ### Talimat:
                {data_point["instruction"]}

                ### Girdi:
                {data_point["input"]}

                ### Çıktı:
                {data_point["output"]}"""
    else:
      return f"""Aşağıda bir görevi açıklayan talimat bulunmaktadır. Talimatı yeterince sağlayan bir çıktı yaz.

              ### Talimat:
              {data_point["instruction"]}

              ### Çıktı:
              {data_point["output"]}"""

EOS_TOKEN = tokenizer.eos_token

def formatting_func(examples):
  texts = []
  text = generate_prompt(examples) + EOS_TOKEN
  texts.append(text)
  return { "text": texts,}

train_dataset = train_dataset.map(lambda x: print(x), batched = True)

train_dataset
tokenizer.pad_token = tokenizer.eos_token

training_arguments = TrainingArguments(
          fp16= not torch.cuda.is_bf16_supported(),
          bf16= torch.cuda.is_bf16_supported(),
          gradient_accumulation_steps=8,
          gradient_checkpointing=True,
          learning_rate=2.0e-05,
          log_level="info",
          logging_steps=5,
          lr_scheduler_type="cosine",
          max_steps=100,
          output_dir=output_dir,
          overwrite_output_dir=True,
          per_device_train_batch_size=8,
          push_to_hub=True,
          hub_model_id=trained_model_id,
          save_total_limit=1,
          load_best_model_at_end=True
          seed=42,
)

torch.cuda.empty_cache()

trainer = SFTTrainer(
        model=model,
        args=training_arguments,
        train_dataset=train_dataset,
        dataset_text_field = "text",
        formatting_func = formatting_func,
        tokenizer=tokenizer,
        peft_config=peft_config,
        max_seq_length= 2048
)

gpu_stats = torch.cuda.get_device_properties(0)
start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
print(f"{start_gpu_memory} GB of memory reserved.")

model.config.use_cache = False
trainer.train()

used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
used_memory_for_lora = round(used_memory - start_gpu_memory, 3)
used_percentage = round(used_memory         /max_memory*100, 3)
lora_percentage = round(used_memory_for_lora/max_memory*100, 3)
print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.")
print(f"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.")
print(f"Peak reserved memory = {used_memory} GB.")
print(f"Peak reserved memory for training = {used_memory_for_lora} GB.")
print(f"Peak reserved memory % of max memory = {used_percentage} %.")
print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.")