# Core dependencies pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 pip install transformers accelerate peft bitsandbytes trl datasets # For quantization pip install auto-gptq optimum # Optional: Weights & Biases for logging pip install wandb 

# For LLaMA/Mistral/Gemma architecture TARGET_MODULES = [ "q_proj", "k_proj", "v_proj", "o_proj", # Attention  "gate_proj", "up_proj", "down_proj" # MLP ] # For smaller models or less VRAM, use attention only TARGET_MODULES_ATTENTION = ["q_proj", "k_proj", "v_proj", "o_proj"] 

// alpaca_format.json [ { "instruction": "Write a Python function to calculate fibonacci numbers", "input": "", "output": "def fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)" }, { "instruction": "Explain the concept of recursion", "input": "In the context of computer science", "output": "Recursion is a programming technique where a function calls itself..." } ] 

// chatml_format.json [ { "messages": [ {"role": "system", "content": "You are a helpful coding assistant."}, {"role": "user", "content": "How do I handle errors in Go?"}, {"role": "assistant", "content": "In Go, errors are values..."} ] } ] 

# data_prep.py from datasets import Dataset, load_dataset from transformers import AutoTokenizer import json def load_and_format_data(file_path: str, format: str = "alpaca") -> Dataset: with open(file_path) as f: data = json.load(f) if format == "alpaca": def format_alpaca(example): if example["input"]: prompt = f"### Instruction:\n{example['instruction']}\n\n### Input:\n{example['input']}\n\n### Response:\n{example['output']}" else: prompt = f"### Instruction:\n{example['instruction']}\n\n### Response:\n{example['output']}" return {"text": prompt} dataset = Dataset.from_list(data).map(format_alpaca) elif format == "chatml": def format_chatml(example): messages = example["messages"] # Apply chat template  text = tokenizer.apply_chat_template(messages, tokenize=False) return {"text": text} dataset = Dataset.from_list(data).map(format_chatml) return dataset def tokenize_dataset(dataset: Dataset, tokenizer, max_length: int = 2048): def tokenize(example): return tokenizer( example["text"], truncation=True, max_length=max_length, padding="max_length", return_tensors="pt" ) return dataset.map(tokenize, batched=True, remove_columns=dataset.column_names) # Train/val split dataset = load_and_format_data("train.json") dataset = dataset.train_test_split(test_size=0.1, seed=42) train_dataset = dataset["train"] eval_dataset = dataset["test"] 

# train_lora.py import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig ) from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training from trl import SFTTrainer from datasets import load_dataset import os # ============ CONFIGURATION ============ MODEL_ID = "meta-llama/Llama-3.2-3B-Instruct" # Or mistralai/Mistral-7B-Instruct-v0.3 DATASET_PATH = "train.json" OUTPUT_DIR = "./llama-3.2-3b-lora-finetuned" MAX_LENGTH = 2048 BATCH_SIZE = 2 GRAD_ACCUM = 8 # Effective batch = 16 LEARNING_RATE = 2e-4 NUM_EPOCHS = 3 LORA_R = 64 LORA_ALPHA = 16 LORA_DROPOUT = 0.1 # ============ QUANTIZATION CONFIG (QLoRA) ============ bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", # NormalFloat 4-bit  bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, # Double quantization ) # ============ LoRA CONFIG ============ lora_config = LoraConfig( r=LORA_R, lora_alpha=LORA_ALPHA, target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj" ], lora_dropout=LORA_DROPOUT, bias="none", task_type="CAUSAL_LM", # For QLoRA: modules_to_save=["embed_tokens", "lm_head"] # If extending vocab ) # ============ LOAD MODEL & TOKENIZER ============ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True) tokenizer.pad_token = tokenizer.eos_token tokenizer.padding_side = "right" model = AutoModelForCausalLM.from_pretrained( MODEL_ID, quantization_config=bnb_config, device_map="auto", trust_remote_code=True, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2", # Requires flash-attn ) # Prepare for k-bit training model = prepare_model_for_kbit_training(model) model = get_peft_model(model, lora_config) model.print_trainable_parameters() # Example output: trainable params: 35,651,584 || all params: 3,200,000,000 || trainable%: 1.11%  # ============ LOAD DATA ============ def formatting_func(example): return example["text"] dataset = load_dataset("json", data_files=DATASET_PATH, split="train") dataset = dataset.train_test_split(test_size=0.1, seed=42) # ============ TRAINING ARGS ============ training_args = TrainingArguments( output_dir=OUTPUT_DIR, num_train_epochs=NUM_EPOCHS, per_device_train_batch_size=BATCH_SIZE, per_device_eval_batch_size=BATCH_SIZE, gradient_accumulation_steps=GRAD_ACCUM, learning_rate=LEARNING_RATE, lr_scheduler_type="cosine", warmup_ratio=0.03, logging_steps=10, eval_strategy="steps", eval_steps=100, save_strategy="steps", save_steps=100, save_total_limit=3, load_best_model_at_end=True, metric_for_best_model="eval_loss", bf16=True, tf32=True, gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, optim="paged_adamw_32bit", # QLoRA optimizer  report_to="wandb", # or "tensorboard" or "none"  run_name=f"llama-3.2-3b-lora-r{LORA_R}", seed=42, ddp_find_unused_parameters=False, ) # ============ TRAINER ============ trainer = SFTTrainer( model=model, args=training_args, train_dataset=dataset["train"], eval_dataset=dataset["test"], formatting_func=formatting_func, max_seq_length=MAX_LENGTH, tokenizer=tokenizer, packing=True, # Efficient packing of sequences ) # ============ TRAIN ============ trainer.train() # Save final model trainer.save_model() tokenizer.save_pretrained(OUTPUT_DIR) print(f"Model saved to {OUTPUT_DIR}") 

# Single GPU python train_lora.py # Multi-GPU with Accelerate accelerate config # First time setup accelerate launch train_lora.py # With DeepSpeed ZeRO-3 (for larger models) accelerate launch --config_file ds_zero3.yaml train_lora.py 

# Automatically used with optim="paged_adamw_32bit" # Moves optimizer states to CPU when not in use # Enables larger models on same VRAM 

# In TrainingArguments: gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, # Saves ~30% VRAM at cost of ~20% slower training 

# Install pip install flash-attn --no-build-isolation # Use in model loading attn_implementation="flash_attention_2" # 2-3x faster, less memory 

# In TrainingArguments report_to="wandb", run_name="my-lora-run", # Before training import wandb wandb.init(project="llm-finetuning", name="llama-3b-lora-r64") 

# Start tensorboard tensorboard --logdir ./llama-3.2-3b-lora-finetuned/runs # In TrainingArguments report_to="tensorboard", 

# merge_lora.py from peft import PeftModel from transformers import AutoModelForCausalLM, AutoTokenizer import torch BASE_MODEL = "meta-llama/Llama-3.2-3B-Instruct" LORA_PATH = "./llama-3.2-3b-lora-finetuned" OUTPUT_PATH = "./llama-3.2-3b-merged" # Load base model in fp16/bf16 (not 4-bit) base_model = AutoModelForCausalLM.from_pretrained( BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True, ) # Load LoRA adapter model = PeftModel.from_pretrained(base_model, LORA_PATH) # Merge and unload merged_model = model.merge_and_unload() # Save merged model merged_model.save_pretrained(OUTPUT_PATH) tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) tokenizer.save_pretrained(OUTPUT_PATH) print(f"Merged model saved to {OUTPUT_PATH}") 

# Install llama.cpp with Python bindings pip install llama-cpp-python # Convert using llama.cpp convert script python llama.cpp/convert_hf_to_gguf.py \ ./llama-3.2-3b-merged \ --outfile ./llama-3.2-3b-finetuned.Q4_K_M.gguf \ --outtype q4_k_m # Or use llama.cpp quantize directly ./llama.cpp/quantize ./llama-3.2-3b-merged/model.fp16.gguf \ ./llama-3.2-3b-finetuned.Q4_K_M.gguf q4_k_m 

# Modelfile FROM ./llama-3.2-3b-finetuned.Q4_K_M.gguf TEMPLATE "{{ .System }}\n{{ .Prompt }}\n{{ .Response }}" PARAMETER temperature 0.7 PARAMETER top_p 0.9 PARAMETER num_ctx 8192 

# Create Ollama model ollama create my-finetuned-model -f Modelfile # Test ollama run my-finetuned-model "Write a Python function for binary search" 

# Experiment with different ranks RANK_CONFIGS = [ {"r": 8, "alpha": 16}, # Minimal, fast, less capacity  {"r": 16, "alpha": 32}, # Good balance  {"r": 32, "alpha": 64}, # More capacity  {"r": 64, "alpha": 16}, # High capacity (alpha often = r/4 to r/2)  {"r": 128, "alpha": 32}, # Maximum capacity ] 

# Test different target module sets TARGET_CONFIGS = { "attention_only": ["q_proj", "k_proj", "v_proj", "o_proj"], "mlp_only": ["gate_proj", "up_proj", "down_proj"], "all_linear": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], "attention_mlp": ["q_proj", "v_proj", "gate_proj", "up_proj"], # Key ones } 

# For LoRA, higher LR often works (2e-4 to 1e-3) # But depends on dataset size: # Small dataset (<1k): 1e-4 to 5e-4 # Medium (1k-10k): 2e-4 # Large (>10k): 1e-4 to 2e-4 

# eval_model.py from transformers import AutoModelForCausalLM, AutoTokenizer from peft import PeftModel import torch from datasets import load_dataset import json def evaluate_model(model_path, test_data_path, base_model_id=None): if base_model_id: base = AutoModelForCausalLM.from_pretrained( base_model_id, torch_dtype=torch.bfloat16, device_map="auto" ) model = PeftModel.from_pretrained(base, model_path) else: model = AutoModelForCausalLM.from_pretrained( model_path, torch_dtype=torch.bfloat16, device_map="auto" ) tokenizer = AutoTokenizer.from_pretrained(model_path) # Load test data  with open(test_data_path) as f: test_data = json.load(f) results = [] for item in test_data[:50]: # Sample 50  prompt = f"### Instruction:\n{item['instruction']}Source：https://dev.to/ipsphil_digital/fine-tuning-llms-with-lora-qlora-complete-practical-guide-560k
My favorites