Original weights: W ∈ ℝ^(d×k) LoRA adapters: A ∈ ℝ^(d×r), B ∈ ℝ^(r×k) where r << min(d,k) Forward pass: h = Wx + BAx = (W + BA)x 

# peft/lora_theory.py import torch import torch.nn as nn from typing import Optional class LoRALinear(nn.Module): """LoRA-adapted linear layer.""" def __init__( self, in_features: int, out_features: int, r: int = 16, alpha: int = 32, dropout: float = 0.1, bias: bool = False ): super().__init__() self.in_features = in_features self.out_features = out_features self.r = r self.alpha = alpha self.scaling = alpha / r # Original frozen weights  self.weight = nn.Parameter(torch.empty(out_features, in_features)) self.weight.requires_grad = False # LoRA adapters  self.lora_A = nn.Parameter(torch.empty(r, in_features)) self.lora_B = nn.Parameter(torch.empty(out_features, r)) self.dropout = nn.Dropout(dropout) if bias: self.bias = nn.Parameter(torch.empty(out_features)) else: self.register_parameter('bias', None) self.reset_parameters() def reset_parameters(self): nn.init.kaiming_uniform_(self.weight, a=5**0.5) nn.init.kaiming_uniform_(self.lora_A, a=5**0.5) nn.init.zeros_(self.lora_B) if self.bias is not None: nn.init.zeros_(self.bias) def forward(self, x: torch.Tensor) -> torch.Tensor: # Base model forward (frozen)  base_output = nn.functional.linear(x, self.weight, self.bias) # LoRA forward (trainable)  lora_output = (self.dropout(x) @ self.lora_A.T @ self.lora_B.T) * self.scaling return base_output + lora_output def merge_weights(self): """Merge LoRA into base weights for inference.""" if self.r > 0: delta_w = (self.lora_B @ self.lora_A) * self.scaling self.weight.data += delta_w self.lora_A.data.zero_() self.lora_B.data.zero_() def apply_lora_to_model( model: nn.Module, target_modules: list = None, r: int = 16, alpha: int = 32, dropout: float = 0.1 ) -> nn.Module: """Apply LoRA to target modules (e.g., q_proj, v_proj).""" if target_modules is None: target_modules = ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"] for name, module in model.named_modules(): if any(target in name for target in target_modules) and isinstance(module, nn.Linear): parent_name = ".".join(name.split(".")[:-1]) child_name = name.split(".")[-1] parent = model.get_submodule(parent_name) if parent_name else model lora_layer = LoRALinear( module.in_features, module.out_features, r=r, alpha=alpha, dropout=dropout, bias=module.bias is not None ) lora_layer.weight.data = module.weight.data.clone() if module.bias is not None: lora_layer.bias.data = module.bias.data.clone() setattr(parent, child_name, lora_layer) return model 

# peft/qlora.py import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig ) from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training def load_qlora_model( model_id: str, r: int = 64, alpha: int = 16, dropout: float = 0.1, target_modules: list = None, device_map: str = "auto" ): """Load model with 4-bit quantization + LoRA.""" # 4-bit NF4 quantization config  bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, ) # Load base model  model = AutoModelForCausalLM.from_pretrained( model_id, quantization_config=bnb_config, device_map=device_map, torch_dtype=torch.bfloat16, trust_remote_code=True, ) # Prepare for k-bit training  model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True) # LoRA config  if target_modules is None: target_modules = ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"] lora_config = LoraConfig( r=r, lora_alpha=alpha, lora_dropout=dropout, target_modules=target_modules, bias="none", task_type="CAUSAL_LM", modules_to_save=None, ) model = get_peft_model(model, lora_config) model.print_trainable_parameters() return model 

NormalFloat4 (NF4) — optimal for normally distributed weights Standard int4: [-8, -7, ..., 7] uniform spacing NF4: 16 quantiles of N(0,1) — more precision near 0 Double Quantization: Quantize the quantization constants (saves 0.37 bits/param) 

# peft/nf4_details.py NF4_QUANTILES = torch.tensor([ -1.0, -0.6961928, -0.525073, -0.394917, -0.284441, -0.184773, -0.091050, 0.0, 0.079580, 0.160930, 0.246112, 0.337915, 0.440709, 0.562617, 0.722957, 1.0 ]) def quantize_nf4(tensor: torch.Tensor) -> tuple: """Quantize to NF4 with double quantization.""" # Block-wise quantization (block_size=64)  block_size = 64 original_shape = tensor.shape tensor = tensor.view(-1, block_size) # Compute absmax per block  absmax = tensor.abs().max(dim=1, keepdim=True).values absmax = absmax.clamp(min=1e-8) # Normalize  normalized = tensor / absmax # Quantize to NF4  distances = (normalized.unsqueeze(-1) - NF4_QUANTILES.to(tensor.device)).abs() quantized = distances.argmin(dim=-1).to(torch.uint8) # Double quantization: quantize absmax  absmax_quantized = quantize_absmax(absmax) return quantized, absmax_quantized, original_shape def dequantize_nf4(quantized, absmax_quantized, shape): """Dequantize NF4 back to float.""" # Implementation for inference  pass 

# peft/training.py from transformers import TrainingArguments, Trainer from peft import PeftModel from datasets import load_dataset import torch def create_training_args( output_dir: str, num_epochs: int = 3, batch_size: int = 4, gradient_accumulation: int = 4, learning_rate: float = 2e-4, warmup_ratio: float = 0.03, max_grad_norm: float = 0.3, logging_steps: int = 10, save_steps: int = 100, eval_steps: int = 100, fp16: bool = False, bf16: bool = True, gradient_checkpointing: bool = True, optim: str = "paged_adamw_32bit", lr_scheduler_type: str = "cosine", ) -> TrainingArguments: """Production-ready training arguments.""" return TrainingArguments( output_dir=output_dir, num_train_epochs=num_epochs, per_device_train_batch_size=batch_size, per_device_eval_batch_size=batch_size, gradient_accumulation_steps=gradient_accumulation, learning_rate=learning_rate, warmup_ratio=warmup_ratio, max_grad_norm=max_grad_norm, logging_steps=logging_steps, save_steps=save_steps, eval_steps=eval_steps, evaluation_strategy="steps", save_strategy="steps", load_best_model_at_end=True, metric_for_best_model="eval_loss", greater_is_better=False, fp16=fp16, bf16=bf16, gradient_checkpointing=gradient_checkpointing, optim=optim, lr_scheduler_type=lr_scheduler_type, report_to="wandb", run_name=f"lora-{model_id.split('/')[-1]}", ddp_find_unused_parameters=False, remove_unused_columns=False, ) def format_dataset(example, tokenizer, max_length=2048): """Format for instruction tuning.""" # Alpaca format  prompt = f"""### Instruction: {example['instruction']} ### Input: {example.get('input', '')} ### Response: {example['output']}""" tokenized = tokenizer( prompt, truncation=True, max_length=max_length, padding=False, return_tensors=None, ) # Labels = input_ids for causal LM  tokenized["labels"] = tokenized["input_ids"].copy() return tokenized def train_lora( model, tokenizer, train_dataset, eval_dataset, training_args: TrainingArguments, ): """Train LoRA adapters.""" trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=lambda x: { "input_ids": torch.nn.utils.rnn.pad_sequence( [torch.tensor(i["input_ids"]) for i in x], batch_first=True, padding_value=tokenizer.pad_token_id ), "attention_mask": torch.nn.utils.rnn.pad_sequence( [torch.tensor(i["attention_mask"]) for i in x], batch_first=True, padding_value=0 ), "labels": torch.nn.utils.rnn.pad_sequence( [torch.tensor(i["labels"]) for i in x], batch_first=True, padding_value=-100 ), }, ) trainer.train() return trainer # Usage if __name__ == "__main__": model_id = "meta-llama/Meta-Llama-3-8B" model = load_qlora_model(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) tokenizer.pad_token = tokenizer.eos_token dataset = load_dataset("tatsu-lab/alpaca", split="train") dataset = dataset.map(lambda x: format_dataset(x, tokenizer)) train_size = int(0.9 * len(dataset)) train_dataset = dataset.select(range(train_size)) eval_dataset = dataset.select(range(train_size, len(dataset))) args = create_training_args("./lora-llama3-8b-alpaca") trainer = train_lora(model, tokenizer, train_dataset, eval_dataset, args) # Save LoRA adapters only  trainer.model.save_pretrained("./lora-adapters") tokenizer.save_pretrained("./lora-adapters") 

# peft/adalora.py from peft import AdaLoraConfig, get_peft_model def create_adalora_model(model, target_modules=None): """AdaLoRA: dynamically allocates rank per layer.""" config = AdaLoraConfig( init_r=12, target_r=8, beta1=0.85, beta2=0.85, tinit=200, tfinal=1000, deltaT=10, lora_alpha=32Source：https://dev.to/ipsphil_digital/fine-tuning-llms-with-lora-qlora-complete-production-guide-46l6
My favorites