# macOS brew install ollama # Linux curl -fsSL https://ollama.com/install.sh | sh # Windows # Download from https://ollama.com/download # Start service ollama serve # Pull model ollama pull llama3.1:8b ollama pull codellama:13b ollama pull nemotron-mini:4b 

# ollama/client.py import ollama from typing import List, Dict, Optional class OllamaClient: def __init__(self, host: str = "http://localhost:11434"): self.client = ollama.Client(host=host) def list_models(self) -> List[Dict]: return self.client.list()["models"] def pull(self, model: str) -> None: self.client.pull(model) def generate( self, model: str, prompt: str, system: str = None, temperature: float = 0.7, top_p: float = 0.9, num_predict: int = -1, stream: bool = False ) -> str: response = self.client.generate( model=model, prompt=prompt, system=system, options={ "temperature": temperature, "top_p": top_p, "num_predict": num_predict, }, stream=stream ) return response["response"] def chat( self, model: str, messages: List[Dict], stream: bool = False ): return self.client.chat(model=model, messages=messages, stream=stream) def embeddings(self, model: str, prompt: str) -> List[float]: return self.client.embeddings(model=model, prompt=prompt)["embedding"] # Usage client = OllamaClient() response = client.generate("llama3.1:8b", "Explain LoRA in 3 sentences") print(response) # Streaming chat for chunk in client.chat("llama3.1:8b", [ {"role": "user", "content": "Write a haiku about GPUs"} ], stream=True): print(chunk["message"]["content"], end="", flush=True) 

# Modelfile for custom model FROM llama3.1:8b # System prompt SYSTEM """You are a senior Python developer. Write clean, typed, documented code.""" # Parameters PARAMETER temperature 0.3 PARAMETER top_p 0.9 PARAMETER num_ctx 8192 PARAMETER num_predict 2048 # Template (optional) TEMPLATE """{{ .System }}User: {{ .Prompt }}Assistant: """ # Save and create # ollama create python-expert -f Modelfile 

# ollama/api_server.py from fastapi import FastAPI from pydantic import BaseModel import ollama app = FastAPI() class GenerateRequest(BaseModel): model: str prompt: str system: str = None temperature: float = 0.7 stream: bool = False @app.post("/generate") async def generate(req: GenerateRequest): if req.stream: return StreamingResponse( stream_generate(req), media_type="text/event-stream" ) response = ollama.generate( model=req.model, prompt=req.prompt, system=req.system, options={"temperature": req.temperature} ) return {"response": response["response"]} async def stream_generate(req: GenerateRequest): stream = ollama.generate( model=req.model, prompt=req.prompt, system=req.system, options={"temperature": req.temperature}, stream=True ) for chunk in stream: yield f"data: {chunk['response']}\n\n" # Run: uvicorn api_server:app --host 0.0.0.0 --port 8000 

# Ubuntu/Debian sudo apt update && sudo apt install -y build-essential cmake git # Clone git clone https://github.com/ggerganov/llama.cpp cd llama.cpp # CPU-only build (AVX2) make LLAMA_CUBLAS=0 # With CUDA (NVIDIA GPU) make LLAMA_CUBLAS=1 # With Metal (Apple Silicon) make LLAMA_METAL=1 # With Vulkan (AMD/Intel GPU) make LLAMA_VULKAN=1 

# Download HF model huggingface-cli download meta-llama/Meta-Llama-3.1-8B --local-dir llama3.1-8b # Convert to GGUF python convert_hf_to_gguf.py llama3.1-8b --outfile llama3.1-8b.f16.gguf --outtype f16 # Quantize (smaller, faster, slight quality loss) # Q4_K_M = best balance (4-bit) ./llama-quantize llama3.1-8b.f16.gguf llama3.1-8b.Q4_K_M.gguf Q4_K_M # Other quantization levels # Q2_K - smallest, lowest quality # Q3_K_M - small, decent # Q4_K_M - recommended default # Q5_K_M - higher quality # Q6_K - near original # Q8_0 - almost lossless 

# Basic server ./llama-server -m llama3.1-8b.Q4_K_M.gguf -c 8192 --host 0.0.0.0 --port 8080 # With GPU offloading (Metal) ./llama-server -m llama3.1-8b.Q4_K_M.gguf -c 8192 --host 0.0.0.0 --port 8080 -ngl 99 # With CUDA ./llama-server -m llama3.1-8b.Q4_K_M.gguf -c 8192 --host 0.0.0.0 --port 8080 -ngl 99 -mgpu 0 # Performance options ./llama-server \ -m model.gguf \ -c 8192 \  # Context size -b 512 \  # Batch size -ub 512 \  # UBatch size -ngl 99 \  # GPU layers (-1 = all) -t 8 \  # Threads --flash-attn \  # Flash attention --mlock \  # Lock in RAM --no-mmap \  # Don't mmap (for containers) 

# llama_cpp/client.py from llama_cpp import Llama from typing: List, Dict import json class LlamaCppClient: def __init__( self, model_path: str, n_ctx: int = 8192, n_gpu_layers: int = -1, n_threads: int = 8, verbose: bool = False ): self.llm = Llama( model_path=model_path, n_ctx=n_ctx, n_gpu_layers=n_gpu_layers, n_threads=n_threads, verbose=verbose, chat_format="llama-3", # or "chatml", "mistral-instruct"  ) def generate( self, prompt: str, max_tokens: int = 512, temperature: float = 0.7, top_p: float = 0.9, stop: List[str] = None, stream: bool = False ): return self.llm( prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, stop=stop, stream=stream ) def chat( self, messages: List[Dict], max_tokens: int = 512, temperature: float = 0.7, stream: bool = False ): return self.llm.create_chat_completion( messages=messages, max_tokens=max_tokens, temperature=temperature, stream=stream ) def embeddings(self, texts: List[str]) -> List[List[float]]: return [self.llm.embed(text) for text in texts] # Usage client = LlamaCppClient("./llama3.1-8b.Q4_K_M.gguf", n_gpu_layers=99) response = client.generate("Explain quantization:") print(response["choices"][0]["text"]) 

# Requires NVIDIA GPU with CUDA 11.8+ pip install vllm # Or with specific CUDA pip install vllm --extra-index-url https://download.pytorch.org/whl/cu118 # For AMD ROCm pip install vllm --extra-index-url https://download.pytorch.org/whl/rocm5.6 

# OpenAI-compatible server vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct \ --host 0.0.0.0 \ --port 8000 \ --dtype auto \ --max-model-len 8192 \ --gpu-memory-utilization 0.9 \ --tensor-parallel-size 1 \ --enforce-eager # With quantization (AWQ/GPTQ) vllm serve TheBloke/Llama-3.1-8B-Instruct-AWQ \ --quantization awq \ --max-model-len 8192 # Multi-GPU tensor parallel vllm serve meta-llama/Meta-Llama-3.1-70B-Instruct \ --tensor-parallel-size 4 \ --gpu-memory-utilization 0.9 

# vllm/client.py from openai import OpenAI from typing: List, Dict import asyncio class VLLMClient: def __init__(self, base_url: str = "http://localhost:8000/v1"): self.client = OpenAI( base_url=base_url, api_key="dummy" # Not used by vLLM  ) def list_models(self): return self.client.models.list() def generate( self, model: str, prompt: str, max_tokens: int = 512, temperature: float = 0.7, top_p: float = 0.9, stop: List[str] = None, stream: bool = False ): return self.client.completions.create( model=model, prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, stop=stop, stream=stream ) def chat( self, model: str, messages: List[Dict], max_tokens: int = 512, temperature: float = 0.7, stream: bool = False ): return self.client.chat.completions.create( model=model, messages=messages, max_tokens=max_tokens, temperature=temperature, stream=stream ) async def achat(self, *args, **kwargs): return await self.client.chat.completions.create(*args, **kwargs) # Usage client = VLLMClient() # Chat completion response = client.chat("meta-llama/Meta-Llama-3.1-8B-Instruct", [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Write a Python decorator for retry logic"} ]) print(response.choices[0].message.content) # Streaming for chunk in client.chat(model, messages, stream=True): if chunk.choices[0].delta.content: print(chunk.choices[0].delta.content, end="", flush=True) 

# vllm/advanced.py from vllm import LLM, SamplingParams from vllm.lora.request import LoRARequest # Offline inference (no server) llm = LLM( model="meta-llama/Meta-Llama-3.1-8B-Instruct", tensor_parallel_size=1, gpu_memory_utilization=0.9, max_model_len=8192, enable_lora=True, max_lora_rank=64, ) sampling_params = SamplingParams( temperature=0.7, top_p=0.9, max_tokens=512, stop=["</s>", "<|eot_id|>"], ) # Batch generation prompts = [ "Write a function for: ", "Explain the concept of: ", "Debug this code: ", ] outputs = llm.generate(prompts, sampling_params) # With LoRA adapter lora_outputs = llm.generate( prompts, sampling_params, lora_request=LoRARequest("sql_adapter", 1, "./sql-lora-adapter") ) # Structured output (guided decoding) from vllm import SamplingParams from pydantic import BaseModel class CodeResponse(BaseModel): explanation: str code: str language: str guided_params = SamplingParams( temperature=0.3, max_tokens=1024, guided_json=CodeResponse.model_json_schema() ) output = llm.generate(["Write a binary search in Python"], guided_params) print(output[0].outputs[0].text) # Valid JSON matching schema 

# vllm/tuning.sh # Key parameters: # --max-model-len: Context window (affects KV cache memory) # --gpu-memory-utilization: Fraction of GPU memory for KV cache (0.8-0.95) # --block-size: KV cache block size (16 or 32) # --max-num-seqs: Max concurrent sequences # --max-num-batched-tokens: Max tokens per batch # --enable-chunked-prefill: For long contexts # --disable-log-stats: Reduce logging overhead # High-throughput config vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct \ --max-model-len 4096 \ --gpu-memory-utilization 0.95 \ --max-num-seqs 256 \ --max-num-batched-tokens 8192 \ --enable-chunked-prefill \ --disable-log-stats \ --block-size 16 # Low-latency config vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct \ --max-model-len 8192 \ --gpu-memory-utilization 0.8 \ --max-num-seqs 16 \ --enforce-eager \ --disable-log-stats 

# Docker (recommended) docker run --gpus all --shm-size 1g \ -p 8080:80 \ -v $PWD/data:/data \ ghcr.io/huggingface/text-generation-inference:latest \ --model-id meta-llama/Meta-Llama-3.1-8B-Instruct \ --max-input-length 4096 \ --max-total-tokens 8192 \ --max-batch-prefill-tokens 4096 # With quantization docker run --gpus all --shm-size 1g \ -p 8080:80 \ ghcr.io/huggingface/text-generation-inference:latest \ --model-id TheBloke/Llama-3.1-8B-Instruct-AWQ \ --quantize awq # Multi-GPU docker run --gpus all --shm-size 1g \ -p 8080:80 \ ghcr.io/huggingface/text-generation-inference:latest \ --model-id meta-llama/Meta-Llama-3.1-70B-Instruct \ --num-shard 4 

# tgi/client.py from text_generation import Client from typing: List, Dict import asyncio class TGIClient: def __init__(self, urlSource：https://dev.to/ipsphil_digital/deploying-llms-locally-ollama-llamacpp-vllm-text-generation-inference-4c0k
My favorites