Ollama GPU Scheduling: Running Inference and ComfyUI on One RTX Without OOM

# Example: Daily schedule for balanced usage # 6:00 AM - 10:00 AM: Ollama priority (code assistant during work hours) # 10:00 AM - 12:00 PM: ComfyUI priority (creative work) # 12:00 PM - 2:00 PM: Shared/lunch break (light usage both) # 2:00 PM - 6:00 PM: Ollama priority (continued work) # 6:00 PM - 10:00 PM: ComfyUI priority (evening creative sessions) # 10:00 PM - 6:00 AM: Ollama priority (background processing, embeddings) # Implementation via systemd timers or cron # Ollama gets GPU during work hours 0 6 * * * /usr/local/bin/gpu-grant ollama 0 10 * * * /usr/local/bin/gpu-revoke ollama 0 12 * * * /usr/local/bin/gpu-grant comfyui 0 14 * * * /usr/local/bin/gpu-revoke comfyui # ... and so on 

# Example: Priority queue implementation # Priority levels (higher number = higher priority) # 10: Interactive user requests (chat, immediate image generation) # 5: Background processing (embeddings, batch jobs) # 1: Maintenance/cleanup tasks # Pseudocode for GPU arbiter class GpuArbiter { constructor() { this.queue = new PriorityQueue(); this.currentHolder = null; } async requestAccess(application, priority, estimatedDuration) { // Add request to queue await this.queue.enqueue({ application, priority, estimatedDuration, timestamp: Date.now() }); // If we're not holding GPU or new request has higher priority if (!this.currentHolder || priority > this.currentHolder.priority) { await this.preemptAndGrant(application, priority); } } async releaseAccess(application) { if (this.currentHolder?.application === application) { this.currentHolder = null; await this.grantNextInQueue(); } } } // Usage in Ollama wrapper class OllamaManager { constructor(gpuArbiter) { this.gpuArbiter = gpuArbiter; } async generate(prompt) { // Request GPU with medium priority await this.gpuArbiter.requestAccess('ollama', 5, 30000); // 30s estimate try { return await this.ollamaInstance.generate(prompt); } finally { await this.gpuArbiter.releaseAccess('ollama'); } } } // Usage in ComfyUI wrapper class ComfyUIManager { constructor(gpuArbiter) { this.gpuArbiter = gpuArbiter; } async generateImage(params) { // Request GPU with high priority for interactive use await this.gpuArbiter.requestAccess('comfyui', 10, 15000); // 15s estimate try { return await this.comfyuiInstance.generateImage(params); } finally { await this.gpuArbiter.releaseAccess('comfyui'); } } } 

# Implementation using CUDA MPS (Multi-Process Service) or similar # Actually, for most users, a simpler approach works better:  # 1. Monitor VRAM usage in real-time # 2. Set soft limits that applications should stay under # 3. Allow temporary exceeding with graceful degradation # 4. Send warnings when approaching hard limits  # Example monitoring script #!/usr/bin/env python3 import subprocess import time import requests def get_vram_usage(): """Get current VRAM usage in MB""" try: result = subprocess.run([ 'nvidia-smi', '--query-gpu=memory.used,memory.total', '--format=csv,noheader,nounits' ], capture_output=True, text=True, check=True) used, total = map(int, result.stdout.strip().split(', ')) return used, total except Exception as e: print(f"Error getting VRAM usage: {e}") return 0, 16384 # 16GB in MB  def check_and_warn(): used, total = get_vram_usage() percent = (used / total) * 100 if percent > 90: print(f"⚠️ HIGH VRAM USAGE: {used}MB / {total}MB ({percent:.1f}%)") # Could trigger notifications or automatic cleanup  elif percent > 80: print(f"⚠️ Elevated VRAM usage: {used}MB / {total}MB ({percent:.1f}%)") return percent > 95 # Return True if approaching critical  # Main monitoring loop if __name__ == "__main__": print("Starting VRAM monitor...") while True: if check_and_warn(): # In a real implementation, you might:  # 1. Send a desktop notification  # 2. Trigger graceful cleanup in applications  # 3. Log the event for analysis  pass time.sleep(5) # Check every 5 seconds 

# Example: Ollama model quantization choices # Pull different quantizations for different use cases # High-quality coding assistant (when you have the VRAM to spare) ollama pull qwen2.5-coder:14b-q5_K_M # General purpose assistant (balanced) ollama pull qwen2.5-coder:14b-q4_K_M # Lightweight assistant (when sharing GPU heavily) ollama pull qwen2.5-coder:14b-q3_K_M # Embedding model (tiny footprint) ollama pull nomic-embed-text:latest # Switch between them based on current needs # You can even run multiple quantizations of same model! 

# Example: ComfyUI launch with VRAM optimizations # Add these to your ComfyUI startup script or command line #!/bin/bash # Launch ComfyUI with memory optimizations export COMMANDLINE_ARGS="--medvram --opencv --disable-numpy-random-seed" # --medvram: Medium VRAM optimization (enables attention slicing, etc.) # --lowvram: More aggressive (slower but uses less VRAM) # --disable-numpy-random-seed: Fixes nondeterminism in some cases # Alternative: Use environment variables export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128 # Helps with memory fragmentation # Start ComfyUI python main.py --listen --port 8188 # For extreme VRAM constraints, consider: # --ckpt sd-v1-5.ckpt # Use smaller model # --no-half-vae # Disable VAE FP16 (saves VRAM, costs quality) 

# Simple request queue for Ollama import asyncio import time from collections import deque from dataclasses import dataclass from typing import Callable, Any @dataclass class OllamaRequest: prompt: str model: str future: asyncio.Future timestamp: float priority: int = 0 # Higher = more urgent  class OllamaRequestQueue: def __init__(self, max_concurrent=1): self.max_concurrent = max_concurrent self.queue = deque() self.processing = 0 self.results = {} async def add_request(self, prompt, model="qwen2.5-coder:14b", priority=0): """Add a request to the queue and return a future for the result""" future = asyncio.Future() request = OllamaRequest(prompt, model, future, time.time(), priority) # Insert in priority order  inserted = False for i, req in enumerate(self.queue): if priority > req.priority: self.queue.insert(i, request) inserted = True break if not inserted: self.queue.append(request) await self._process_queue() return future async def _process_queue(self): """Process requests from the queue""" while (self.queue and self.processing < self.max_concurrent): request = self.queue.popleft() self.processing += 1 # Process the request (this would call your Ollama client)  asyncio.create_task(self._handle_request(request)) async def _handle_request(self, request): """Actually process a single request""" try: # Simulate calling Ollama - replace with actual implementation  start = time.time() # result = await ollama_client.generate(  # model=request.model,  # prompt=request.prompt  # )  await asyncio.sleep(2) # Simulate processing time  result = f"Response to: {request.prompt[:50]}..." # Calculate and store result  request.future.set_result({ 'response': result, 'processing_time': time.time() - start }) except Exception as e: request.future.set_exception(e) finally: self.processing -= 1 await self._process_queue() # Process next in queue  # Usage queue = OllamaRequestQueue(max_concurrent=1) # Sequential processing  # Add requests - they'll be processed sequentially future1 = queue.add_request("Explain quantum computing") future2 = queue.add_request("Write a Python function to sort a list") # Wait for results result1 = await future1 result2 = await future2 

# Simple ComfyUI job queue import asyncio import uuid from dataclasses import dataclass from enum import Enum class JobStatus(Enum): QUEUED = "queued" PROCESSING = "processing" COMPLETED = "completed" FAILED = "failed" @dataclass class ComfyUIJob: id: str prompt: str params: dict status: JobStatus result: any = None error: str = None created_at: float = 0 started_at: float = 0 completed_at: float = 0 class ComfyUIJobQueue: def __init__(self, max_concurrent=1): self.max_concurrent = max_concurrent self.queue = [] # List of ComfyUIJob  self.processing = {} # id -> ComfyUIJob  self.results = {} # id -> result  async def add_job(self, prompt, params=None, priority=0): """Add a job to the queue""" if params is None: params = {} job = ComfyUIJob( id=str(uuid.uuid4()), prompt=prompt, params=params, status=JobStatus.QUEUED, created_at=time.time() ) # Add to queue (simple FIFO for now - could add priority)  self.queue.append(job) await self._process_queue() return job.id async def _process_queue(self): """Start processing queued jobs""" while (len(self.queue) > 0 and len(self.processing) < self.max_concurrent): job = self.queue.pop(0) job.status = JobStatus.PROCESSING job.started_at = time.time() self.processing[job.id] = job # Process the job  asyncio.create_task(self._handle_job(job)) async def _handle_job(self, job): """Actually process a ComfyUI job""" try: # Simulate calling ComfyUI - replace with actual API call  await asyncio.sleep(5) # Simulate image generation time  # In reality, you'd call ComfyUI's API here  result = { 'image_path': f"/tmp/comfyui_{job.id}.png", 'prompt': job.prompt, 'params': job.params, 'generation_time': time.time() - job.started_at } job.status = JobStatus.COMPLETED job.result = result job.completed_at = time.time() self.results[job.id] = result except Exception as e: job.status = JobStatus.FAILED job.error = str(e) job.completed_at = time.time() finally: # Clean up processing tracking  if job.id in self.processing: del self.processing[job.id] # Process next job  await self._process_queue() def get_job_status(self, job_id):Source：https://dev.to/ayraix/ollama-gpu-scheduling-running-inference-and-comfyui-on-one-rtx-without-oom-1j75
My favorites