vllm: cooperative VRAM co-tenancy + GPU selection (full engine parity)

- _get_model_used_vram_gb reports a vLLM model's real footprint (gpu_memory_utilization
  x total VRAM via new _total_vram_gb), so the standard evict-before-load path frees the
  right room before starting vLLM (gmu~0.9 => effectively exclusive; low gmu co-tenants).
  Eviction OF vLLM to free room for others is already handled by normal LRU (backend
  .cleanup -> stop_service); thermal throttling is inherited via request_model/wait_until_safe.
- vllm.gpu (CUDA_VISIBLE_DEVICES) to pin the instance to specific NVIDIA GPU(s); wired in
  vllm_worker env + config + admin card + routes. vLLM is CUDA-only (capability on nvidia/
  cuda nodes); AMD would need a ROCm vLLM build.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Mw2KQiswmD69T45fTfjKwW

codai/admin/routes.py

@@ -3510,6 +3510,7 @@ def build_settings_dict(c, gpu_cards):
             "venv": c.vllm.venv,
             "model_path": c.vllm.model_path,
             "model_id": c.vllm.model_id,
+            "gpu": c.vllm.gpu,
             "host": c.vllm.host,
             "port": c.vllm.port,
             "ctx": c.vllm.ctx,
@@ -4014,6 +4015,7 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
         if "venv" in d: v.venv = (d.get("venv") or "").strip()
         if "model_path" in d: v.model_path = (d.get("model_path") or "").strip()
         if "model_id" in d: v.model_id = (d.get("model_id") or v.model_id or "vllm").strip()
+        if "gpu" in d: v.gpu = (d.get("gpu") or "").strip()
         if "host" in d: v.host = (d.get("host") or "127.0.0.1").strip()
         if "port" in d:
             try: v.port = int(d.get("port") or 0)

codai/admin/templates/settings.html

@@ -776,6 +776,11 @@
       <label class="form-label">Model path (HF dir or repo id)</label>
       <input type="text" id="s-vllm-model-path" class="form-input" placeholder="/nvme/Qwen3-8B or Qwen/Qwen3-8B">
     </div>
+    <div class="form-group">
+      <label class="form-label">GPU(s) <span class="muted">(CUDA_VISIBLE_DEVICES; blank = all NVIDIA)</span></label>
+      <input type="text" id="s-vllm-gpu" class="form-input" placeholder="e.g. 0  or  0,1">
+      <span class="form-hint">vLLM is CUDA/NVIDIA-only. Pin it to a specific card (e.g. <code>0</code>) to dedicate a GPU, or list several (<code>0,1</code>) with a matching TP size.</span>
+    </div>
     <div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
       <div class="form-group" style="flex:1;min-width:130px"><label class="form-label">Max model len</label><input type="number" id="s-vllm-ctx" class="form-input" min="512" placeholder="32768"></div>
       <div class="form-group" style="flex:1;min-width:130px"><label class="form-label">GPU mem util</label><input type="number" id="s-vllm-gmu" class="form-input" min="0.1" max="1" step="0.05" placeholder="0.90"></div>
@@ -1643,6 +1648,7 @@ async function loadSettings(){
     document.getElementById('s-vllm-enabled').checked = !!vll.enabled;
     document.getElementById('s-vllm-model-id').value = vll.model_id ?? 'vllm';
     document.getElementById('s-vllm-model-path').value = vll.model_path ?? '';
+    document.getElementById('s-vllm-gpu').value = vll.gpu ?? '';
     document.getElementById('s-vllm-ctx').value = vll.ctx ?? 32768;
     document.getElementById('s-vllm-gmu').value = vll.gpu_memory_utilization ?? 0.90;
     document.getElementById('s-vllm-tp').value = vll.tensor_parallel_size ?? 1;
@@ -1821,6 +1827,7 @@ async function saveSettings(){
       enabled: document.getElementById('s-vllm-enabled').checked,
       model_id: document.getElementById('s-vllm-model-id').value.trim() || 'vllm',
       model_path: document.getElementById('s-vllm-model-path').value.trim(),
+      gpu: document.getElementById('s-vllm-gpu').value.trim(),
       ctx: parseInt(document.getElementById('s-vllm-ctx').value) || 32768,
       gpu_memory_utilization: parseFloat(document.getElementById('s-vllm-gmu').value) || 0.90,
       tensor_parallel_size: parseInt(document.getElementById('s-vllm-tp').value) || 1,

codai/api/vllm_worker.py

@@ -180,6 +180,10 @@ def ensure_service(cfg, model_path: Optional[str] = None,
         # via extra_env. (Set before extra_env so an explicit override wins.)
         env.setdefault("VLLM_USE_FLASHINFER", "0")
         env.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
+        # Pin this vLLM instance to specific NVIDIA GPU(s) if configured (CUDA-only).
+        gpu = (getattr(cfg, "gpu", "") or "").strip()
+        if gpu:
+            env["CUDA_VISIBLE_DEVICES"] = gpu
         extra_env = (getattr(cfg, "extra_env", "") or "").strip()
         applied = {}
         if extra_env:

codai/config.py

@@ -534,6 +534,8 @@ class VllmConfig:
     venv: str = ""                       # isolated venv dir; blank = auto (baked/cache/home)
     model_path: str = ""                 # HF model dir or repo id (--model)
     model_id: str = "vllm"               # id/alias that routes to vllm (and --served-model-name)
+    gpu: str = ""                        # CUDA device(s) for this vLLM instance (CUDA_VISIBLE_DEVICES,
+                                         # e.g. "0" or "0,1"); blank = all visible NVIDIA GPUs. CUDA-only.
     host: str = "127.0.0.1"
     port: int = 0                        # 0 = auto-pick a free port
     ctx: int = 32768                     # --max-model-len
@@ -1074,6 +1076,7 @@ class ConfigManager:
                 "venv": self.config.vllm.venv,
                 "model_path": self.config.vllm.model_path,
                 "model_id": self.config.vllm.model_id,
+                "gpu": self.config.vllm.gpu,
                 "host": self.config.vllm.host,
                 "port": self.config.vllm.port,
                 "ctx": self.config.vllm.ctx,

codai/models/manager.py

@@ -2759,6 +2759,30 @@ class MultiModelManager:
         except Exception:
             pass
 
+    def _total_vram_gb(self) -> float:
+        """Total VRAM (GB) of the largest visible NVIDIA GPU — used to size a vLLM
+        instance's gpu_memory_utilization footprint. Falls back to 24 GB."""
+        try:
+            import pynvml
+            pynvml.nvmlInit()
+            best = 0.0
+            for i in range(pynvml.nvmlDeviceGetCount()):
+                h = pynvml.nvmlDeviceGetHandleByIndex(i)
+                tot = pynvml.nvmlDeviceGetMemoryInfo(h).total / (1024 ** 3)
+                best = max(best, tot)
+            pynvml.nvmlShutdown()
+            if best > 0:
+                return best
+        except Exception:
+            pass
+        try:
+            import torch
+            if torch.cuda.is_available():
+                return torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)
+        except Exception:
+            pass
+        return 24.0
+
     def _get_free_vram_gb(self) -> float:
         """Return estimated free VRAM in GB across the cards this engine can use.
 
@@ -3665,6 +3689,20 @@ class MultiModelManager:
                 except (TypeError, ValueError):
                     pass
                 return 14.0                          # modest default until measured
+            # vLLM (external, isolated venv) pre-allocates gpu_memory_utilization × total
+            # VRAM up front and manages it in its own process. Report THAT as its footprint
+            # so the standard evict-before-load path frees enough room (with gmu=0.9 this
+            # evicts ~everything, i.e. effectively exclusive; a low gmu co-tenants).
+            if vllm_should_handle(model_key) or (resolved_name and vllm_should_handle(resolved_name)):
+                measured = self._measured_vram_gb.get(model_key)
+                if not measured and resolved_name:
+                    measured = self._measured_vram_gb.get(resolved_name)
+                if measured:
+                    return float(measured)
+                vcfg = get_active_vllm_config()
+                gmu = float(getattr(vcfg, "gpu_memory_utilization", 0.9) or 0.9) if vcfg else 0.9
+                total = self._total_vram_gb()
+                return round(gmu * total, 1) if total > 0 else 20.0
         except Exception:
             pass
         # Resolve by basename/alias too — a model requested by basename would