gpu: context-free VRAM query so idle/GGUF engine pins no CUDA context

Reporting VRAM via torch.cuda.mem_get_info lazily creates the CUDA primary
context (~256 MiB on an RTX 3090). An engine that never loads a torch model
(the GGUF/llama.cpp engine) therefore pinned ~256 MiB just to answer health
polls, the capability probe and the load-path eviction check — and that stray
context was enough to tip a borderline 4-bit Wan2.2 A14B video load into OOM.

New codai/models/gpu_query.py queries VRAM without a context: pynvml first,
nvidia-smi fallback, torch only if a context already exists. visible_gpu_memory()
scopes to the engine's cards via CUDA_VISIBLE_DEVICES (matched by index OR UUID;
empty value -> no CUDA cards, e.g. the Vulkan/Radeon engine).

Wired into the idle health poll (api/app.py), the capability probe
(broker/capabilities.py) and the load-path free-VRAM checks
(models/manager.py: _get_free_vram_gb / _free_vram_snapshot). Adds nvidia-ml-py
to requirements-nvidia.txt and the update overlay. Bumps to 0.1.28.

Net: the GGUF engine sits at 0 MiB while idle (its real context comes from
llama.cpp on model load and is freed on unload), returning the headroom that
makes the A14B load fit instead of OOM-ing.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RdMufYvtTbtGDWsiZVoXce

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.27"
+__version__ = "0.1.28"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/api/app.py

@@ -250,19 +250,23 @@ def _cached_engine_vram():
         return _ENGINE_VRAM_CACHE["vram"]
     vram = None
     try:
-        import torch
-        if torch.cuda.is_available():
-            n = torch.cuda.device_count()
+        # Query VRAM context-free (pynvml → nvidia-smi → torch-if-inited) so an
+        # engine that never loads a torch model (the GGUF/llama.cpp engine) doesn't
+        # pin a ~256 MiB CUDA context just to answer this health poll.
+        from codai.models.gpu_query import gpu_memory
+        gpus = gpu_memory()
+        if gpus:
             used = free = total = 0
             devs = []
             _names = _ENGINE_VRAM_CACHE["names"]
-            for i in range(n):
-                f, t = torch.cuda.mem_get_info(i)
+            for d in gpus:
+                i, f, t = d["index"], d["free"], d["total"]
                 used += (t - f); free += f; total += t
                 if i not in _names:
-                    _names[i] = torch.cuda.get_device_name(i)
+                    _names[i] = d["name"]
                 devs.append({"index": i, "name": _names[i],
                              "free": round(f / 1e9, 2), "total": round(t / 1e9, 2)})
+            n = len(devs)
             label = (_names.get(0, "CUDA") if n == 1 else f"{n}× CUDA")
             vram = {"used": round(used / 1e9, 2), "free": round(free / 1e9, 2),
                     "total": round(total / 1e9, 2), "gpu": label,

codai/broker/capabilities.py

@@ -49,36 +49,24 @@ def build_hardware_summary() -> Dict[str, Any]:
     total_vram_mb = 0
     available_vram_mb = 0
 
-    # Only use torch if it's ALREADY loaded (i.e. we're in an engine). Never import
-    # it here — the front is torch-free and must stay that way (importing torch in
-    # the front is heavy and would initialise CUDA in the wrong process).
-    import sys as _sys
+    # Prefer a context-free query (pynvml → nvidia-smi → torch-if-inited). This
+    # never creates a CUDA primary context, so probing capabilities in an engine
+    # that hasn't loaded a torch model (the GGUF/llama.cpp engine) doesn't pin a
+    # ~256 MiB context. It also works in the torch-free front (NVML/nvidia-smi
+    # only — it imports torch solely if a context already exists in-process).
     try:
-        if "torch" not in _sys.modules:
-            raise ImportError("torch not loaded (front) — using torch-free path")
-        import torch
-
-        if torch.cuda.is_available():
-            gpu_count = torch.cuda.device_count()
-            for index in range(gpu_count):
-                props = torch.cuda.get_device_properties(index)
-                device_total_mb = int(props.total_memory / (1024 * 1024))
-                if index == torch.cuda.current_device():
-                    free_bytes, total_bytes = torch.cuda.mem_get_info()
-                    total_vram_mb = int(total_bytes / (1024 * 1024))
-                    available_vram_mb = int(free_bytes / (1024 * 1024))
-                gpus.append(
-                    {
-                        "index": index,
-                        "name": torch.cuda.get_device_name(index),
-                        "total_vram_mb": device_total_mb,
-                    }
-                )
-            if gpus:
-                if total_vram_mb == 0:
-                    total_vram_mb = sum(gpu["total_vram_mb"] for gpu in gpus)
-                if available_vram_mb == 0 and total_vram_mb:
-                    available_vram_mb = total_vram_mb
+        from codai.models.gpu_query import gpu_memory
+        _gpus = gpu_memory()
+        if _gpus:
+            for d in _gpus:
+                device_total_mb = int(d["total"] / (1024 * 1024))
+                gpus.append({
+                    "index": d["index"],
+                    "name": d["name"],
+                    "total_vram_mb": device_total_mb,
+                })
+                total_vram_mb += device_total_mb
+                available_vram_mb += int(d["free"] / (1024 * 1024))
     except Exception:
         pass
 

codai/models/gpu_query.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Context-free GPU memory query.
+
+Reporting VRAM (for /admin/api/status, the capability document, eviction
+decisions, …) must NOT create a CUDA primary context. ``torch.cuda.mem_get_info``
+lazily initialises the device's primary context (~256 MiB on an RTX 3090) the
+first time it's called — so an engine that never loads a torch model (the
+GGUF/llama.cpp engine) would still pin ~256 MiB just to answer a health poll.
+That stray context can be the few hundred MB that tips a borderline torch-engine
+video load (e.g. Wan2.2 A14B at 4-bit) into OOM.
+
+NVML (pynvml / nvidia-ml-py) and ``nvidia-smi`` both read memory straight from
+the driver WITHOUT a context. Query order:
+
+  1. pynvml      — fast in-process NVML binding (no context, no fork)
+  2. nvidia-smi  — subprocess fallback when the binding isn't installed
+  3. torch       — ONLY if a CUDA context already exists in this process
+                   (``torch.cuda.is_initialized()``), so we never create one here
+
+The GGUF engine therefore sits at 0 MiB while idle; its real context is created
+by llama.cpp when a model loads and released on unload — independent of this.
+"""
+
+import os
+import shutil
+import subprocess
+from typing import List, Optional, Dict, Any, Set
+
+_pynvml = None            # the imported+initialised module, or None
+_pynvml_tried = False     # have we attempted import/init yet?
+
+
+def _pynvml_module():
+    global _pynvml, _pynvml_tried
+    if _pynvml_tried:
+        return _pynvml
+    _pynvml_tried = True
+    try:
+        import pynvml  # provided by the `nvidia-ml-py` package
+        pynvml.nvmlInit()
+        _pynvml = pynvml
+    except Exception:
+        _pynvml = None
+    return _pynvml
+
+
+def _via_pynvml() -> Optional[List[Dict[str, Any]]]:
+    p = _pynvml_module()
+    if p is None:
+        return None
+    try:
+        out: List[Dict[str, Any]] = []
+        for i in range(p.nvmlDeviceGetCount()):
+            h = p.nvmlDeviceGetHandleByIndex(i)
+            m = p.nvmlDeviceGetMemoryInfo(h)
+            name = p.nvmlDeviceGetName(h)
+            if isinstance(name, bytes):
+                name = name.decode("utf-8", "replace")
+            try:
+                uuid = p.nvmlDeviceGetUUID(h)
+                if isinstance(uuid, bytes):
+                    uuid = uuid.decode("utf-8", "replace")
+            except Exception:
+                uuid = None
+            out.append({"index": i, "name": str(name), "uuid": uuid,
+                        "free": int(m.free), "total": int(m.total)})
+        return out or None
+    except Exception:
+        return None
+
+
+def _via_nvidia_smi() -> Optional[List[Dict[str, Any]]]:
+    exe = shutil.which("nvidia-smi")
+    if not exe:
+        return None
+    try:
+        r = subprocess.run(
+            [exe, "--query-gpu=index,name,uuid,memory.free,memory.total",
+             "--format=csv,noheader,nounits"],
+            capture_output=True, text=True, timeout=5)
+        if r.returncode != 0:
+            return None
+        out: List[Dict[str, Any]] = []
+        for line in r.stdout.strip().splitlines():
+            parts = [x.strip() for x in line.split(",")]
+            if len(parts) < 5:
+                continue
+            try:
+                out.append({"index": int(parts[0]), "name": parts[1],
+                            "uuid": parts[2] or None,
+                            "free": int(float(parts[3])) * 1024 * 1024,
+                            "total": int(float(parts[4])) * 1024 * 1024})
+            except (TypeError, ValueError):
+                continue
+        return out or None
+    except Exception:
+        return None
+
+
+def _via_torch_if_inited() -> Optional[List[Dict[str, Any]]]:
+    """Last resort — only when a CUDA context ALREADY exists in this process, so
+    we never create one here."""
+    import sys
+    if "torch" not in sys.modules:
+        return None
+    try:
+        import torch
+        if not (torch.cuda.is_available() and torch.cuda.is_initialized()):
+            return None
+        out: List[Dict[str, Any]] = []
+        for i in range(torch.cuda.device_count()):
+            f, t = torch.cuda.mem_get_info(i)
+            try:
+                uuid = "GPU-" + str(torch.cuda.get_device_properties(i).uuid)
+            except Exception:
+                uuid = None
+            out.append({"index": i, "name": torch.cuda.get_device_name(i),
+                        "uuid": uuid, "free": int(f), "total": int(t)})
+        return out or None
+    except Exception:
+        return None
+
+
+def gpu_memory() -> Optional[List[Dict[str, Any]]]:
+    """Per-device VRAM for ALL physical NVIDIA cards, without creating a CUDA
+    context.
+
+    Returns a list of ``{"index", "name", "uuid", "free", "total"}`` (bytes)
+    ordered by device index, or ``None`` if no NVIDIA GPU could be queried (e.g.
+    non-NVIDIA host — callers fall back to their own vendor-agnostic detection).
+
+    NOTE: NVML/nvidia-smi ignore ``CUDA_VISIBLE_DEVICES`` and report every card on
+    the node. For an engine-scoped view use ``visible_gpu_memory()``."""
+    return _via_pynvml() or _via_nvidia_smi() or _via_torch_if_inited()
+
+
+def _visible_tokens() -> Optional[Set[str]]:
+    """Parse ``CUDA_VISIBLE_DEVICES`` into a set of selector tokens (physical
+    index strings and/or ``GPU-<uuid>`` strings).
+
+      - unset            → None  (no scoping; all cards visible)
+      - "" (empty)       → set() (NO CUDA cards visible — e.g. a Vulkan/Radeon
+                                   engine; the front sets this explicitly)
+      - "0,1" / UUIDs    → {"0", "1"} / {"GPU-…", …}
+    """
+    raw = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if raw is None:
+        return None
+    return {tok.strip() for tok in raw.split(",") if tok.strip() != ""}
+
+
+def visible_gpu_memory() -> Optional[List[Dict[str, Any]]]:
+    """Like :func:`gpu_memory` but restricted to the cards THIS process may use,
+    per ``CUDA_VISIBLE_DEVICES`` (which NVML/nvidia-smi otherwise ignore).
+
+    Matches each card by physical index OR UUID, so it works whether the front
+    pinned the engine by index ("0,1") or by UUID ("GPU-…"). An explicit empty
+    ``CUDA_VISIBLE_DEVICES`` yields ``[]`` (a Radeon/Vulkan engine sees no CUDA
+    card); an unset variable yields all cards."""
+    gpus = gpu_memory()
+    if not gpus:
+        return gpus
+    want = _visible_tokens()
+    if want is None:
+        return gpus
+    if not want:
+        return []
+    return [g for g in gpus
+            if str(g.get("index")) in want or (g.get("uuid") and g["uuid"] in want)]

codai/models/manager.py

@@ -2306,15 +2306,14 @@ class MultiModelManager:
         cuda_free = 0.0
         cuda_count = 0
         try:
-            import torch
-            if torch.cuda.is_available():
-                for i in range(torch.cuda.device_count()):
-                    try:
-                        free, _ = torch.cuda.mem_get_info(i)
-                        cuda_free += free / 1e9
-                        cuda_count += 1
-                    except Exception:
-                        pass
+            # Context-free (pynvml → nvidia-smi → torch-if-inited), scoped to this
+            # engine's cards via CUDA_VISIBLE_DEVICES — so the eviction/offload check
+            # doesn't pin a ~256 MiB CUDA context in an engine (e.g. GGUF/llama.cpp)
+            # that wouldn't otherwise create one.
+            from codai.models.gpu_query import visible_gpu_memory
+            for d in (visible_gpu_memory() or []):
+                cuda_free += d["free"] / 1e9
+                cuda_count += 1
         except Exception:
             pass
         total_free = cuda_free
@@ -2353,15 +2352,11 @@ class MultiModelManager:
         total_free = 0.0
         cuda_count = 0
         try:
-            import torch
-            if torch.cuda.is_available():
-                for i in range(torch.cuda.device_count()):
-                    try:
-                        free, _ = torch.cuda.mem_get_info(i)
-                        total_free += float(free)
-                        cuda_count += 1
-                    except Exception:
-                        pass
+            # Context-free + CUDA_VISIBLE_DEVICES-scoped (see _get_free_vram_gb).
+            from codai.models.gpu_query import visible_gpu_memory
+            for d in (visible_gpu_memory() or []):
+                total_free += float(d["free"])
+                cuda_count += 1
         except Exception:
             pass
         got = cuda_count > 0

packaging/linux/Dockerfile.update

@@ -40,4 +40,9 @@ RUN set -eux; \
     find /opt/coderai/app -type d -name __pycache__ -prune -exec rm -rf '{}' +; \
     /opt/coderai/python/bin/python3 -c "import importlib.util, sys; m=[n for n in ('fastapi','uvicorn','torch') if importlib.util.find_spec(n) is None]; sys.exit('base image missing: '+', '.join(m) if m else 0)"
 
+# Context-free GPU memory query dependency (provides `pynvml`). Pure-python and
+# tiny, so install it in the overlay too — the base may predate it. The code
+# falls back to nvidia-smi when absent, hence `|| true` keeps offline builds OK.
+RUN /opt/coderai/python/bin/python3 -m pip install --no-input --no-cache-dir "nvidia-ml-py>=12.0.0" || true
+
 # ENTRYPOINT / EXPOSE / VOLUME / ENV / WORKDIR are inherited from the base image.

requirements-nvidia.txt

@@ -12,6 +12,11 @@ accelerate>=0.24.0
 # System resource detection
 psutil>=5.9.0
 # procname>=0.3.0  # optional - uncomment to set process name (requires libproc2-dev)
+# Context-free GPU memory query (provides `pynvml`). Lets engines report VRAM via
+# NVML without creating a CUDA primary context — so an engine that never loads a
+# torch model (the GGUF/llama.cpp engine) stays at 0 MiB while idle instead of
+# pinning ~256 MiB. Falls back to `nvidia-smi` when absent.
+nvidia-ml-py>=12.0.0
 
 # Optional: Audio transcription dependencies
 faster-whisper>=0.10.0  # For NVIDIA/CUDA whisper transcription