llama pod on the light core: the GGUF backend asks NVML about the card, not torch

coderai-llama came out at 11.6 GB — the CUDA wheel for five GPU
generations on top of the full core — over RunPod's size line. The only
thing the CUDA GGUF backend needed torch for was three questions about
the card (is there one, total memory, free memory); gpu_probe answers
them from torch when present and NVML otherwise. The image drops to the
light core plus NVIDIA's runtime, cuBLAS and NCCL wheels.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/backends/cuda.py

@@ -174,29 +174,20 @@ class NvidiaBackend(ModelBackend):
     
     def _detect_device(self) -> str:
         """Auto-detect available GPU or fall back to CPU."""
-        import torch
-        if torch.cuda.is_available():
-            if hasattr(torch.version, 'hip') and torch.version.hip is not None:
-                print(f"ROCm/HIP detected: {torch.version.hip}")
-                return "cuda"
-            else:
-                print(f"CUDA detected: {torch.version.cuda}")
-                return "cuda"
-        else:
-            print("No GPU detected, using CPU")
-            return "cpu"
+        # torch when present, NVML otherwise: a GGUF pod image carries no torch
+        # (see codai/backends/gpu_probe.py), and llama.cpp drives the card itself.
+        from codai.backends import gpu_probe
+        if gpu_probe.cuda_available():
+            print(f"{gpu_probe.runtime_label() or 'GPU'} detected")
+            return "cuda"
+        print("No GPU detected, using CPU")
+        return "cpu"
     
     def _get_available_vram(self) -> int:
         """Get available VRAM in bytes."""
-        import torch
-        if not torch.cuda.is_available():
-            return 0
+        from codai.backends import gpu_probe
         try:
-            total_vram = 0
-            for i in range(torch.cuda.device_count()):
-                props = torch.cuda.get_device_properties(i)
-                total_vram += props.total_memory
-            return total_vram
+            return sum(gpu_probe.total_memory_bytes())
         except Exception as e:
             print(f"Warning: Could not detect VRAM: {e}")
             return 0
@@ -447,8 +438,10 @@ class NvidiaBackend(ModelBackend):
         fixed 8 GB if it can't be read.
         """
         try:
-            import torch
-            free, _ = torch.cuda.mem_get_info()
+            from codai.backends import gpu_probe
+            free = gpu_probe.free_memory_bytes()
+            if free is None:
+                raise RuntimeError("no GPU memory probe")
             # Leave ~2 GB for activations/compute; KV above that goes to CPU.
             return max(int(2 * 1024 ** 3), int(free - 2 * 1024 ** 3))
         except Exception:

codai/backends/gpu_probe.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+"""What the CUDA GGUF backend needs to know about the cards, with or without torch.
+
+llama-cpp-python drives the GPU itself; the only thing the backend asked
+torch for was "is there a CUDA device, how much memory has it, how much is
+free". That made torch — 5 GB with its CUDA libraries — a hard requirement
+of a pod image whose model never touches it, and put the llama pod image
+over RunPod's size line. NVML answers the same three questions from a
+1 MB package. torch is still preferred when present: it is what the local
+install has, and its answers are the ones every number here was tuned on.
+"""
+
+from typing import List, Optional
+
+
+def _torch():
+    try:
+        import torch
+        if torch.cuda.is_available():
+            return torch
+    except Exception:
+        pass
+    return None
+
+
+def _nvml():
+    try:
+        import pynvml
+        pynvml.nvmlInit()
+        return pynvml
+    except Exception:
+        return None
+
+
+def cuda_available() -> bool:
+    if _torch() is not None:
+        return True
+    nv = _nvml()
+    try:
+        return bool(nv and nv.nvmlDeviceGetCount() > 0)
+    except Exception:
+        return False
+
+
+def runtime_label() -> str:
+    """'CUDA 12.8' / 'ROCm 6.2' / 'CUDA (NVML, driver 570.x)' / '' — for the log line."""
+    t = _torch()
+    if t is not None:
+        hip = getattr(t.version, "hip", None)
+        return f"ROCm/HIP {hip}" if hip else f"CUDA {t.version.cuda}"
+    nv = _nvml()
+    if nv is None:
+        return ""
+    try:
+        drv = nv.nvmlSystemGetDriverVersion()
+        drv = drv.decode() if isinstance(drv, bytes) else str(drv)
+        return f"CUDA (NVML, driver {drv})"
+    except Exception:
+        return "CUDA (NVML)"
+
+
+def total_memory_bytes() -> List[int]:
+    """Total memory of each card, in order."""
+    t = _torch()
+    if t is not None:
+        return [t.cuda.get_device_properties(i).total_memory
+                for i in range(t.cuda.device_count())]
+    nv = _nvml()
+    if nv is None:
+        return []
+    out = []
+    try:
+        for i in range(nv.nvmlDeviceGetCount()):
+            h = nv.nvmlDeviceGetHandleByIndex(i)
+            out.append(int(nv.nvmlDeviceGetMemoryInfo(h).total))
+    except Exception:
+        pass
+    return out
+
+
+def free_memory_bytes(index: int = 0) -> Optional[int]:
+    """Free memory on one card, or None when nothing can say."""
+    t = _torch()
+    if t is not None:
+        try:
+            free, _total = t.cuda.mem_get_info(index)
+            return int(free)
+        except Exception:
+            return None
+    nv = _nvml()
+    if nv is None:
+        return None
+    try:
+        h = nv.nvmlDeviceGetHandleByIndex(index)
+        return int(nv.nvmlDeviceGetMemoryInfo(h).free)
+    except Exception:
+        return None

packaging/runpod/Dockerfile.capability-llama

@@ -5,9 +5,9 @@
 # no local LoRA upload, nothing on the Tasks page. This image is coderai's
 # text path with llama-cpp-python compiled for CUDA in a toolchain stage —
 # SASS for Ampere through Blackwell plus PTX for what comes next — on the
-# full core, because the CUDA GGUF backend asks torch about the card.
+# LIGHT core: the GGUF backend asks NVML about the card, not torch.
 
-ARG CORE_BASE=coderai-capability-base:latest
+ARG CORE_BASE=coderai-capability-base-light:latest
 ARG CUDA_DEVEL=nvidia/cuda:12.8.1-devel-ubuntu24.04
 ARG LLAMA_CPP_PYTHON=0.3.30
 
@@ -40,12 +40,14 @@ COPY --from=wheel /wheels/ /tmp/wheels/
 RUN --mount=type=cache,target=/root/.cache/pip set -eux; \
     python -m pip install --extra-index-url ${TORCH_INDEX} -r /tmp/profiles/${PROFILE}.txt; \
     python -m pip install --no-cache-dir /tmp/wheels/llama_cpp_python-*.whl; \
-    python -c "import torch; print('torch', torch.__version__, 'cuda', torch.version.cuda)"; \
+    if python -c "import torch" 2>/dev/null; then \
+        echo "ERROR: llama is a light-core image but torch reached its main venv" >&2; exit 1; \
+    fi; echo "main venv: no torch (light core, as intended)"; \
     rm -rf /tmp/profiles /tmp/wheels
 
-# The CUDA runtime the wheel links, from torch's own wheels; fail the BUILD if
-# libllama cannot find a library, rather than the pod at its first load.
-ENV LD_LIBRARY_PATH=/usr/local/lib/python3.12/site-packages/nvidia/cuda_runtime/lib:/usr/local/lib/python3.12/site-packages/nvidia/cublas/lib:${LD_LIBRARY_PATH}
+# The CUDA runtime, cuBLAS and NCCL the wheel links, from NVIDIA's wheels; fail
+# the BUILD if libllama cannot find a library, rather than the pod at its load.
+ENV LD_LIBRARY_PATH=/usr/local/lib/python3.12/site-packages/nvidia/cuda_runtime/lib:/usr/local/lib/python3.12/site-packages/nvidia/cublas/lib:/usr/local/lib/python3.12/site-packages/nvidia/nccl/lib:${LD_LIBRARY_PATH}
 COPY packaging/runpod/check_llama_wheel.py /tmp/check_llama_wheel.py
 RUN python /tmp/check_llama_wheel.py && rm /tmp/check_llama_wheel.py
 

packaging/runpod/check_llama_wheel.py

@@ -5,9 +5,23 @@ import subprocess
 
 libs = glob.glob("/usr/local/lib/python3.12/site-packages/llama_cpp/lib/*.so*")
 assert libs, "no llama_cpp shared libraries installed"
-bad = [l for l in libs
-       if "not found" in subprocess.run(["ldd", l], capture_output=True, text=True).stdout]
+# libcuda.so.1 (and libnvidia-ml) belong to the DRIVER: the container runtime
+# mounts them from the host at run time, and no image ever carries them.
+DRIVER = ("libcuda.so", "libnvidia-ml.so")
+
+
+def _unresolved(lib):
+    out = subprocess.run(["ldd", lib], capture_output=True, text=True).stdout
+    return [ln.split()[0] for ln in out.splitlines()
+            if "not found" in ln and not ln.strip().startswith(DRIVER)]
+
+
+bad = {l: m for l in libs if (m := _unresolved(l))}
 assert not bad, f"unresolved libraries in {bad}"
-import llama_cpp  # noqa: E402
-print("llama-cpp-python", llama_cpp.__version__, "with", len(libs), "libs, all resolved")
+# Not `import llama_cpp`: loading libggml-cuda needs the driver, which a build
+# host without a GPU does not have. Linkage is proven above; the version comes
+# from the package metadata.
+import importlib.metadata  # noqa: E402
+print("llama-cpp-python", importlib.metadata.version("llama_cpp_python"),
+      "with", len(libs), "libs, all resolved")
 assert any("ggml-cuda" in l for l in libs), "the wheel was built WITHOUT CUDA"

packaging/runpod/profiles/llama.txt

@@ -2,10 +2,17 @@
 # GPU generation, driving GGUF models through coderai's own CUDA backend —
 # the same kernels as the upstream ghcr.io/ggml-org/llama.cpp image, plus what
 # a coderai pod gives (TLS on the direct path, /boot, seeding, local LoRAs,
-# the Tasks page). FULL core: the GGUF backend asks torch about the card.
+# the Tasks page). LIGHT core: the GGUF backend used to ask torch about the
+# card; it asks NVML now (codai/backends/gpu_probe.py), so the 5 GB of torch
+# that put this image at 11.6 GB is gone. The wheel links the CUDA runtime,
+# cuBLAS and NCCL, which NVIDIA ships as wheels at the 12.8 it was built with.
 #
 # The wheel itself is compiled in Dockerfile.capability-llama's toolchain
 # stage (see the `llama.dockerfile` marker); this file lists what else the
 # text path needs.
+nvidia-cuda-runtime-cu12==12.8.90
+nvidia-cublas-cu12==12.8.4.1
+nvidia-nccl-cu12>=2.26
+nvidia-ml-py>=12.0.0
 sentencepiece>=0.1.99
 protobuf>=3.20.0

tests/test_burst_placement.py

@@ -1184,3 +1184,41 @@ def test_the_llm_servers_can_run_as_coderai_pods_instead_of_upstream_images(monk
     assert "GGML_CUDA=on" in df and "80;86;89;90;120" in df and "check_llama_wheel" in df
     assert (prof / "vllm.light").exists() and (prof / "vllm.venv-vllm.uv").exists()
     assert "vllm==" in (prof / "vllm.venv-vllm.txt").read_text()
+
+
+def test_the_gguf_backend_can_size_the_card_without_torch(monkeypatch):
+    """The llama pod image carried 5 GB of torch so the CUDA GGUF backend
+    could ask it three questions about the card — 11.6 GB, over RunPod's
+    line. NVML answers the same three from a 1 MB package; torch stays
+    preferred where it exists."""
+    import sys, types
+    from codai.backends import gpu_probe as gp
+
+    # No torch: NVML answers.
+    monkeypatch.setattr(gp, "_torch", lambda: None)
+    class _Mem:  total = 80 * 1024**3; free = 70 * 1024**3
+    nv = types.SimpleNamespace(
+        nvmlInit=lambda: None, nvmlDeviceGetCount=lambda: 2,
+        nvmlDeviceGetHandleByIndex=lambda i: i,
+        nvmlDeviceGetMemoryInfo=lambda h: _Mem(),
+        nvmlSystemGetDriverVersion=lambda: b"570.86")
+    monkeypatch.setitem(sys.modules, "pynvml", nv)
+    assert gp.cuda_available() is True
+    assert gp.total_memory_bytes() == [80 * 1024**3] * 2
+    assert gp.free_memory_bytes() == 70 * 1024**3
+    assert "NVML" in gp.runtime_label() and "570.86" in gp.runtime_label()
+
+    # Neither: honest zeros, not a crash.
+    monkeypatch.setitem(sys.modules, "pynvml", None)
+    assert gp.cuda_available() is False and gp.total_memory_bytes() == []
+    assert gp.free_memory_bytes() is None
+
+    # The backend goes through the probe, not torch.
+    from pathlib import Path
+    src = Path("codai/backends/cuda.py").read_text()
+    i = src.index("def _detect_device"); j = src.index("def _get_gpu_memory_map")
+    assert "import torch" not in src[i:src.index("def _get_available_vram")]
+    assert "gpu_probe" in src[i:j]
+    prof = Path("packaging/runpod/profiles")
+    assert (prof / "llama.light").exists()
+    assert "nvidia-ml-py" in (prof / "llama.txt").read_text()