gpu: make VRAM accounting/eviction ALWAYS multi-device, not just under split

Previously pooling was gated on gpu_split, so a 2-card same-backend engine (e.g.
2× 3090, or 2× Radeon) still measured only one device for the fit/eviction math.

Now both manager._get_free_vram_gb() and vulkan._pooled_free_vram_gb():
- ALWAYS sum every visible CUDA device (torch honours CUDA_VISIBLE_DEVICES, so it
  is scoped to this engine's NVIDIA cards) → same-backend split is accounted for
  with no flag.
- add AMD card(s) (amdgpu sysfs) only when cross-backend split is on OR no CUDA
  device is visible (a Radeon/Vulkan engine), so a Radeon engine counts its own
  cards and an NVIDIA engine only reaches across to Radeon when split is enabled.

So: 2× NVIDIA → summed across both NVIDIAs; 2× Radeon → summed across both Radeons;
split on → summed across all NVIDIA + Radeon.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/backends/vulkan.py

@@ -253,24 +253,32 @@ def _free_vram_gb(device: int = 0) -> float:
     return _amd_free_vram_gb(device)
 
 
-def _pooled_free_vram_gb() -> float:
-    """Sum of free VRAM (GB) across EVERY GPU this process can see — all visible
-    CUDA devices plus all AMD cards (amdgpu sysfs). Used for cross-GPU split
-    (gpu_split), where the usable capacity is the POOL, not a single card. CUDA
-    and amdgpu sysfs never describe the same physical card, so there's no double
+def _pooled_free_vram_gb(cross: bool = False) -> float:
+    """Sum of free VRAM (GB) across the GPUs this engine can actually use.
+
+    Always sums EVERY visible CUDA device — torch honours CUDA_VISIBLE_DEVICES, so
+    this is automatically scoped to this engine's NVIDIA cards (e.g. both 3090s),
+    making same-backend split accounting correct without any flag. AMD cards
+    (amdgpu sysfs) are added when ``cross`` (cross-backend split) is on, OR when no
+    CUDA device is visible (a Radeon/Vulkan engine) so its own card(s) are counted.
+    CUDA and amdgpu sysfs never name the same physical card, so there's no double
     count (NVIDIA cards don't expose mem_info_vram_*)."""
-    total = 0.0
+    cuda_free = 0.0
+    cuda_count = 0
     try:
         import torch
         if torch.cuda.is_available():
             for i in range(torch.cuda.device_count()):
                 try:
                     free, _ = torch.cuda.mem_get_info(i)
-                    total += free / (1024 ** 3)
+                    cuda_free += free / (1024 ** 3)
+                    cuda_count += 1
                 except Exception:
                     pass
     except Exception:
         pass
+    total = cuda_free
+    if cross or cuda_count == 0:
         try:
             import glob
             for tp in sorted(glob.glob('/sys/class/drm/card*/device/mem_info_vram_total')):
@@ -909,14 +917,12 @@ class VulkanBackend(ModelBackend):
             try:
                 _exp = kwargs.get('expected_vram_gb')
                 _nlayers = _gguf_block_count(model_path)
-                # When this model is split across GPUs, the usable VRAM is the POOL
-                # across every visible card (e.g. 3090 + RX 580), not just main_gpu —
-                # otherwise we'd needlessly offload layers to CPU thinking only one
+                # Usable VRAM is the POOL across the cards this engine can use, not
+                # just main_gpu: all same-backend cards always (e.g. both 3090s), plus
+                # the other backend's cards when cross-split is on (3090 + RX 580).
+                # Otherwise we'd needlessly offload layers to CPU thinking only one
                 # card's free VRAM is available.
-                if kwargs.get('gpu_split'):
-                    _free = _pooled_free_vram_gb()
-                else:
-                    _free = _free_vram_gb(self.main_gpu if isinstance(self.main_gpu, int) else 0)
+                _free = _pooled_free_vram_gb(cross=bool(kwargs.get('gpu_split')))
                 if _exp and _exp > 0 and _nlayers and _free > 0 and _exp > _free * 0.95:
                     # Scale layers on GPU by the VRAM ratio (weights + KV roughly
                     # scale per-layer). The estimate tends to undercount the KV

codai/models/manager.py

@@ -2214,32 +2214,32 @@ class MultiModelManager:
             return False
 
     def _get_free_vram_gb(self) -> float:
-        """Return estimated free VRAM in GB, or a large number if unavailable.
-
-        With cross-GPU pooling (gpu_split) on, a model can be split across every
-        visible card, so report the POOL: free CUDA VRAM (all devices) + free AMD
-        VRAM (sysfs). CUDA and amdgpu sysfs never name the same card, so there's no
-        double count. Off → just the primary CUDA device (legacy single-GPU math)."""
-        pooled = self._cross_gpu_pooling_enabled()
-        total_free = 0.0
-        got = False
+        """Return estimated free VRAM in GB across the cards this engine can use.
+
+        ALWAYS multi-device: sums every visible CUDA device (torch honours
+        CUDA_VISIBLE_DEVICES, so it's scoped to this engine's NVIDIA cards — e.g.
+        both 3090s — making same-backend eviction math correct even without split).
+        AMD card(s) (amdgpu sysfs) are added when cross-backend pooling (gpu_split)
+        is on, OR when no CUDA device is visible (a Radeon/Vulkan engine). CUDA and
+        amdgpu sysfs never name the same card, so there's no double count."""
+        cross = self._cross_gpu_pooling_enabled()
+        cuda_free = 0.0
+        cuda_count = 0
         try:
             import torch
             if torch.cuda.is_available():
-                if pooled:
                 for i in range(torch.cuda.device_count()):
                     try:
                         free, _ = torch.cuda.mem_get_info(i)
-                            total_free += free / 1e9
-                            got = True
+                        cuda_free += free / 1e9
+                        cuda_count += 1
                     except Exception:
                         pass
-                else:
-                    free, total = torch.cuda.mem_get_info()
-                    return free / 1e9
         except Exception:
             pass
-        # AMD GPU(s) via sysfs (covers Vulkan/ROCm on Linux).
+        total_free = cuda_free
+        got = cuda_count > 0
+        if cross or cuda_count == 0:
             try:
                 import glob
                 for total_path in sorted(glob.glob('/sys/class/drm/card*/device/mem_info_vram_total')):
@@ -2248,15 +2248,11 @@ class MultiModelManager:
                         total = int(f.read().strip())
                     with open(used_path) as f:
                         used = int(f.read().strip())
-                free_amd = (total - used) / 1e9
-                if pooled:
-                    total_free += free_amd
+                    total_free += (total - used) / 1e9
                     got = True
-                else:
-                    return free_amd
             except Exception:
                 pass
-        if pooled and got:
+        if got:
             return total_free
         return 999.0  # Unknown — assume enough