gpu: cross-GPU VRAM accounting for split models (Phase 3)

When cross-backend pooling (offload.gpu_split) is on a model can span several
cards, so VRAM math must consider the pool, not one device:

- vulkan.py: new _pooled_free_vram_gb() sums free VRAM across every visible card
  (all CUDA devices + AMD via amdgpu sysfs; no double count). The GGUF auto-offload
  fit decision uses the pooled figure when the model is gpu_split, so layers aren't
  needlessly pushed to CPU just because one card's free VRAM looks small.
- manager._get_free_vram_gb(): when gpu_split is enabled, report pooled free across
  all GPUs (used by the eviction loop + budget checks), so eviction frees/measures
  capacity across both cards. Off → unchanged single-device behavior.

Pooling is aggregate (sum); strict per-device fitting/targeted eviction can be a
later refinement.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/backends/vulkan.py

@@ -253,6 +253,41 @@ def _free_vram_gb(device: int = 0) -> float:
     return _amd_free_vram_gb(device)
 
 
+def _pooled_free_vram_gb() -> float:
+    """Sum of free VRAM (GB) across EVERY GPU this process can see — all visible
+    CUDA devices plus all AMD cards (amdgpu sysfs). Used for cross-GPU split
+    (gpu_split), where the usable capacity is the POOL, not a single card. CUDA
+    and amdgpu sysfs never describe the same physical card, so there's no double
+    count (NVIDIA cards don't expose mem_info_vram_*)."""
+    total = 0.0
+    try:
+        import torch
+        if torch.cuda.is_available():
+            for i in range(torch.cuda.device_count()):
+                try:
+                    free, _ = torch.cuda.mem_get_info(i)
+                    total += free / (1024 ** 3)
+                except Exception:
+                    pass
+    except Exception:
+        pass
+    try:
+        import glob
+        for tp in sorted(glob.glob('/sys/class/drm/card*/device/mem_info_vram_total')):
+            up = tp.replace('vram_total', 'vram_used')
+            try:
+                with open(tp) as f:
+                    _t = int(f.read().strip())
+                with open(up) as f:
+                    _u = int(f.read().strip())
+                total += (_t - _u) / (1024 ** 3)
+            except Exception:
+                pass
+    except Exception:
+        pass
+    return total if total > 0 else _free_vram_gb(0)
+
+
 def _ggml_kv_type(name):
     """Map a KV-cache quant name to the llama.cpp GGML type int, or None.
 
@@ -874,7 +909,14 @@ class VulkanBackend(ModelBackend):
             try:
                 _exp = kwargs.get('expected_vram_gb')
                 _nlayers = _gguf_block_count(model_path)
-                _free = _free_vram_gb(self.main_gpu if isinstance(self.main_gpu, int) else 0)
+                # When this model is split across GPUs, the usable VRAM is the POOL
+                # across every visible card (e.g. 3090 + RX 580), not just main_gpu —
+                # otherwise we'd needlessly offload layers to CPU thinking only one
+                # card's free VRAM is available.
+                if kwargs.get('gpu_split'):
+                    _free = _pooled_free_vram_gb()
+                else:
+                    _free = _free_vram_gb(self.main_gpu if isinstance(self.main_gpu, int) else 0)
                 if _exp and _exp > 0 and _nlayers and _free > 0 and _exp > _free * 0.95:
                     # Scale layers on GPU by the VRAM ratio (weights + KV roughly
                     # scale per-layer). The estimate tends to undercount the KV

codai/models/manager.py

@@ -2203,16 +2203,43 @@ class MultiModelManager:
         except Exception as e:
             print(f"  Warning during VRAM load of '{model_key}': {e}")
 
+    @staticmethod
+    def _cross_gpu_pooling_enabled() -> bool:
+        """True when cross-backend GPU pooling (offload.gpu_split) is on, so a model
+        may span several cards and free-VRAM/eviction math must sum across them."""
+        try:
+            from codai.api.state import get_global_args
+            return bool(getattr(get_global_args(), "gpu_split", False))
+        except Exception:
+            return False
+
     def _get_free_vram_gb(self) -> float:
-        """Return estimated free VRAM in GB, or a large number if unavailable."""
+        """Return estimated free VRAM in GB, or a large number if unavailable.
+
+        With cross-GPU pooling (gpu_split) on, a model can be split across every
+        visible card, so report the POOL: free CUDA VRAM (all devices) + free AMD
+        VRAM (sysfs). CUDA and amdgpu sysfs never name the same card, so there's no
+        double count. Off → just the primary CUDA device (legacy single-GPU math)."""
+        pooled = self._cross_gpu_pooling_enabled()
+        total_free = 0.0
+        got = False
         try:
             import torch
             if torch.cuda.is_available():
-                free, total = torch.cuda.mem_get_info()
-                return free / 1e9
+                if pooled:
+                    for i in range(torch.cuda.device_count()):
+                        try:
+                            free, _ = torch.cuda.mem_get_info(i)
+                            total_free += free / 1e9
+                            got = True
+                        except Exception:
+                            pass
+                else:
+                    free, total = torch.cuda.mem_get_info()
+                    return free / 1e9
         except Exception:
             pass
-        # AMD GPU via sysfs (covers Vulkan/ROCm on Linux)
+        # AMD GPU(s) via sysfs (covers Vulkan/ROCm on Linux).
         try:
             import glob
             for total_path in sorted(glob.glob('/sys/class/drm/card*/device/mem_info_vram_total')):
@@ -2221,9 +2248,16 @@ class MultiModelManager:
                     total = int(f.read().strip())
                 with open(used_path) as f:
                     used = int(f.read().strip())
-                return (total - used) / 1e9
+                free_amd = (total - used) / 1e9
+                if pooled:
+                    total_free += free_amd
+                    got = True
+                else:
+                    return free_amd
         except Exception:
             pass
+        if pooled and got:
+            return total_free
         return 999.0  # Unknown — assume enough
 
     @staticmethod