gpu: global VRAM cap is now per physical card (nvidia/radeon), auto-enumerated

The global "secondary card VRAM cap" was a single number applied to every
non-lead card. Replace it with a per-CARD map so each physical card on the
machine can be capped independently — e.g. RX 580 → 4 GB, RTX 3090 → 20 GB.

- gpu_detect: card_key() (nvidia:<uuid> / amd:<pci>) + gpu_cards() enumerating
  every physical GPU. Stable keys the front and engines both compute identically.
- vulkan: _per_device_card_key() (parallel to _per_device_free_vram_gb, same
  llama.cpp order) + _resolve_device_caps() combining the per-model secondary cap
  (non-main devices) with the global per-card map, lowest wins per device. Applied
  in both the auto-offload pool and the auto-split ratio.
- config: OffloadConfig.split_card_caps_gb {key: gb}; per-model
  split_secondary_cap_gb stays a single value (one cap is enough per model).
- manager: pass _global_card_caps to the engine; keep per-model scalar.
- settings UI: render one cap input per detected card (name + vendor), keyed by
  the stable card key; GET returns gpu_cards + saved caps, POST saves the map.

Applies on the next engine (re)load (global args are read at engine startup),
matching the previous global cap's behavior.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011DDv7BchtZQWsnPG6Jm49m

codai/admin/routes.py

@@ -3397,6 +3397,16 @@ from datetime import datetime
 
 # --- Settings page ---
 
+def _detect_gpu_cards() -> list:
+    """Every physical GPU on the machine ([{key, vendor, name}]) for the per-card
+    VRAM-cap settings UI. Best-effort: returns [] if detection is unavailable."""
+    try:
+        from codai.frontproxy.gpu_detect import gpu_cards
+        return gpu_cards()
+    except Exception:
+        return []
+
+
 @router.get("/admin/settings", response_class=HTMLResponse, summary="Settings page")
 async def settings_page(request: Request, username: str = Depends(require_admin)):
     return _tmpl(request, "settings.html", {"username": username, "is_admin": True})
@@ -3456,6 +3466,10 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "tensor_split": c.offload.tensor_split,
             "split_strategy": c.offload.split_strategy,
             "split_secondary_cap_gb": c.offload.split_secondary_cap_gb,
+            "split_card_caps_gb": c.offload.split_card_caps_gb,
+            # Every physical card on the machine, so the UI can render a per-card cap
+            # row (key + name) and the user can cap each independently.
+            "gpu_cards": _detect_gpu_cards(),
         },
         "vulkan": {
             "n_gpu_layers": c.vulkan.n_gpu_layers,
@@ -3644,6 +3658,18 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
                 c.offload.split_secondary_cap_gb = float(_v) if _v not in (None, "", 0, "0") else None
             except (TypeError, ValueError):
                 c.offload.split_secondary_cap_gb = None
+        if "split_card_caps_gb" in off:
+            # {card_key: gb}. Drop blank/zero entries so an unset card means uncapped.
+            _raw = off["split_card_caps_gb"] or {}
+            _caps = {}
+            if isinstance(_raw, dict):
+                for _k, _v in _raw.items():
+                    try:
+                        if _v not in (None, "", 0, "0"):
+                            _caps[str(_k)] = float(_v)
+                    except (TypeError, ValueError):
+                        pass
+            c.offload.split_card_caps_gb = _caps
         # Push the RAM-cap settings to live global_args so the watcher, per-load
         # budget clamp and eviction honour them without a restart.
         try:
@@ -3660,6 +3686,7 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
                 ga.tensor_split = c.offload.tensor_split
                 ga.split_strategy = c.offload.split_strategy
                 ga.split_secondary_cap_gb = c.offload.split_secondary_cap_gb
+                ga.split_card_caps_gb = c.offload.split_card_caps_gb
         except Exception:
             pass
 

codai/admin/templates/settings.html

@@ -130,9 +130,9 @@
     <span class="form-hint">Default for split models when no explicit ratio is set. <b>VRAM</b> fits the largest model; <b>Speed</b> keeps most layers on the fast card so the slow one bottlenecks less. Per-model setting overrides this.</span>
   </div>
   <div class="form-row" style="margin:0">
-    <label class="form-label">Secondary card VRAM cap <span class="muted">(GB per non-lead card)</span></label>
-    <input type="number" step="0.5" min="0" id="s-split-secondary-cap" class="form-input" placeholder="no cap">
-    <span class="form-hint">Global cap on how much VRAM the auto-split may place on each secondary (slow) card — applies to every engine. e.g. <b>4</b> keeps at most 4 GB on an RX 580; the rest stays on the fast card or spills to CPU. Per-model setting overrides this. Blank/0 = no cap.</span>
+    <label class="form-label">Per-card VRAM cap <span class="muted">(GB per physical card)</span></label>
+    <div id="s-split-card-caps"><span class="form-hint">Detecting cards…</span></div>
+    <span class="form-hint">Cap how much VRAM the auto-split may place on each specific card on this machine — applies to every engine. e.g. set the <b>RX 580</b> to 4 GB and the <b>RTX 3090</b> to 20 GB independently; the rest stays on another card or spills to CPU. Blank/0 = no cap for that card.</span>
   </div>
   <div class="form-row" style="margin:0">
     <label class="form-label">Default weight distribution <span class="muted">(when splitting)</span></label>
@@ -747,6 +747,43 @@ function showAlert(type, msg){
   if(type !== 'error') setTimeout(()=>{ el.style.display='none'; }, 4000);
 }
 
+function renderCardCaps(cards, caps){
+  const box = document.getElementById('s-split-card-caps');
+  if(!box) return;
+  if(!cards || !cards.length){
+    box.innerHTML = '<span class="form-hint">No GPUs detected on this machine.</span>';
+    return;
+  }
+  box.innerHTML = '';
+  cards.forEach(c => {
+    const row = document.createElement('div');
+    row.style.cssText = 'display:flex;align-items:center;gap:8px;margin:4px 0';
+    const lbl = document.createElement('span');
+    lbl.style.cssText = 'flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
+    lbl.textContent = c.name + ' (' + (c.vendor||'') + ')';
+    lbl.title = c.key;
+    const inp = document.createElement('input');
+    inp.type = 'number'; inp.step = '0.5'; inp.min = '0';
+    inp.className = 'form-input s-card-cap'; inp.placeholder = 'no cap';
+    inp.style.cssText = 'width:120px;flex:none';
+    inp.dataset.cardKey = c.key;
+    const v = (caps && caps[c.key] != null) ? caps[c.key] : '';
+    inp.value = v;
+    const unit = document.createElement('span'); unit.className='muted'; unit.textContent='GB';
+    row.appendChild(lbl); row.appendChild(inp); row.appendChild(unit);
+    box.appendChild(row);
+  });
+}
+
+function collectCardCaps(){
+  const out = {};
+  document.querySelectorAll('#s-split-card-caps .s-card-cap').forEach(inp => {
+    const v = (inp.value||'').trim();
+    if(v !== ''){ const n = parseFloat(v); if(!isNaN(n) && n > 0) out[inp.dataset.cardKey] = n; }
+  });
+  return out;
+}
+
 async function loadSettings(){
   try{
     const d = await fetch(ROOT_PATH + '/admin/api/settings').then(r=>r.json());
@@ -789,8 +826,7 @@ async function loadSettings(){
     document.getElementById('s-evict-idle-ram').checked = d.offload?.evict_idle_on_ram !== false;
     document.getElementById('s-gpu-split').checked = !!d.offload?.gpu_split;
     document.getElementById('s-split-strategy').value = d.offload?.split_strategy ?? 'vram';
-    document.getElementById('s-split-secondary-cap').value =
-      (d.offload?.split_secondary_cap_gb != null ? d.offload.split_secondary_cap_gb : '');
+    renderCardCaps(d.offload?.gpu_cards || [], d.offload?.split_card_caps_gb || {});
     document.getElementById('s-tensor-split').value = d.offload?.tensor_split ?? '';
     document.getElementById('s-ram-leak-watch').checked = d.offload?.ram_leak_watch !== false;
     document.getElementById('s-ram-watch-cuda').checked = d.offload?.ram_watch_cuda !== false;
@@ -907,7 +943,7 @@ async function saveSettings(){
       evict_idle_on_ram: document.getElementById('s-evict-idle-ram').checked,
       gpu_split: document.getElementById('s-gpu-split').checked,
       split_strategy: document.getElementById('s-split-strategy').value,
-      split_secondary_cap_gb: (function(){ const v=document.getElementById('s-split-secondary-cap').value.trim(); return v===''?null:parseFloat(v); })(),
+      split_card_caps_gb: collectCardCaps(),
       tensor_split: (document.getElementById('s-tensor-split').value.trim() || null),
       ram_leak_watch: document.getElementById('s-ram-leak-watch').checked,
       ram_watch_cuda: document.getElementById('s-ram-watch-cuda').checked,

codai/backends/vulkan.py

@@ -349,6 +349,70 @@ def _per_device_free_vram_gb() -> list:
     return out
 
 
+def _per_device_card_key() -> list:
+    """Stable card key per device in the SAME order as :func:`_per_device_free_vram_gb`
+    (CUDA devices first by index, then AMD/Vulkan via amdgpu sysfs sorted). Keys match
+    the front's ``gpu_detect.card_key`` (nvidia:<uuid> / amd:<pci>) so a global per-card
+    VRAM cap configured in the UI can be applied to the right physical device here."""
+    keys = []
+    try:
+        import torch
+        if torch.cuda.is_available():
+            for i in range(torch.cuda.device_count()):
+                u = ""
+                try:
+                    u = str(getattr(torch.cuda.get_device_properties(i), "uuid", "") or "")
+                except Exception:
+                    u = ""
+                if u.upper().startswith("GPU-"):
+                    u = u[4:]
+                keys.append(f"nvidia:{u}" if u else f"nvidia:idx{i}")
+    except Exception:
+        pass
+    try:
+        import glob as _glob
+        for tp in sorted(_glob.glob('/sys/class/drm/card*/device/mem_info_vram_total')):
+            dev = os.path.dirname(tp)   # .../cardN/device
+            try:
+                pci = os.path.basename(os.path.realpath(dev))
+            except Exception:
+                pci = "?"
+            keys.append(f"amd:{pci}")
+    except Exception:
+        pass
+    return keys
+
+
+def _resolve_device_caps(n_devices: int, main_gpu: int,
+                         per_model_cap, global_caps) -> list:
+    """Return a per-device VRAM cap (GB) list (None = uncapped), length ``n_devices``.
+
+    Effective cap on a device = the LOWEST of:
+      * the per-MODEL secondary cap (``per_model_cap``), applied to NON-main devices;
+      * the GLOBAL per-card cap for that physical card (``global_caps[key]``).
+    """
+    try:
+        pmc = float(per_model_cap) if per_model_cap not in (None, '', 0, '0') else None
+    except (TypeError, ValueError):
+        pmc = None
+    gc = global_caps if isinstance(global_caps, dict) else {}
+    keys = _per_device_card_key() if gc else []
+    out = []
+    for i in range(n_devices):
+        vals = []
+        if pmc and pmc > 0 and i != main_gpu:
+            vals.append(pmc)
+        if gc and i < len(keys):
+            try:
+                _v = gc.get(keys[i])
+                if _v not in (None, '', 0, '0'):
+                    vals.append(float(_v))
+            except (TypeError, ValueError):
+                pass
+        out.append(min(vals) if vals else None)
+    return out
+
+
 def _ggml_kv_type(name):
     """Map a KV-cache quant name to the llama.cpp GGML type int, or None.
 
@@ -976,21 +1040,20 @@ class VulkanBackend(ModelBackend):
                 # Otherwise we'd needlessly offload layers to CPU thinking only one
                 # card's free VRAM is available.
                 _free = _pooled_free_vram_gb(cross=bool(kwargs.get('gpu_split')))
-                # Apply the secondary-card VRAM cap to the usable pool too, so that
-                # capping the slow card proactively offloads the remainder to CPU here
-                # (rather than only reacting to an OOM in the load-retry loop).
-                try:
-                    _sc = kwargs.get('split_secondary_cap_gb',
-                                     (kwargs.get('_raw_cfg') or {}).get('split_secondary_cap_gb'))
-                    _sc = float(_sc) if _sc not in (None, '', 0, '0') else None
-                except (TypeError, ValueError):
-                    _sc = None
-                if _sc and _sc > 0 and bool(kwargs.get('gpu_split')):
+                # Apply the VRAM caps to the usable pool too, so that capping a card
+                # proactively offloads the remainder to CPU here (rather than only
+                # reacting to an OOM in the load-retry loop). Per-model secondary cap
+                # plus the global per-card caps; effective per device = the lowest.
+                _pmc = kwargs.get('split_secondary_cap_gb',
+                                  (kwargs.get('_raw_cfg') or {}).get('split_secondary_cap_gb'))
+                _gcc = kwargs.get('_global_card_caps') or {}
+                if bool(kwargs.get('gpu_split')) and (_pmc or _gcc):
                     _mgi = self.main_gpu if isinstance(self.main_gpu, int) else 0
                     _pd = _per_device_free_vram_gb()
                     if len(_pd) > 1:
-                        _free = sum((d if i == _mgi else min(d, _sc))
-                                    for i, d in enumerate(_pd))
+                        _caps = _resolve_device_caps(len(_pd), _mgi, _pmc, _gcc)
+                        _free = sum((min(d, c) if c else d)
+                                    for d, c in zip(_pd, _caps))
                 # A vision projector (mmproj) loads ON TOP of the model on main_gpu;
                 # subtract it (weights + compute margin) from the usable pool so that
                 # when the model+KV+projector exceed BOTH cards combined, we reduce
@@ -1212,22 +1275,25 @@ class VulkanBackend(ModelBackend):
                 _adj = list(_free_dev)
                 if _reserve > 0 and 0 <= _mg < len(_adj):
                     _adj[_mg] = max(0.5, _adj[_mg] - _reserve)
-                # Secondary-card VRAM cap: limit how much of each NON-main device's
-                # free VRAM the auto-split may use. Keeps a slow second GPU (e.g. an
-                # RX 580) lightly loaded so it bottlenecks throughput less — the rest
-                # stays on the fast card or spills to CPU. Applies to both strategies.
-                try:
-                    _sec_cap = kwargs.get('split_secondary_cap_gb',
-                                          _raw_cfg.get('split_secondary_cap_gb'))
-                    _sec_cap = float(_sec_cap) if _sec_cap not in (None, '', 0, '0') else None
-                except (TypeError, ValueError):
-                    _sec_cap = None
-                if _sec_cap and _sec_cap > 0:
-                    for _i in range(len(_adj)):
-                        if _i != _mg:
-                            _adj[_i] = min(_adj[_i], _sec_cap)
-                    print(f"  gpu split    : secondary card VRAM capped at "
-                          f"{_sec_cap:.1f} GB/card")
+                # VRAM caps: limit how much of each device's free VRAM the auto-split
+                # may use. Two sources, combined as the LOWEST per device:
+                #   * per-MODEL secondary cap — caps every NON-main device (keeps a slow
+                #     second GPU, e.g. an RX 580, lightly loaded so it bottlenecks less);
+                #   * GLOBAL per-card caps — cap a SPECIFIC physical card by its key,
+                #     so you can independently limit each card on the machine.
+                # The remainder stays on the fast card or spills to CPU. Both strategies.
+                _pmc = kwargs.get('split_secondary_cap_gb',
+                                  _raw_cfg.get('split_secondary_cap_gb'))
+                _gcc = kwargs.get('_global_card_caps') or {}
+                if _pmc or _gcc:
+                    _devcaps = _resolve_device_caps(len(_adj), _mg, _pmc, _gcc)
+                    _keys = _per_device_card_key()
+                    for _i, _c in enumerate(_devcaps):
+                        if _c and _c > 0:
+                            _adj[_i] = min(_adj[_i], _c)
+                            _lbl = _keys[_i] if _i < len(_keys) else f"dev{_i}"
+                            print(f"  gpu split    : VRAM capped at {_c:.1f} GB on "
+                                  f"{_lbl}")
                 # Split strategy: "vram" (default) = proportional to free VRAM (max
                 # capacity); "performance" = fill the FAST lead card (main_gpu) first
                 # and spill only the overflow to the slower card(s), so the weak GPU

codai/config.py

@@ -140,10 +140,16 @@ class OffloadConfig:
     #   "performance" → fill the fast lead card first, spill only the overflow to the
     #                   slower card(s) so the weak GPU gates throughput the least.
     split_strategy: str = "vram"
-    # Cap (GB) on how much VRAM the auto-split may place on EACH secondary (non-main)
-    # card. Keeps a slow second GPU lightly loaded so it bottlenecks less; the
-    # remainder stays on the fast card or spills to CPU. None/0 = no cap.
+    # Per-MODEL cap (GB) on how much VRAM the auto-split may place on the secondary
+    # (non-main) card. For a 2-card split one cap is enough — which card is secondary
+    # is implicit. Lives in each model's config; None/0 = no cap.
     split_secondary_cap_gb: Optional[float] = None
+    # GLOBAL per-CARD VRAM caps: {card_key: gb}. Unlike the per-model scalar above,
+    # this caps a SPECIFIC physical card by a stable key (nvidia:<uuid> / amd:<pci>)
+    # regardless of which engine/model uses it — so you can independently limit, say,
+    # the Radeon to 4 GB and the NVIDIA to 20 GB. Effective cap on a device = lowest
+    # of its global per-card cap and the per-model secondary cap (when it applies).
+    split_card_caps_gb: Dict[str, float] = field(default_factory=dict)
 
 
 @dataclass
@@ -630,7 +636,8 @@ class ConfigManager:
                 "gpu_split": self.config.offload.gpu_split,
                 "tensor_split": self.config.offload.tensor_split,
                 "split_strategy": self.config.offload.split_strategy,
-                "split_secondary_cap_gb": self.config.offload.split_secondary_cap_gb
+                "split_secondary_cap_gb": self.config.offload.split_secondary_cap_gb,
+                "split_card_caps_gb": self.config.offload.split_card_caps_gb
             },
             "vulkan": {
                 "n_gpu_layers": self.config.vulkan.n_gpu_layers,

codai/frontproxy/gpu_detect.py

@@ -379,6 +379,51 @@ def engine_gpu_stats() -> list:
             if c.get("vendor") in sels or (c.get("uuid") and c["uuid"] in sels)]
 
 
+def card_key(vendor: str, uuid: str = "", pci: str = "") -> str:
+    """Stable identifier for one physical GPU, computable identically by the front
+    (which sees every card) and by an engine (which sees only its own, in llama.cpp
+    device order). NVIDIA → its UUID (stable across reboots); AMD/others → PCI bus
+    address. Used to key per-card VRAM caps so a cap follows the actual card."""
+    vendor = _norm_vendor(vendor)
+    if vendor == "nvidia" and uuid:
+        u = uuid.strip()
+        if u.upper().startswith("GPU-"):
+            u = u[4:]
+        return f"nvidia:{u}"
+    if pci:
+        return f"{vendor}:{pci.strip()}"
+    return f"{vendor}:?"
+
+
+def gpu_cards() -> list:
+    """Every physical GPU on the machine as ``[{key, vendor, name}]`` for a per-card
+    settings UI. Keys are stable (see :func:`card_key`) so a saved cap maps back to
+    the same card on the engine side. Software rasterizers are excluded."""
+    cards = []
+    for g in nvidia_gpus():
+        cards.append({"key": card_key("nvidia", uuid=g.get("uuid", "")),
+                      "vendor": "nvidia",
+                      "name": g.get("name") or "NVIDIA GPU"})
+    # AMD (and any non-NVIDIA) via sysfs DRM, keyed by PCI bus address.
+    for card in sorted(glob.glob("/sys/class/drm/card*")):
+        base = os.path.basename(card)
+        if not re.match(r"^card\d+$", base):
+            continue
+        dev = os.path.join(card, "device")
+        try:
+            with open(os.path.join(dev, "vendor")) as f:
+                vid = f.read().strip().lower()
+        except OSError:
+            continue
+        vendor = _PCI_VENDOR.get(vid)
+        if not vendor or vendor == "nvidia":   # nvidia handled above via nvidia-smi
+            continue
+        pci = os.path.basename(os.path.realpath(dev))
+        name = _amd_gpu_name(dev, base) if vendor == "amd" else f"{vendor} GPU ({base})"
+        cards.append({"key": card_key(vendor, pci=pci), "vendor": vendor, "name": name})
+    return cards
+
+
 def summary() -> dict:
     """Detected hardware, for a 'detect engines' UI / debugging."""
     return {"nvidia": nvidia_gpus(), "vulkan": vulkan_devices(),

codai/main.py

@@ -1062,6 +1062,7 @@ def main():
     global_args.tensor_split = getattr(config.offload, "tensor_split", None)
     global_args.split_strategy = getattr(config.offload, "split_strategy", "vram")
     global_args.split_secondary_cap_gb = getattr(config.offload, "split_secondary_cap_gb", None)
+    global_args.split_card_caps_gb = getattr(config.offload, "split_card_caps_gb", None) or {}
     # Thermal protection settings (read live by codai.models.thermal).
     global_args.thermal_cpu_enabled = config.thermal.cpu_enabled
     global_args.thermal_gpu_enabled = config.thermal.gpu_enabled

codai/models/manager.py

@@ -1112,9 +1112,11 @@ class MultiModelManager:
                 if _ts_val:
                     kwargs['tensor_split'] = _ts_val
                 kwargs['split_strategy'] = _cfg_or_global('split_strategy', 'split_strategy', 'vram')
-                # Secondary-card VRAM cap = the LOWEST of the global and per-model
-                # caps (whichever are set). A per-model cap can only tighten the
-                # global one — a per-model value higher than the global is ignored.
+                # Per-MODEL secondary cap (single value; the legacy global scalar still
+                # tightens it as a fallback) AND the GLOBAL per-card cap map. The engine
+                # combines them per device, taking the lowest. The per-card map keys
+                # physical cards (nvidia:<uuid> / amd:<pci>) so each card is capped
+                # independently regardless of which model/engine uses it.
                 _mcap = config.get('split_secondary_cap_gb') if isinstance(config, dict) else None
                 _gcap = getattr(_ga, 'split_secondary_cap_gb', None) if _ga else None
                 _caps = []
@@ -1125,6 +1127,7 @@ class MultiModelManager:
                     except (TypeError, ValueError):
                         pass
                 kwargs['split_secondary_cap_gb'] = min(_caps) if _caps else None
+                kwargs['_global_card_caps'] = getattr(_ga, 'split_card_caps_gb', None) or {}
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')
@@ -1250,9 +1253,11 @@ class MultiModelManager:
                 if _ts_val:
                     kwargs['tensor_split'] = _ts_val
                 kwargs['split_strategy'] = _cfg_or_global('split_strategy', 'split_strategy', 'vram')
-                # Secondary-card VRAM cap = the LOWEST of the global and per-model
-                # caps (whichever are set). A per-model cap can only tighten the
-                # global one — a per-model value higher than the global is ignored.
+                # Per-MODEL secondary cap (single value; the legacy global scalar still
+                # tightens it as a fallback) AND the GLOBAL per-card cap map. The engine
+                # combines them per device, taking the lowest. The per-card map keys
+                # physical cards (nvidia:<uuid> / amd:<pci>) so each card is capped
+                # independently regardless of which model/engine uses it.
                 _mcap = config.get('split_secondary_cap_gb') if isinstance(config, dict) else None
                 _gcap = getattr(_ga, 'split_secondary_cap_gb', None) if _ga else None
                 _caps = []
@@ -1263,6 +1268,7 @@ class MultiModelManager:
                     except (TypeError, ValueError):
                         pass
                 kwargs['split_secondary_cap_gb'] = min(_caps) if _caps else None
+                kwargs['_global_card_caps'] = getattr(_ga, 'split_card_caps_gb', None) or {}
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')