gpu: secondary-card VRAM cap (per-model + global), effective = lowest of the two

Add a cap on how much VRAM the auto-split may place on EACH secondary (non-main)
card, so a slow second GPU (e.g. RX 580) stays lightly loaded and bottlenecks
throughput less — the remainder stays on the fast card or spills to CPU.

- config offload.split_secondary_cap_gb: global default, persisted + pushed live to
  every engine's global_args (each engine reads config at startup, so it's global).
- per-model split_secondary_cap_gb overrides — but only to TIGHTEN: the effective
  cap is min(global, per-model) of whichever are set; a per-model value higher than
  the global is ignored.
- applied in vulkan auto-split: caps each non-main device's free VRAM in the ratio
  (both vram & performance strategies), and in the auto-offload pool so the excess
  proactively spills to CPU instead of only reacting to an OOM.
- UI: model config modal "Secondary card VRAM cap" + global Settings field.
- added to _LOAD_AFFECTING so changes re-apply live on the next request.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -2748,7 +2748,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                 "component_quantization", "output_crf", "force_vram_update",
                 "balanced_gpu_percent", "acceleration",
                 "cache_type_k", "cache_type_v", "kv_offload", "n_batch", "n_ubatch", "n_seq_max",
-                "gpu_split", "tensor_split", "split_strategy",
+                "gpu_split", "tensor_split", "split_strategy", "split_secondary_cap_gb",
                 "turboquant", "engine", "engine_fallback",
                 "quant_backend", "kv_cache_budget_mb", "kv_cache_slots", "mmproj",
                 "auto_compact", "auto_compact_pct", "auto_compact_strategy",
@@ -3455,6 +3455,7 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "gpu_split": c.offload.gpu_split,
             "tensor_split": c.offload.tensor_split,
             "split_strategy": c.offload.split_strategy,
+            "split_secondary_cap_gb": c.offload.split_secondary_cap_gb,
         },
         "vulkan": {
             "n_gpu_layers": c.vulkan.n_gpu_layers,
@@ -3637,6 +3638,12 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
             c.offload.tensor_split = off["tensor_split"] or None
         if "split_strategy" in off:
             c.offload.split_strategy = off["split_strategy"] or "vram"
+        if "split_secondary_cap_gb" in off:
+            _v = off["split_secondary_cap_gb"]
+            try:
+                c.offload.split_secondary_cap_gb = float(_v) if _v not in (None, "", 0, "0") else None
+            except (TypeError, ValueError):
+                c.offload.split_secondary_cap_gb = None
         # Push the RAM-cap settings to live global_args so the watcher, per-load
         # budget clamp and eviction honour them without a restart.
         try:
@@ -3652,6 +3659,7 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
                 ga.gpu_split = c.offload.gpu_split
                 ga.tensor_split = c.offload.tensor_split
                 ga.split_strategy = c.offload.split_strategy
+                ga.split_secondary_cap_gb = c.offload.split_secondary_cap_gb
         except Exception:
             pass
 

codai/admin/templates/models.html

@@ -580,6 +580,9 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
           <option value="performance">Maximize speed — fill the fast card first, least on the slow one</option>
         </select>
         <span class="form-hint" style="font-size:11px"><b>VRAM</b> packs the most onto every card (best for fitting a large model) but the slow card bottlenecks throughput. <b>Speed</b> keeps as much as possible on the lead/fast card and spills only the overflow to the slower one — faster, as long as it still fits.</span>
+        <label class="form-label" style="font-weight:600;margin-top:.5rem">Secondary card VRAM cap <span class="muted" style="font-weight:400">— GB per non-lead card</span></label>
+        <input type="number" step="0.5" min="0" id="cfg-split-secondary-cap" class="form-input" placeholder="no cap (uses global default)">
+        <span class="form-hint" style="font-size:11px">Max VRAM the auto-split may put on each secondary (slow) card, e.g. <b>4</b> = at most 4 GB on the RX 580; the remainder stays on the lead card or spills to CPU. Lower = the slow card bottlenecks less. Blank = use the global default; 0 = no cap.</span>
         <label class="form-label" style="font-weight:600;margin-top:.5rem">Weight distribution <span class="muted" style="font-weight:400">— layers per GPU (manual override)</span></label>
         <input type="text" id="cfg-tensor-split" class="form-input" placeholder="auto (uses the strategy above)">
         <span class="form-hint" style="font-size:11px">Optional explicit ratio for the lead card then the rest, e.g. <b>0.8,0.2</b>. Overrides the strategy. Blank = automatic per the strategy above.</span>
@@ -3293,6 +3296,8 @@ function openCfgModal(idx, cfgIdx){
   document.getElementById('cfg-engine-fallback').checked = !!s.engine_fallback;
   document.getElementById('cfg-tensor-split').value = s.tensor_split || '';
   document.getElementById('cfg-split-strategy').value = s.split_strategy || 'vram';
+  document.getElementById('cfg-split-secondary-cap').value =
+    (s.split_secondary_cap_gb != null ? s.split_secondary_cap_gb : '');
   _populateDs4(m, s);
   document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
   document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
@@ -3714,6 +3719,10 @@ async function saveModelConfig(){
                           ? (document.getElementById('cfg-tensor-split').value.trim() || null) : null),
     split_strategy:    (document.getElementById('cfg-engine').value.trim().startsWith('__split__:')
                           ? document.getElementById('cfg-split-strategy').value : null),
+    split_secondary_cap_gb: (function(){
+                          if (!document.getElementById('cfg-engine').value.trim().startsWith('__split__:')) return null;
+                          const v = document.getElementById('cfg-split-secondary-cap').value.trim();
+                          return v === '' ? null : parseFloat(v); })(),
     ds4:               _collectDs4(),
     system_prompt:     document.getElementById('cfg-sysprompt').value.trim() || null,
     parser:            document.getElementById('cfg-parser').value,

codai/admin/templates/settings.html

@@ -129,6 +129,11 @@
     </select>
     <span class="form-hint">Default for split models when no explicit ratio is set. <b>VRAM</b> fits the largest model; <b>Speed</b> keeps most layers on the fast card so the slow one bottlenecks less. Per-model setting overrides this.</span>
   </div>
+  <div class="form-row" style="margin:0">
+    <label class="form-label">Secondary card VRAM cap <span class="muted">(GB per non-lead card)</span></label>
+    <input type="number" step="0.5" min="0" id="s-split-secondary-cap" class="form-input" placeholder="no cap">
+    <span class="form-hint">Global cap on how much VRAM the auto-split may place on each secondary (slow) card — applies to every engine. e.g. <b>4</b> keeps at most 4 GB on an RX 580; the rest stays on the fast card or spills to CPU. Per-model setting overrides this. Blank/0 = no cap.</span>
+  </div>
   <div class="form-row" style="margin:0">
     <label class="form-label">Default weight distribution <span class="muted">(when splitting)</span></label>
     <input type="text" id="s-tensor-split" class="form-input" placeholder="auto (by strategy)">
@@ -784,6 +789,8 @@ async function loadSettings(){
     document.getElementById('s-evict-idle-ram').checked = d.offload?.evict_idle_on_ram !== false;
     document.getElementById('s-gpu-split').checked = !!d.offload?.gpu_split;
     document.getElementById('s-split-strategy').value = d.offload?.split_strategy ?? 'vram';
+    document.getElementById('s-split-secondary-cap').value =
+      (d.offload?.split_secondary_cap_gb != null ? d.offload.split_secondary_cap_gb : '');
     document.getElementById('s-tensor-split').value = d.offload?.tensor_split ?? '';
     document.getElementById('s-ram-leak-watch').checked = d.offload?.ram_leak_watch !== false;
     document.getElementById('s-ram-watch-cuda').checked = d.offload?.ram_watch_cuda !== false;
@@ -900,6 +907,7 @@ async function saveSettings(){
       evict_idle_on_ram: document.getElementById('s-evict-idle-ram').checked,
       gpu_split: document.getElementById('s-gpu-split').checked,
       split_strategy: document.getElementById('s-split-strategy').value,
+      split_secondary_cap_gb: (function(){ const v=document.getElementById('s-split-secondary-cap').value.trim(); return v===''?null:parseFloat(v); })(),
       tensor_split: (document.getElementById('s-tensor-split').value.trim() || null),
       ram_leak_watch: document.getElementById('s-ram-leak-watch').checked,
       ram_watch_cuda: document.getElementById('s-ram-watch-cuda').checked,

codai/backends/vulkan.py

@@ -976,6 +976,21 @@ class VulkanBackend(ModelBackend):
                 # Otherwise we'd needlessly offload layers to CPU thinking only one
                 # card's free VRAM is available.
                 _free = _pooled_free_vram_gb(cross=bool(kwargs.get('gpu_split')))
+                # Apply the secondary-card VRAM cap to the usable pool too, so that
+                # capping the slow card proactively offloads the remainder to CPU here
+                # (rather than only reacting to an OOM in the load-retry loop).
+                try:
+                    _sc = kwargs.get('split_secondary_cap_gb',
+                                     (kwargs.get('_raw_cfg') or {}).get('split_secondary_cap_gb'))
+                    _sc = float(_sc) if _sc not in (None, '', 0, '0') else None
+                except (TypeError, ValueError):
+                    _sc = None
+                if _sc and _sc > 0 and bool(kwargs.get('gpu_split')):
+                    _mgi = self.main_gpu if isinstance(self.main_gpu, int) else 0
+                    _pd = _per_device_free_vram_gb()
+                    if len(_pd) > 1:
+                        _free = sum((d if i == _mgi else min(d, _sc))
+                                    for i, d in enumerate(_pd))
                 # A vision projector (mmproj) loads ON TOP of the model on main_gpu;
                 # subtract it (weights + compute margin) from the usable pool so that
                 # when the model+KV+projector exceed BOTH cards combined, we reduce
@@ -1197,6 +1212,22 @@ class VulkanBackend(ModelBackend):
                 _adj = list(_free_dev)
                 if _reserve > 0 and 0 <= _mg < len(_adj):
                     _adj[_mg] = max(0.5, _adj[_mg] - _reserve)
+                # Secondary-card VRAM cap: limit how much of each NON-main device's
+                # free VRAM the auto-split may use. Keeps a slow second GPU (e.g. an
+                # RX 580) lightly loaded so it bottlenecks throughput less — the rest
+                # stays on the fast card or spills to CPU. Applies to both strategies.
+                try:
+                    _sec_cap = kwargs.get('split_secondary_cap_gb',
+                                          _raw_cfg.get('split_secondary_cap_gb'))
+                    _sec_cap = float(_sec_cap) if _sec_cap not in (None, '', 0, '0') else None
+                except (TypeError, ValueError):
+                    _sec_cap = None
+                if _sec_cap and _sec_cap > 0:
+                    for _i in range(len(_adj)):
+                        if _i != _mg:
+                            _adj[_i] = min(_adj[_i], _sec_cap)
+                    print(f"  gpu split    : secondary card VRAM capped at "
+                          f"{_sec_cap:.1f} GB/card")
                 # Split strategy: "vram" (default) = proportional to free VRAM (max
                 # capacity); "performance" = fill the FAST lead card (main_gpu) first
                 # and spill only the overflow to the slower card(s), so the weak GPU

codai/config.py

@@ -140,6 +140,10 @@ class OffloadConfig:
     #   "performance" → fill the fast lead card first, spill only the overflow to the
     #                   slower card(s) so the weak GPU gates throughput the least.
     split_strategy: str = "vram"
+    # Cap (GB) on how much VRAM the auto-split may place on EACH secondary (non-main)
+    # card. Keeps a slow second GPU lightly loaded so it bottlenecks less; the
+    # remainder stays on the fast card or spills to CPU. None/0 = no cap.
+    split_secondary_cap_gb: Optional[float] = None
 
 
 @dataclass
@@ -625,7 +629,8 @@ class ConfigManager:
                 "ram_watch_cuda": self.config.offload.ram_watch_cuda,
                 "gpu_split": self.config.offload.gpu_split,
                 "tensor_split": self.config.offload.tensor_split,
-                "split_strategy": self.config.offload.split_strategy
+                "split_strategy": self.config.offload.split_strategy,
+                "split_secondary_cap_gb": self.config.offload.split_secondary_cap_gb
             },
             "vulkan": {
                 "n_gpu_layers": self.config.vulkan.n_gpu_layers,

codai/main.py

@@ -156,6 +156,7 @@ def build_runtime_kwargs(model_cfg, model_type):
         'gpu_split':       bool(model_cfg.get('gpu_split', False)),
         'tensor_split':    model_cfg.get('tensor_split'),
         'split_strategy':  model_cfg.get('split_strategy'),
+        'split_secondary_cap_gb': model_cfg.get('split_secondary_cap_gb'),
         '_raw_cfg':        dict(model_cfg) if isinstance(model_cfg, dict) else {},
     }
     if model_type == "text":
@@ -239,7 +240,7 @@ def apply_model_entry_live(entry, model_types) -> int:
         "n_seq_max", "flash_attention", "flash_attn", "load_in_4bit", "load_in_8bit",
         "offload_strategy", "no_ram", "max_gpu_percent", "mmproj", "quant_backend",
         "engine", "precision", "vae_path", "vae_tiling", "component_quantization",
-        "gpu_split", "tensor_split", "split_strategy",
+        "gpu_split", "tensor_split", "split_strategy", "split_secondary_cap_gb",
     )
 
     def _load_sig(c):
@@ -1048,6 +1049,7 @@ def main():
     global_args.gpu_split = getattr(config.offload, "gpu_split", False)
     global_args.tensor_split = getattr(config.offload, "tensor_split", None)
     global_args.split_strategy = getattr(config.offload, "split_strategy", "vram")
+    global_args.split_secondary_cap_gb = getattr(config.offload, "split_secondary_cap_gb", None)
     # Thermal protection settings (read live by codai.models.thermal).
     global_args.thermal_cpu_enabled = config.thermal.cpu_enabled
     global_args.thermal_gpu_enabled = config.thermal.gpu_enabled

codai/models/manager.py

@@ -1112,6 +1112,19 @@ class MultiModelManager:
                 if _ts_val:
                     kwargs['tensor_split'] = _ts_val
                 kwargs['split_strategy'] = _cfg_or_global('split_strategy', 'split_strategy', 'vram')
+                # Secondary-card VRAM cap = the LOWEST of the global and per-model
+                # caps (whichever are set). A per-model cap can only tighten the
+                # global one — a per-model value higher than the global is ignored.
+                _mcap = config.get('split_secondary_cap_gb') if isinstance(config, dict) else None
+                _gcap = getattr(_ga, 'split_secondary_cap_gb', None) if _ga else None
+                _caps = []
+                for _c in (_mcap, _gcap):
+                    try:
+                        if _c not in (None, '', 0, '0'):
+                            _caps.append(float(_c))
+                    except (TypeError, ValueError):
+                        pass
+                kwargs['split_secondary_cap_gb'] = min(_caps) if _caps else None
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')
@@ -1237,6 +1250,19 @@ class MultiModelManager:
                 if _ts_val:
                     kwargs['tensor_split'] = _ts_val
                 kwargs['split_strategy'] = _cfg_or_global('split_strategy', 'split_strategy', 'vram')
+                # Secondary-card VRAM cap = the LOWEST of the global and per-model
+                # caps (whichever are set). A per-model cap can only tighten the
+                # global one — a per-model value higher than the global is ignored.
+                _mcap = config.get('split_secondary_cap_gb') if isinstance(config, dict) else None
+                _gcap = getattr(_ga, 'split_secondary_cap_gb', None) if _ga else None
+                _caps = []
+                for _c in (_mcap, _gcap):
+                    try:
+                        if _c not in (None, '', 0, '0'):
+                            _caps.append(float(_c))
+                    except (TypeError, ValueError):
+                        pass
+                kwargs['split_secondary_cap_gb'] = min(_caps) if _caps else None
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')