gpu: performance split strategy (fast card first) — per-model + global, UI-configurable

The auto split is VRAM-proportional, which hands a big share to a slow second GPU
(e.g. RX 580) so it bottlenecks throughput (Radeon ~100%, 3090 ~30%). Add a
"performance" strategy that fills the fast lead card (main_gpu) first and spills
only the overflow to the slower card(s), using the model's expected size — so the
weak GPU holds the fewest layers (and none at all if the model fits on the lead
card). "vram" (default) keeps the capacity-maximizing behavior.

- config offload.split_strategy ("vram"|"performance") global default, persisted +
  pushed to live global_args; per-model split_strategy overrides it.
- plumbed via build_runtime_kwargs + manager _cfg_or_global into vulkan auto-split.
- model config modal: "Split strategy" select shown with the split options; global
  Settings: default split strategy select.
- split_strategy added to the live-reload (_LOAD_AFFECTING) set so changing it
  re-applies on the next request without a restart.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -2748,7 +2748,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                 "component_quantization", "output_crf", "force_vram_update",
                 "balanced_gpu_percent", "acceleration",
                 "cache_type_k", "cache_type_v", "kv_offload", "n_batch", "n_ubatch", "n_seq_max",
-                "gpu_split", "tensor_split",
+                "gpu_split", "tensor_split", "split_strategy",
                 "turboquant", "engine", "engine_fallback",
                 "quant_backend", "kv_cache_budget_mb", "kv_cache_slots", "mmproj",
                 "auto_compact", "auto_compact_pct", "auto_compact_strategy",
@@ -3454,6 +3454,7 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "ram_watch_cuda": c.offload.ram_watch_cuda,
             "gpu_split": c.offload.gpu_split,
             "tensor_split": c.offload.tensor_split,
+            "split_strategy": c.offload.split_strategy,
         },
         "vulkan": {
             "n_gpu_layers": c.vulkan.n_gpu_layers,
@@ -3634,6 +3635,8 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
         c.offload.gpu_split = bool(off.get("gpu_split", c.offload.gpu_split))
         if "tensor_split" in off:
             c.offload.tensor_split = off["tensor_split"] or None
+        if "split_strategy" in off:
+            c.offload.split_strategy = off["split_strategy"] or "vram"
         # Push the RAM-cap settings to live global_args so the watcher, per-load
         # budget clamp and eviction honour them without a restart.
         try:
@@ -3648,6 +3651,7 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
                 ga.ram_watch_cuda = c.offload.ram_watch_cuda
                 ga.gpu_split = c.offload.gpu_split
                 ga.tensor_split = c.offload.tensor_split
+                ga.split_strategy = c.offload.split_strategy
         except Exception:
             pass
 

codai/admin/templates/models.html

@@ -574,9 +574,15 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
         <span class="form-hint" style="font-size:11px">When off (default), a request fails if the pinned engine is down/busy-unreachable instead of running on a different card.</span>
       </div>
       <div class="form-row" id="cfg-gpu-split-row" style="margin-top:.75rem;display:none;border:1px solid var(--border,#333);border-radius:8px;padding:.6rem">
-        <label class="form-label" style="font-weight:600">Weight distribution <span class="muted" style="font-weight:400">— layers per GPU</span></label>
-        <input type="text" id="cfg-tensor-split" class="form-input" placeholder="auto (by free VRAM)">
-        <span class="form-hint" style="font-size:11px">Comma-separated ratio for the chosen "first" card then the rest, e.g. <b>0.8,0.2</b> = 80% of layers on the lead card, 20% on the other. <b>Blank = automatic</b>, split proportionally to each card's free VRAM. The slower card bottlenecks each token, so the lead/faster GPU usually gets the larger share.</span>
+        <label class="form-label" style="font-weight:600">Split strategy <span class="muted" style="font-weight:400">(when ratio is auto)</span></label>
+        <select id="cfg-split-strategy" class="form-input">
+          <option value="vram">Maximize VRAM — proportional to free memory (biggest model)</option>
+          <option value="performance">Maximize speed — fill the fast card first, least on the slow one</option>
+        </select>
+        <span class="form-hint" style="font-size:11px"><b>VRAM</b> packs the most onto every card (best for fitting a large model) but the slow card bottlenecks throughput. <b>Speed</b> keeps as much as possible on the lead/fast card and spills only the overflow to the slower one — faster, as long as it still fits.</span>
+        <label class="form-label" style="font-weight:600;margin-top:.5rem">Weight distribution <span class="muted" style="font-weight:400">— layers per GPU (manual override)</span></label>
+        <input type="text" id="cfg-tensor-split" class="form-input" placeholder="auto (uses the strategy above)">
+        <span class="form-hint" style="font-size:11px">Optional explicit ratio for the lead card then the rest, e.g. <b>0.8,0.2</b>. Overrides the strategy. Blank = automatic per the strategy above.</span>
       </div>
       <div class="form-row" id="cfg-ds4-row" style="margin-top:.75rem;display:none;border:1px solid var(--border,#333);border-radius:8px;padding:.6rem">
         <label class="form-label" style="font-weight:600">ds4 (DeepSeek V4) streaming <span class="muted" style="font-weight:400">— per-model overrides</span></label>
@@ -3286,6 +3292,7 @@ function openCfgModal(idx, cfgIdx){
   _populateEnginePin(s.gpu_split ? ('__split__:' + ((s.engine || '').trim() || (_engineNames[0] || ''))) : (s.engine || ''));
   document.getElementById('cfg-engine-fallback').checked = !!s.engine_fallback;
   document.getElementById('cfg-tensor-split').value = s.tensor_split || '';
+  document.getElementById('cfg-split-strategy').value = s.split_strategy || 'vram';
   _populateDs4(m, s);
   document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
   document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
@@ -3705,6 +3712,8 @@ async function saveModelConfig(){
     gpu_split:         document.getElementById('cfg-engine').value.trim().startsWith('__split__:'),
     tensor_split:      (document.getElementById('cfg-engine').value.trim().startsWith('__split__:')
                           ? (document.getElementById('cfg-tensor-split').value.trim() || null) : null),
+    split_strategy:    (document.getElementById('cfg-engine').value.trim().startsWith('__split__:')
+                          ? document.getElementById('cfg-split-strategy').value : null),
     ds4:               _collectDs4(),
     system_prompt:     document.getElementById('cfg-sysprompt').value.trim() || null,
     parser:            document.getElementById('cfg-parser').value,

codai/admin/templates/settings.html

@@ -121,10 +121,18 @@
     </label>
     <span class="form-hint">Default for every model: pool one model's layers across all GPUs of all backends (e.g. NVIDIA 3090 + Radeon) for more total VRAM. Off = each model stays on its own backend's card(s) (multiple same-backend cards still split). A per-model "Engine / card → All GPUs" setting overrides this. The slower card bottlenecks each token.</span>
   </div>
+  <div class="form-row" style="margin:0">
+    <label class="form-label">Default split strategy</label>
+    <select id="s-split-strategy" class="form-input">
+      <option value="vram">Maximize VRAM — proportional to free memory (biggest model)</option>
+      <option value="performance">Maximize speed — fill the fast card first, least on the slow one</option>
+    </select>
+    <span class="form-hint">Default for split models when no explicit ratio is set. <b>VRAM</b> fits the largest model; <b>Speed</b> keeps most layers on the fast card so the slow one bottlenecks less. Per-model setting overrides this.</span>
+  </div>
   <div class="form-row" style="margin:0">
     <label class="form-label">Default weight distribution <span class="muted">(when splitting)</span></label>
-    <input type="text" id="s-tensor-split" class="form-input" placeholder="auto (by free VRAM)">
-    <span class="form-hint">Comma-separated ratio in llama.cpp device order (CUDA cards first, then Vulkan), e.g. <b>0.8,0.2</b>. Blank = automatic, proportional to each card's free VRAM.</span>
+    <input type="text" id="s-tensor-split" class="form-input" placeholder="auto (by strategy)">
+    <span class="form-hint">Comma-separated ratio in llama.cpp device order (CUDA cards first, then Vulkan), e.g. <b>0.8,0.2</b>. Blank = automatic per the strategy above.</span>
   </div>
   <div class="form-row" style="margin:0">
     <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
@@ -775,6 +783,7 @@ async function loadSettings(){
     document.getElementById('s-max-ram').value = d.offload?.max_ram_gb ?? '';
     document.getElementById('s-evict-idle-ram').checked = d.offload?.evict_idle_on_ram !== false;
     document.getElementById('s-gpu-split').checked = !!d.offload?.gpu_split;
+    document.getElementById('s-split-strategy').value = d.offload?.split_strategy ?? 'vram';
     document.getElementById('s-tensor-split').value = d.offload?.tensor_split ?? '';
     document.getElementById('s-ram-leak-watch').checked = d.offload?.ram_leak_watch !== false;
     document.getElementById('s-ram-watch-cuda').checked = d.offload?.ram_watch_cuda !== false;
@@ -890,6 +899,7 @@ async function saveSettings(){
       max_ram_gb: (parseFloat(document.getElementById('s-max-ram').value) || null),
       evict_idle_on_ram: document.getElementById('s-evict-idle-ram').checked,
       gpu_split: document.getElementById('s-gpu-split').checked,
+      split_strategy: document.getElementById('s-split-strategy').value,
       tensor_split: (document.getElementById('s-tensor-split').value.trim() || null),
       ram_leak_watch: document.getElementById('s-ram-leak-watch').checked,
       ram_watch_cuda: document.getElementById('s-ram-watch-cuda').checked,

codai/backends/vulkan.py

@@ -1197,13 +1197,41 @@ class VulkanBackend(ModelBackend):
                 _adj = list(_free_dev)
                 if _reserve > 0 and 0 <= _mg < len(_adj):
                     _adj[_mg] = max(0.5, _adj[_mg] - _reserve)
-                _sum = sum(_adj)
-                if len(_adj) > 1 and _sum > 0:
-                    _parsed = [round(f / _sum, 3) for f in _adj]
-                    _note = (f" (reserved ~{_reserve:.1f} GB on GPU{_mg} for mmproj)"
-                             if _reserve > 0 else "")
-                    print(f"  gpu split    : auto tensor_split={_parsed} (by free VRAM "
-                          f"{[round(f,1) for f in _free_dev]} GB){_note}")
+                # Split strategy: "vram" (default) = proportional to free VRAM (max
+                # capacity); "performance" = fill the FAST lead card (main_gpu) first
+                # and spill only the overflow to the slower card(s), so the weak GPU
+                # holds the fewest layers and gates throughput the least.
+                _strategy = (kwargs.get('split_strategy', _raw_cfg.get('split_strategy'))
+                             or kwargs.get('_global_split_strategy') or 'vram')
+                _strategy = str(_strategy).lower()
+                _exp_total = kwargs.get('expected_vram_gb')
+                if (_strategy == 'performance' and len(_adj) > 1
+                        and _exp_total and _exp_total > 0):
+                    _w = [0.0] * len(_adj)
+                    _main_cap = _adj[_mg]
+                    if _exp_total <= _main_cap:
+                        _w[_mg] = 1.0   # fits on the fast card alone — no real spill
+                    else:
+                        _overflow = _exp_total - _main_cap
+                        _others = sum(_adj[i] for i in range(len(_adj)) if i != _mg)
+                        _w[_mg] = _main_cap
+                        for i in range(len(_adj)):
+                            if i != _mg and _others > 0:
+                                _w[i] = _overflow * (_adj[i] / _others)
+                    _sw = sum(_w)
+                    if _sw > 0:
+                        _parsed = [round(x / _sw, 3) for x in _w]
+                        print(f"  gpu split    : PERFORMANCE tensor_split={_parsed} "
+                              f"(fast card GPU{_mg} first; free {[round(f,1) for f in _free_dev]} GB, "
+                              f"model ~{_exp_total:.1f} GB)")
+                if not _parsed:
+                    _sum = sum(_adj)
+                    if len(_adj) > 1 and _sum > 0:
+                        _parsed = [round(f / _sum, 3) for f in _adj]
+                        _note = (f" (reserved ~{_reserve:.1f} GB on GPU{_mg} for mmproj)"
+                                 if _reserve > 0 else "")
+                        print(f"  gpu split    : VRAM tensor_split={_parsed} (by free VRAM "
+                              f"{[round(f,1) for f in _free_dev]} GB){_note}")
             else:
                 print(f"  gpu split    : tensor_split={_parsed} (multi-GPU pool)")
             if _parsed and _llama_accepts('tensor_split'):

codai/config.py

@@ -133,6 +133,11 @@ class OffloadConfig:
     # order (CUDA devices first, then Vulkan). e.g. "0.8,0.2" = 80% on the first GPU
     # (3090), 20% on the second (RX 580). Blank = even split across the devices.
     tensor_split: Optional[str] = None
+    # Auto-split strategy when no explicit tensor_split is given:
+    #   "vram"        → proportional to each card's free VRAM (max capacity)
+    #   "performance" → fill the fast lead card first, spill only the overflow to the
+    #                   slower card(s) so the weak GPU gates throughput the least.
+    split_strategy: str = "vram"
 
 
 @dataclass
@@ -617,7 +622,8 @@ class ConfigManager:
                 "ram_watch_soft_fraction": self.config.offload.ram_watch_soft_fraction,
                 "ram_watch_cuda": self.config.offload.ram_watch_cuda,
                 "gpu_split": self.config.offload.gpu_split,
-                "tensor_split": self.config.offload.tensor_split
+                "tensor_split": self.config.offload.tensor_split,
+                "split_strategy": self.config.offload.split_strategy
             },
             "vulkan": {
                 "n_gpu_layers": self.config.vulkan.n_gpu_layers,

codai/main.py

@@ -155,6 +155,7 @@ def build_runtime_kwargs(model_cfg, model_type):
         # confined to one card and spilled to CPU instead of using the 2nd GPU.
         'gpu_split':       bool(model_cfg.get('gpu_split', False)),
         'tensor_split':    model_cfg.get('tensor_split'),
+        'split_strategy':  model_cfg.get('split_strategy'),
         '_raw_cfg':        dict(model_cfg) if isinstance(model_cfg, dict) else {},
     }
     if model_type == "text":
@@ -238,6 +239,7 @@ def apply_model_entry_live(entry, model_types) -> int:
         "n_seq_max", "flash_attention", "flash_attn", "load_in_4bit", "load_in_8bit",
         "offload_strategy", "no_ram", "max_gpu_percent", "mmproj", "quant_backend",
         "engine", "precision", "vae_path", "vae_tiling", "component_quantization",
+        "gpu_split", "tensor_split", "split_strategy",
     )
 
     def _load_sig(c):
@@ -1045,6 +1047,7 @@ def main():
     # the GGUF loader to set llama.cpp split_mode/tensor_split.
     global_args.gpu_split = getattr(config.offload, "gpu_split", False)
     global_args.tensor_split = getattr(config.offload, "tensor_split", None)
+    global_args.split_strategy = getattr(config.offload, "split_strategy", "vram")
     # Thermal protection settings (read live by codai.models.thermal).
     global_args.thermal_cpu_enabled = config.thermal.cpu_enabled
     global_args.thermal_gpu_enabled = config.thermal.gpu_enabled

codai/models/manager.py

@@ -1111,6 +1111,7 @@ class MultiModelManager:
                 _ts_val = _cfg_or_global('tensor_split', 'tensor_split', None)
                 if _ts_val:
                     kwargs['tensor_split'] = _ts_val
+                kwargs['split_strategy'] = _cfg_or_global('split_strategy', 'split_strategy', 'vram')
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')
@@ -1235,6 +1236,7 @@ class MultiModelManager:
                 _ts_val = _cfg_or_global('tensor_split', 'tensor_split', None)
                 if _ts_val:
                     kwargs['tensor_split'] = _ts_val
+                kwargs['split_strategy'] = _cfg_or_global('split_strategy', 'split_strategy', 'vram')
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')