ui: cross-GPU split controls (global + per-model) with auto VRAM-proportional ratio

Per-model (model config modal): the "Engine / card" selector gains an "All GPUs
(split across every backend)" option. Choosing it sets gpu_split and reveals a
"Weight distribution" field (tensor_split, e.g. 0.8,0.2); blank = auto. A real
engine name still pins as before.

Global (Settings): "Split models across all GPUs" checkbox + default weight
distribution field, persisted to offload.gpu_split / offload.tensor_split and
pushed to live global_args. Per-model "All GPUs" overrides the global default.

api: settings GET/POST now expose+accept offload.gpu_split / tensor_split.

backend: when gpu_split is on and no ratio is given, auto-derive tensor_split
proportional to each device's FREE VRAM (new _per_device_free_vram_gb, llama.cpp
device order: CUDA first, then Vulkan/AMD) — e.g. 24 GB 3090 + 8 GB RX 580 ->
~0.75/0.25 — so the bigger card carries more without the user computing it.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -3452,6 +3452,8 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "ram_watch_poll_seconds": c.offload.ram_watch_poll_seconds,
             "ram_watch_soft_fraction": c.offload.ram_watch_soft_fraction,
             "ram_watch_cuda": c.offload.ram_watch_cuda,
+            "gpu_split": c.offload.gpu_split,
+            "tensor_split": c.offload.tensor_split,
         },
         "vulkan": {
             "n_gpu_layers": c.vulkan.n_gpu_layers,
@@ -3629,6 +3631,9 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
         if "ram_watch_soft_fraction" in off:
             c.offload.ram_watch_soft_fraction = float(off["ram_watch_soft_fraction"] or c.offload.ram_watch_soft_fraction)
         c.offload.ram_watch_cuda = bool(off.get("ram_watch_cuda", c.offload.ram_watch_cuda))
+        c.offload.gpu_split = bool(off.get("gpu_split", c.offload.gpu_split))
+        if "tensor_split" in off:
+            c.offload.tensor_split = off["tensor_split"] or None
         # Push the RAM-cap settings to live global_args so the watcher, per-load
         # budget clamp and eviction honour them without a restart.
         try:
@@ -3641,6 +3646,8 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
                 ga.ram_watch_poll_seconds = c.offload.ram_watch_poll_seconds
                 ga.ram_watch_soft_fraction = c.offload.ram_watch_soft_fraction
                 ga.ram_watch_cuda = c.offload.ram_watch_cuda
+                ga.gpu_split = c.offload.gpu_split
+                ga.tensor_split = c.offload.tensor_split
         except Exception:
             pass
 

codai/admin/templates/models.html

@@ -563,16 +563,22 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
       </div>
       <div class="form-row" id="cfg-engine-row" style="margin-top:.75rem;display:none">
         <label class="form-label">Engine / card</label>
-        <select id="cfg-engine" class="form-input">
+        <select id="cfg-engine" class="form-input" onchange="_updateGpuSplitVisibility()">
           <option value="">Default (auto — by capability)</option>
+          <option value="__all__">All GPUs (split across every backend)</option>
         </select>
-        <span class="form-hint" style="font-size:11px">Pin this model to a specific engine/card. Overrides the default engine. Only shown when multiple engines are running.</span>
+        <span class="form-hint" style="font-size:11px">Pin this model to a specific engine/card, or <b>All GPUs</b> to split one model across every card (e.g. NVIDIA 3090 + Radeon) for more total VRAM. Only shown when multiple engines are running.</span>
         <label style="display:flex;align-items:center;gap:.4rem;margin-top:.4rem;font-size:12px">
           <input type="checkbox" id="cfg-engine-fallback" style="width:auto;margin:0">
           Fall back to another engine if the pinned one is unavailable
         </label>
         <span class="form-hint" style="font-size:11px">When off (default), a request fails if the pinned engine is down/busy-unreachable instead of running on a different card.</span>
       </div>
+      <div class="form-row" id="cfg-gpu-split-row" style="margin-top:.75rem;display:none;border:1px solid var(--border,#333);border-radius:8px;padding:.6rem">
+        <label class="form-label" style="font-weight:600">Weight distribution <span class="muted" style="font-weight:400">— layers per GPU</span></label>
+        <input type="text" id="cfg-tensor-split" class="form-input" placeholder="auto (by free VRAM)">
+        <span class="form-hint" style="font-size:11px">Comma-separated ratio in llama.cpp device order (CUDA cards first, then Vulkan), e.g. <b>0.8,0.2</b> = 80% of layers on the first GPU (3090), 20% on the second (RX 580). <b>Blank = automatic</b>, split proportionally to each card's free VRAM. The slower card bottlenecks each token, so bias toward the faster GPU.</span>
+      </div>
       <div class="form-row" id="cfg-ds4-row" style="margin-top:.75rem;display:none;border:1px solid var(--border,#333);border-radius:8px;padding:.6rem">
         <label class="form-label" style="font-weight:600">ds4 (DeepSeek V4) streaming <span class="muted" style="font-weight:400">— per-model overrides</span></label>
         <span class="form-hint" style="font-size:11px;margin-bottom:.4rem">These tune ds4-server for THIS model (they vary by quant/size/context). Leave blank to inherit the global ds4 settings. Context window is set by <b>n_ctx</b> above.</span>
@@ -3266,8 +3272,11 @@ function openCfgModal(idx, cfgIdx){
   document.getElementById('cfg-offload-strategy').value = s.offload_strategy || 'auto';
   document.getElementById('cfg-offload-dir').value = s.offload_dir || _defaultOffloadDir;
   _applyBackendCfgVisibility(m);
-  _populateEnginePin(s.engine || '');
+  // "All GPUs" cross-split (gpu_split) takes precedence over an engine pin in the
+  // selector; otherwise show the pinned engine. Weight ratio populates the field.
+  _populateEnginePin(s.gpu_split ? '__all__' : (s.engine || ''));
   document.getElementById('cfg-engine-fallback').checked = !!s.engine_fallback;
+  document.getElementById('cfg-tensor-split').value = s.tensor_split || '';
   _populateDs4(m, s);
   document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
   document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
@@ -3565,19 +3574,30 @@ async function removeThisConfig(){
   } catch(e) { showAlert('Error: ' + e.message); }
 }
 
+// Show the weight-distribution row only when "All GPUs" (cross-GPU split) is chosen.
+function _updateGpuSplitVisibility(){
+  const sel = document.getElementById('cfg-engine');
+  const row = document.getElementById('cfg-gpu-split-row');
+  if(!sel || !row) return;
+  row.style.display = (sel.value === '__all__') ? '' : 'none';
+}
+
 // Engine-pin field: populate the datalist from declared engines and only show the
 // row when more than one engine is configured (single-engine setups don't need it).
+// The special "__all__" option (added in the HTML) means split across all GPUs.
 async function _populateEnginePin(desired){
   const row = document.getElementById('cfg-engine-row');
   const sel = document.getElementById('cfg-engine');
   try {
     if (!_engineNames || !_engineNames.length) await _loadEngineInfo();
     const want = (desired !== undefined) ? desired : sel.value;
-    sel.querySelectorAll('option:not([value=""])').forEach(o => o.remove());
-    _engineNames.forEach(n => { const o=document.createElement('option'); o.value=n; o.textContent=n; sel.appendChild(o); });
+    sel.querySelectorAll('option[value]:not([value=""]):not([value="__all__"])').forEach(o => o.remove());
+    const allOpt = sel.querySelector('option[value="__all__"]');
+    _engineNames.forEach(n => { const o=document.createElement('option'); o.value=n; o.textContent=n; sel.insertBefore(o, allOpt); });
     sel.value = want || '';   // set AFTER options exist so the selection sticks
     row.style.display = _engineNames.length > 1 ? '' : 'none';
   } catch(e) { row.style.display = 'none'; }
+  _updateGpuSplitVisibility();
 }
 
 // Per-model ds4 overrides: shown only when the ds4 engine is enabled globally and
@@ -3658,8 +3678,13 @@ async function saveModelConfig(){
     no_ram:            document.getElementById('cfg-noram').checked,
     offload_strategy:  document.getElementById('cfg-offload-strategy').value,
     offload_dir:       document.getElementById('cfg-offload-dir').value.trim() || './offload',
-    engine:            document.getElementById('cfg-engine').value.trim() || null,
+    // "All GPUs" → gpu_split (no single-engine pin); any real engine name → pin.
+    engine:            (document.getElementById('cfg-engine').value.trim() === '__all__'
+                          ? null : document.getElementById('cfg-engine').value.trim() || null),
     engine_fallback:   document.getElementById('cfg-engine-fallback').checked,
+    gpu_split:         document.getElementById('cfg-engine').value.trim() === '__all__',
+    tensor_split:      (document.getElementById('cfg-engine').value.trim() === '__all__'
+                          ? (document.getElementById('cfg-tensor-split').value.trim() || null) : null),
     ds4:               _collectDs4(),
     system_prompt:     document.getElementById('cfg-sysprompt').value.trim() || null,
     parser:            document.getElementById('cfg-parser').value,

codai/admin/templates/settings.html

@@ -114,6 +114,18 @@
     </label>
     <span class="form-hint">Unload least-recently-used idle models to free real RAM before forcing disk offload.</span>
   </div>
+  <div class="form-row" style="margin:0">
+    <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
+      <input type="checkbox" id="s-gpu-split">
+      <span>Split models across all GPUs (cross-backend pooling)</span>
+    </label>
+    <span class="form-hint">Default for every model: pool one model's layers across all GPUs of all backends (e.g. NVIDIA 3090 + Radeon) for more total VRAM. Off = each model stays on its own backend's card(s) (multiple same-backend cards still split). A per-model "Engine / card → All GPUs" setting overrides this. The slower card bottlenecks each token.</span>
+  </div>
+  <div class="form-row" style="margin:0">
+    <label class="form-label">Default weight distribution <span class="muted">(when splitting)</span></label>
+    <input type="text" id="s-tensor-split" class="form-input" placeholder="auto (by free VRAM)">
+    <span class="form-hint">Comma-separated ratio in llama.cpp device order (CUDA cards first, then Vulkan), e.g. <b>0.8,0.2</b>. Blank = automatic, proportional to each card's free VRAM.</span>
+  </div>
   <div class="form-row" style="margin:0">
     <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
       <input type="checkbox" id="s-ram-leak-watch">
@@ -762,6 +774,8 @@ async function loadSettings(){
     document.getElementById('s-offload-dir').value = d.offload?.directory ?? './offload';
     document.getElementById('s-max-ram').value = d.offload?.max_ram_gb ?? '';
     document.getElementById('s-evict-idle-ram').checked = d.offload?.evict_idle_on_ram !== false;
+    document.getElementById('s-gpu-split').checked = !!d.offload?.gpu_split;
+    document.getElementById('s-tensor-split').value = d.offload?.tensor_split ?? '';
     document.getElementById('s-ram-leak-watch').checked = d.offload?.ram_leak_watch !== false;
     document.getElementById('s-ram-watch-cuda').checked = d.offload?.ram_watch_cuda !== false;
     document.getElementById('s-ram-watch-poll').value = d.offload?.ram_watch_poll_seconds ?? '';
@@ -875,6 +889,8 @@ async function saveSettings(){
       directory: document.getElementById('s-offload-dir').value.trim() || './offload',
       max_ram_gb: (parseFloat(document.getElementById('s-max-ram').value) || null),
       evict_idle_on_ram: document.getElementById('s-evict-idle-ram').checked,
+      gpu_split: document.getElementById('s-gpu-split').checked,
+      tensor_split: (document.getElementById('s-tensor-split').value.trim() || null),
       ram_leak_watch: document.getElementById('s-ram-leak-watch').checked,
       ram_watch_cuda: document.getElementById('s-ram-watch-cuda').checked,
       ram_watch_poll_seconds: (parseFloat(document.getElementById('s-ram-watch-poll').value) || null),

codai/backends/vulkan.py

@@ -296,6 +296,40 @@ def _pooled_free_vram_gb(cross: bool = False) -> float:
     return total if total > 0 else _free_vram_gb(0)
 
 
+def _per_device_free_vram_gb() -> list:
+    """Free VRAM (GB) per device in llama.cpp's device order — CUDA devices first
+    (by index), then Vulkan/AMD cards (amdgpu sysfs, sorted) — matching how
+    tensor_split is interpreted. Used to auto-derive a VRAM-proportional split when
+    the user enables cross-GPU pooling without specifying a ratio."""
+    out = []
+    try:
+        import torch
+        if torch.cuda.is_available():
+            for i in range(torch.cuda.device_count()):
+                try:
+                    free, _ = torch.cuda.mem_get_info(i)
+                    out.append(free / (1024 ** 3))
+                except Exception:
+                    out.append(0.0)
+    except Exception:
+        pass
+    try:
+        import glob
+        for tp in sorted(glob.glob('/sys/class/drm/card*/device/mem_info_vram_total')):
+            up = tp.replace('vram_total', 'vram_used')
+            try:
+                with open(tp) as f:
+                    _t = int(f.read().strip())
+                with open(up) as f:
+                    _u = int(f.read().strip())
+                out.append((_t - _u) / (1024 ** 3))
+            except Exception:
+                pass
+    except Exception:
+        pass
+    return out
+
+
 def _ggml_kv_type(name):
     """Map a KV-cache quant name to the llama.cpp GGML type int, or None.
 
@@ -1082,19 +1116,31 @@ class VulkanBackend(ModelBackend):
         _ts = kwargs.get('tensor_split', _raw_cfg.get('tensor_split'))
         if _ts is None:
             _ts = kwargs.get('_global_tensor_split')
-        if _gpu_split and _ts:
+        if _gpu_split:
             _parsed = []
-            for _p in str(_ts).replace(' ', '').split(','):
-                if _p == '':
-                    continue
-                try:
-                    _parsed.append(float(_p))
-                except ValueError:
-                    _parsed = []
-                    break
+            if _ts:
+                # Explicit ratio.
+                for _p in str(_ts).replace(' ', '').split(','):
+                    if _p == '':
+                        continue
+                    try:
+                        _parsed.append(float(_p))
+                    except ValueError:
+                        _parsed = []
+                        break
+            if not _parsed:
+                # Auto: distribute proportionally to each device's free VRAM (a 24 GB
+                # 3090 + 8 GB RX 580 → ~0.75/0.25), so the bigger card carries more.
+                _free_dev = _per_device_free_vram_gb()
+                _sum = sum(_free_dev)
+                if len(_free_dev) > 1 and _sum > 0:
+                    _parsed = [round(f / _sum, 3) for f in _free_dev]
+                    print(f"  gpu split    : auto tensor_split={_parsed} (by free VRAM "
+                          f"{[round(f,1) for f in _free_dev]} GB)")
+            else:
+                print(f"  gpu split    : tensor_split={_parsed} (multi-GPU pool, CUDA-first order)")
             if _parsed and _llama_accepts('tensor_split'):
                 llama_kwargs['tensor_split'] = _parsed
-                print(f"  gpu split    : tensor_split={_parsed} (multi-GPU pool, CUDA-first order)")
 
         # Multimodal projector (mmproj): pairs a CLIP/vision projector GGUF with
         # this text model so it can accept images — the llama.cpp `--mmproj`