gpu: per-engine backend isolation (fix docker cross-GPU split) + opt-in...

gpu: per-engine backend isolation (fix docker cross-GPU split) + opt-in per-model cross-backend pooling

Phase 1 — fix the docker-only "model loads on both GPUs":
- gpu_detect.vendor_env detects each Vulkan device's vendor and pins each engine
  to ONLY its own backend's cards by real indices (not assumed 0..n-1). When a
  vendor has no Vulkan device (e.g. NVIDIA in a container that lacks nvidia_icd.json
  because the toolkit only injects it with the graphics capability), the engine
  gets ZERO Vulkan and runs CUDA-only instead of falling back to all ICDs and
  grabbing the Radeon via RADV. Same-backend split (e.g. 2x 3090) is preserved.

Phase 2 — opt-in cross-backend GPU pooling, per model:
- OffloadConfig.gpu_split (default off) + tensor_split ("0.8,0.2", llama.cpp
  device order: CUDA first then Vulkan); global default + per-model override.
- vendor_env(allow_cross=…) exposes the foreign card when enabled; the engine
  supervisor passes it from config.
- manager threads gpu_split/tensor_split (per-model via _raw_cfg, else global via
  global_args) into the GGUF loader; vulkan.py sets llama.cpp tensor_split when on
  and otherwise leaves split_mode=LAYER so same-backend split still works.
- admin model-configure accepts gpu_split + tensor_split per model.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -2748,6 +2748,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                 "component_quantization", "output_crf", "force_vram_update",
                 "balanced_gpu_percent", "acceleration",
                 "cache_type_k", "cache_type_v", "kv_offload", "n_batch", "n_ubatch", "n_seq_max",
+                "gpu_split", "tensor_split",
                 "turboquant", "engine", "engine_fallback",
                 "quant_backend", "kv_cache_budget_mb", "kv_cache_slots", "mmproj",
                 "auto_compact", "auto_compact_pct", "auto_compact_strategy",

codai/backends/vulkan.py

@@ -1013,6 +1013,41 @@ class VulkanBackend(ModelBackend):
                     print(f"  slots        : n_seq_max={_n_seq} requested but llama-cpp-python "
                           f"{getattr(_llama_cpp, '__version__', '?')} doesn't expose it — ignoring")
 
+        # Cross-GPU layer split (opt-in, per-model). gpu_split=True lets llama.cpp
+        # spread this model's layers across multiple visible GPU devices (e.g. an
+        # NVIDIA 3090 via CUDA + a Radeon via Vulkan) for more total VRAM. The
+        # engine's device visibility is set by the front (GGML_VK_VISIBLE_DEVICES /
+        # CUDA_VISIBLE_DEVICES); here we only choose HOW to distribute:
+        #   - gpu_split off → split_mode NONE: keep the whole model on main_gpu (one
+        #     card), so a foreign card the engine can see isn't used by accident.
+        #   - gpu_split on  → split_mode LAYER + optional tensor_split ratio
+        #     ("0.8,0.2" in llama.cpp device order: CUDA devices first, then Vulkan).
+        # By default (gpu_split off) we DON'T touch split_mode: llama.cpp's default
+        # LAYER split spreads the model across the engine's visible cards — which the
+        # front has already isolated to this engine's own backend (e.g. both 3090s),
+        # never the foreign Radeon. When gpu_split is on AND a tensor_split ratio is
+        # given, we set the per-device ratio (llama.cpp device order: CUDA first,
+        # then Vulkan), e.g. "0.8,0.2" = 80% on the 3090, 20% on the RX 580.
+        _gpu_split = kwargs.get('gpu_split', _raw_cfg.get('gpu_split'))
+        if _gpu_split is None:
+            _gpu_split = bool(kwargs.get('_global_gpu_split', False))
+        _ts = kwargs.get('tensor_split', _raw_cfg.get('tensor_split'))
+        if _ts is None:
+            _ts = kwargs.get('_global_tensor_split')
+        if _gpu_split and _ts:
+            _parsed = []
+            for _p in str(_ts).replace(' ', '').split(','):
+                if _p == '':
+                    continue
+                try:
+                    _parsed.append(float(_p))
+                except ValueError:
+                    _parsed = []
+                    break
+            if _parsed and _llama_accepts('tensor_split'):
+                llama_kwargs['tensor_split'] = _parsed
+                print(f"  gpu split    : tensor_split={_parsed} (multi-GPU pool, CUDA-first order)")
+
         # Multimodal projector (mmproj): pairs a CLIP/vision projector GGUF with
         # this text model so it can accept images — the llama.cpp `--mmproj`
         # equivalent, which adds vision capability (e.g. gemma). Uses llama.cpp's

codai/config.py

@@ -122,6 +122,17 @@ class OffloadConfig:
     ram_watch_poll_seconds: float = 15.0    # how often the watcher samples RSS
     ram_watch_soft_fraction: float = 0.90   # mitigate at/above this fraction of the cap
     ram_watch_cuda: bool = True             # allow mitigation to call CUDA empty_cache()
+    # Cross-backend GPU pooling. OFF by default: each engine uses only its own
+    # backend's GPUs (CUDA for an NVIDIA engine, Vulkan for a Radeon engine) and a
+    # model still splits across multiple SAME-backend cards (e.g. 2× 3090). When ON,
+    # an engine may ALSO pool a model across a different backend's card (e.g. NVIDIA
+    # 3090 via CUDA + Radeon RX 580 via Vulkan) for more total VRAM — slower, since
+    # the weakest card bottlenecks each token.
+    gpu_split: bool = False
+    # llama.cpp per-device layer ratio when a model is split, in llama.cpp device
+    # order (CUDA devices first, then Vulkan). e.g. "0.8,0.2" = 80% on the first GPU
+    # (3090), 20% on the second (RX 580). Blank = even split across the devices.
+    tensor_split: Optional[str] = None
 
 
 @dataclass

codai/frontproxy/engine_supervisor.py

@@ -193,6 +193,7 @@ class EngineSupervisor:
         specs = getattr(srv, "engine_specs", None)
         engines = []
 
+        _cross = bool(getattr(getattr(self.config, "offload", None), "gpu_split", False))
         if specs:
             from codai.frontproxy.gpu_detect import vendor_env
             for idx, spec in enumerate(specs):
@@ -203,7 +204,7 @@ class EngineSupervisor:
                 gpus_kw = (spec.get("gpus") or "").strip().lower()
                 if not gpus_kw and not spec.get("env") and backend == "nvidia":
                     gpus_kw = "nvidia"
-                detected = vendor_env(gpus_kw) if gpus_kw else {}
+                detected = vendor_env(gpus_kw, allow_cross=_cross) if gpus_kw else {}
                 explicit = {str(k): str(v) for k, v in (spec.get("env") or {}).items()}
                 env = {**detected, **explicit}     # explicit overrides detected
                 # Tell the engine which physical cards it owns, so thermal
@@ -241,7 +242,7 @@ class EngineSupervisor:
             return engines
 
         for idx, (name, vkw, backend) in enumerate(plan):
-            env = vendor_env(vkw)
+            env = vendor_env(vkw, allow_cross=_cross)
             sels = _gpu_selectors({"backend": backend, "gpus": vkw}, env)
             if sels:
                 env["CODERAI_ENGINE_GPUS"] = ",".join(sels)

codai/frontproxy/gpu_detect.py

@@ -158,12 +158,17 @@ def find_vulkan_icd(vendor: str) -> str:
     return ""
 
 
-def vendor_env(vendor: str) -> dict:
+def vendor_env(vendor: str, allow_cross: bool = False) -> dict:
     """Env that pins an engine to **all** of ``vendor``'s cards on this machine.
 
     NVIDIA: CUDA visible = all NVIDIA UUIDs (+ PCI_BUS_ID order), Vulkan ICD = nvidia.
     AMD/Intel: CUDA hidden (""), Vulkan ICD = that vendor's, so it sees only those
-    cards. Missing tools degrade gracefully (the key is simply omitted)."""
+    cards. Missing tools degrade gracefully (the key is simply omitted).
+
+    ``allow_cross`` (opt-in) lets this engine ALSO use other vendors' Vulkan cards
+    — e.g. an NVIDIA engine pooling a model across the 3090 (CUDA) and a Radeon
+    (Vulkan) for more total VRAM. Off by default: each engine stays on its own
+    backend's GPUs (and still splits across multiple same-backend cards)."""
     vendor = _norm_vendor(vendor)
     env = {}
     if vendor == "nvidia":
@@ -178,16 +183,41 @@ def vendor_env(vendor: str) -> dict:
     # non-AMD engine can't pick up a Radeon (mirrors CUDA hiding for non-NVIDIA).
     if vendor != "amd":
         env["RADEON_VISIBLE_DEVICES"] = ""
+    # Vulkan isolation. Detect each Vulkan device's vendor (vulkan_devices() reads
+    # vendorid and preserves Vulkan index order) and pin this engine to ONLY its
+    # vendor's cards — by their REAL indices, never an assumed 0..n-1 order.
     icd = find_vulkan_icd(vendor)
-    if icd:
+    vk_devs = vulkan_devices()
+    vendor_idx = [i for i, d in enumerate(vk_devs) if d.get("vendor") == vendor]
+    if allow_cross:
+        # Opt-in cross-backend pooling: expose EVERY enumerated Vulkan device (any
+        # vendor) so llama.cpp can split a model across backends. Don't pin the ICD.
+        if vk_devs:
+            env["GGML_VK_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(len(vk_devs)))
+    elif icd:
+        # The vendor ICD exists: point the loader at it so ONLY this vendor's cards
+        # are visible, then they are reindexed 0..n-1.
         env["VK_ICD_FILENAMES"] = icd
-    # After ICD isolation only THIS vendor's card(s) are visible to Vulkan, as
-    # indices 0..n-1. Pin GGML_VK_VISIBLE_DEVICES to those indices so an inherited
-    # value (e.g. a launcher exporting GGML_VK_VISIBLE_DEVICES=1 from the old
-    # multi-vendor enumeration) can't select an invalid index and silently fall back
-    # to CPU. Default to "0" when the count can't be determined (single card).
-    _n = sum(1 for d in vulkan_devices() if d.get("vendor") == vendor)
-    env["GGML_VK_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(max(1, _n)))
+        n = len(vendor_idx)
+        if n == 0 and vendor in gpu_vendors():
+            n = 1  # vulkaninfo couldn't enumerate (e.g. missing) but the card exists
+        env["GGML_VK_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(n)) if n else ""
+    elif vendor_idx:
+        # No vendor ICD to filter with, but this vendor DOES have Vulkan device(s):
+        # select their ACTUAL indices in the unfiltered enumeration (detected, not
+        # assumed), so we never grab a different vendor's card sitting at index 0.
+        env["GGML_VK_VISIBLE_DEVICES"] = ",".join(str(i) for i in vendor_idx)
+    else:
+        # No vendor ICD AND no Vulkan device for this vendor — e.g. an NVIDIA engine
+        # in a container that lacks nvidia_icd.json (the toolkit only injects it with
+        # the graphics capability). Previously VK_ICD_FILENAMES was left unset, so the
+        # loader fell back to ALL ICDs and the engine grabbed another vendor's card
+        # (the RX 580 via RADV) — splitting the model across the wrong GPU. Give this
+        # engine ZERO Vulkan devices instead (it runs on CUDA only); point the loader
+        # at a non-existent ICD so no Vulkan driver loads. Per-engine, so the AMD
+        # engine still keeps its Vulkan.
+        env["VK_ICD_FILENAMES"] = os.path.join(_ICD_DIRS[0], "_coderai_no_vulkan.json")
+        env["GGML_VK_VISIBLE_DEVICES"] = ""
     return env
 
 

codai/main.py

@@ -1012,6 +1012,10 @@ def main():
     global_args.ram_watch_poll_seconds = config.offload.ram_watch_poll_seconds
     global_args.ram_watch_soft_fraction = config.offload.ram_watch_soft_fraction
     global_args.ram_watch_cuda = config.offload.ram_watch_cuda
+    # Cross-backend GPU pooling defaults (per-model config can override). Read by
+    # the GGUF loader to set llama.cpp split_mode/tensor_split.
+    global_args.gpu_split = getattr(config.offload, "gpu_split", False)
+    global_args.tensor_split = getattr(config.offload, "tensor_split", None)
     # Thermal protection settings (read live by codai.models.thermal).
     global_args.thermal_cpu_enabled = config.thermal.cpu_enabled
     global_args.thermal_gpu_enabled = config.thermal.gpu_enabled

codai/models/manager.py

@@ -1106,6 +1106,11 @@ class MultiModelManager:
                         kwargs[_kvk] = _kvv
                 if _raw and '_raw_cfg' not in kwargs:
                     kwargs['_raw_cfg'] = _raw
+                # Cross-backend GPU pooling: per-model overrides global (global_args).
+                kwargs['gpu_split'] = _cfg_or_global('gpu_split', 'gpu_split', False)
+                _ts_val = _cfg_or_global('tensor_split', 'tensor_split', None)
+                if _ts_val:
+                    kwargs['tensor_split'] = _ts_val
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')
@@ -1225,6 +1230,11 @@ class MultiModelManager:
                         kwargs[_kvk] = _kvv
                 if _raw and '_raw_cfg' not in kwargs:
                     kwargs['_raw_cfg'] = _raw
+                # Cross-backend GPU pooling: per-model overrides global (global_args).
+                kwargs['gpu_split'] = _cfg_or_global('gpu_split', 'gpu_split', False)
+                _ts_val = _cfg_or_global('tensor_split', 'tensor_split', None)
+                if _ts_val:
+                    kwargs['tensor_split'] = _ts_val
                 no_ram = _cfg_or_global('no_ram', 'no_ram', False)
                 kwargs['no_ram'] = bool(no_ram)
                 offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')