ds4: configurable CUDA env knobs (expert-cache reserve + free-form extra_env)

ds4-server exposes several CUDA tunables only via environment, not CLI flags.
By default ds4 reserves half the card for non-cache use and allocates the model
weight arena in 1792 MiB chunks — both starve / OOM the streaming expert cache
on small-weight MoE models served from SSD.

Pass an explicit env to ds4-server (Popen now sets env=) with:
  - expert_cache_reserve_gb: typed knob -> DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB
    (0 = leave ds4's default).
  - extra_env: free-form KEY=VALUE passthrough for the rest, e.g.
    DS4_CUDA_WEIGHT_ARENA_CHUNK_MB=512 to shrink the weight-arena chunk so it
    fits a heap fragmented by the expert cache.

Both surfaced in Settings (config + admin GET/POST + UI), default to no-op so
behaviour is unchanged unless set.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -3403,6 +3403,8 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "ctx": c.ds4.ctx,
             "ssd_streaming": c.ds4.ssd_streaming,
             "extra_args": c.ds4.extra_args,
+            "expert_cache_reserve_gb": c.ds4.expert_cache_reserve_gb,
+            "extra_env": c.ds4.extra_env,
             "auto_build": c.ds4.auto_build,
         },
         "broker": {
@@ -3683,6 +3685,13 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
             c.ds4.ctx = max(1024, int(d.get("ctx") or c.ds4.ctx))
         if "extra_args" in d:
             c.ds4.extra_args = (d.get("extra_args") or "").strip()
+        if "expert_cache_reserve_gb" in d:
+            try:
+                c.ds4.expert_cache_reserve_gb = max(0, int(d.get("expert_cache_reserve_gb") or 0))
+            except (TypeError, ValueError):
+                c.ds4.expert_cache_reserve_gb = 0
+        if "extra_env" in d:
+            c.ds4.extra_env = (d.get("extra_env") or "").strip()
         if "auto_build" in d:
             c.ds4.auto_build = bool(d["auto_build"])
 

codai/admin/templates/settings.html

@@ -469,6 +469,16 @@
       <input type="text" id="s-ds4-extra-args" class="form-input" placeholder="--ssd-streaming-cache-experts 12GB --ssd-streaming-cold">
       <span class="form-hint">Passed verbatim to <code>ds4-server</code>. <code>--kv-disk-dir</code> is auto-set to coderai's offload dir unless you override it here. With SSD streaming on CUDA, sizing the routed expert cache helps, e.g. <code>--ssd-streaming-cache-experts 12GB</code>. Note: extremely low quants (Q2_K) can fail ds4's CUDA prefill — prefer Q4_K+.</span>
     </div>
+    <div class="form-row">
+      <label class="form-label">CUDA expert-cache VRAM reserve (GiB)</label>
+      <input type="number" id="s-ds4-expert-cache-reserve" class="form-input" min="0" placeholder="0 = ds4 default (½ the card)">
+      <span class="form-hint">Exports <code>DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB</code> — VRAM ds4 keeps free for non-cache use under SSD streaming. ds4 defaults to <b>half the card</b>, which over-reserves for small-weight MoE models and caps the expert cache far below your <code>--ssd-streaming-cache-experts</code> request. Set just above the model's resident weights (+~2 GiB headroom) to hand the rest to the cache; <code>0</code> leaves ds4's default. There is no CLI flag for this — it's env-only.</span>
+    </div>
+    <div class="form-row">
+      <label class="form-label">Extra ds4-server env</label>
+      <input type="text" id="s-ds4-extra-env" class="form-input" placeholder="DS4_CUDA_WEIGHT_ARENA_CHUNK_MB=512">
+      <span class="form-hint">Whitespace-separated <code>KEY=VALUE</code> pairs added to ds4-server's environment — for the many CUDA tunables it exposes only via env. Notably <code>DS4_CUDA_WEIGHT_ARENA_CHUNK_MB</code> (default 1792): lower it (e.g. <code>512</code>) so the weight arena allocates in smaller chunks that fit a heap fragmented by the streaming expert cache, fixing <i>model arena alloc failed … out of memory</i>.</span>
+    </div>
   </div>
 </div>
 {% endblock %}
@@ -686,6 +696,8 @@ async function loadSettings(){
     document.getElementById('s-ds4-ctx').value = ds4.ctx ?? 100000;
     document.getElementById('s-ds4-repo-url').value = ds4.repo_url ?? 'https://github.com/antirez/ds4';
     document.getElementById('s-ds4-extra-args').value = ds4.extra_args ?? '';
+    document.getElementById('s-ds4-expert-cache-reserve').value = ds4.expert_cache_reserve_gb ?? 0;
+    document.getElementById('s-ds4-extra-env').value = ds4.extra_env ?? '';
     toggleDs4Fields();
   }catch(e){ showAlert('error','Failed to load settings: '+e.message); }
 }
@@ -761,6 +773,8 @@ async function saveSettings(){
       ctx: parseInt(document.getElementById('s-ds4-ctx').value) || 100000,
       repo_url: document.getElementById('s-ds4-repo-url').value.trim() || 'https://github.com/antirez/ds4',
       extra_args: document.getElementById('s-ds4-extra-args').value.trim(),
+      expert_cache_reserve_gb: parseInt(document.getElementById('s-ds4-expert-cache-reserve').value) || 0,
+      extra_env: document.getElementById('s-ds4-extra-env').value.trim(),
     },
     broker:{
       enabled: document.getElementById('s-broker-enabled').checked,

codai/api/ds4_worker.py

@@ -419,10 +419,41 @@ def ensure_service(cfg, model_file: Optional[str] = None,
             import shlex
             cmd += shlex.split(extra)
 
-        print(f"[ds4] launching ds4-server: {' '.join(cmd)}", flush=True)
+        # ds4-server inherits our environment. The CUDA streaming expert-cache
+        # reserve (VRAM kept free for non-cache use) is tunable ONLY via env, not a
+        # CLI flag; ds4 defaults it to half the card, which starves the cache for
+        # small-weight MoE models. Export the configured override when set (>0).
+        env = os.environ.copy()
+        try:
+            reserve_gb = int(getattr(cfg, "expert_cache_reserve_gb", 0) or 0)
+        except (TypeError, ValueError):
+            reserve_gb = 0
+        if reserve_gb > 0 and "DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB" not in env:
+            env["DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB"] = str(reserve_gb)
+
+        # Free-form env passthrough (KEY=VALUE pairs) for ds4's many CUDA tunables;
+        # an explicit pair here overrides the dedicated reserve knob above.
+        applied_env = {}
+        extra_env = (getattr(cfg, "extra_env", "") or "").strip()
+        if extra_env:
+            import shlex
+            for tok in shlex.split(extra_env):
+                if "=" in tok:
+                    k, v = tok.split("=", 1)
+                    k = k.strip()
+                    if k:
+                        env[k] = v
+                        applied_env[k] = v
+        if reserve_gb > 0:
+            applied_env.setdefault("DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB",
+                                   env["DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB"])
+
+        env_note = ("  (" + " ".join(f"{k}={v}" for k, v in applied_env.items()) + ")"
+                    if applied_env else "")
+        print(f"[ds4] launching ds4-server: {' '.join(cmd)}{env_note}", flush=True)
         proc = subprocess.Popen(
             cmd, cwd=str(install_dir), stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT, text=True, bufsize=1,
+            stderr=subprocess.STDOUT, text=True, bufsize=1, env=env,
         )
         tail = collections.deque(maxlen=15)
         threading.Thread(target=_pump_logs, args=(proc, tail), daemon=True).start()

codai/config.py

@@ -242,6 +242,18 @@ class Ds4Config:
     ctx: int = 100000                      # ds4-server --ctx context window
     ssd_streaming: bool = False            # ds4-server --ssd-streaming: stream experts from SSD/disk
     extra_args: str = ""                   # extra flags passed to ds4-server
+    # VRAM (GiB) ds4-server keeps free for non-cache use on CUDA, exported as
+    # DS4_CUDA_STREAMING_EXPERT_CACHE_RESERVE_GB. ds4 defaults this to half the
+    # card, which over-reserves for small-weight MoE models and starves the
+    # streaming expert cache. 0 = leave ds4's default. Set just above the model's
+    # resident weights (+~2 GiB headroom) to hand the rest to the expert cache.
+    expert_cache_reserve_gb: int = 0
+    # Free-form environment for ds4-server, as whitespace/newline-separated
+    # KEY=VALUE pairs. ds4 exposes many CUDA tunables only via env, e.g.
+    # DS4_CUDA_WEIGHT_ARENA_CHUNK_MB (default 1792) — lower it (e.g. 512) so the
+    # model-weight arena allocates in smaller chunks that fit a heap fragmented by
+    # the streaming expert cache, avoiding "model arena alloc failed … OOM".
+    extra_env: str = ""
     auto_build: bool = True                # clone+build the binary if it's missing
 
 
@@ -609,6 +621,8 @@ class ConfigManager:
                 "ctx": self.config.ds4.ctx,
                 "ssd_streaming": self.config.ds4.ssd_streaming,
                 "extra_args": self.config.ds4.extra_args,
+                "expert_cache_reserve_gb": self.config.ds4.expert_cache_reserve_gb,
+                "extra_env": self.config.ds4.extra_env,
                 "auto_build": self.config.ds4.auto_build,
             },
             "broker": {