ds4: on-disk KV-cache age cleanup + HF search on Enter

- ds4: configurable janitor that age-prunes the on-disk KV-checkpoint cache
  (--kv-disk-dir, defaulted to <offload>/ds4-kv). New Ds4Config fields
  kv_cache_cleanup_enabled / kv_cache_max_age_hours (7d) /
  kv_cache_cleanup_interval_minutes (6h); new codai/api/ds4_kv_janitor.py
  reuses the tmp_janitor sweep (newest-mtime, so active sessions are spared),
  started from main.py only when ds4 + cleanup are both on. Settings UI +
  get/save wired.
- ds4: corrected the perf note — i-quants (IQ2/IQ3) and Q2_K load but fail
  ds4's CUDA prefill (gpu layer 0 ffn batch encode failed → empty reply);
  use K-quants Q4_K and up.
- models: pressing Enter in the HuggingFace search field now runs the search.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -3432,6 +3432,9 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "expert_cache_reserve_gb": c.ds4.expert_cache_reserve_gb,
             "extra_env": c.ds4.extra_env,
             "auto_build": c.ds4.auto_build,
+            "kv_cache_cleanup_enabled": c.ds4.kv_cache_cleanup_enabled,
+            "kv_cache_max_age_hours": c.ds4.kv_cache_max_age_hours,
+            "kv_cache_cleanup_interval_minutes": c.ds4.kv_cache_cleanup_interval_minutes,
         },
         "compaction": {
             "enabled": c.compaction.enabled,
@@ -3726,6 +3729,18 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
             c.ds4.extra_env = (d.get("extra_env") or "").strip()
         if "auto_build" in d:
             c.ds4.auto_build = bool(d["auto_build"])
+        if "kv_cache_cleanup_enabled" in d:
+            c.ds4.kv_cache_cleanup_enabled = bool(d["kv_cache_cleanup_enabled"])
+        if "kv_cache_max_age_hours" in d:
+            try:
+                c.ds4.kv_cache_max_age_hours = max(0.0, float(d["kv_cache_max_age_hours"]))
+            except (TypeError, ValueError):
+                pass
+        if "kv_cache_cleanup_interval_minutes" in d:
+            try:
+                c.ds4.kv_cache_cleanup_interval_minutes = max(1.0, float(d["kv_cache_cleanup_interval_minutes"]))
+            except (TypeError, ValueError):
+                pass
 
     if "compaction" in data:
         cp = data["compaction"] or {}

codai/admin/templates/models.html

@@ -153,7 +153,7 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
 
     <!-- query row -->
     <div class="search-bar" style="margin-bottom:.75rem">
-      <input type="text" id="search-q" class="form-input" placeholder="Search models (e.g. llama, mistral, qwen…)">
+      <input type="text" id="search-q" class="form-input" placeholder="Search models (e.g. llama, mistral, qwen…)" onkeydown="if(event.key==='Enter'){event.preventDefault();doSearch();}">
       <button class="btn btn-secondary" onclick="doSearch()">Search</button>
     </div>
 

codai/admin/templates/settings.html

@@ -488,9 +488,28 @@
         <br>• <b>Reserve</b> ≈ resident weights + ~2 GiB (field above). ds4 defaults to half the card, which starves the cache.
         <br>• Add <code>DS4_CUDA_WEIGHT_ARENA_CHUNK_MB=512</code> (env) to stop the weight-arena OOM under heavy caching.
         <br>• <b>Avoid</b> <code>DS4_CUDA_WEIGHT_CACHE</code> — it disables ds4's zero-copy path and was slower in testing.
-        <br>• Decode of a model far larger than VRAM is streaming-bound (~0.5 tok/s here); the only real fix is a <b>smaller quant</b> (IQ2/IQ3) that mostly fits VRAM+RAM. Context window is set per-model via <b>n_ctx</b>.
+        <br>• Decode of a model far larger than VRAM is streaming-bound (~0.5 tok/s here); a <b>smaller quant</b> that mostly fits VRAM+RAM is the only real decode fix. But <b>i-quants (IQ2/IQ3) and Q2_K fail ds4's CUDA prefill</b> (<i>gpu layer 0 ffn batch encode failed</i> → empty reply) — they load but can't generate. Use <b>K-quants Q4_K and up</b>. Context window is set per-model via <b>n_ctx</b>.
       </span>
     </div>
+    <div class="form-row">
+      <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
+        <input type="checkbox" id="s-ds4-kv-cleanup" onchange="toggleDs4KvCleanup()">
+        <span style="font-size:13px;font-weight:500">Auto-clean the on-disk KV cache</span>
+      </label>
+      <span class="form-hint">ds4 writes prompt KV checkpoints to its <code>--kv-disk-dir</code> (<code>&lt;offload&gt;/ds4-kv</code>) to reuse prefixes across requests; abandoned sessions accumulate large files that never self-prune. When on, a janitor deletes entries whose newest file is older than the age below. Active (recently-reused) checkpoints are spared.</span>
+    </div>
+    <div id="ds4-kv-cleanup-fields" style="display:none">
+      <div class="form-row" style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem">
+        <div>
+          <label class="form-label">Delete older than <span class="muted">(hours)</span></label>
+          <input type="number" id="s-ds4-kv-max-age" class="form-input" min="0" step="1" placeholder="168 (7 days)">
+        </div>
+        <div>
+          <label class="form-label">Sweep every <span class="muted">(minutes)</span></label>
+          <input type="number" id="s-ds4-kv-interval" class="form-input" min="1" step="1" placeholder="360 (6 hours)">
+        </div>
+      </div>
+    </div>
   </div>
 </div>
 
@@ -599,6 +618,10 @@ function toggleCompactFields(){
   document.getElementById('compact-fields').style.display =
     document.getElementById('s-compact-enabled').checked ? 'block' : 'none';
 }
+function toggleDs4KvCleanup(){
+  document.getElementById('ds4-kv-cleanup-fields').style.display =
+    document.getElementById('s-ds4-kv-cleanup').checked ? 'block' : 'none';
+}
 function toggleHttps(){
   document.getElementById('https-fields').style.display =
     document.getElementById('s-https').checked ? 'block' : 'none';
@@ -755,6 +778,10 @@ async function loadSettings(){
     document.getElementById('s-ds4-extra-args').value = ds4.extra_args ?? '';
     document.getElementById('s-ds4-expert-cache-reserve').value = ds4.expert_cache_reserve_gb ?? 0;
     document.getElementById('s-ds4-extra-env').value = ds4.extra_env ?? '';
+    document.getElementById('s-ds4-kv-cleanup').checked = !!ds4.kv_cache_cleanup_enabled;
+    document.getElementById('s-ds4-kv-max-age').value = ds4.kv_cache_max_age_hours ?? 168;
+    document.getElementById('s-ds4-kv-interval').value = ds4.kv_cache_cleanup_interval_minutes ?? 360;
+    toggleDs4KvCleanup();
     toggleDs4Fields();
   }catch(e){ showAlert('error','Failed to load settings: '+e.message); }
 }
@@ -832,6 +859,9 @@ async function saveSettings(){
       extra_args: document.getElementById('s-ds4-extra-args').value.trim(),
       expert_cache_reserve_gb: parseInt(document.getElementById('s-ds4-expert-cache-reserve').value) || 0,
       extra_env: document.getElementById('s-ds4-extra-env').value.trim(),
+      kv_cache_cleanup_enabled: document.getElementById('s-ds4-kv-cleanup').checked,
+      kv_cache_max_age_hours: parseFloat(document.getElementById('s-ds4-kv-max-age').value) || 168,
+      kv_cache_cleanup_interval_minutes: parseFloat(document.getElementById('s-ds4-kv-interval').value) || 360,
     },
     compaction:{
       enabled: document.getElementById('s-compact-enabled').checked,

codai/api/ds4_kv_janitor.py

+"""Periodic cleanup of the ds4 on-disk KV-checkpoint cache.
+
+ds4-server writes prompt KV checkpoints to its ``--kv-disk-dir`` (coderai defaults
+this to ``<offload>/ds4-kv``) so a prefix can be reused across requests without
+recomputing it. Those checkpoints are never self-pruned, so abandoned sessions
+accumulate on disk indefinitely (a streamed MoE's KV files are large).
+
+This background janitor age-prunes that directory: every
+``interval_minutes`` it deletes top-level entries whose most-recent mtime is older
+than ``max_age_hours``. Age is taken from the *newest* file in each entry, so an
+actively-reused checkpoint (touched recently) is spared while stale ones are
+reclaimed.
+
+It reuses the proven sweep primitives from ``codai.models.tmp_janitor`` and keeps
+its own module-level state (a separate, independently-configured janitor). Started
+once from ``codai.main`` in the engine process that owns the ds4 worker.
+"""
+import os
+import threading
+import time
+import logging
+from typing import Optional, Dict, Any
+
+from codai.models.tmp_janitor import _sweep, _FORBIDDEN
+
+_log = logging.getLogger(__name__)
+
+_state_lock = threading.Lock()
+_state: Dict[str, Any] = {
+    "enabled": False,
+    "kv_dir": None,
+    "max_age_hours": None,
+    "interval_minutes": None,
+    "last_run_ts": 0.0,
+    "last_removed": 0,
+    "total_removed": 0,
+    "last_freed_bytes": 0,
+    "runs": 0,
+}
+_thread: Optional[threading.Thread] = None
+_started = False
+
+
+def get_status() -> Dict[str, Any]:
+    """Snapshot for the admin status endpoint / dashboard."""
+    with _state_lock:
+        return dict(_state)
+
+
+def _run(kv_dir: str, max_age_hours: float, interval_minutes: float) -> None:
+    max_age_seconds = max(0.0, max_age_hours) * 3600.0
+    interval = max(60.0, interval_minutes * 60.0)
+    while True:
+        try:
+            # The dir may not exist until ds4-server first starts — tolerate that.
+            if os.path.isdir(kv_dir):
+                removed, freed = _sweep(kv_dir, max_age_seconds)
+                with _state_lock:
+                    _state["last_run_ts"] = time.time()
+                    _state["last_removed"] = removed
+                    _state["total_removed"] += removed
+                    _state["last_freed_bytes"] = freed
+                    _state["runs"] += 1
+                if removed:
+                    _log.info("ds4 kv janitor: removed %d stale entr%s (%.1f MB) from %s",
+                              removed, "y" if removed == 1 else "ies",
+                              freed / (1024 * 1024), kv_dir)
+        except Exception as e:  # never let the janitor die
+            _log.warning("ds4 kv janitor sweep failed: %s", e)
+        time.sleep(interval)
+
+
+def start(kv_dir: Optional[str], enabled: bool = False,
+          max_age_hours: float = 168.0, interval_minutes: float = 360.0) -> bool:
+    """Start the janitor for the ds4 KV-disk dir. No-op (returns False) when
+    disabled, when no dir is given, or when the path is a shared system dir."""
+    global _thread, _started
+    if _started:
+        return True
+    if not enabled or not kv_dir:
+        return False
+    real = os.path.abspath(os.path.expanduser(kv_dir)).rstrip("/") or "/"
+    if real in _FORBIDDEN:
+        _log.info("ds4 kv janitor: refusing to prune shared dir %s", real)
+        return False
+    with _state_lock:
+        _state.update({
+            "enabled": True, "kv_dir": real,
+            "max_age_hours": max_age_hours, "interval_minutes": interval_minutes,
+        })
+    _thread = threading.Thread(target=_run, args=(real, max_age_hours, interval_minutes),
+                               name="ds4-kv-janitor", daemon=True)
+    _thread.start()
+    _started = True
+    _log.info("ds4 kv janitor: pruning %s every %.0f min (entries older than %.1f h)",
+              real, interval_minutes, max_age_hours)
+    return True
+
+
+def sweep_once(kv_dir: str, max_age_hours: float = 168.0) -> tuple[int, int]:
+    """Run a single prune pass and return (removed, freed_bytes). For cron use."""
+    real = os.path.abspath(os.path.expanduser(kv_dir)).rstrip("/") or "/"
+    if real in _FORBIDDEN or not os.path.isdir(real):
+        raise SystemExit(f"refusing to prune {real!r} (not a valid ds4 kv dir)")
+    return _sweep(real, max(0.0, max_age_hours) * 3600.0)
+
+
+if __name__ == "__main__":
+    import argparse
+    p = argparse.ArgumentParser(description="Prune the ds4 on-disk KV cache.")
+    p.add_argument("--kv-dir", required=True, help="the ds4 --kv-disk-dir to prune")
+    p.add_argument("--max-age-hours", type=float, default=168.0,
+                   help="delete entries whose newest file is older than this")
+    a = p.parse_args()
+    n, b = sweep_once(a.kv_dir, a.max_age_hours)
+    print(f"ds4 kv janitor: removed {n} entr{'y' if n == 1 else 'ies'} "
+          f"({b / (1024 * 1024):.1f} MB) from {a.kv_dir}")

codai/config.py

@@ -273,6 +273,15 @@ class Ds4Config:
     # the streaming expert cache, avoiding "model arena alloc failed … OOM".
     extra_env: str = ""
     auto_build: bool = True                # clone+build the binary if it's missing
+    # On-disk KV-checkpoint cache (ds4-server --kv-disk-dir, defaulted to
+    # <offload>/ds4-kv). ds4 writes prompt KV checkpoints here to reuse across
+    # requests; abandoned sessions accumulate and never self-prune. When enabled, a
+    # background janitor deletes cache entries whose newest file is older than
+    # kv_cache_max_age_hours every kv_cache_cleanup_interval_minutes. Active
+    # sessions (recently touched) are spared (age is by newest mtime).
+    kv_cache_cleanup_enabled: bool = False
+    kv_cache_max_age_hours: float = 168.0          # 7 days
+    kv_cache_cleanup_interval_minutes: float = 360.0  # 6 hours
 
 
 @dataclass
@@ -644,6 +653,9 @@ class ConfigManager:
                 "expert_cache_reserve_gb": self.config.ds4.expert_cache_reserve_gb,
                 "extra_env": self.config.ds4.extra_env,
                 "auto_build": self.config.ds4.auto_build,
+                "kv_cache_cleanup_enabled": self.config.ds4.kv_cache_cleanup_enabled,
+                "kv_cache_max_age_hours": self.config.ds4.kv_cache_max_age_hours,
+                "kv_cache_cleanup_interval_minutes": self.config.ds4.kv_cache_cleanup_interval_minutes,
             },
             "compaction": {
                 "enabled": self.config.compaction.enabled,

codai/main.py

@@ -354,6 +354,36 @@ def main():
         except Exception as _e:
             print(f"WARNING: tmp janitor failed to start: {_e}")
 
+    # Age-prune the ds4 on-disk KV-checkpoint cache (<offload>/ds4-kv, or an
+    # explicit --kv-disk-dir in ds4 extra_args). Abandoned ds4 sessions leave large
+    # KV files that never self-prune. Only when ds4 + this cleanup are both enabled.
+    try:
+        _ds4 = getattr(config, "ds4", None)
+        if _ds4 and getattr(_ds4, "enabled", False) and getattr(_ds4, "kv_cache_cleanup_enabled", False):
+            _kv_dir = ""
+            _extra = (getattr(_ds4, "extra_args", "") or "")
+            if "--kv-disk-dir" in _extra:
+                import shlex as _shlex
+                _toks = _shlex.split(_extra)
+                for _i, _t in enumerate(_toks):
+                    if _t == "--kv-disk-dir" and _i + 1 < len(_toks):
+                        _kv_dir = _toks[_i + 1]
+                        break
+            if not _kv_dir:
+                _off = (getattr(config.offload, "directory", "") or "").strip()
+                if _off:
+                    _kv_dir = os.path.join(os.path.expanduser(_off), "ds4-kv")
+            if _kv_dir:
+                from codai.api.ds4_kv_janitor import start as _start_ds4_kv_janitor
+                _start_ds4_kv_janitor(
+                    _kv_dir,
+                    enabled=True,
+                    max_age_hours=getattr(_ds4, "kv_cache_max_age_hours", 168.0),
+                    interval_minutes=getattr(_ds4, "kv_cache_cleanup_interval_minutes", 360.0),
+                )
+    except Exception as _e:
+        print(f"WARNING: ds4 kv janitor failed to start: {_e}")
+
     # Apply cache directory overrides from config before any cache module is used.
     # We set env vars AND patch huggingface_hub.constants in case the library was
     # already imported (constants are computed once at import time from env vars).