embeddings: eviction must wait for in-flight encodes; engines card shows per-model VRAM/RAM

Eviction cleanup() closed the llama ctx / killed the dinov2 subprocess
while another thread was mid-encode (embeddings hold no pool ref, so
_is_key_busy saw idle) — use-after-free segfaults that crash-looped the
radeon engine. cleanup() now takes the same per-model lock the embed
paths hold, so it waits for the in-flight call.

Engines card: the loaded-models hover now lists each model with its
measured VRAM footprint and, separately, the host-RAM slice when
CPU-offloading. Plumbed engine-state loaded_info → supervisor →
registry → engines_list → tooltip.

dinov2cpp: build the quantize tool too (dinov2-large-Q8_0 is lossless
at cos 0.9973 and drops 612→329 MB).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014S8VtAvG499SsCbeESRK7V

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.44"
+__version__ = "0.1.45"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/admin/templates/tasks.html

@@ -315,7 +315,22 @@ async function loadEngines(){
     const proc = e.processing ? ` <span class="badge badge-admin" style="font-size:9px" title="actively processing a request">● processing</span>` : '';
     const loaded = e.loaded_models||[];
     const models = loaded.length;
-    const mtip = models ? esc(loaded.join('\n')) : 'no models loaded';
+    // Tooltip: one line per model with its memory footprint — VRAM, plus the
+    // host-RAM slice separately when the model is CPU-offloading.
+    const info = e.loaded_info||[];
+    const infoByBase = {};
+    info.forEach(m => {
+      const b = (m.model||'').split('/').pop().replace(/\.gguf$/i,'');
+      infoByBase[b] = m; infoByBase[m.model] = m;
+    });
+    const mtip = models ? esc(loaded.map(name => {
+      const b = (name||'').split('/').pop().replace(/\.gguf$/i,'');
+      const m = infoByBase[name] || infoByBase[b];
+      if (!m || (!m.vram_gb && !m.ram_gb)) return name;
+      let s = `${name} — VRAM ${m.vram_gb} GB`;
+      if (m.ram_gb) s += ` · RAM ${m.ram_gb} GB (offload)`;
+      return s;
+    }).join('\n')) : 'no models loaded';
     return `<div class="sys-tile">
       <div class="sys-head">
         <span class="sys-name">${esc(e.name)} <span class="dim" style="text-transform:none">(${esc(e.backend)})</span>${prim}${proc}${cool}</span>

codai/api/app.py

@@ -357,7 +357,38 @@ async def internal_engine_state():
         paused = _therm.external_pause_active()
     except Exception:
         paused = False
+    # Per-model memory footprint for the engines card tooltip: measured VRAM
+    # (in-memory truth from record_vram_delta, falling back to the config
+    # estimate) plus the separately-measured host-RAM slice when the model is
+    # CPU-offloading.
+    loaded_info = []
+    try:
+        _seen_li = set()
+        for _k in loaded:
+            if _k in _seen_li:
+                continue
+            _seen_li.add(_k)
+            try:
+                _vg = (multi_model_manager._measured_vram_gb.get(_k)
+                       or multi_model_manager._get_model_used_vram_gb(_k) or 0)
+            except Exception:
+                _vg = 0
+            _cfg = {}
+            try:
+                _cfg = multi_model_manager._config_for_model_key(_k) or {}
+            except Exception:
+                pass
+            _rg = _cfg.get("measured_ram_gb") or 0
+            loaded_info.append({
+                "model": _k,
+                "vram_gb": round(float(_vg), 2) if _vg else 0,
+                "ram_gb": round(float(_rg), 2) if _rg else 0,
+                "device": multi_model_manager.model_devices.get(_k) or "",
+            })
+    except Exception:
+        loaded_info = []
     return {"ok": True, "pid": _os.getpid(), "loaded_models": loaded,
+            "loaded_info": loaded_info,
             "vram": vram, "tasks": tasks, "cooling": cooling, "paused": paused}
 
 

codai/api/embeddings.py

@@ -76,6 +76,21 @@ class _EmbeddingModel:
         yield self.model
 
     def cleanup(self):
+        # Eviction can fire while a request is mid-encode (embeddings hold no
+        # pool ref, so _is_key_busy sees idle). Closing a llama ctx / killing
+        # the subprocess under a running embed segfaults the engine — take the
+        # same per-model lock the embed paths hold so cleanup WAITS for the
+        # in-flight call to finish.
+        _l = getattr(self, 'lock', None)
+        if _l is not None:
+            _l.acquire()
+        try:
+            self._cleanup_locked()
+        finally:
+            if _l is not None:
+                _l.release()
+
+    def _cleanup_locked(self):
         try:
             if self.backend == 'dinov2cpp':
                 # persistent embed server subprocess — terminating it frees

codai/frontproxy/app.py

@@ -1116,6 +1116,7 @@ class FrontProxy:
                         "thermal_paused": bool(getattr(e, "therm_paused", False)),
                         "thermal_frozen": bool(getattr(e, "therm_sigstopped", False)),
                         "loaded_models": self._canonical_loaded(e.loaded_models),
+                        "loaded_info": list(getattr(e, "loaded_info", []) or []),
                         "inflight": int(getattr(e, "inflight", 0) or 0),
                         "processing": (int(getattr(e, "inflight", 0) or 0) > 0
                                        or bool(getattr(e, "loading", None))),

codai/frontproxy/engine_supervisor.py

@@ -807,6 +807,7 @@ class EngineSupervisor:
                         self.registry.update_state(
                             engine.id, healthy=True,
                             loaded_models=loaded,
+                            loaded_info=d.get("loaded_info") or [],
                             vram=d.get("vram"),
                             tasks=d.get("tasks") or [],
                             cooling=d.get("cooling"),

codai/frontproxy/registry.py

@@ -64,6 +64,9 @@ class Engine:
     url: str = ""
     healthy: bool = False
     loaded_models: Set[str] = field(default_factory=set)
+    # Per-model memory info from /internal/engine-state:
+    # [{model, vram_gb, ram_gb, device}, …] for the engines-card tooltip.
+    loaded_info: list = field(default_factory=list)
     vram: Optional[dict] = None
     tasks: list = field(default_factory=list)   # running/queued tasks on this engine
     cooling: Optional[dict] = None  # thermal cooldown state, or None when not cooling
@@ -191,8 +194,8 @@ class EngineRegistry:
         return match
 
     def update_state(self, engine_id: int, *, healthy: bool,
-                     loaded_models=None, vram=None, tasks=None,
-                     cooling=False) -> None:
+                     loaded_models=None, loaded_info=None, vram=None,
+                     tasks=None, cooling=False) -> None:
         with self._lock:
             e = self._engines.get(engine_id)
             if not e:
@@ -204,6 +207,8 @@ class EngineRegistry:
                 e.last_ok = time.monotonic()
             if loaded_models is not None:
                 e.loaded_models = set(loaded_models)
+            if loaded_info is not None:
+                e.loaded_info = list(loaded_info)
             if vram is not None:
                 e.vram = vram
             if tasks is not None:

packaging/dinov2cpp/build.sh

@@ -70,6 +70,6 @@ cp "$HERE/embed.cpp" .
 mkdir -p build && cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON \
       -DBUILD_SHARED_LIBS=OFF \
-      -DBUILD_REALTIME=OFF -DBUILD_QUANTIZE=OFF ..
-make -j"$(nproc)" embed
+      -DBUILD_REALTIME=OFF -DBUILD_QUANTIZE=ON ..
+make -j"$(nproc)" embed quantize
 echo "BUILT: $(pwd)/bin/embed"