perf: cache the cache-scans + gpu-stats so the models page paints instantly

The models page was slow because the work itself is slow, not because of routing:
cached-models walks the HF/GGUF cache doing per-model capability detection, and
cache-stats walks every file summing sizes — many seconds each on a large cache.

- routes.py: cached-models + cache-stats are now stale-while-revalidate (TTL 25s).
  First call computes; afterwards they return the last result instantly and refresh
  in a background thread. Mutating ops (clear-cache, delete cached model, free-disk)
  force the next refresh. The coderai-system worker pre-warms both scans on startup,
  so even the first page load is fast.
- gpu_detect.gpu_stats(): short 1.5s TTL cache — the Tasks page polls it ~every
  second and nvidia-smi is slow on a saturated GPU, so collapse repeats.

Verified: _cached returns stale instantly and refreshes in the background
(2 underlying calls across first+expire); all modules compile.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011DDv7BchtZQWsnPG6Jm49m

codai/admin/routes.py

@@ -1503,25 +1503,66 @@ def _do_delete_model(model_id: str, cache_type: str) -> dict:
     return {"success": False, "detail": "Unknown cache_type"}
 
 
+# Cache scans walk the whole HF/GGUF cache on disk (sizes + per-model capability
+# detection), which can take many seconds for a large cache. They change only when
+# a model is added/removed/downloaded, so serve them stale-while-revalidate: return
+# the last result instantly and refresh in the background once it's older than TTL.
+# First call computes synchronously; mutating ops force the next refresh.
+import threading as _cache_thr
+_SCAN_TTL = 25.0
+_scan_state = {"data": None, "at": 0.0, "busy": False}
+_stats_state = {"data": None, "at": 0.0, "busy": False}
+
+
+def _cached(state, fn):
+    import time as _t
+    now = _t.time()
+    if state["data"] is None:
+        state["data"] = fn()
+        state["at"] = _t.time()
+        return state["data"]
+    if now - state["at"] >= _SCAN_TTL and not state["busy"]:
+        state["busy"] = True
+
+        def _bg():
+            try:
+                d = fn()
+                state["data"] = d
+                state["at"] = _t.time()
+            finally:
+                state["busy"] = False
+        _cache_thr.Thread(target=_bg, daemon=True).start()
+    return state["data"]
+
+
+def _invalidate_cache_scan():
+    """Force the next cached-models/cache-stats read to refresh (after a model is
+    deleted/downloaded/freed)."""
+    _scan_state["at"] = 0.0
+    _stats_state["at"] = 0.0
+
+
 @router.get("/admin/api/cached-models", summary="List cached models")
 async def api_cached_models(username: str = Depends(require_admin)):
-    """Scan both caches and return all locally stored models."""
+    """Scan both caches and return all locally stored models (stale-while-revalidate)."""
     import asyncio
-    return await asyncio.to_thread(_scan_caches)
+    return await asyncio.to_thread(_cached, _scan_state, _scan_caches)
 
 
 @router.get("/admin/api/cache-stats", summary="Model cache statistics")
 async def api_cache_stats(username: str = Depends(require_admin)):
-    """Return disk-usage statistics for each cache."""
+    """Return disk-usage statistics for each cache (stale-while-revalidate)."""
     import asyncio
-    return await asyncio.to_thread(_get_cache_stats)
+    return await asyncio.to_thread(_cached, _stats_state, _get_cache_stats)
 
 
 @router.delete("/admin/api/cache", summary="Clear the model cache")
 async def api_clear_cache(cache_type: str = "all", username: str = Depends(require_admin)):
     """Bulk-delete cache. cache_type: all | hf | gguf"""
     import asyncio
-    return await asyncio.to_thread(_do_clear_cache, cache_type)
+    r = await asyncio.to_thread(_do_clear_cache, cache_type)
+    _invalidate_cache_scan()
+    return r
 
 
 @router.delete("/admin/api/cached-models/{model_id:path}", summary="Evict a cached model")
@@ -1532,7 +1573,9 @@ async def api_delete_cached_model(
 ):
     """Delete a specific cached model (HF repo ID or GGUF filename)."""
     import asyncio
-    return await asyncio.to_thread(_do_delete_model, model_id, cache_type)
+    r = await asyncio.to_thread(_do_delete_model, model_id, cache_type)
+    _invalidate_cache_scan()
+    return r
 
 
 @router.post("/admin/api/model-free-disk", summary="Delete a model's files but keep its config")
@@ -1574,6 +1617,7 @@ async def api_model_free_disk(request: Request, username: str = Depends(require_
             config_manager.save_models()
 
     result = await asyncio.to_thread(_do_delete_model, path, cache_type)
+    _invalidate_cache_scan()
     _broker_notify_models_updated(request)
     return result
 

codai/frontproxy/gpu_detect.py

@@ -354,13 +354,28 @@ def _amd_stats() -> list:
     return cards
 
 
+_gpu_stats_cache = {"data": None, "at": 0.0}
+_GPU_STATS_TTL = 1.5
+
+
 def gpu_stats() -> list:
     """Live per-card stats for EVERY physical GPU installed (vendor-agnostic):
     ``[{vendor, index, name, util%, mem_used GB, mem_total GB, temp °C, uuid}]``.
 
     Independent of engine ownership — nvidia-smi and sysfs report all cards
-    regardless of CUDA_VISIBLE_DEVICES — so this shows the whole machine."""
-    return _nvidia_stats() + _amd_stats()
+    regardless of CUDA_VISIBLE_DEVICES — so this shows the whole machine.
+
+    Cached for a short TTL: the dashboard polls this ~every second and nvidia-smi
+    can be slow on a saturated GPU, so collapse rapid repeats onto one query."""
+    import time as _t
+    now = _t.time()
+    c = _gpu_stats_cache
+    if c["data"] is not None and now - c["at"] < _GPU_STATS_TTL:
+        return c["data"]
+    data = _nvidia_stats() + _amd_stats()
+    c["data"] = data
+    c["at"] = now
+    return data
 
 
 def engine_gpu_stats() -> list:

codai/system_app.py

@@ -100,5 +100,23 @@ def build_system_app(config, config_dir, internal_port: int = 0) -> FastAPI:
                     out.append({"id": str(getattr(m, "id", m))})
         return JSONResponse({"object": "list", "data": out})
 
+    @app.on_event("startup")
+    async def _prewarm():
+        # Warm the cache scans in the background so the first models-page load is
+        # already fast (the scans walk the whole HF/GGUF cache on disk).
+        import threading
+
+        def _warm():
+            try:
+                from codai.admin.routes import (
+                    _cached, _scan_state, _stats_state, _scan_caches,
+                    _get_cache_stats)
+                _cached(_scan_state, _scan_caches)
+                _cached(_stats_state, _get_cache_stats)
+                print("[system] cache scan pre-warmed", flush=True)
+            except Exception as exc:
+                print(f"[system] cache pre-warm skipped: {exc}", flush=True)
+        threading.Thread(target=_warm, daemon=True).start()
+
     app.include_router(admin_router, tags=["Admin"])
     return app