video: stop the offload retry death-spiral; log swallowed load/gen errors; robust pipeline cache

Three fixes for the Wan2.2-VACE-Fun OOM cascade:

1. Offload retry death-spiral (the OOM driver). When from_pretrained OOMs
   MID-LOAD, `pipe` is never assigned, so the ~20 GB already placed on the GPU is
   pinned by the EXCEPTION'S TRACEBACK frames (their locals hold the partial
   model), not by `pipe`. _clear_mem() frees via `pipe` (None) so it reclaims
   nothing, and the next balanced step recomputes its GPU budget from a card still
   ~20 GB full (70%->17.5, 60%->3.4, 40%->2.3 GiB — all doomed). Drop
   `e.__traceback__` and run _clear_mem() OUTSIDE the except block (where
   sys.exc_info no longer pins the frames) so the stranded tensors are collectable
   before the next attempt. Applied to the balanced chain and the full-GPU and
   model-offload handlers.

2. Observability. The load-failure and generation-failure handlers raised
   HTTPException(500) with the cause only in the HTTP detail — debug.log showed a
   bare "Response status: 500". Log the full traceback in all three handlers.

3. Pipeline cache robustness. diffusers refuses save_pretrained on a CPU/seq
   offloaded pipeline, and a device_map/disk-offloaded one has meta tensors —
   either way the save half-writes a .building dir then a killed process leaves it
   orphaned (tens of GB). Add _unsavable_reason() to skip those cleanly (no junk),
   and sweep_stale() to delete orphaned *.building dirs. A full-GPU load stays
   savable and still caches the 4-bit pipeline.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RdMufYvtTbtGDWsiZVoXce

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.30"
+__version__ = "0.1.31"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/api/video.py

@@ -1186,11 +1186,13 @@ def _load_video_pipeline(model_name: str, device: str, mode: str, offload: str =
             _pcts = sorted({_gpu_pct} | {p for p in (80.0, 60.0, 40.0)
                                          if p < _gpu_pct}, reverse=True)
             for _i, _pct in enumerate(_pcts):
+                _oom = False
                 try:
                     return _load_balanced(_pct)
                 except (RuntimeError, MemoryError) as _e:
                     if not _is_oom(_e):
                         raise
+                    _oom = True
                     _nxt = _pcts[_i + 1] if _i + 1 < len(_pcts) else None
                     if _nxt is not None:
                         print(f"  Video: balanced {_pct:.0f}% GPU OOM — "
@@ -1198,6 +1200,17 @@ def _load_video_pipeline(model_name: str, device: str, mode: str, offload: str =
                     else:
                         print(f"  Video: balanced {_pct:.0f}% GPU OOM — "
                               f"falling back to sequential CPU offload…")
+                    # When from_pretrained OOMs MID-LOAD, `pipe` is never assigned —
+                    # the ~20 GB already placed on the GPU is pinned by this
+                    # exception's traceback frames (their locals hold the partial
+                    # model). Drop the traceback so those frames become collectable.
+                    _e.__traceback__ = None
+                # CRITICAL: reclaim OUTSIDE the except block. While we're still
+                # inside it, sys.exc_info() keeps the traceback (hence the stranded
+                # GPU tensors) alive, so _clear_mem()'s gc/empty_cache would free
+                # nothing — and the next step would recompute its GPU budget from a
+                # card still ~20 GB full (the 70%→60%→40% shrinking death-spiral).
+                if _oom:
                     _clear_mem()
             return None
 
@@ -1329,7 +1342,8 @@ def _load_video_pipeline(model_name: str, device: str, mode: str, offload: str =
                 # then group offload + stream, then sequential, then disk. Balanced
                 # is reserved for an explicit offload_strategy=balanced.
                 print(f"  Video: full-GPU OOM ({e}) — trying model CPU offload…")
-                _clear_mem()
+                e.__traceback__ = None  # release frames pinning the partial model's GPU tensors
+            _clear_mem()
 
         # ── Attempt 1: model CPU offload ─────────────────────────────────────
         if offload not in ('group', 'sequential', 'disk'):
@@ -1349,7 +1363,8 @@ def _load_video_pipeline(model_name: str, device: str, mode: str, offload: str =
                     raise
                 print(f"  Video: model CPU offload OOM ({e})"
                       f" — trying group offload + stream…")
-                _clear_mem()
+                e.__traceback__ = None  # release frames pinning the partial model's GPU tensors
+            _clear_mem()
 
         # ── Attempt 1.5: group offload + stream → sequential ─────────────────
         # Block-level offloading with CUDA-stream prefetch: lowest VRAM after
@@ -3235,6 +3250,14 @@ async def video_generations(request: VideoGenerationRequest,
                 pipe = await asyncio.get_event_loop().run_in_executor(
                     None, _load_video_pipeline, model_name, device, request.mode, _offload, _model_cfg)
         except Exception as e:
+            # Log the FULL traceback — otherwise the only trace of a post-load
+            # failure (e.g. enable_sequential_cpu_offload rejecting a bnb-4bit pipe,
+            # or an OOM on the first forward) is a bare "Response status: 500" with
+            # the cause swallowed into the HTTP detail. We need the stack to debug.
+            import traceback as _tb
+            print(f"  [video] model load FAILED for '{model_name}' "
+                  f"(strategy={_offload or 'auto'}): {str(e).splitlines()[0] if str(e) else type(e).__name__}\n"
+                  + _tb.format_exc())
             multi_model_manager._mark_cuda_poisoned_if_fatal(e)
             if getattr(multi_model_manager, 'cuda_context_poisoned', False):
                 raise HTTPException(status_code=503, detail=(
@@ -3342,6 +3365,10 @@ async def video_generations(request: VideoGenerationRequest,
                 frames, fps = await asyncio.get_event_loop().run_in_executor(
                     None, _generate_video, pipe, request)
             except Exception as e2:
+                import traceback as _tb
+                print(f"  [video] generation FAILED on OOM-retry for '{model_name}': "
+                      f"{str(e2).splitlines()[0] if str(e2) else type(e2).__name__}\n"
+                      + _tb.format_exc())
                 multi_model_manager.models.pop(model_key, None)
                 multi_model_manager.model_pools.pop(model_key, None)
                 # Release the failed retry pipeline too, so the NEXT request
@@ -3352,6 +3379,10 @@ async def video_generations(request: VideoGenerationRequest,
         else:
             # Non-OOM failure: evict the cached pipeline so the next request
             # attempts a clean reload rather than reusing a broken object.
+            import traceback as _tb
+            print(f"  [video] generation FAILED (non-OOM) for '{model_name}': "
+                  f"{str(e).splitlines()[0] if str(e) else type(e).__name__}\n"
+                  + _tb.format_exc())
             multi_model_manager.models.pop(model_key, None)
             multi_model_manager.model_pools.pop(model_key, None)
             _free_pipeline_vram(pipe)

codai/models/pipeline_cache.py

@@ -32,6 +32,7 @@ Everything here is best-effort: any failure (save or load) is swallowed and the
 caller falls back to a normal build, so the cache can never break generation.
 """
 
+import glob
 import hashlib
 import json
 import os
@@ -131,6 +132,50 @@ def invalidate(model_name: str, model_cfg: Optional[dict]) -> None:
         pass
 
 
+def sweep_stale(max_age_s: float = 1800.0) -> None:
+    """Delete orphaned ``*.building`` temp dirs left by a save that was killed
+    mid-write (the atomic ``os.replace`` never ran). They waste tens of GB and,
+    being incomplete, are never valid() anyway. Best-effort; only removes dirs
+    older than ``max_age_s`` so it never races a save in progress."""
+    try:
+        root = cache_root()
+        now = time.time()
+        for d in glob.glob(os.path.join(root, "*.building")):
+            try:
+                if now - os.path.getmtime(d) >= max_age_s:
+                    shutil.rmtree(d, ignore_errors=True)
+                    print(f"  [pipeline-cache] swept stale {os.path.basename(d)}")
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+
+def _unsavable_reason(pipe) -> Optional[str]:
+    """Return why ``pipe`` cannot be serialized, or None if it's savable.
+
+    diffusers refuses save_pretrained on a pipeline with CPU-offload hooks
+    (enable_model_cpu_offload / enable_sequential_cpu_offload), and a device_map
+    pipeline that spilled to disk holds meta tensors that serialize to garbage.
+    Detect both so we skip cleanly (no half-written .building) instead of failing
+    deep inside save_pretrained — the cache must be built from a savable
+    (pre-offload, fully-materialised) pipeline."""
+    try:
+        if getattr(pipe, "_all_hooks", None) or getattr(pipe, "hf_device_map", None):
+            return "accelerate offload active (cpu/sequential/device_map)"
+        comps = getattr(pipe, "components", {}) or {}
+        for name, comp in comps.items():
+            if not hasattr(comp, "parameters"):
+                continue
+            for prm in comp.parameters():
+                if getattr(prm, "is_meta", False) or str(getattr(prm, "device", "")) == "meta":
+                    return f"component '{name}' has meta/offloaded tensors"
+                break  # one param per component is enough to classify
+    except Exception:
+        return None  # can't tell — let save_pretrained try
+    return None
+
+
 def save(pipe, p: str, *, model_name: str = "", model_cfg: Optional[dict] = None) -> bool:
     """Serialize ``pipe`` to the cache dir ``p`` (atomic via a temp dir).
 
@@ -138,9 +183,15 @@ def save(pipe, p: str, *, model_name: str = "", model_cfg: Optional[dict] = None
     keeps the freshly built in-memory pipeline regardless."""
     if not p:
         return False
+    _why = _unsavable_reason(pipe)
+    if _why:
+        print(f"  [pipeline-cache] not caching {os.path.basename(p)}: {_why} "
+              f"— cache must be built from a pre-offload pipeline")
+        return False
     tmp = p + ".building"
     try:
         os.makedirs(cache_root(), exist_ok=True)
+        sweep_stale()
         if os.path.exists(tmp):
             shutil.rmtree(tmp, ignore_errors=True)
         print(f"  [pipeline-cache] saving quantized pipeline → {p}")