LoRA training base cache + thermal averaging + scoped debug flags; live model config

- LoRA trainer: cache the SD/SDXL base on CPU between jobs so back-to-back
  trainings against the same base skip the disk reload, and the base holds no
  VRAM between jobs (moved to GPU only while training). Fixes the post-training
  eviction failure that forced the next image request into CPU/disk offload.
- Model manager: add register_external_vram_releaser() + last-resort eviction
  pass so a generation can reclaim the trainer's cached base when needed (skips
  while a job runs).
- Thermal: average 3 CPU samples spread across a 3s budget for the resume/
  cooldown decision (CPU sensors swing +/-10C); pause stays single-read to react
  fast. Bounded so it never blocks past 3s of the poll interval.
- Debug flags: --debug-web (uvicorn access lines), --debug-thermal ([thermal]
  [debug] checks), --debug-lora (per-step training loss to terminal); all off by
  default and independent of --debug.
- Admin: lora_train_base_model field on the Models page; saves apply live to the
  running server (build_runtime_kwargs/apply_model_entry_live) with no restart.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -47,6 +47,15 @@ _download_status: dict = {}   # session_id → latest progress state (survives S
 _download_cancelled: set = set()  # session_ids the user has requested to cancel
 
 
+def get_active_download_model_ids() -> set:
+    """Return the set of model IDs whose download is currently in progress."""
+    return {
+        s["model_id"]
+        for s in _download_status.values()
+        if s.get("status") == "downloading"
+    }
+
+
 def _url(request: Request, path: str) -> str:
     """Return a proxy-aware absolute path (root_path prefix + path)."""
     from codai.api.urlutils import get_public_prefix
@@ -54,11 +63,19 @@ def _url(request: Request, path: str) -> str:
 
 
 def _tmpl(request: Request, name: str, ctx: dict = None):
-    """Render a template with root_path injected into the context."""
+    """Render a template with root_path injected into the context.
+
+    Admin pages are served with no-cache headers so template/UI changes are
+    picked up on a normal reload instead of being masked by the browser cache.
+    """
     from codai.api.urlutils import get_public_prefix
     c = ctx or {}
     c.setdefault("root_path", get_public_prefix(request))
-    return templates.TemplateResponse(request, name, c)
+    resp = templates.TemplateResponse(request, name, c)
+    resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    return resp
 
 
 def init_session_manager(config_dir: Path, port: int = 0):
@@ -1919,7 +1936,10 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                 "max_instances", "preload_all_instances", "capabilities",
                 "model_template", "vae_path", "t5xxl_path", "clip_l_path",
                 "clip_g_path", "clip_vision_path", "lora_path", "lora_model_dir",
-                "max_vram", "sdcpp_flash_attn", "sdcpp_diffusion_flash_attn", "vae_tiling"):
+                "lora_train_base_model",
+                "max_vram", "sdcpp_flash_attn", "sdcpp_diffusion_flash_attn", "vae_tiling",
+                "component_quantization", "output_crf", "force_vram_update",
+                "balanced_gpu_percent"):
         if key in data:
             entry[key] = data[key]
 
@@ -1927,7 +1947,27 @@ async def api_model_configure(request: Request, username: str = Depends(require_
     for mtype in model_types:
         config_manager.models_data.setdefault(mtype, []).append(entry)
     config_manager.save_models()
-    return {"success": True}
+
+    # Apply to the running server immediately so config changes (e.g.
+    # lora_train_base_model, vae_path, quant flags) take effect without a
+    # restart. Only the live config dict is updated — loaded weights are left
+    # alone; weight-level changes (quantization) still apply on next (re)load.
+    applied = 0
+    try:
+        from codai.main import apply_model_entry_live, _CATEGORY_TYPE_PREFIX
+        # Drop stale live-config keys for any path we removed (e.g. a rename),
+        # so the running server doesn't keep serving the old entry's config.
+        from codai.models.manager import multi_model_manager as _mmm
+        stale = {p for p in paths_to_remove if p and p != path}
+        if stale:
+            prefixes = {pref for (_t, pref) in _CATEGORY_TYPE_PREFIX.values()}
+            for sp in stale:
+                for pref in prefixes:
+                    _mmm.config.pop(f"{pref}{sp}", None)
+        applied = apply_model_entry_live(entry, model_types)
+    except Exception as e:
+        print(f"  [admin] live config apply failed (restart to apply): {e}")
+    return {"success": True, "applied_live": applied}
 
 
 # --- System endpoints ---
@@ -2011,6 +2051,15 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "directory": c.archive.directory,
             "retention": c.archive.retention,
         },
+        "thermal": {
+            "cpu_enabled": c.thermal.cpu_enabled,
+            "gpu_enabled": c.thermal.gpu_enabled,
+            "cpu_high": c.thermal.cpu_high,
+            "cpu_resume": c.thermal.cpu_resume,
+            "gpu_high": c.thermal.gpu_high,
+            "gpu_resume": c.thermal.gpu_resume,
+            "poll_seconds": c.thermal.poll_seconds,
+        },
         "broker": {
             "enabled": c.broker.enabled,
             "base_url": c.broker.base_url,
@@ -2119,6 +2168,30 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
         resolved = raw_dir if raw_dir and _os.path.isabs(raw_dir) else _os.path.join(cfg_dir, raw_dir or "archive")
         archive_manager.configure(c.archive.enabled, resolved, c.archive.retention)
 
+    if "thermal" in data:
+        th = data["thermal"]
+        c.thermal.cpu_enabled = bool(th.get("cpu_enabled", c.thermal.cpu_enabled))
+        c.thermal.gpu_enabled = bool(th.get("gpu_enabled", c.thermal.gpu_enabled))
+        c.thermal.cpu_high = float(th.get("cpu_high", c.thermal.cpu_high))
+        c.thermal.cpu_resume = float(th.get("cpu_resume", c.thermal.cpu_resume))
+        c.thermal.gpu_high = float(th.get("gpu_high", c.thermal.gpu_high))
+        c.thermal.gpu_resume = float(th.get("gpu_resume", c.thermal.gpu_resume))
+        c.thermal.poll_seconds = max(1.0, float(th.get("poll_seconds", c.thermal.poll_seconds)))
+        # Push to the live global_args so changes apply without a restart.
+        try:
+            from codai.api.state import get_global_args
+            ga = get_global_args()
+            if ga is not None:
+                ga.thermal_cpu_enabled = c.thermal.cpu_enabled
+                ga.thermal_gpu_enabled = c.thermal.gpu_enabled
+                ga.thermal_cpu_high = c.thermal.cpu_high
+                ga.thermal_cpu_resume = c.thermal.cpu_resume
+                ga.thermal_gpu_high = c.thermal.gpu_high
+                ga.thermal_gpu_resume = c.thermal.gpu_resume
+                ga.thermal_poll_seconds = c.thermal.poll_seconds
+        except Exception:
+            pass
+
     if "broker" in data:
         bro = data["broker"]
         c.broker.enabled = bool(bro.get("enabled", c.broker.enabled))

codai/admin/templates/models.html

@@ -320,8 +320,9 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
           <label class="form-label">HuggingFace repo ID or URL</label>
           <div style="display:flex;gap:.5rem">
             <input type="text" id="dl-id" class="form-input" style="flex:1" placeholder="e.g. bartowski/Llama-3.1-8B-Instruct-GGUF" onkeydown="if(event.key==='Enter'){event.preventDefault();browseHfFiles()}">
-            <button class="btn btn-ghost btn-sm" id="dl-browse-btn" onclick="browseHfFiles()" style="white-space:nowrap" title="Fetch available files from HuggingFace">Browse</button>
+            <button type="button" class="btn btn-ghost btn-sm" id="dl-browse-btn" onclick="browseHfFiles()" style="white-space:nowrap" title="Fetch available files from HuggingFace">Browse</button>
           </div>
+          <div id="dl-browse-status" style="font-size:11px;margin-top:.3rem;min-height:1em"></div>
         </div>
         <!-- Quant picker: shown when GGUF files are found via Browse -->
         <div id="dl-quant-row" style="display:none;margin-top:.5rem">
@@ -581,6 +582,11 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
           <label class="form-label">Used VRAM <span class="muted">(GB)</span></label>
           <input type="number" id="cfg-used-vram" class="form-input" min="0" step="0.1" placeholder="auto-estimate">
           <span class="form-hint" id="cfg-used-vram-hint" style="font-size:11px;color:var(--text-3)"></span>
+          <label style="display:flex;align-items:center;gap:.4rem;cursor:pointer;margin-top:.4rem">
+            <input type="checkbox" id="cfg-force-vram-update">
+            <span style="font-size:12px">Force VRAM update</span>
+          </label>
+          <span class="form-hint" style="font-size:11px;color:var(--text-3)">When on, the real measured VRAM is re-recorded every run and used for eviction even if "Used VRAM" is set above.</span>
         </div>
         <div class="form-row" style="margin:0">
           <label class="form-label">Max instances
@@ -626,6 +632,17 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
         <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer;font-size:13px"><input type="checkbox" id="cfg-noram"> No RAM fallback</label>
       </div>
 
+      <!-- Per-component quantization (diffusers image/video pipelines) -->
+      <div id="cfg-compquant-wrap">
+        <div class="card-title" style="margin-top:1.25rem">Component Quantization <span class="muted" style="font-weight:normal">(image / video pipelines)</span></div>
+        <div style="font-size:12px;color:var(--text-muted,#888);margin:.25rem 0 .5rem">
+          Quantize individual pipeline components. <b>Default</b> uses the global 4-bit/8-bit checkboxes above.
+          <b>2-bit</b> needs optimum-quanto; <b>GGUF file</b> (Q5_K/Q6_K…) needs a pre-quantized .gguf path/URL.
+          Tip: backbone <code>4-bit</code> + text encoder <code>8-bit</code> + VAE <code>None</code> = best quality-per-GB.
+        </div>
+        <div id="cfg-compquant-rows" style="display:grid;grid-template-columns:repeat(auto-fill,minmax(230px,1fr));gap:.5rem"></div>
+      </div>
+
       <!-- offload -->
       <div class="card-title" style="margin-top:1.25rem">Offload</div>
       <div style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem">
@@ -658,6 +675,20 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
         <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer;font-size:13px"><input type="checkbox" id="cfg-vae-tiling"> VAE Tiling <span class="muted">(reduces VRAM at decode)</span></label>
       </div>
 
+      <!-- video output quality (CRF) — only relevant for video models -->
+      <div class="form-row" style="margin-top:.75rem;max-width:340px">
+        <label class="form-label">Video output quality — CRF
+          <span class="muted">(video models; blank = default)</span></label>
+        <input type="number" id="cfg-output-crf" class="form-input" min="0" max="51" step="1"
+               placeholder="e.g. 20 (lower = better quality, bigger file)">
+      </div>
+      <div class="form-row" style="margin-top:.75rem;max-width:340px">
+        <label class="form-label">Balanced GPU % <span class="muted">(video models; default 80)</span></label>
+        <input type="number" id="cfg-balanced-gpu-pct" class="form-input" min="10" max="99" step="1"
+               placeholder="80">
+        <span class="form-hint">When the model won't fit entirely in VRAM, fill this % of free VRAM then spill the rest to CPU RAM. Also the cap used by auto-balanced fallback.</span>
+      </div>
+
       <!-- components -->
       <div class="card-title" style="margin-top:1.25rem">Components</div>
       <div class="form-row">
@@ -725,6 +756,16 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
         <label class="form-label">LoRA directory <span class="muted">(optional — for prompt-based loras)</span></label>
         <input type="text" id="cfg-lora-dir" class="form-input" placeholder="e.g. /models/loras">
       </div>
+      <div class="form-row">
+        <label class="form-label">LoRA training base model <span class="muted">(optional — SD1.x/SDXL to train LoRAs against)</span></label>
+        <input type="text" id="cfg-lora-train-base" class="form-input"
+          placeholder="e.g. stabilityai/stable-diffusion-xl-base-1.0">
+        <div class="muted small" style="margin-top:.25rem">
+          When this image model is a transformer/DiT (Z-Image, Flux, SD3) the in-process LoRA
+          trainer can't target it. Set a UNet-based SD1.x/SDXL model here to train identity LoRAs
+          against, while generation still uses this model. Leave empty to train on this model.
+        </div>
+      </div>
 
       <!-- generation -->
       <div class="card-title" style="margin-top:1.25rem">Generation</div>
@@ -1181,6 +1222,9 @@ function openDownloadFor(modelId, filePattern){
   document.getElementById('dl-progress').style.display='none';
   document.getElementById('dl-quant-row').style.display='none';
   document.getElementById('dl-start-btn').textContent='Download';
+  const _browseBtn = document.getElementById('dl-browse-btn');
+  if(_browseBtn){ _browseBtn.textContent='Browse'; _browseBtn.disabled=false; }
+  _dlStatus('');
 
   document.getElementById('dl-id').value = modelId || '';
   document.getElementById('dl-pattern').value = filePattern || '';
@@ -1407,11 +1451,22 @@ function renderQuantPicker(ggufFiles, initialPattern){
   document.getElementById('dl-start-btn').textContent = 'Download selected';
 }
 
+function _dlStatus(msg, color){
+  const el = document.getElementById('dl-browse-status');
+  if(!el) return;
+  el.textContent = msg;
+  el.style.color = color || 'var(--text-2)';
+}
+
 async function browseHfFiles(){
   const id = document.getElementById('dl-id').value.trim();
-  if(!id){ document.getElementById('dl-id').focus(); return; }
+  if(!id){
+    _dlStatus('Enter a HuggingFace repo ID first, e.g. bartowski/Llama-3.1-8B-Instruct-GGUF', 'var(--red,#f87171)');
+    document.getElementById('dl-id').focus();
+    return;
+  }
   if(!id.includes('/')){
-    // Local path or bare name — can't browse, use pattern fallback
+    _dlStatus('Use the format owner/model-name (e.g. bartowski/Llama-3.1-8B-Instruct-GGUF)', 'var(--red,#f87171)');
     document.getElementById('dl-quant-row').style.display='none';
     const isGguf = looksLikeGguf(id,'');
     document.getElementById('dl-pattern-row').style.display = isGguf?'block':'none';
@@ -1419,34 +1474,40 @@ async function browseHfFiles(){
     return;
   }
   const btn = document.getElementById('dl-browse-btn');
-  btn.textContent='…'; btn.disabled=true;
+  btn.textContent='Fetching…'; btn.disabled=true;
+  _dlStatus('Fetching file list from HuggingFace…');
+  const controller = new AbortController();
+  const _browseTimeout = setTimeout(()=>controller.abort(), 20000);
   try{
-    const r = await fetch(ROOT_PATH+'/admin/api/hf-files?repo_id='+encodeURIComponent(id));
-    if(!r.ok){ throw new Error((await r.json()).detail||'Request failed'); }
+    const r = await fetch(ROOT_PATH+'/admin/api/hf-files?repo_id='+encodeURIComponent(id), {signal:controller.signal});
+    if(!r.ok){
+      let detail='Request failed (HTTP '+r.status+')';
+      try{ detail=(await r.json()).detail||detail; }catch(_){}
+      throw new Error(detail);
+    }
     const {files, error} = await r.json();
     if(error){ throw new Error(error); }
     const gguf = (files||[]).filter(f=>f.name.toLowerCase().endsWith('.gguf'));
     if(gguf.length){
+      _dlStatus(gguf.length+' GGUF file'+(gguf.length!==1?'s':'')+' found — select quantizations below');
       renderQuantPicker(gguf, document.getElementById('dl-pattern').value);
     } else {
-      // Non-GGUF repo — hide picker, show snapshot note
+      _dlStatus('No GGUF files — will download as full repository snapshot');
       _dlQuantFiles=[];
       document.getElementById('dl-quant-row').style.display='none';
       document.getElementById('dl-pattern-row').style.display='none';
       document.getElementById('dl-snapshot-note').style.display='flex';
       document.getElementById('dl-start-btn').textContent='Download';
-      const note = document.getElementById('dl-quant-note');
-      note.textContent='';
     }
   }catch(e){
-    // Fall back to manual pattern input
+    const errMsg = e.name==='AbortError' ? 'Timed out — HuggingFace did not respond in 20s' : e.message;
+    _dlStatus('Error: '+errMsg, 'var(--red,#f87171)');
     _dlQuantFiles=[];
     document.getElementById('dl-quant-row').style.display='none';
     document.getElementById('dl-pattern-row').style.display='block';
     document.getElementById('dl-snapshot-note').style.display='none';
-    document.getElementById('dl-start-btn').textContent='Download';
-    alert('Could not browse repo: '+e.message);
   }finally{
+    clearTimeout(_browseTimeout);
     btn.textContent='Browse'; btn.disabled=false;
   }
 }
@@ -1798,6 +1859,57 @@ function _readCompField(slug){
   return sel.value || '';
 }
 
+// Per-component quantization (diffusers image/video pipelines).
+// Common component names across SD/SDXL/Flux/Wan; 'default' = use global flag.
+const COMPONENT_QUANT_NAMES = [
+  'transformer', 'transformer_2', 'unet',
+  'text_encoder', 'text_encoder_2', 'text_encoder_3', 'vae'
+];
+function _isGgufVal(v){ return typeof v==='string' && v.trim().toLowerCase().endsWith('.gguf'); }
+function _renderComponentQuant(current){
+  current = current || {};
+  const host = document.getElementById('cfg-compquant-rows');
+  if(!host) return;
+  host.innerHTML = COMPONENT_QUANT_NAMES.map(name=>{
+    const raw = (current[name] || 'default').toString();
+    const isGguf = _isGgufVal(raw);
+    const mode = isGguf ? 'gguf'
+      : (['2bit','4bit','8bit','none'].includes(raw.toLowerCase()) ? raw.toLowerCase() : 'default');
+    const opt = (val,lbl)=>`<option value="${val}"${mode===val?' selected':''}>${lbl}</option>`;
+    const ggufPath = isGguf ? raw : '';
+    return `<div style="display:flex;flex-direction:column;gap:2px;font-size:12px">
+      <label style="display:flex;align-items:center;justify-content:space-between;gap:.5rem">
+        <span style="font-family:monospace">${name}</span>
+        <select class="form-input cfg-compquant" data-comp="${name}" style="width:120px;padding:2px 6px"
+          onchange="_toggleGgufInput(this)">
+          ${opt('default','Default')}${opt('2bit','2-bit (quanto)')}${opt('4bit','4-bit')}${opt('8bit','8-bit')}${opt('gguf','GGUF file…')}${opt('none','None')}
+        </select>
+      </label>
+      <input type="text" class="form-input cfg-compquant-gguf" data-comp="${name}"
+        placeholder="path or URL to .gguf (Q5_K, Q6_K…)" value="${ggufPath}"
+        style="padding:2px 6px;${mode==='gguf'?'':'display:none'}">
+    </div>`;
+  }).join('');
+}
+function _toggleGgufInput(sel){
+  const inp = sel.closest('div').querySelector('.cfg-compquant-gguf');
+  if(inp) inp.style.display = (sel.value==='gguf') ? '' : 'none';
+}
+function _collectComponentQuant(){
+  const out = {};
+  document.querySelectorAll('.cfg-compquant').forEach(sel=>{
+    const comp = sel.dataset.comp;
+    if(sel.value === 'gguf'){
+      const inp = document.querySelector('.cfg-compquant-gguf[data-comp="'+comp+'"]');
+      const path = inp ? inp.value.trim() : '';
+      if(path) out[comp] = path;            // store the .gguf path/URL
+    } else if(sel.value && sel.value !== 'default'){
+      out[comp] = sel.value;                // '2bit' | '4bit' | '8bit' | 'none'
+    }
+  });
+  return Object.keys(out).length ? out : null;
+}
+
 function _renderWhisperServerRows(models){
   if(!models.length) return '';
   const rows = models.map(m=>{
@@ -2471,6 +2583,7 @@ function openCfgModal(idx, cfgIdx){
   const nCtxForEst = s.n_ctx || 2048;
   const estVram = _estimateVram(m, nCtxForEst);
   document.getElementById('cfg-used-vram-hint').textContent = estVram ? `Estimated: ~${estVram.toFixed(1)} GB` : '';
+  document.getElementById('cfg-force-vram-update').checked = !!s.force_vram_update;
   document.getElementById('cfg-gpu-layers').value = s.n_gpu_layers !== undefined ? s.n_gpu_layers : -1;
   document.getElementById('cfg-n-ctx').value = nCtxForEst;
   document.getElementById('cfg-max-instances').value = s.max_instances != null ? s.max_instances : 1;
@@ -2492,6 +2605,7 @@ function openCfgModal(idx, cfgIdx){
   document.getElementById('cfg-ram-gb').value = s.manual_ram_gb != null ? s.manual_ram_gb : '';
   document.getElementById('cfg-4bit').checked = !!s.load_in_4bit;
   document.getElementById('cfg-8bit').checked = !!s.load_in_8bit;
+  _renderComponentQuant(s.component_quantization || {});
   document.getElementById('cfg-flash').checked = !!s.flash_attention;
   document.getElementById('cfg-noram').checked = !!s.no_ram;
   document.getElementById('cfg-offload-strategy').value = s.offload_strategy || 'auto';
@@ -2505,6 +2619,8 @@ function openCfgModal(idx, cfgIdx){
   document.getElementById('cfg-sdcpp-flash').checked = !!s.sdcpp_flash_attn;
   document.getElementById('cfg-sdcpp-diff-flash').checked = !!s.sdcpp_diffusion_flash_attn;
   document.getElementById('cfg-vae-tiling').checked = !!s.vae_tiling;
+  document.getElementById('cfg-output-crf').value = (s.output_crf != null ? s.output_crf : '');
+  document.getElementById('cfg-balanced-gpu-pct').value = (s.balanced_gpu_percent != null ? s.balanced_gpu_percent : '');
   _populateCompSelects();
   _setCompField('vae',        s.vae_path        || '');
   _setCompField('t5xxl',      s.t5xxl_path      || '');
@@ -2513,6 +2629,7 @@ function openCfgModal(idx, cfgIdx){
   _setCompField('clip-vision',s.clip_vision_path|| '');
   _setCompField('lora',       s.lora_path       || '');
   document.getElementById('cfg-lora-dir').value = s.lora_model_dir || '';
+  document.getElementById('cfg-lora-train-base').value = s.lora_train_base_model || '';
   openModal('cfg-modal');
 }
 
@@ -2577,6 +2694,7 @@ async function saveModelConfig(){
     backend:           document.getElementById('cfg-backend').value,
     load_mode:         document.getElementById('cfg-load-mode').value,
     used_vram_gb:      isNaN(usedVram) ? null : usedVram,
+    force_vram_update: document.getElementById('cfg-force-vram-update').checked,
     max_instances:     parseInt(document.getElementById('cfg-max-instances').value) || 1,
     preload_all_instances: document.getElementById('cfg-preload-all-instances').checked,
     n_gpu_layers:      parseInt(document.getElementById('cfg-gpu-layers').value) || -1,
@@ -2585,6 +2703,7 @@ async function saveModelConfig(){
     manual_ram_gb:     isNaN(ramGb) ? null : ramGb,
     load_in_4bit:      document.getElementById('cfg-4bit').checked,
     load_in_8bit:      document.getElementById('cfg-8bit').checked,
+    component_quantization: _collectComponentQuant(),
     flash_attention:   document.getElementById('cfg-flash').checked,
     no_ram:            document.getElementById('cfg-noram').checked,
     offload_strategy:  document.getElementById('cfg-offload-strategy').value,
@@ -2602,10 +2721,15 @@ async function saveModelConfig(){
     clip_vision_path:  _readCompField('clip-vision') || null,
     lora_path:         _readCompField('lora')        || null,
     lora_model_dir:    document.getElementById('cfg-lora-dir').value.trim() || null,
+    lora_train_base_model: document.getElementById('cfg-lora-train-base').value.trim() || null,
     max_vram:          parseFloat(document.getElementById('cfg-max-vram').value) || 0,
     sdcpp_flash_attn:  document.getElementById('cfg-sdcpp-flash').checked,
     sdcpp_diffusion_flash_attn: document.getElementById('cfg-sdcpp-diff-flash').checked,
     vae_tiling:        document.getElementById('cfg-vae-tiling').checked,
+    output_crf:        (document.getElementById('cfg-output-crf').value.trim() === ''
+                          ? null : parseInt(document.getElementById('cfg-output-crf').value)),
+    balanced_gpu_percent: (document.getElementById('cfg-balanced-gpu-pct').value.trim() === ''
+                          ? null : parseFloat(document.getElementById('cfg-balanced-gpu-pct').value)),
   };
   try{
     const r = await fetch(ROOT_PATH + '/admin/api/model-configure',{

codai/api/loras.py

@@ -149,6 +149,26 @@ class LoraTrainRequest(BaseModel):
 
 # ── Base-model resolution ─────────────────────────────────────────────────────
 
+def _configured_train_base(base_model: str) -> Optional[str]:
+    """Read a per-model `lora_train_base_model` override from the image model's
+    config (models.json entry), if any.  Lets a deployment declare once that e.g.
+    a Z-Image generation model should train LoRAs against an SDXL model, so no
+    client has to know.  Returns the configured model key/path or None."""
+    try:
+        from codai.models.manager import multi_model_manager
+        for key in (f"image:{base_model}", base_model):
+            cfg = multi_model_manager.config.get(key)
+            if not cfg:
+                continue
+            for src in (cfg, cfg.get('_raw_cfg') or {}):
+                v = src.get('lora_train_base_model') if isinstance(src, dict) else None
+                if v and isinstance(v, str) and v.strip():
+                    return v.strip()
+    except Exception:
+        pass
+    return None
+
+
 def _resolve_base_model_path(base_model: str) -> str:
     """Resolve an image model key (or path/HF id) to a diffusers model directory."""
     try:
@@ -210,6 +230,148 @@ def _set_progress(**kw):
         _progress.update(kw)
 
 
+def _lora_debug_enabled() -> bool:
+    """LoRA training step logging to the terminal is gated on --debug-lora."""
+    try:
+        from codai.api.state import get_global_args
+        return bool(getattr(get_global_args(), "debug_lora", False))
+    except Exception:
+        return False
+
+
+def _dbg_lora(msg: str) -> None:
+    if _lora_debug_enabled():
+        print(f"  [lora][debug] {msg}", flush=True)
+
+
+def _free_vram_gb() -> float:
+    try:
+        import torch
+        if torch.cuda.is_available():
+            return torch.cuda.mem_get_info()[0] / (1024 ** 3)
+    except Exception:
+        pass
+    return 0.0
+
+
+def _free_train_vram() -> None:
+    """Fully release training-base-model VRAM after a job.
+
+    The trainer loads the SD1.x/SDXL base directly (not via the model manager),
+    so eviction can't see it. peft adapter hooks create reference cycles that
+    keep the modules alive until a gc pass runs, so empty_cache() alone leaves
+    the memory pinned. Collect twice, then drop the allocator cache.
+    """
+    import gc
+    try:
+        import torch
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            torch.cuda.synchronize()
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        _dbg_lora(f"freed training VRAM — {_free_vram_gb():.1f} GB now free")
+    except Exception:
+        pass
+
+
+# ── Training base-model cache ────────────────────────────────────────────────
+# The SD1.x/SDXL base used for LoRA training is expensive to load from disk.
+# We keep its components cached *on CPU* between consecutive training jobs so a
+# back-to-back job against the same base skips the reload. Holding it on CPU
+# (not GPU) means it consumes no VRAM between jobs, so image/video generation
+# can use the GPU freely — components are moved to the GPU only while a job is
+# actually running, then moved back to CPU when it finishes.
+_base_lock = threading.RLock()
+_base_cache = {"path": None, "arch": None, "components": None}
+
+
+def _load_base_components(base_path, arch, dtype):
+    """Load all base components on CPU at the given dtype (no GPU placement)."""
+    from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel
+    if arch == "sdxl":
+        from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
+        return {
+            "tokenizer_1": CLIPTokenizer.from_pretrained(base_path, subfolder="tokenizer"),
+            "tokenizer_2": CLIPTokenizer.from_pretrained(base_path, subfolder="tokenizer_2"),
+            "text_encoder_1": CLIPTextModel.from_pretrained(base_path, subfolder="text_encoder").to(dtype=dtype),
+            "text_encoder_2": CLIPTextModelWithProjection.from_pretrained(base_path, subfolder="text_encoder_2").to(dtype=dtype),
+            "vae": AutoencoderKL.from_pretrained(base_path, subfolder="vae").to(dtype=dtype),
+            "unet": UNet2DConditionModel.from_pretrained(base_path, subfolder="unet").to(dtype=dtype),
+            "noise_scheduler": DDPMScheduler.from_pretrained(base_path, subfolder="scheduler"),
+        }
+    from transformers import CLIPTextModel, CLIPTokenizer
+    return {
+        "tokenizer": CLIPTokenizer.from_pretrained(base_path, subfolder="tokenizer"),
+        "text_encoder": CLIPTextModel.from_pretrained(base_path, subfolder="text_encoder").to(dtype=dtype),
+        "vae": AutoencoderKL.from_pretrained(base_path, subfolder="vae").to(dtype=dtype),
+        "unet": UNet2DConditionModel.from_pretrained(base_path, subfolder="unet").to(dtype=dtype),
+        "noise_scheduler": DDPMScheduler.from_pretrained(base_path, subfolder="scheduler"),
+    }
+
+
+def _acquire_base(base_path, arch, dtype):
+    """Return cached CPU-resident base components, loading from disk if needed.
+
+    A change of base_path/arch drops the previous cache first. Must be called
+    inside a training job (holds the train lock for the job's duration)."""
+    with _base_lock:
+        c = _base_cache
+        if c["components"] is not None and c["path"] == base_path and c["arch"] == arch:
+            _dbg_lora(f"reusing cached base model (CPU): {base_path}")
+            return c["components"]
+        if c["components"] is not None:
+            _dbg_lora(f"base model changed ({c['path']} → {base_path}); dropping cache")
+            _drop_base_cache_locked()
+        _dbg_lora(f"loading base model from disk: {base_path} ({arch})")
+        comps = _load_base_components(base_path, arch, dtype)
+        c.update(path=base_path, arch=arch, components=comps)
+        return comps
+
+
+def _drop_base_cache_locked() -> None:
+    _base_cache["components"] = None
+    _base_cache["path"] = None
+    _base_cache["arch"] = None
+
+
+def _drop_base_cache() -> None:
+    with _base_lock:
+        had = _base_cache["components"] is not None
+        _drop_base_cache_locked()
+    if had:
+        _free_train_vram()
+
+
+def _release_base_cache(needed_gb: float = 0.0) -> float:
+    """External VRAM releaser (registered with the model manager).
+
+    Drops the cached training base on demand. Skips while a training job is
+    running (the base is in use). Between jobs the base lives on CPU, so this
+    mainly reclaims host RAM; it returns the VRAM delta it observed."""
+    if not _train_lock.acquire(blocking=False):
+        return 0.0
+    try:
+        with _base_lock:
+            if _base_cache["components"] is None:
+                return 0.0
+        before = _free_vram_gb()
+        _drop_base_cache()
+        return max(0.0, _free_vram_gb() - before)
+    finally:
+        _train_lock.release()
+
+
+# Let the model manager reclaim this cache when a generation needs VRAM.
+try:
+    from codai.models.manager import multi_model_manager as _mmm
+    _mmm.register_external_vram_releaser(_release_base_cache)
+except Exception:
+    pass
+
+
 # ── Training core ─────────────────────────────────────────────────────────────
 
 def _train_lora_sync(req: LoraTrainRequest) -> dict:
@@ -226,9 +388,16 @@ def _train_lora_sync(req: LoraTrainRequest) -> dict:
     from transformers import CLIPTextModel, CLIPTokenizer
 
     name = req.name
-    # Train against a dedicated UNet model when provided (the generation model
-    # may be a DiT this trainer can't target); otherwise use base_model.
-    base_path = _resolve_base_model_path(req.train_base_model or req.base_model)
+    # Resolve the model to TRAIN against (the generation model may be a DiT this
+    # trainer can't target). Precedence: explicit request override > per-model
+    # `lora_train_base_model` from models.json config > the base_model itself.
+    train_base = (req.train_base_model
+                  or _configured_train_base(req.base_model)
+                  or req.base_model)
+    if train_base != req.base_model:
+        print(f"  [lora] training '{name}' against base '{train_base}' "
+              f"(generation model: '{req.base_model}')")
+    base_path = _resolve_base_model_path(train_base)
     steps = max(50, min(5000, int(req.steps or 800)))
     rank = max(2, min(128, int(req.rank or 16)))
     resolution = int(req.resolution or 512)
@@ -335,11 +504,13 @@ def _train_sd15(req, base_path, images, instance_prompt,
 
     # Consistent fp32 precision (see _train_sdxl) to avoid mixed-dtype crashes.
     weight_dtype = torch.float32
-    tokenizer = CLIPTokenizer.from_pretrained(base_path, subfolder="tokenizer")
-    text_encoder = CLIPTextModel.from_pretrained(base_path, subfolder="text_encoder").to(device, dtype=weight_dtype)
-    vae = AutoencoderKL.from_pretrained(base_path, subfolder="vae").to(device, dtype=weight_dtype)
-    unet = UNet2DConditionModel.from_pretrained(base_path, subfolder="unet").to(device, dtype=weight_dtype)
-    noise_scheduler = DDPMScheduler.from_pretrained(base_path, subfolder="scheduler")
+    # Components come from the cross-job CPU cache; move to GPU for this job.
+    comps = _acquire_base(base_path, "sd15", weight_dtype)
+    tokenizer = comps["tokenizer"]
+    text_encoder = comps["text_encoder"].to(device, dtype=weight_dtype)
+    vae = comps["vae"].to(device, dtype=weight_dtype)
+    unet = comps["unet"].to(device, dtype=weight_dtype)
+    noise_scheduler = comps["noise_scheduler"]
 
     vae.requires_grad_(False)
     text_encoder.requires_grad_(False)
@@ -349,7 +520,7 @@ def _train_sd15(req, base_path, images, instance_prompt,
         r=rank, lora_alpha=rank, init_lora_weights="gaussian",
         target_modules=["to_k", "to_q", "to_v", "to_out.0"],
     )
-    unet.add_adapter(lora_cfg)
+    unet.add_adapter(lora_cfg, adapter_name="default")
     lora_params = [p for p in unet.parameters() if p.requires_grad]
     optimizer = torch.optim.AdamW(lora_params, lr=lr)
 
@@ -362,8 +533,10 @@ def _train_sd15(req, base_path, images, instance_prompt,
                         return_tensors="pt").input_ids.to(device)
         encoder_hidden_states = text_encoder(tok)[0]
 
-    # VAE + text encoder are done; free them so only the UNet trains resident.
-    del vae, text_encoder
+    # VAE + text encoder are done; move them back to CPU (keeps them cached for
+    # the next job) so only the UNet stays resident during training.
+    vae.to("cpu")
+    text_encoder.to("cpu")
     try:
         torch.cuda.empty_cache()
     except Exception:
@@ -393,6 +566,7 @@ def _train_sd15(req, base_path, images, instance_prompt,
 
         if step % 10 == 0 or step == steps - 1:
             _set_progress(step=step + 1, message=f"step {step+1}/{steps} loss={loss.item():.4f}")
+            _dbg_lora(f"SD1.5 step {step+1}/{steps} loss={loss.item():.4f}")
         # Mid-training thermal checkpoint (pauses if CPU/GPU too hot).
         try:
             from codai.models.thermal import checkpoint as _thermal_checkpoint
@@ -409,12 +583,19 @@ def _train_sd15(req, base_path, images, instance_prompt,
                                               safe_serialization=True)
     _write_meta(name, req, base_path, len(images), "sd15", instance_prompt)
 
-    # Release training tensors.
-    del unet, optimizer, latents_list
+    # Job done: drop this job's adapter + transients and move the UNet back to
+    # CPU. The base stays cached on CPU (reused by the next job); no VRAM is held
+    # afterwards, so the next image/video request gets the full GPU.
     try:
-        torch.cuda.empty_cache()
+        unet.delete_adapters("default")
     except Exception:
         pass
+    unet.to("cpu")
+    try:
+        del optimizer, latents_list, lora_params, encoder_hidden_states
+    except Exception:
+        pass
+    _free_train_vram()
 
     path = _lora_weight_file(name) or save_dir
     _set_progress(active=False, status="done", message="done", path=path)
@@ -443,13 +624,15 @@ def _train_sdxl(req, base_path, images, instance_prompt,
     # LoRA fine-tuning.
     weight_dtype = torch.float32
 
-    tokenizer_1 = CLIPTokenizer.from_pretrained(base_path, subfolder="tokenizer")
-    tokenizer_2 = CLIPTokenizer.from_pretrained(base_path, subfolder="tokenizer_2")
-    text_encoder_1 = CLIPTextModel.from_pretrained(base_path, subfolder="text_encoder").to(device, dtype=weight_dtype)
-    text_encoder_2 = CLIPTextModelWithProjection.from_pretrained(base_path, subfolder="text_encoder_2").to(device, dtype=weight_dtype)
-    vae = AutoencoderKL.from_pretrained(base_path, subfolder="vae").to(device, dtype=weight_dtype)
-    unet = UNet2DConditionModel.from_pretrained(base_path, subfolder="unet").to(device, dtype=weight_dtype)
-    noise_scheduler = DDPMScheduler.from_pretrained(base_path, subfolder="scheduler")
+    # Components come from the cross-job CPU cache; move to GPU for this job.
+    comps = _acquire_base(base_path, "sdxl", weight_dtype)
+    tokenizer_1 = comps["tokenizer_1"]
+    tokenizer_2 = comps["tokenizer_2"]
+    text_encoder_1 = comps["text_encoder_1"].to(device, dtype=weight_dtype)
+    text_encoder_2 = comps["text_encoder_2"].to(device, dtype=weight_dtype)
+    vae = comps["vae"].to(device, dtype=weight_dtype)
+    unet = comps["unet"].to(device, dtype=weight_dtype)
+    noise_scheduler = comps["noise_scheduler"]
 
     for m in (vae, text_encoder_1, text_encoder_2, unet):
         m.requires_grad_(False)
@@ -458,7 +641,7 @@ def _train_sdxl(req, base_path, images, instance_prompt,
         r=rank, lora_alpha=rank, init_lora_weights="gaussian",
         target_modules=["to_k", "to_q", "to_v", "to_out.0"],
     )
-    unet.add_adapter(lora_cfg)
+    unet.add_adapter(lora_cfg, adapter_name="default")
     lora_params = [p for p in unet.parameters() if p.requires_grad]
     optimizer = torch.optim.AdamW(lora_params, lr=lr)
 
@@ -496,9 +679,12 @@ def _train_sdxl(req, base_path, images, instance_prompt,
         device=device, dtype=prompt_embeds.dtype,
     )
 
-    # VAE + text encoders are no longer needed during the training loop; free
-    # them so only the UNet stays resident (keeps SDXL fp32 training in budget).
-    del vae, text_encoder_1, text_encoder_2
+    # VAE + text encoders are no longer needed during the training loop; move
+    # them back to CPU (keeps them cached for the next job) so only the UNet
+    # stays resident — keeps SDXL fp32 training in VRAM budget.
+    vae.to("cpu")
+    text_encoder_1.to("cpu")
+    text_encoder_2.to("cpu")
     try:
         torch.cuda.empty_cache()
     except Exception:
@@ -529,6 +715,7 @@ def _train_sdxl(req, base_path, images, instance_prompt,
 
         if step % 10 == 0 or step == steps - 1:
             _set_progress(step=step + 1, message=f"step {step+1}/{steps} loss={loss.item():.4f}")
+            _dbg_lora(f"SDXL step {step+1}/{steps} loss={loss.item():.4f}")
         try:
             from codai.models.thermal import checkpoint as _thermal_checkpoint
             _thermal_checkpoint(context="lora-train", throttle_seconds=2.0)
@@ -544,11 +731,20 @@ def _train_sdxl(req, base_path, images, instance_prompt,
                                                 safe_serialization=True)
     _write_meta(name, req, base_path, len(images), "sdxl", instance_prompt)
 
-    del unet, optimizer, latents_list
+    # Job done: drop this job's adapter + transients and move the UNet back to
+    # CPU. The base stays cached on CPU for the next job; no VRAM held afterwards
+    # so the next image/video request gets the full GPU (see _train_sd15 note).
     try:
-        torch.cuda.empty_cache()
+        unet.delete_adapters("default")
+    except Exception:
+        pass
+    unet.to("cpu")
+    try:
+        del optimizer, latents_list, lora_params
+        del prompt_embeds, pooled, add_time_ids
     except Exception:
         pass
+    _free_train_vram()
 
     path = _lora_weight_file(name) or save_dir
     _set_progress(active=False, status="done", message="done", path=path)
@@ -559,7 +755,9 @@ def _write_meta(name, req, base_path, n_images, arch, instance_prompt):
     meta = {
         "name": name,
         "base_model": req.base_model,
-        "train_base_model": req.train_base_model or req.base_model,
+        "train_base_model": (req.train_base_model
+                             or _configured_train_base(req.base_model)
+                             or req.base_model),
         "base_path": base_path,
         "arch": arch,
         "instance_prompt": instance_prompt,
@@ -597,6 +795,9 @@ async def train_lora(req: LoraTrainRequest, _auth=Depends(_require_api_auth)):
             import traceback
             traceback.print_exc()
             _set_progress(active=False, status="error", message=str(e))
+            # On error the base may be in a half-moved / inconsistent state — drop
+            # the cache entirely (and reclaim its VRAM) rather than reuse it.
+            _drop_base_cache()
             raise HTTPException(status_code=500, detail=f"LoRA training failed: {e}")
     finally:
         _train_lock.release()

codai/cli.py

@@ -189,6 +189,26 @@ configuration directory (--config DIR, default: OS-specific CoderAI directory).
         action="store_true",
         help="Enable debug mode - dumps full request/response to stdout for troubleshooting",
     )
+    parser.add_argument(
+        "--debug-ws",
+        action="store_true",
+        help="Enable WebSocket debug logging (websockets library). Requires --debug or standalone.",
+    )
+    parser.add_argument(
+        "--debug-web",
+        action="store_true",
+        help="Enable web/HTTP access logging (uvicorn per-request lines, e.g. /v1/loras/progress polling). Off by default.",
+    )
+    parser.add_argument(
+        "--debug-thermal",
+        action="store_true",
+        help="Enable thermal-protection debug logging ([thermal][debug] temperature checks). Off by default.",
+    )
+    parser.add_argument(
+        "--debug-lora",
+        action="store_true",
+        help="Enable LoRA training step logging to the terminal (per-step loss/progress). Off by default.",
+    )
     parser.add_argument(
         "--dump",
         action="store_true",

codai/main.py

@@ -108,6 +108,116 @@ def _migrate_hf_gguf_to_gguf_cache() -> None:
         )
 
 
+# Category → (type string used by build_runtime_kwargs, manager config-key prefix)
+_CATEGORY_TYPE_PREFIX = {
+    "text_models":      ("text", ""),
+    "gguf_models":      ("text", ""),
+    "image_models":     ("image", "image:"),
+    "audio_models":     ("audio", "audio:"),
+    "tts_models":       ("tts", "tts:"),
+    "vision_models":    ("vision", "vision:"),
+    "video_models":     ("video", "video:"),
+    "audio_gen_models": ("audio_gen", "audio_gen:"),
+    "embedding_models": ("embedding", "embedding:"),
+    "spatial_models":   ("spatial", "spatial:"),
+}
+
+
+def build_runtime_kwargs(model_cfg, model_type):
+    """Translate a models.json entry into the runtime kwargs stored in
+    ``multi_model_manager.config``. Pure function (no external state) so it can
+    be reused both at startup and when applying an admin config change live."""
+    _flash = bool(
+        model_cfg.get('flash_attention', model_cfg.get('flash_attn', False))
+        or model_cfg.get('sdcpp_flash_attn', False)
+        or model_cfg.get('sdcpp_diffusion_flash_attn', False)
+    )
+    kwargs = {
+        'load_in_4bit':    model_cfg.get('load_in_4bit', False),
+        'load_in_8bit':    model_cfg.get('load_in_8bit', False),
+        'flash_attn':      _flash,
+        'offload_strategy': model_cfg.get('offload_strategy', 'auto'),
+        'offload_dir':     model_cfg.get('offload_dir'),
+        'max_gpu_percent': model_cfg.get('max_gpu_percent'),
+        'manual_ram_gb':   model_cfg.get('manual_ram_gb'),
+        'no_ram':          model_cfg.get('no_ram', False),
+        'n_gpu_layers':    model_cfg.get('n_gpu_layers', -1),
+        'force_vram_update': model_cfg.get('force_vram_update', False),
+        '_raw_cfg':        dict(model_cfg) if isinstance(model_cfg, dict) else {},
+    }
+    if model_type == "text":
+        kwargs['ctx'] = model_cfg.get('n_ctx', model_cfg.get('context_size'))
+    elif model_type == "image":
+        kwargs['llm_path'] = model_cfg.get('llm_path')
+        kwargs['vae_path'] = model_cfg.get('vae_path')
+        kwargs['sample_method'] = model_cfg.get('sample_method', 'res_multistep')
+        kwargs['steps'] = model_cfg.get('steps', 4)
+        kwargs['width'] = model_cfg.get('width', 512)
+        kwargs['height'] = model_cfg.get('height', 512)
+        kwargs['cfg_scale'] = model_cfg.get('cfg_scale', 1.0)
+        kwargs['precision'] = model_cfg.get('precision', 'f32')
+        kwargs['cpu_offload'] = model_cfg.get('cpu_offload', False)
+        kwargs['seed'] = model_cfg.get('seed')
+        kwargs['vae_tiling'] = model_cfg.get('vae_tiling', False)
+        kwargs['clip_on_cpu'] = model_cfg.get('clip_on_cpu', False)
+    elif model_type == "audio":
+        kwargs['ctx'] = model_cfg.get('context_ms')
+        kwargs['offload'] = model_cfg.get('offload') or model_cfg.get('offload_strategy')
+        kwargs['vulkan_device'] = model_cfg.get('vulkan_device', 0)
+    elif model_type == "vision":
+        kwargs['ctx'] = model_cfg.get('n_ctx', model_cfg.get('context_size'))
+        kwargs['offload'] = model_cfg.get('offload') or model_cfg.get('offload_strategy')
+    elif model_type == "video":
+        kwargs['precision'] = model_cfg.get('precision', 'bf16')
+        kwargs['vae_tiling'] = model_cfg.get('vae_tiling', True)
+        kwargs['balanced_gpu_percent'] = model_cfg.get('balanced_gpu_percent', 80)
+        kwargs['output_crf'] = model_cfg.get('output_crf')
+    return kwargs
+
+
+def build_runtime_model_cfg(entry, model_type):
+    """build_runtime_kwargs + the extra passthrough keys main() applies."""
+    cfg = build_runtime_kwargs(entry, model_type) if isinstance(entry, dict) else {}
+    if isinstance(entry, dict):
+        for k in ("load_mode", "used_vram_gb", "alias", "max_instances"):
+            if k in entry:
+                cfg[k] = entry[k]
+    return cfg
+
+
+def apply_model_entry_live(entry, model_types) -> int:
+    """Update the running ``multi_model_manager.config`` for one models.json
+    entry so config changes (e.g. lora_train_base_model, vae_path, quant flags)
+    take effect immediately without a restart. Only the config dict is touched —
+    already-loaded weights and VRAM bookkeeping are left alone, and no downloads
+    are triggered. Returns the number of live config keys updated."""
+    try:
+        from codai.models.manager import multi_model_manager
+    except Exception:
+        return 0
+    if not isinstance(entry, dict):
+        return 0
+    mid = entry.get("path") or entry.get("id") or ""
+    if not mid:
+        return 0
+    updated = 0
+    for cat in (model_types or []):
+        tp = _CATEGORY_TYPE_PREFIX.get(cat)
+        if not tp:
+            continue
+        type_str, prefix = tp
+        cfg = build_runtime_model_cfg(entry, type_str)
+        multi_model_manager.config[f"{prefix}{mid}"] = cfg
+        updated += 1
+        alias = entry.get("alias")
+        if alias:
+            try:
+                multi_model_manager.set_model_alias(alias, mid)
+            except Exception:
+                pass
+    return updated
+
+
 def main():
     """Main entry point for the codai server."""
     # Suppress unraisable exceptions from LlamaModel.__del__
@@ -400,42 +510,10 @@ def main():
                 return m
         return {}
     
-    # Helper to build kwargs from model config
-    def build_kwargs_from_config(model_cfg, model_type):
-        kwargs = {}
-        if model_type == "text":
-            kwargs['ctx'] = model_cfg.get('context_size')
-            kwargs['n_gpu_layers'] = model_cfg.get('n_gpu_layers', -1)
-            kwargs['load_in_4bit'] = model_cfg.get('load_in_4bit', False)
-            kwargs['load_in_8bit'] = model_cfg.get('load_in_8bit', False)
-            kwargs['flash_attn'] = model_cfg.get('flash_attn', False)
-            kwargs['offload_strategy'] = model_cfg.get('offload_strategy', 'auto')
-            kwargs['manual_ram_gb'] = model_cfg.get('manual_ram_gb')
-            kwargs['max_gpu_percent'] = model_cfg.get('max_gpu_percent')
-            kwargs['no_ram'] = model_cfg.get('no_ram', False)
-        elif model_type == "image":
-            kwargs['llm_path'] = model_cfg.get('llm_path')
-            kwargs['vae_path'] = model_cfg.get('vae_path')
-            kwargs['sample_method'] = model_cfg.get('sample_method', 'res_multistep')
-            kwargs['steps'] = model_cfg.get('steps', 4)
-            kwargs['width'] = model_cfg.get('width', 512)
-            kwargs['height'] = model_cfg.get('height', 512)
-            kwargs['cfg_scale'] = model_cfg.get('cfg_scale', 1.0)
-            kwargs['precision'] = model_cfg.get('precision', 'f32')
-            kwargs['cpu_offload'] = model_cfg.get('cpu_offload', False)
-            kwargs['seed'] = model_cfg.get('seed')
-            kwargs['vae_tiling'] = model_cfg.get('vae_tiling', False)
-            kwargs['clip_on_cpu'] = model_cfg.get('clip_on_cpu', False)
-        elif model_type == "audio":
-            kwargs['ctx'] = model_cfg.get('context_ms')
-            kwargs['offload'] = model_cfg.get('offload')
-            kwargs['vulkan_device'] = model_cfg.get('vulkan_device', 0)
-        elif model_type == "vision":
-            kwargs['ctx'] = model_cfg.get('context_size')
-            kwargs['offload'] = model_cfg.get('offload')
-            kwargs['n_gpu_layers'] = model_cfg.get('n_gpu_layers', -1)
-        return kwargs
-    
+    # Helper to build kwargs from model config (module-level build_runtime_kwargs
+    # is the single source of truth; admin live-apply reuses the same function).
+    build_kwargs_from_config = build_runtime_kwargs
+
     # =========================================================================
     # Register and optionally pre-load all configured models
     # Models with load_mode == "load" are pre-loaded at startup.
@@ -670,6 +748,14 @@ def main():
     global_args.load_in_8bit = config.offload.load_in_8bit
     global_args.flash_attn = config.offload.flash_attention
     global_args.max_gpu_percent = config.offload.max_gpu_percent
+    # Thermal protection settings (read live by codai.models.thermal).
+    global_args.thermal_cpu_enabled = config.thermal.cpu_enabled
+    global_args.thermal_gpu_enabled = config.thermal.gpu_enabled
+    global_args.thermal_cpu_high = config.thermal.cpu_high
+    global_args.thermal_cpu_resume = config.thermal.cpu_resume
+    global_args.thermal_gpu_high = config.thermal.gpu_high
+    global_args.thermal_gpu_resume = config.thermal.gpu_resume
+    global_args.thermal_poll_seconds = config.thermal.poll_seconds
     global_args.n_gpu_layers = config.vulkan.n_gpu_layers
     global_args.n_ctx = [config.vulkan.n_ctx]
     global_args.vulkan_device = config.vulkan.device_id
@@ -688,6 +774,9 @@ def main():
     global_args.tools_closer_prompt = config.tools_closer_prompt
     global_args.grammar_guided_gen = config.grammar_guided
     global_args.debug = global_debug
+    global_args.debug_web = getattr(args, 'debug_web', False)
+    global_args.debug_thermal = getattr(args, 'debug_thermal', False)
+    global_args.debug_lora = getattr(args, 'debug_lora', False)
     global_args.dump = global_dump
     global_args.file_path = config.file_path
     global_args.parser = config.parser
@@ -774,6 +863,10 @@ def main():
     from codai.api.characters import set_global_args as set_chars_global_args
     set_chars_global_args(global_args)
 
+    # Set LoRA training module global args
+    from codai.api.loras import set_global_args as set_loras_global_args
+    set_loras_global_args(global_args)
+
     # Set environment profiles module global args
     from codai.api.environments import set_global_args as set_envs_global_args
     set_envs_global_args(global_args)
@@ -821,18 +914,41 @@ def main():
     # Configure Python logging so broker/API log calls reach the terminal.
     # uvicorn is started with log_config=None to keep our config in place.
     _log_level = logging.DEBUG if global_debug else logging.INFO
+
+    class _RenameUvicornError(logging.Filter):
+        """uvicorn names its general-purpose lifecycle logger 'uvicorn.error' even
+        for non-error messages (startup, listening address, shutdown).  Rename it
+        to 'uvicorn' in the output so INFO lines don't look like errors."""
+        def filter(self, record):
+            if record.name == "uvicorn.error":
+                record.name = "uvicorn"
+            return True
+
     logging.basicConfig(
         level=_log_level,
         format="%(asctime)s [%(levelname)-8s] %(name)s: %(message)s",
         stream=sys.stdout,
         force=True,
     )
+    logging.getLogger("uvicorn.error").addFilter(_RenameUvicornError())
     # Suppress noisy third-party libraries at WARNING unless in debug mode.
     for _noisy in ("httpx", "httpcore", "urllib3", "multipart", "PIL"):
         logging.getLogger(_noisy).setLevel(logging.WARNING)
     if not global_debug:
-        logging.getLogger("websockets").setLevel(logging.WARNING)
         logging.getLogger("asyncio").setLevel(logging.WARNING)
+    # WebSocket + broker-client debug logs are suppressed unless --debug-ws is given.
+    # --debug alone does NOT enable them; they are too noisy for general debugging.
+    _debug_ws = getattr(args, 'debug_ws', False)
+    if not _debug_ws:
+        logging.getLogger("websockets").setLevel(logging.WARNING)
+        logging.getLogger("websockets.client").setLevel(logging.WARNING)
+        logging.getLogger("websockets.server").setLevel(logging.WARNING)
+        logging.getLogger("codai.broker.client").setLevel(logging.INFO)
+    # Per-request HTTP access lines (uvicorn.access) are noisy — e.g. the web UI
+    # polls /v1/loras/progress every ~1.5s. Suppress them unless --debug-web.
+    _debug_web = getattr(args, 'debug_web', False)
+    if not _debug_web:
+        logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
 
     # Start the server
     import uvicorn

codai/models/manager.py

@@ -36,6 +36,24 @@ from codai.models.utils import FuzzyToolBreaker
 from codai.pydantic.textrequest import ModelInfo
 
 
+def _trim_cpu_ram() -> None:
+    """Return freed CPU heap memory to the OS (and let the kernel reclaim swap).
+
+    Python frees evicted-model tensors, but glibc's allocator keeps the pages in
+    the process arena (RSS stays high; swap is not returned).  malloc_trim(0)
+    releases the top of the heap back to the OS.  No-op on non-glibc platforms.
+    """
+    try:
+        gc.collect()
+        import ctypes, ctypes.util
+        _libc_name = ctypes.util.find_library('c') or 'libc.so.6'
+        _libc = ctypes.CDLL(_libc_name)
+        if hasattr(_libc, 'malloc_trim'):
+            _libc.malloc_trim(0)
+    except Exception:
+        pass
+
+
 class ModelManager:
     """Manages the loaded model and tokenizer."""
     
@@ -170,13 +188,20 @@ class ModelManager:
     def generate_chat(self, messages: List[Dict], max_tokens: Optional[int] = None,
                       temperature: float = 0.7, top_p: float = 1.0,
                       stop: Optional[List[str]] = None, tools: Optional[List] = None,
-                      response_format: Optional[Dict] = None):
+                      response_format: Optional[Dict] = None, enable_thinking: bool = False):
         """Generate chat completion non-streaming."""
         if self.backend is None:
             raise RuntimeError("No model loaded")
         # Use generate_chat if available (Vulkan backend), otherwise format and use generate
         if hasattr(self.backend, 'generate_chat'):
-            return self.backend.generate_chat(messages, max_tokens, temperature, top_p, stop, tools, response_format)
+            try:
+                return self.backend.generate_chat(messages, max_tokens, temperature, top_p,
+                                                  stop, tools, response_format,
+                                                  enable_thinking=enable_thinking)
+            except TypeError:
+                # Backend doesn't accept enable_thinking (e.g. Vulkan) — call plainly.
+                return self.backend.generate_chat(messages, max_tokens, temperature, top_p,
+                                                  stop, tools, response_format)
         else:
             # Fallback for NVIDIA backend
             from codai.pydantic.textrequest import ChatMessage
@@ -195,13 +220,20 @@ class ModelManager:
     async def generate_chat_stream(self, messages: List[Dict], max_tokens: Optional[int] = None,
                                     temperature: float = 0.7, top_p: float = 1.0,
                                     stop: Optional[List[str]] = None, tools: Optional[List] = None,
-                                    response_format: Optional[Dict] = None):
+                                    response_format: Optional[Dict] = None, enable_thinking: bool = False):
         """Generate chat completion streaming."""
         if self.backend is None:
             raise RuntimeError("No model loaded")
         # Use generate_chat_stream if available (Vulkan backend), otherwise format and use generate_stream
         if hasattr(self.backend, 'generate_chat_stream'):
-            async for chunk in self.backend.generate_chat_stream(messages, max_tokens, temperature, top_p, stop, tools, response_format):
+            try:
+                _gen = self.backend.generate_chat_stream(
+                    messages, max_tokens, temperature, top_p, stop, tools,
+                    response_format, enable_thinking=enable_thinking)
+            except TypeError:
+                _gen = self.backend.generate_chat_stream(
+                    messages, max_tokens, temperature, top_p, stop, tools, response_format)
+            async for chunk in _gen:
                 yield chunk
         else:
             # Fallback for NVIDIA backend
@@ -524,11 +556,63 @@ class MultiModelManager:
         self.model_backend_types: Dict[str, str] = {}
         self.tool_breaker = FuzzyToolBreaker(threshold=3)  # Circuit breaker for repetitive tool calls
         self._load_lock = threading.Lock()  # Prevents duplicate on-demand model loads
+        # Set when NO model switch/load is in progress; cleared during load.
+        # Callers that need to wait for a load to complete can poll this event.
+        self._model_ready_event = threading.Event()
+        self._model_ready_event.set()   # initially ready (nothing loading)
         self.model_pools: Dict[str, ModelInstancePool] = {}  # per-key instance pools
+        # Set once a CUDA device-side assert / unrecoverable CUDA error is seen.
+        # The CUDA context is corrupted process-wide after such an error, so all
+        # further GPU work is futile until the server is restarted.  We surface
+        # this immediately instead of retrying loads dozens of times.
+        self.cuda_context_poisoned: bool = False
+        self.cuda_poison_reason: Optional[str] = None
         self._pending_new_instance: set = set()  # keys awaiting a second+ instance load
         self._global_max_instances: int = 1  # set from config at startup
         self._measured_vram_gb: Dict[str, float] = {}  # actual measured VRAM delta per model key
-    
+        # Callbacks that free VRAM held *outside* the model manager (e.g. the
+        # LoRA trainer caches its SD/SDXL base model between jobs). Each returns
+        # the GB it freed (or None). Invoked as a last resort during eviction.
+        self._external_vram_releasers: List[Any] = []
+
+    def register_external_vram_releaser(self, fn) -> None:
+        """Register a callback that frees VRAM held outside the manager.
+
+        Called during on-request eviction when freeing tracked models isn't
+        enough. The callback must be safe to call when there's nothing to free
+        (return 0.0 / None) and must not block on in-flight work it owns.
+        """
+        if callable(fn) and fn not in self._external_vram_releasers:
+            self._external_vram_releasers.append(fn)
+
+    @staticmethod
+    def _is_cuda_context_fatal(err: Exception) -> bool:
+        """True for CUDA errors that corrupt the whole process context."""
+        s = str(err).lower()
+        return (
+            'device-side assert' in s
+            or 'cuda error' in s
+            or 'cublas' in s and 'not initialized' in s
+            or 'an illegal memory access' in s
+            or 'misaligned address' in s
+        )
+
+    def _mark_cuda_poisoned_if_fatal(self, err: Exception) -> bool:
+        """Record process-wide CUDA corruption. Returns True if poisoned."""
+        if self.cuda_context_poisoned:
+            return True
+        if self._is_cuda_context_fatal(err):
+            self.cuda_context_poisoned = True
+            self.cuda_poison_reason = str(err).split('\n', 1)[0][:200]
+            print("\n" + "=" * 72)
+            print("FATAL: CUDA context corrupted (device-side assert / CUDA error).")
+            print(f"  Reason: {self.cuda_poison_reason}")
+            print("  All GPU operations will fail until coderai is RESTARTED.")
+            print("  Failing fast instead of retrying. Restart the server.")
+            print("=" * 72 + "\n")
+            return True
+        return False
+
     @property
     def image_model(self) -> Optional[str]:
         """Return the first image model or None."""
@@ -655,9 +739,11 @@ class MultiModelManager:
         if self.default_model in self.models and self.default_model not in self._pending_new_instance:
             return self._get_least_busy_instance(self.default_model)
 
+        self._model_ready_event.clear()   # signal: load in progress
         with self._load_lock:
             # Re-check inside the lock to avoid duplicate loads from concurrent requests
             if self.default_model in self.models and self.default_model not in self._pending_new_instance:
+                self._model_ready_event.set()
                 return self._get_least_busy_instance(self.default_model)
             self._pending_new_instance.discard(self.default_model)
 
@@ -672,46 +758,73 @@ class MultiModelManager:
 
             model_manager = ModelManager()
             try:
+                # Build kwargs: per-model config takes precedence over global_args.
+                # This ensures settings configured per-model in coderai (4-bit quant,
+                # flash attention, offload strategy, …) are honoured on every load,
+                # including on-demand reloads after VRAM eviction.
                 kwargs = {}
-                if 'ctx' in config:
-                    kwargs['ctx'] = config['ctx']
-                if global_args:
-                    if hasattr(global_args, 'n_gpu_layers'):
-                        kwargs['n_gpu_layers'] = global_args.n_gpu_layers
-                    if hasattr(global_args, 'offload_dir'):
-                        kwargs['offload_dir'] = global_args.offload_dir
-                    if hasattr(global_args, 'ram'):
-                        kwargs['manual_ram_gb'] = global_args.ram
-                    if hasattr(global_args, 'flash_attn'):
-                        kwargs['flash_attn'] = global_args.flash_attn
-                    if hasattr(global_args, 'no_ram'):
-                        kwargs['no_ram'] = global_args.no_ram
-                    if hasattr(global_args, 'offload_strategy'):
-                        kwargs['offload_strategy'] = global_args.offload_strategy
-                    if hasattr(global_args, 'load_in_4bit'):
-                        kwargs['load_in_4bit'] = global_args.load_in_4bit
-                    if hasattr(global_args, 'load_in_8bit'):
-                        kwargs['load_in_8bit'] = global_args.load_in_8bit
-                    if hasattr(global_args, 'max_gpu_percent'):
-                        kwargs['max_gpu_percent'] = global_args.max_gpu_percent
+                _ga = global_args  # shorthand
+
+                def _cfg_or_global(cfg_key, ga_attr, default=None):
+                    if cfg_key in config and config[cfg_key] is not None:
+                        return config[cfg_key]
+                    if _ga and hasattr(_ga, ga_attr):
+                        v = getattr(_ga, ga_attr)
+                        if v is not None:
+                            return v
+                    return default
+
+                ctx = _cfg_or_global('ctx', 'n_ctx')
+                if ctx:
+                    kwargs['ctx'] = ctx
+                n_gpu_layers = _cfg_or_global('n_gpu_layers', 'n_gpu_layers')
+                if n_gpu_layers is not None:
+                    kwargs['n_gpu_layers'] = n_gpu_layers
+                offload_dir = _cfg_or_global('offload_dir', 'offload_dir')
+                if offload_dir:
+                    kwargs['offload_dir'] = offload_dir
+                manual_ram = _cfg_or_global('manual_ram_gb', 'ram')
+                if manual_ram is not None:
+                    kwargs['manual_ram_gb'] = manual_ram
+                # flash_attn comes from the per-model config (source of truth).
+                # build_kwargs_from_config populates it from the model's
+                # 'flash_attention' setting; CLI/global is NOT consulted here.
+                kwargs['flash_attn'] = bool(config.get('flash_attn', False))
+                no_ram = _cfg_or_global('no_ram', 'no_ram', False)
+                kwargs['no_ram'] = bool(no_ram)
+                offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')
+                if offload_strategy:
+                    kwargs['offload_strategy'] = offload_strategy
+                load_in_4bit = _cfg_or_global('load_in_4bit', 'load_in_4bit', False)
+                kwargs['load_in_4bit'] = bool(load_in_4bit)
+                load_in_8bit = _cfg_or_global('load_in_8bit', 'load_in_8bit', False)
+                kwargs['load_in_8bit'] = bool(load_in_8bit)
+                max_gpu_pct = _cfg_or_global('max_gpu_percent', 'max_gpu_percent')
+                if max_gpu_pct is not None:
+                    kwargs['max_gpu_percent'] = max_gpu_pct
 
                 print(f"Loading default model on demand: {self.default_model}")
                 _snap = self.vram_before_load()
+                kwargs['expected_vram_gb'] = self._get_model_used_vram_gb(self.default_model)
                 model_manager.load_model(self.default_model, backend_type=backend_type, **kwargs)
                 self.add_model(self.default_model, model_manager)
                 self.record_vram_delta(self.default_model, _snap)
                 self.current_model_key = self.default_model
                 print(f"Model loaded successfully: {self.default_model}")
+                self._model_ready_event.set()
                 return model_manager
             except Exception as e:
                 print(f"Error loading model {self.default_model}: {e}")
+                self._mark_cuda_poisoned_if_fatal(e)
+                self._model_ready_event.set()
                 return None
-    
+
     def _load_model_by_name(self, model_name: str):
         """Load a model by name on demand (thread-safe)."""
         if model_name in self.models and model_name not in self._pending_new_instance:
             return self._get_least_busy_instance(model_name)
 
+        self._model_ready_event.clear()   # signal: load in progress
         with self._load_lock:
             # Re-check inside lock to prevent duplicate loads, but honour pending new instance.
             if model_name in self.models and model_name not in self._pending_new_instance:
@@ -729,43 +842,70 @@ class MultiModelManager:
 
             model_manager = ModelManager()
             try:
+                # Per-model config takes precedence over global_args (same logic as
+                # _load_default_model so on-demand reloads respect per-model settings).
                 kwargs = {}
-                if 'ctx' in config:
-                    kwargs['ctx'] = config['ctx']
-                if global_args:
-                    if hasattr(global_args, 'n_gpu_layers'):
-                        kwargs['n_gpu_layers'] = global_args.n_gpu_layers
-                    if hasattr(global_args, 'offload_dir'):
-                        kwargs['offload_dir'] = global_args.offload_dir
-                    if hasattr(global_args, 'ram'):
-                        kwargs['manual_ram_gb'] = global_args.ram
-                    if hasattr(global_args, 'flash_attn'):
-                        kwargs['flash_attn'] = global_args.flash_attn
-                    if hasattr(global_args, 'no_ram'):
-                        kwargs['no_ram'] = global_args.no_ram
-                    if hasattr(global_args, 'offload_strategy'):
-                        kwargs['offload_strategy'] = global_args.offload_strategy
-                    if hasattr(global_args, 'load_in_4bit'):
-                        kwargs['load_in_4bit'] = global_args.load_in_4bit
-                    if hasattr(global_args, 'load_in_8bit'):
-                        kwargs['load_in_8bit'] = global_args.load_in_8bit
-                    if hasattr(global_args, 'max_gpu_percent'):
-                        kwargs['max_gpu_percent'] = global_args.max_gpu_percent
+                _ga = global_args
+
+                def _cfg_or_global(cfg_key, ga_attr, default=None):
+                    if cfg_key in config and config[cfg_key] is not None:
+                        return config[cfg_key]
+                    if _ga and hasattr(_ga, ga_attr):
+                        v = getattr(_ga, ga_attr)
+                        if v is not None:
+                            return v
+                    return default
+
+                ctx = _cfg_or_global('ctx', 'n_ctx')
+                if ctx:
+                    kwargs['ctx'] = ctx
+                n_gpu_layers = _cfg_or_global('n_gpu_layers', 'n_gpu_layers')
+                if n_gpu_layers is not None:
+                    kwargs['n_gpu_layers'] = n_gpu_layers
+                offload_dir = _cfg_or_global('offload_dir', 'offload_dir')
+                if offload_dir:
+                    kwargs['offload_dir'] = offload_dir
+                manual_ram = _cfg_or_global('manual_ram_gb', 'ram')
+                if manual_ram is not None:
+                    kwargs['manual_ram_gb'] = manual_ram
+                # flash_attn comes from the per-model config (source of truth).
+                # build_kwargs_from_config populates it from the model's
+                # 'flash_attention' setting; CLI/global is NOT consulted here.
+                kwargs['flash_attn'] = bool(config.get('flash_attn', False))
+                no_ram = _cfg_or_global('no_ram', 'no_ram', False)
+                kwargs['no_ram'] = bool(no_ram)
+                offload_strategy = _cfg_or_global('offload_strategy', 'offload_strategy', 'auto')
+                if offload_strategy:
+                    kwargs['offload_strategy'] = offload_strategy
+                load_in_4bit = _cfg_or_global('load_in_4bit', 'load_in_4bit', False)
+                kwargs['load_in_4bit'] = bool(load_in_4bit)
+                load_in_8bit = _cfg_or_global('load_in_8bit', 'load_in_8bit', False)
+                kwargs['load_in_8bit'] = bool(load_in_8bit)
+                max_gpu_pct = _cfg_or_global('max_gpu_percent', 'max_gpu_percent')
+                if max_gpu_pct is not None:
+                    kwargs['max_gpu_percent'] = max_gpu_pct
 
                 pool = self.model_pools.get(model_name)
                 inst_num = pool.count + 1 if pool else 1
                 print(f"Loading model on demand: {model_name}"
                       + (f" (instance {inst_num})" if inst_num > 1 else ""))
                 _snap = self.vram_before_load()
+                # Tell the backend how much VRAM this model is expected to need so
+                # it can decide whether Flash-Attention-2 is safe (FA2 requires the
+                # whole model on GPU; it device-side-asserts when layers offload).
+                kwargs['expected_vram_gb'] = self._get_model_used_vram_gb(model_name)
                 model_manager.load_model(model_name, backend_type=backend_type, **kwargs)
                 self.add_model(model_name, model_manager)
                 self.record_vram_delta(model_name, _snap)
                 self.current_model_key = model_name
                 print(f"Model loaded successfully: {model_name}"
                       + (f" (instance {inst_num})" if inst_num > 1 else ""))
+                self._model_ready_event.set()   # signal: ready
                 return model_manager
             except Exception as e:
                 print(f"Error loading model {model_name}: {e}")
+                self._mark_cuda_poisoned_if_fatal(e)
+                self._model_ready_event.set()   # signal: ready (even on failure)
                 return None
 
     def set_audio_model(self, model_name: str, config: Dict = None):
@@ -1492,42 +1632,60 @@ class MultiModelManager:
         if delta_gb <= 0:
             return
 
-        measured = round(delta_gb, 3)
-        # Read the pre-existing estimate BEFORE storing the measurement so that
-        # _get_model_used_vram_gb doesn't return the measured value as the baseline.
-        estimated = self._get_model_used_vram_gb(model_key)
+        # The delta is the resident WEIGHT footprint measured at load time. Add
+        # the runtime reserve (KV cache / activations / VAE-decode spike) so the
+        # value we cache and persist reflects the model's PEAK runtime need — not
+        # just its loaded weights — and future eviction frees enough headroom.
+        cfg = self.config.get(model_key, {})
+        reserve_gb = self._runtime_reserve_gb(
+            cfg if isinstance(cfg, dict) else {}, model_key, delta_gb)
+        measured = round(delta_gb + reserve_gb, 3)
+        # In-memory truth for the rest of this run (used by eviction estimates).
         self._measured_vram_gb[model_key] = measured
-        print(f"Measured VRAM for '{model_key}': {measured:.2f} GB")
-
-        # Persist the real value when it's meaningfully larger than the current estimate
+        print(f"Measured VRAM for '{model_key}': {delta_gb:.2f} GB weights "
+              f"+ {reserve_gb:.2f} GB runtime reserve = {measured:.2f} GB")
+
+        # Persist the real value into the SEPARATE, system-owned
+        # `measured_vram_gb` field (never the user's `used_vram_gb`). Rules:
+        #   * force_vram_update set → refresh it EVERY run, even when
+        #     used_vram_gb is configured.
+        #   * otherwise → only when used_vram_gb is NOT configured (a configured
+        #     value is authoritative and must remain unchanged).
+        force_update = bool(cfg.get("force_vram_update")) if isinstance(cfg, dict) else False
+        if not force_update:
+            if isinstance(cfg, dict) and cfg.get("used_vram_gb") is not None:
+                return  # configured & not forced → leave it exactly as-is
+            # Avoid churning models.json when the measurement hasn't meaningfully
+            # changed from what's already persisted.
+            prev = cfg.get("measured_vram_gb") if isinstance(cfg, dict) else None
+            try:
+                if prev is not None and abs(measured - float(prev)) <= max(0.5, float(prev) * 0.10):
+                    return
+            except (TypeError, ValueError):
+                pass
         try:
-            if estimated <= 0 or measured > estimated * 1.10:
-                label = f"{estimated:.2f} GB estimated" if estimated > 0 else "no estimate"
-                print(f"  Updating used_vram_gb for '{model_key}': {label} → {measured:.2f} GB (real)")
-                # Update in-memory config
-                cfg = self.config.get(model_key, {})
-                cfg["used_vram_gb"] = measured
-                self.config[model_key] = cfg
-                # Persist to models.json via config_manager
-                try:
-                    from codai.admin.routes import config_manager
-                    if config_manager is not None:
-                        # Strip type prefix to get the bare path/id used in models_data
-                        bare = model_key.split(":", 1)[1] if ":" in model_key else model_key
-                        for cat in ("text_models", "image_models", "audio_models", "tts_models",
-                                    "vision_models", "video_models", "audio_gen_models",
-                                    "embedding_models", "spatial_models"):
-                            for entry in config_manager.models_data.get(cat, []):
-                                if not isinstance(entry, dict):
-                                    continue
-                                epath = entry.get("path") or entry.get("id") or ""
-                                if epath == bare or epath.split("/")[-1] == bare.split("/")[-1]:
-                                    entry["used_vram_gb"] = measured
-                        config_manager.save_models()
-                except Exception as e:
-                    print(f"  Warning: could not persist used_vram_gb: {e}")
+            _why = "force_vram_update" if force_update else "no used_vram_gb configured"
+            print(f"  Saving measured_vram_gb for '{model_key}': {measured:.2f} GB ({_why})")
+            cfg = dict(cfg) if isinstance(cfg, dict) else {}
+            cfg["measured_vram_gb"] = measured
+            self.config[model_key] = cfg
+            # Persist to models.json via config_manager (separate field; the
+            # user's own keys are never touched).
+            from codai.admin.routes import config_manager
+            if config_manager is not None:
+                bare = model_key.split(":", 1)[1] if ":" in model_key else model_key
+                for cat in ("text_models", "image_models", "audio_models", "tts_models",
+                            "vision_models", "video_models", "audio_gen_models",
+                            "embedding_models", "spatial_models"):
+                    for entry in config_manager.models_data.get(cat, []):
+                        if not isinstance(entry, dict):
+                            continue
+                        epath = entry.get("path") or entry.get("id") or ""
+                        if epath == bare or epath.split("/")[-1] == bare.split("/")[-1]:
+                            entry["measured_vram_gb"] = measured
+                config_manager.save_models()
         except Exception as e:
-            print(f"  Warning: VRAM update check failed: {e}")
+            print(f"  Warning: could not persist measured_vram_gb: {e}")
 
     # In-process cache: model_id -> size_bytes (never stale within a run)
     _hf_size_cache: Dict[str, int] = {}
@@ -1574,11 +1732,27 @@ class MultiModelManager:
                     revs = sorted(repo.revisions, key=lambda r: r.last_modified, reverse=True)
                     if revs:
                         rev = revs[0]
-                        total = sum(
-                            f.size_on_disk
-                            for f in rev.files
-                            if os.path.splitext(f.file_name)[1].lower() in weight_exts
+                        # Separate GGUF files (alternative quantizations of the same model)
+                        # from dense weight files (safetensors/bin shards that collectively
+                        # represent one model variant).  Summing all GGUFs produces wildly
+                        # inflated estimates; instead take the largest single GGUF as the
+                        # worst-case for that format, and sum the dense shards separately.
+                        gguf_exts = {'.gguf', '.ggml'}
+                        dense_exts = {'.safetensors', '.bin', '.pt'}
+                        gguf_sizes = [
+                            f.size_on_disk for f in rev.files
+                            if os.path.splitext(f.file_name)[1].lower() in gguf_exts
+                        ]
+                        dense_total = sum(
+                            f.size_on_disk for f in rev.files
+                            if os.path.splitext(f.file_name)[1].lower() in dense_exts
                         )
+                        # Use whichever single-load format is smaller (prefer the real
+                        # on-disk format that will actually be used).
+                        largest_gguf = max(gguf_sizes) if gguf_sizes else 0
+                        total = dense_total if dense_total > 0 else largest_gguf
+                        if total == 0:
+                            total = largest_gguf  # fallback
                         # LoRA adapter: add base model size
                         for f in rev.files:
                             if f.file_name == "adapter_config.json":
@@ -1607,18 +1781,25 @@ class MultiModelManager:
             req = urllib.request.Request(url, headers={"User-Agent": "coderai/1.0"})
             with urllib.request.urlopen(req, timeout=10) as resp:
                 data = _json.loads(resp.read())
-            total = 0
+            gguf_sizes_api = []
+            dense_total_api = 0
             has_adapter_config = False
             for sib in data.get("siblings", []):
                 name = sib.get("rfilename", "")
                 if name == "adapter_config.json":
                     has_adapter_config = True
                     continue
-                if os.path.splitext(name)[1].lower() not in weight_exts:
+                ext = os.path.splitext(name)[1].lower()
+                if ext not in weight_exts:
                     continue
                 lfs = sib.get("lfs") or {}
                 size = lfs.get("size") or sib.get("size") or 0
-                total += size
+                if ext in {'.gguf', '.ggml'}:
+                    gguf_sizes_api.append(size)
+                else:
+                    dense_total_api += size
+            largest_gguf_api = max(gguf_sizes_api) if gguf_sizes_api else 0
+            total = dense_total_api if dense_total_api > 0 else largest_gguf_api
             # LoRA adapter: fetch adapter_config.json to get the base model
             if has_adapter_config:
                 try:
@@ -1643,24 +1824,429 @@ class MultiModelManager:
         MultiModelManager._hf_size_cache[model_id] = 0
         return 0
 
+    @staticmethod
+    def _quant_divisor(mode) -> float:
+        """Approx. VRAM reduction factor (vs fp16) for a quantization mode.
+
+        Accepts the strings used in coderai configs / component_quantization,
+        including GGUF file paths (the Qn level is sniffed from the name).
+        Returns 1.0 for full precision / unknown.
+        """
+        s = str(mode or "").strip().lower()
+        if not s or s in ("none", "full", "fp16", "bf16", "f16", "f32", "fp32", "default"):
+            return 1.0
+        # GGUF file or label → infer from the Qn tag (Q2..Q8).
+        if "gguf" in s or s.endswith(".gguf") or s.startswith("q"):
+            if "q2" in s or "2bit" in s:
+                return 6.0
+            if "q3" in s or "3bit" in s:
+                return 5.0
+            if "q4" in s or "4bit" in s:
+                return 4.0
+            if "q5" in s or "5bit" in s:
+                return 3.2
+            if "q6" in s or "6bit" in s:
+                return 2.67
+            if "q8" in s or "8bit" in s:
+                return 2.0
+            return 3.0  # unknown GGUF level — moderate default
+        norm = s.replace("_", "").replace("-", "")
+        if norm in ("4bit", "int4", "nf4", "fp4"):
+            return 4.0
+        if norm in ("8bit", "int8", "fp8"):
+            return 2.0
+        if norm in ("2bit", "int2"):
+            return 6.0
+        if norm in ("3bit", "int3"):
+            return 5.0
+        if norm in ("5bit",):
+            return 3.2
+        if norm in ("6bit",):
+            return 2.67
+        return 1.0
+
+    def _effective_quant_multiplier(self, cfg: dict,
+                                    load_in_4bit: bool, load_in_8bit: bool) -> float:
+        """Fraction of full-precision weight size that stays resident.
+
+        Per-component quantization wins when present: the big weight-bearing
+        components (transformer(s), unet, text encoders) dominate, so we assume
+        the quantized components carry ~80 % of the weight and the rest (VAE,
+        etc.) stays dense. Falls back to the global 4/8-bit flags.
+        """
+        comp_q = cfg.get("component_quantization") or {}
+        divisors = [self._quant_divisor(v) for v in comp_q.values()]
+        divisors = [d for d in divisors if d > 1.0]
+        if divisors:
+            avg_div = sum(divisors) / len(divisors)
+            # Lean slightly HIGH: assume ~70 % of the footprint is shrunk by
+            # quantization and ~30 % (VAE, embeddings, fp16 compute buffers,
+            # bitsandbytes scale/absmax tensors) stays dense. Better to slightly
+            # over-estimate and evict enough than to under-estimate and OOM —
+            # but not so high it exceeds the card and forces needless offload.
+            # (e.g. all-4bit → ~0.475× raw, ×1.15 overhead ≈ 0.55× → ~+10% over
+            # the measured resident size.)
+            quantized_share = 0.7
+            return quantized_share / avg_div + (1.0 - quantized_share)
+        if load_in_4bit:
+            # 4-bit nf4 keeps scale/absmax + fp16 buffers → effective ~0.3, not 0.25.
+            return 0.3
+        if load_in_8bit:
+            return 0.55
+        return 1.0
+
+    @staticmethod
+    def _model_type_of(model_key: str) -> str:
+        """Extract the model type from a 'type:name' key (default 'text')."""
+        return model_key.split(":", 1)[0] if ":" in model_key else "text"
+
+    # bytes-per-element for safetensors dtype strings
+    _DTYPE_BYTES = {
+        'F64': 8, 'F32': 4, 'F16': 2, 'BF16': 2,
+        'F8_E4M3': 1, 'F8_E5M2': 1, 'I64': 8, 'I32': 4,
+        'I16': 2, 'I8': 1, 'U8': 1, 'BOOL': 1,
+    }
+    _storage_dtype_cache: Dict[str, float] = {}
+
+    @classmethod
+    def _safetensors_numel_bytes(cls, path: str):
+        """Return (total_numel, total_weight_bytes) for a .safetensors file.
+
+        Only the header (8-byte length + JSON) is read, never the weights. Used
+        to compute a parameter-weighted average bytes/elem across a model so a
+        MIXED-precision model (e.g. fp32 transformer + bf16 text encoder) isn't
+        misjudged from a single file.
+        """
+        try:
+            import json as _json
+            with open(path, 'rb') as f:
+                n = int.from_bytes(f.read(8), 'little')
+                if n <= 0 or n > 100_000_000:
+                    return 0, 0
+                header = _json.loads(f.read(n))
+        except Exception:
+            return 0, 0
+        total_numel = 0
+        total_bytes = 0
+        for name, meta in header.items():
+            if name == '__metadata__' or not isinstance(meta, dict):
+                continue
+            dt = meta.get('dtype')
+            b = cls._DTYPE_BYTES.get(dt)
+            if not b:
+                continue
+            numel = 1
+            for d in (meta.get('shape') or []):
+                numel *= max(1, int(d))
+            total_numel += numel
+            total_bytes += numel * b
+        return total_numel, total_bytes
+
+    def _storage_bytes_per_elem(self, model_key: str, resolved_name: str, cfg: dict) -> float:
+        """Detect the on-disk weight precision (bytes/elem) for a model.
+
+        Computes the PARAMETER-WEIGHTED average bytes/elem across all of the
+        model's safetensors files (total weight bytes / total params), so a
+        mixed-precision model is handled correctly. Defaults to 2.0 (fp16) when
+        it can't be determined, so the normalization is a no-op rather than a
+        wrong guess. Cached per model.
+        """
+        ck = resolved_name or model_key
+        cached = self._storage_dtype_cache.get(ck)
+        if cached is not None:
+            return cached
+
+        import os
+        result = 2.0  # safe default: treat unknown as fp16 (normalization no-op)
+        candidates = []
+        for v in (resolved_name, cfg.get('path'), cfg.get('model_path'), cfg.get('model')):
+            if v and isinstance(v, str):
+                candidates.append(v)
+
+        st_files = []  # (size, path)
+        try:
+            # Local file or directory.
+            for c in candidates:
+                if os.path.isfile(c) and c.endswith('.safetensors'):
+                    st_files.append((os.path.getsize(c), c))
+                elif os.path.isdir(c):
+                    for root, _, files in os.walk(c):
+                        for fn in files:
+                            if fn.endswith('.safetensors'):
+                                p = os.path.join(root, fn)
+                                try:
+                                    st_files.append((os.path.getsize(p), p))
+                                except OSError:
+                                    pass
+            # HuggingFace hub cache.
+            if not st_files:
+                from huggingface_hub import scan_cache_dir
+                from codai.models.cache import get_all_cache_dirs, is_huggingface_model_id
+                hf_dir = get_all_cache_dirs().get("huggingface")
+                for c in candidates:
+                    repo_id = c.split(":", 1)[1] if ":" in c else c
+                    if not (hf_dir and is_huggingface_model_id(repo_id)):
+                        continue
+                    info = scan_cache_dir(hf_dir)
+                    for repo in info.repos:
+                        if repo.repo_id != repo_id:
+                            continue
+                        revs = sorted(repo.revisions, key=lambda r: r.last_modified, reverse=True)
+                        if revs:
+                            for fobj in revs[0].files:
+                                if str(fobj.file_name).endswith('.safetensors'):
+                                    st_files.append((fobj.size_on_disk, str(fobj.file_path)))
+                    if st_files:
+                        break
+        except Exception:
+            pass
+
+        if st_files:
+            # Parameter-weighted average across ALL files (mixed-precision aware).
+            st_files.sort(reverse=True)  # largest first (cap the count we read)
+            agg_numel = 0
+            agg_bytes = 0
+            for _, p in st_files[:64]:
+                numel, wbytes = self._safetensors_numel_bytes(p)
+                agg_numel += numel
+                agg_bytes += wbytes
+            if agg_numel > 0:
+                result = agg_bytes / agg_numel
+
+        self._storage_dtype_cache[ck] = result
+        return result
+
+    @staticmethod
+    def _load_bytes_per_elem(cfg: dict) -> float:
+        """Bytes/elem for the configured LOAD precision (default fp16 = 2)."""
+        prec = str((cfg or {}).get('precision') or '').lower()
+        if prec in ('f32', 'fp32', 'float32'):
+            return 4.0
+        if prec in ('f16', 'fp16', 'bf16', 'float16', 'bfloat16', 'half'):
+            return 2.0
+        return 2.0  # default load precision for HF/diffusers is half
+
+    # LoRA adapters load into VRAM alongside the base weights; applying/fusing
+    # them also briefly holds extra copies. Scale the raw adapter size to cover
+    # that transient spike.
+    _LORA_SPIKE_FACTOR = 1.3
+
+    @classmethod
+    def _path_or_hf_size_bytes(cls, spec: str) -> int:
+        """Best-effort on-disk size for a LoRA/adapter spec (file, dir, or HF id)."""
+        import os
+        if not spec or not isinstance(spec, str):
+            return 0
+        try:
+            if os.path.isfile(spec):
+                return os.path.getsize(spec)
+            if os.path.isdir(spec):
+                total = 0
+                for root, _, files in os.walk(spec):
+                    for f in files:
+                        if f.endswith((".safetensors", ".bin", ".pt", ".ckpt")):
+                            try:
+                                total += os.path.getsize(os.path.join(root, f))
+                            except OSError:
+                                pass
+                return total
+        except OSError:
+            pass
+        # HuggingFace repo id → scan local hub cache.
+        try:
+            from codai.models.cache import is_huggingface_model_id
+            if is_huggingface_model_id(spec):
+                return cls._hf_cached_model_size_bytes(spec)
+        except Exception:
+            pass
+        return 0
+
+    def _lora_vram_gb(self, specs) -> float:
+        """Approx. extra VRAM (GB) for one or more LoRA adapters (× spike factor).
+
+        `specs` may be a path/id string, a list of those, or a list of objects
+        exposing a `.model` attribute (LoraConfig) / dicts with a 'model' key.
+        """
+        if not specs:
+            return 0.0
+        if isinstance(specs, (str, bytes)):
+            specs = [specs]
+        total_bytes = 0
+        for s in specs:
+            path = None
+            if isinstance(s, str):
+                path = s
+            elif isinstance(s, dict):
+                path = s.get("model") or s.get("path") or s.get("lora") or s.get("name")
+            else:
+                path = getattr(s, "model", None) or getattr(s, "path", None)
+            if path:
+                total_bytes += self._path_or_hf_size_bytes(path)
+        if total_bytes <= 0:
+            return 0.0
+        return (total_bytes / 1e9) * self._LORA_SPIKE_FACTOR
+
+    def _config_lora_vram_gb(self, cfg: dict) -> float:
+        """Extra VRAM for LoRA(s) declared in the model config (lora_path/dir)."""
+        if not isinstance(cfg, dict):
+            return 0.0
+        specs = []
+        for key in ("lora_path", "lora_model_dir"):
+            v = cfg.get(key)
+            if v:
+                specs.append(v)
+        return self._lora_vram_gb(specs)
+
+    def _runtime_reserve_gb(self, cfg: dict, model_key: str, base_gb: float) -> float:
+        """Extra VRAM the model needs at RUNTIME beyond its resident weights.
+
+        A model occupies its weights at load time, but inference allocates more:
+        the KV cache (autoregressive text/vision, grows with context length),
+        attention/temporal activations and the VAE-decode spike (diffusion
+        image/video). The eviction estimate and the persisted measurement must
+        account for this so we reserve enough headroom and don't OOM mid-run.
+        Returned as GB to add on top of `base_gb` (the weight footprint).
+        """
+        if base_gb <= 0:
+            return 0.0
+        mtype = self._model_type_of(model_key)
+        if mtype in ("text", "vision"):
+            # KV cache + activations scale with context length.
+            try:
+                n_ctx = int(cfg.get("n_ctx") or cfg.get("context_size") or 2048)
+            except (TypeError, ValueError):
+                n_ctx = 2048
+            ctx_factor = min(3.0, max(1.0, n_ctx / 4096.0))
+            return base_gb * (0.10 + 0.08 * ctx_factor)   # ~0.18–0.34×
+        if mtype == "video":
+            return base_gb * 0.08   # temporal activations + VAE decode (tiling-capped)
+        if mtype == "image":
+            return base_gb * 0.08
+        return base_gb * 0.08
+
     def _get_model_used_vram_gb(self, model_key: str, resolved_name: str = None) -> float:
         """Return VRAM requirement in GB for a model.
 
         Priority:
         1. Explicit ``used_vram_gb`` in the model's config entry.
-        2. File size of a local weight file (GGUF/GGML/bin/safetensors).
-        3. File size of matching weight files in the HuggingFace hub cache.
+        2. Measured actual VRAM delta from a previous load of this model.
+        3. File size of a local weight file, adjusted for quantization/offload.
+        4. HuggingFace hub cache size (dense shards or largest GGUF), adjusted.
         Returns 0 when the requirement cannot be determined.
         """
         cfg = self.config.get(model_key, {})
+        # Unwrap a forwarded `_raw_cfg` so we see the ORIGINAL model entry the
+        # same way the loaders do (build_kwargs_from_config only copies a few
+        # keys to the top level — component_quantization lives ONLY in _raw_cfg).
+        # Without this the per-component quant map is invisible here and the
+        # estimate stays at full precision (e.g. 54 GB instead of ~25 GB).
+        if isinstance(cfg, dict) and isinstance(cfg.get('_raw_cfg'), dict):
+            _merged = dict(cfg)
+            for _k, _v in cfg['_raw_cfg'].items():
+                _merged.setdefault(_k, _v)
+            cfg = _merged
+
+        # --- Read quantization / offload settings from model config ---
+        # These come from the coderai model configuration and dramatically
+        # change how much VRAM the model actually occupies.  Read them FIRST so
+        # they apply to every estimate path, including the explicit value below.
+        load_in_4bit = cfg.get("load_in_4bit", False)
+        load_in_8bit = cfg.get("load_in_8bit", False)
+        offload_strategy = cfg.get("offload_strategy") or "auto"
+        max_gpu_percent = cfg.get("max_gpu_percent")  # explicit cap, 0-100
+
         explicit = cfg.get("used_vram_gb")
-        if explicit:
-            return float(explicit)
 
-        # Measured actual VRAM delta from when this model was loaded
+        # Effective quantization multiplier (fraction of full-precision weight
+        # size that remains in memory). Honours BOTH the global load_in_4/8bit
+        # flags AND per-component quantization (component_quantization map), so a
+        # diffusion pipeline whose transformer/text_encoder are 4-bit isn't
+        # wildly over-estimated at full precision.
+        quant_mult = self._effective_quant_multiplier(
+            cfg, load_in_4bit, load_in_8bit)
+
+        # Precision normalization: used_vram_gb / disk-scan baselines are STORAGE
+        # sizes at the on-disk dtype (e.g. Wan2.2 ships fp32 → 4 bytes/elem), but
+        # the quant divisors are fp16-relative and the model loads at `precision`
+        # (bf16/fp16 → 2 bytes). Without this a 4-bit fp32 model is ~2× over-
+        # estimated (151 GB → 49 instead of ~24). factor = load_bytes/storage_bytes.
+        _storage_bpe = self._storage_bytes_per_elem(model_key, resolved_name, cfg)
+        _load_bpe = self._load_bytes_per_elem(cfg)
+        prec_factor = (_load_bpe / _storage_bpe) if _storage_bpe > 0 else 1.0
+
+        def _dbg_est(source: str, value: float) -> float:
+            try:
+                from codai.api.state import get_global_debug
+                if get_global_debug():
+                    print(f"  [vram-est][debug] key='{model_key}' source={source} "
+                          f"-> {value:.2f} GB | used_vram_gb={explicit} "
+                          f"measured_cfg={cfg.get('measured_vram_gb')} "
+                          f"component_quantization={cfg.get('component_quantization')} "
+                          f"load_in_4bit={load_in_4bit} load_in_8bit={load_in_8bit} "
+                          f"quant_mult={quant_mult:.3f} "
+                          f"prec(load{_load_bpe:.0f}/disk{_storage_bpe:.0f})={prec_factor:.2f}")
+            except Exception:
+                pass
+            return value
+
+        def _apply_factors(raw_weights_gb: float) -> float:
+            """Adjust raw weight size for quantization and GPU-offload settings."""
+            # Normalize storage precision → load precision, then apply the
+            # (fp16-relative) quantization multiplier.
+            gpu_gb = raw_weights_gb * prec_factor * quant_mult
+
+            # When offload is active the model is split across GPU + CPU RAM.
+            # Only the GPU portion counts toward free-VRAM requirements.
+            if offload_strategy and offload_strategy != "none":
+                if max_gpu_percent is not None:
+                    # User has set an explicit GPU cap — honour it.
+                    gpu_gb = gpu_gb * max(0.0, min(1.0, float(max_gpu_percent) / 100.0))
+                else:
+                    # Conservative default: assume the model will use at most
+                    # 93 % of whatever GPU headroom exists.  Since we don't know
+                    # the exact GPU size here, cap the estimate at gpu_gb itself
+                    # (no reduction) so eviction is still triggered when needed.
+                    # The actual split is decided by transformers/accelerate at
+                    # load time.
+                    pass  # no reduction — be conservative
+
+            # Add the runtime reserve (KV cache / activations / VAE-decode spike)
+            # so the estimate reflects the model's PEAK footprint, not just its
+            # resident weights — otherwise eviction frees too little and the run
+            # OOMs mid-generation. Type-aware (ctx-scaled for text).
+            return gpu_gb + self._runtime_reserve_gb(cfg, model_key, gpu_gb)
+
+        # Resolve a real measurement (this run via `_measured_vram_gb`, or the
+        # persisted, system-owned `measured_vram_gb` field from a prior load). It
+        # already reflects quant/precision/offload, so it's used WITHOUT factors.
         measured = self._measured_vram_gb.get(model_key)
+        if not measured:
+            _cm = cfg.get("measured_vram_gb")
+            if _cm:
+                try:
+                    measured = float(_cm)
+                except (TypeError, ValueError):
+                    measured = None
+
+        # force_vram_update: the real measurement ALWAYS wins — even over a
+        # configured used_vram_gb — and is refreshed from reality every run.
+        if bool(cfg.get("force_vram_update")) and measured:
+            return _dbg_est("measured(force)", float(measured))
+
+        # used_vram_gb set in config is AUTHORITATIVE — use it as-is (with the
+        # usual quant/offload factors) and never override it with a measurement.
+        # Per user intent: a configured value must remain unchanged even if the
+        # resulting estimate is imperfect (unless force_vram_update is set).
+        # Config-declared LoRA(s) are added on top — they load into VRAM beyond
+        # the base weights. (Not added to forced/measured: forced is the user's
+        # final number; a measurement already reflects any load-time adapters.)
+        if explicit is not None:
+            return _dbg_est("used_vram_gb",
+                            _apply_factors(float(explicit)) + self._config_lora_vram_gb(cfg))
+
+        # No configured used_vram_gb → the real measurement is GROUND TRUTH.
         if measured:
-            return measured
+            return _dbg_est("measured", float(measured))
 
         # Build a list of candidate paths/IDs to check
         candidates = []
@@ -1687,9 +2273,8 @@ class MultiModelManager:
             try:
                 size_bytes = os.path.getsize(path)
                 weights_gb = size_bytes / 1e9
-                n_ctx = cfg.get("n_ctx") or 2048
-                kv_cache_gb = (n_ctx / 1024) * 0.5 / 1024  # 0.5 MB per 1 K ctx → GB
-                return weights_gb * 1.15 + kv_cache_gb
+                # _apply_factors already adds the runtime reserve (incl. KV cache).
+                return _apply_factors(weights_gb) + self._config_lora_vram_gb(cfg)
             except OSError:
                 continue
 
@@ -1705,35 +2290,99 @@ class MultiModelManager:
             size_bytes = self._hf_cached_model_size_bytes(repo_id)
             if size_bytes > 0:
                 weights_gb = size_bytes / 1e9
-                # Diffusers / transformers models don't have a KV-cache term;
-                # use 20% overhead for activations, optimizer states, scratch buffers.
-                return weights_gb * 1.2
+                return _apply_factors(weights_gb) + self._config_lora_vram_gb(cfg)
 
         return 0
 
+    def _is_key_busy(self, key: str) -> bool:
+        """True if any instance for this key is currently serving a request.
+
+        Evicting a busy model would move its tensors to CPU mid-forward-pass,
+        which triggers a CUDA device-side assert in the in-flight request and
+        corrupts the whole context.  Such models must NOT be evicted until idle.
+        """
+        pool = self.model_pools.get(key)
+        if pool is not None:
+            with pool._lock:
+                return any(r > 0 for r in pool.ref_counts)
+        return False
+
+    def _wait_until_idle(self, key: str, timeout: float = 120.0) -> bool:
+        """Block until no instance of `key` is busy, or until timeout. Returns idle?"""
+        import time as _time
+        deadline = _time.time() + timeout
+        while self._is_key_busy(key):
+            if _time.time() >= deadline:
+                return False
+            _time.sleep(0.25)
+        return True
+
     def _evict_models_for_vram(self, needed_gb: float):
-        """Unload loaded models (LRU first) until we have at least needed_gb free VRAM."""
+        """Unload loaded models (LRU first) until we have at least needed_gb free VRAM.
+
+        Never evicts a model that is actively serving a request — doing so would
+        move its weights off the GPU mid-generation and crash the CUDA context.
+        Busy models are waited on (briefly) before eviction.
+        """
         if needed_gb <= 0:
             return
 
         def _evict_key(key):
+            # Make absolutely sure nothing is mid-request on this model before we
+            # pull its tensors off the GPU.
+            if self._is_key_busy(key):
+                if not self._wait_until_idle(key):
+                    print(f"  Skipping eviction of '{key}' — still busy after wait")
+                    return
             model_obj = self.models.pop(key, None)
             self.models_in_vram.discard(key)
+            # Debug-only: ground-truth the object state before cleanup so an
+            # orphaned/detached backend (VRAM that won't free) is visible.
+            try:
+                from codai.api.state import get_global_debug
+                if get_global_debug():
+                    _be = getattr(model_obj, 'backend', '∅')
+                    _has_model = (getattr(_be, 'model', None) is not None) if _be not in ('∅', None) else False
+                    _pool = self.model_pools.get(key)
+                    print(f"  evict-debug '{key}': obj_id={id(model_obj)} "
+                          f"backend={'None' if _be is None else ('missing' if _be=='∅' else 'set')} "
+                          f"backend.model={'set' if _has_model else 'None'} "
+                          f"pool_instances={_pool.count if _pool else 0}")
+            except Exception:
+                pass
+            # Clean every instance in the pool (not just the primary) so extra
+            # instances don't leak VRAM, then drop the pool.
+            pool = self.model_pools.pop(key, None)
+            if pool is not None:
+                try:
+                    pool.cleanup_all()
+                except Exception as e:
+                    print(f"  Warning cleaning pool for '{key}': {e}")
             if model_obj is not None:
                 try:
                     if hasattr(model_obj, 'cleanup'):
                         model_obj.cleanup()
                     elif hasattr(model_obj, 'to'):
+                        # Diffusers pipeline: move all components to CPU explicitly
+                        # before dropping the reference so VRAM is freed promptly.
                         model_obj.to('cpu')
                 except Exception as e:
                     print(f"  Warning during eviction of '{key}': {e}")
-            gc.collect()
+            del model_obj
+            for _ in range(3):
+                gc.collect()
             try:
                 import torch
                 if torch.cuda.is_available():
+                    torch.cuda.synchronize()
                     torch.cuda.empty_cache()
             except Exception:
                 pass
+            # Return freed CPU heap (the evicted model's host-side copy / offloaded
+            # weights) to the OS, and let the kernel reclaim any swap backing it —
+            # otherwise RSS stays high across evict/load cycles and the machine
+            # slowly fills RAM + swap.
+            _trim_cpu_ram()
 
         def _size_label(key: str) -> str:
             m = self._measured_vram_gb.get(key)
@@ -1744,23 +2393,86 @@ class MultiModelManager:
                 return f"estimated {e:.2f} GB"
             return "size unknown"
 
-        # First pass: evict non-active models in LRU order
+        _free_before = self._get_free_vram_gb()
+
+        # First pass: evict idle non-active models in LRU order.
         for key in list(self.models.keys()):
             if key == self.active_in_vram:
                 continue
             if self._get_free_vram_gb() >= needed_gb:
                 break
+            if self._is_key_busy(key):
+                print(f"  '{key}' is busy serving a request — not evicting it")
+                continue
             print(f"On-request VRAM eviction: unloading '{key}' ({_size_label(key)}) to free VRAM")
             _evict_key(key)
 
-        # Second pass: evict active model if still not enough
-        if self._get_free_vram_gb() < needed_gb and self.active_in_vram and self.active_in_vram in self.models:
-            print(f"On-request VRAM eviction: unloading active model '{self.active_in_vram}' "
-                  f"({_size_label(self.active_in_vram)}) to free VRAM")
-            _evict_key(self.active_in_vram)
-            self.active_in_vram = None
+        # Second pass: evict the active model if still not enough AND it is idle.
+        if (self._get_free_vram_gb() < needed_gb and self.active_in_vram
+                and self.active_in_vram in self.models):
+            _active = self.active_in_vram
+            if self._is_key_busy(_active):
+                print(f"  Active model '{_active}' is busy — waiting for it to go idle "
+                      f"before eviction…")
+            if self._wait_until_idle(_active):
+                print(f"On-request VRAM eviction: unloading active model '{_active}' "
+                      f"({_size_label(_active)}) to free VRAM")
+                _evict_key(_active)
+                self.active_in_vram = None
+            else:
+                print(f"  Active model '{_active}' still busy after wait — leaving it loaded")
+
+        # Last resort: ask external holders (e.g. the LoRA trainer's cached base
+        # model) to release their VRAM. These aren't tracked models, so eviction
+        # above can't see them — this is what stops a cached training base from
+        # forcing the next image request into CPU/disk offload.
+        if self._get_free_vram_gb() < needed_gb and self._external_vram_releasers:
+            for _rel in list(self._external_vram_releasers):
+                try:
+                    freed = _rel(needed_gb)
+                    if freed:
+                        print(f"On-request VRAM eviction: external releaser freed {freed:.1f} GB")
+                except Exception as e:
+                    print(f"  Warning in external VRAM releaser: {e}")
+                if self._get_free_vram_gb() >= needed_gb:
+                    break
 
-    def request_model(self, requested_model: str, model_type: str = None) -> Dict[str, Any]:
+        # Report what we actually freed so a failure to free is visible.
+        _free_after = self._get_free_vram_gb()
+        if _free_after < needed_gb:
+            print(f"  ⚠ Eviction freed {_free_after - _free_before:.1f} GB "
+                  f"(now {_free_after:.1f} GB free, needed {needed_gb:.1f} GB). "
+                  f"Remaining models are busy or VRAM is held elsewhere — the new "
+                  f"model will load with CPU/disk offload.")
+
+    def ensure_vram_for(self, model_key: str, resolved_name: str = None,
+                        extra_vram_gb: float = 0.0) -> None:
+        """Make room in VRAM for a model about to load — ANY type, ANY load mode.
+
+        Universal guarantee: before loading a model that is not already resident,
+        if its estimated VRAM need exceeds the free VRAM, evict other loaded
+        models (LRU first, then the active one) until there is enough room.
+        `_evict_models_for_vram` already frees one-or-more models as needed and
+        never touches a model that is busy serving a request.
+
+        `extra_vram_gb` is added to the model's own estimate — used to reserve
+        room for per-request additions the base estimate can't know about (e.g.
+        dynamically-applied LoRA adapters).
+        """
+        needed_gb = self._get_model_used_vram_gb(model_key, resolved_name)
+        if extra_vram_gb and extra_vram_gb > 0:
+            needed_gb += extra_vram_gb
+        if needed_gb <= 0:
+            return
+        free_gb = self._get_free_vram_gb()
+        if free_gb < needed_gb:
+            _extra = f" (+{extra_vram_gb:.1f} GB LoRA/extra)" if extra_vram_gb else ""
+            print(f"Loading '{model_key}': need {needed_gb:.1f} GB VRAM{_extra}, "
+                  f"have {free_gb:.1f} GB free — evicting models to make room")
+            self._evict_models_for_vram(needed_gb)
+
+    def request_model(self, requested_model: str, model_type: str = None,
+                      extra_vram_gb: float = 0.0) -> Dict[str, Any]:
         """
         Central method for API modules to request a model.
         
@@ -1789,9 +2501,32 @@ class MultiModelManager:
                 - 'config': The stored configuration for this model
                 - 'already_loaded': True if the model is already loaded and ready in VRAM
         """
+        # Fail fast if the CUDA context is already corrupted — loading anything
+        # onto a poisoned context just re-triggers the same async assert.
+        if self.cuda_context_poisoned:
+            return {
+                'model_key': None,
+                'model_name': requested_model,
+                'model_object': None,
+                'config': {},
+                'already_loaded': False,
+                'error': ("CUDA context corrupted by an earlier device-side assert "
+                          f"({self.cuda_poison_reason}). Restart coderai to recover."),
+            }
+
+        # Thermal protection: before serving a request, wait if the CPU/GPU are
+        # too hot so a long sequence of heavy generations can't overheat the
+        # machine and trip its power-off protection. No-op when disabled or cool.
+        try:
+            from codai.models.thermal import wait_until_safe
+            wait_until_safe(context=f"{model_type or 'model'}:{requested_model}")
+        except Exception as _therr:
+            # Never let thermal monitoring failures block serving.
+            print(f"[thermal] monitor unavailable ({_therr}) — serving without wait")
+
         from codai.api.state import get_load_mode
         mode = get_load_mode()
-        
+
         # Step 1: Resolve the model name from aliases
         resolved_name = None
         model_key = None
@@ -1967,13 +2702,8 @@ class MultiModelManager:
                     'config': per_model_cfg,
                     'already_loaded': True,
                 }
-            # Not loaded — check VRAM and evict if needed
-            needed_gb = self._get_model_used_vram_gb(model_key, resolved_name)
-            if needed_gb > 0:
-                free_gb = self._get_free_vram_gb()
-                if free_gb < needed_gb:
-                    print(f"On-request: need {needed_gb:.1f} GB VRAM, have {free_gb:.1f} GB free — evicting models")
-                    self._evict_models_for_vram(needed_gb)
+            # Not loaded — make room in VRAM (evict other models as needed).
+            self.ensure_vram_for(model_key, resolved_name, extra_vram_gb)
             return {
                 'model_key': model_key,
                 'model_name': resolved_name,
@@ -1994,6 +2724,10 @@ class MultiModelManager:
                     'config': per_model_cfg,
                     'already_loaded': True,
                 }
+            # A "load" model that is NOT currently resident was evicted to make
+            # room for an on-request model (e.g. the text model bumped for a video
+            # model).  Make room again so it returns to VRAM instead of CPU.
+            self.ensure_vram_for(model_key, resolved_name, extra_vram_gb)
             return {
                 'model_key': model_key,
                 'model_name': resolved_name,
@@ -2019,6 +2753,7 @@ class MultiModelManager:
             # Model not loaded yet in loadall mode - caller needs to load it
             # (this happens for models not pre-loaded at startup, e.g., image models)
             print(f"Loadall mode: Model '{model_key}' not pre-loaded, will load now")
+            self.ensure_vram_for(model_key, resolved_name, extra_vram_gb)
             return {
                 'model_key': model_key,
                 'model_name': resolved_name,
@@ -2154,16 +2889,16 @@ class MultiModelManager:
                     print(f"Ondemand mode - keeping '{loaded_canonical}' in VRAM alongside new model "
                           f"(need {needed_gb:.1f} GB + {headroom_gb:.1f} GB headroom, have {free_gb:.1f} GB free)")
                 else:
-                    print(f"Ondemand mode - model switch detected:")
-                    print(f"  Requested: '{model_key}' (resolved: '{resolved_name}')")
-                    print(f"  Currently loaded: '{loaded_canonical}'")
-                    print(f"  -> Unloading current model(s) before loading new model...")
-                    self.unload_all_models()
-
-                    # Also cleanup the legacy singleton if it has a model
-                    if has_legacy_model:
+                    print(f"Ondemand mode - model switch detected (requested '{model_key}', "
+                          f"loaded '{loaded_canonical}'):")
+                    # Evict one-or-more models (LRU first) until there is enough
+                    # room — preserves coexistence of models that still fit rather
+                    # than nuking everything.
+                    self._evict_models_for_vram(needed_gb + headroom_gb)
+                    # If still short, fall back to cleaning the legacy singleton.
+                    if has_legacy_model and self._get_free_vram_gb() < needed_gb:
                         try:
-                            print(f"  -> Cleaning up legacy model_manager...")
+                            print(f"  -> Cleaning up legacy model_manager…")
                             _legacy_mm.cleanup()
                         except Exception as e:
                             print(f"  Warning: Error cleaning up legacy model_manager: {e}")
@@ -2381,13 +3116,12 @@ class MultiModelManager:
                         setattr(caps, f, True)
             else:
                 caps = detect_model_capabilities(model_id)
-                # If heuristic detection missed the type (e.g. custom/vendor model IDs
-                # that don't match any keyword), ensure the minimum capability for the
-                # config-declared type is set so badges display correctly.
-                if model_type and model_type in TYPE_MIN_CAP:
-                    min_cap = TYPE_MIN_CAP[model_type]
-                    if not getattr(caps, min_cap, False):
-                        setattr(caps, min_cap, True)
+            # Apply minimum capability for the config-declared category only when
+            # detection found nothing at all (unrecognized model name). If any caps
+            # were detected (e.g. image_upscaling for latent-upscaler models), trust
+            # the detection result — TYPE_MIN_CAP must not override it.
+            if model_type and model_type in TYPE_MIN_CAP and not caps.to_list():
+                setattr(caps, TYPE_MIN_CAP[model_type], True)
             resolved_type = model_type or (caps.to_list()[0].split("_")[0] if caps.to_list() else "text")
             models.append(ModelInfo(
                 id=model_id,

codai/models/thermal.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Thermal protection.
+
+Before running a request against a loaded model, wait until CPU and GPU
+temperatures are within safe limits.  This prevents a long sequence of heavy
+generations (e.g. batched video) from driving the hardware hot enough that the
+machine's own protection trips and powers off.
+
+The guard is model-agnostic and backend-agnostic: it only reads temperatures
+(NVIDIA via ``nvidia-smi``, AMD via ``rocm-smi``, CPU via ``psutil`` /
+``/sys`` / ``sensors``) and sleeps.  All thresholds are configurable and the
+feature can be enabled/disabled independently for CPU and GPU.
+
+Semantics (per sensor, when enabled):
+  * If the temperature is at or above ``high`` °C when a request is about to
+    run, block until it drops to ``resume`` °C or below, then proceed.
+  * If a temperature can't be read, that sensor is treated as safe (we never
+    block on missing data).
+"""
+import os
+import shutil
+import subprocess
+import time
+from typing import Optional, Tuple
+
+
+# ---------------------------------------------------------------------------
+# Temperature readers
+# ---------------------------------------------------------------------------
+
+# Short cache so a tight wait-loop doesn't spawn nvidia-smi dozens of times.
+_CACHE_TTL = 2.0
+_gpu_cache: Tuple[float, Optional[float]] = (0.0, None)
+_cpu_cache: Tuple[float, Optional[float]] = (0.0, None)
+
+_NVIDIA_SMI = shutil.which("nvidia-smi")
+_ROCM_SMI = shutil.which("rocm-smi")
+
+
+def _run(cmd, timeout=4.0) -> Optional[str]:
+    try:
+        out = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=timeout, check=False
+        )
+        if out.returncode == 0:
+            return out.stdout
+    except Exception:
+        pass
+    return None
+
+
+def _read_gpu_temp_uncached() -> Optional[float]:
+    """Hottest GPU temperature in °C, or None if unreadable."""
+    # NVIDIA — the inference GPU on CUDA backends.
+    if _NVIDIA_SMI:
+        out = _run([
+            _NVIDIA_SMI,
+            "--query-gpu=temperature.gpu",
+            "--format=csv,noheader,nounits",
+        ])
+        if out:
+            temps = []
+            for line in out.splitlines():
+                line = line.strip()
+                if line:
+                    try:
+                        temps.append(float(line))
+                    except ValueError:
+                        pass
+            if temps:
+                return max(temps)
+    # AMD ROCm GPUs.
+    if _ROCM_SMI:
+        out = _run([_ROCM_SMI, "--showtemp"])
+        if out:
+            temps = []
+            for line in out.splitlines():
+                if "temperature" in line.lower() and ":" in line:
+                    tail = line.rsplit(":", 1)[-1]
+                    tail = tail.replace("C", "").replace("c", "").strip()
+                    try:
+                        temps.append(float(tail))
+                    except ValueError:
+                        pass
+            if temps:
+                return max(temps)
+    # psutil amdgpu fallback.
+    temp = _psutil_temp(("amdgpu", "radeon"))
+    return temp
+
+
+def _psutil_temp(prefer_keys) -> Optional[float]:
+    """Max current temp among the preferred psutil sensor groups."""
+    try:
+        import psutil
+    except Exception:
+        return None
+    try:
+        sensors = psutil.sensors_temperatures()
+    except Exception:
+        return None
+    if not sensors:
+        return None
+    best = None
+    for key in prefer_keys:
+        if key in sensors:
+            for entry in sensors[key]:
+                cur = getattr(entry, "current", None)
+                if cur is not None:
+                    best = cur if best is None else max(best, cur)
+    return best
+
+
+def _read_cpu_temp_uncached() -> Optional[float]:
+    """CPU package temperature in °C, or None if unreadable."""
+    # psutil covers AMD (k10temp: Tctl/Tdie) and Intel (coretemp: Package).
+    temp = _psutil_temp(("k10temp", "coretemp", "zenpower", "cpu_thermal", "k8temp"))
+    if temp is not None:
+        return temp
+    # /sys thermal zones — pick an x86_pkg / cpu zone if present.
+    try:
+        base = "/sys/class/thermal"
+        best = None
+        for name in os.listdir(base):
+            if not name.startswith("thermal_zone"):
+                continue
+            zpath = os.path.join(base, name)
+            try:
+                with open(os.path.join(zpath, "type")) as f:
+                    ztype = f.read().strip().lower()
+                with open(os.path.join(zpath, "temp")) as f:
+                    raw = int(f.read().strip())
+            except Exception:
+                continue
+            if any(k in ztype for k in ("x86_pkg", "cpu", "core", "tctl", "tdie")):
+                val = raw / 1000.0
+                best = val if best is None else max(best, val)
+        if best is not None:
+            return best
+    except Exception:
+        pass
+    # `sensors` text parse as a last resort.
+    smi = shutil.which("sensors")
+    if smi:
+        out = _run([smi])
+        if out:
+            best = None
+            for line in out.splitlines():
+                low = line.lower()
+                if any(k in low for k in ("tctl", "tdie", "package id", "cpu temp")):
+                    # e.g. "Tctl:         +55.0°C"
+                    for tok in line.replace("+", " ").split():
+                        tok = tok.replace("°C", "").replace("C", "").strip()
+                        try:
+                            v = float(tok)
+                        except ValueError:
+                            continue
+                        best = v if best is None else max(best, v)
+                        break
+            if best is not None:
+                return best
+    return None
+
+
+def read_gpu_temp() -> Optional[float]:
+    global _gpu_cache
+    now = time.monotonic()
+    ts, val = _gpu_cache
+    if now - ts < _CACHE_TTL:
+        return val
+    val = _read_gpu_temp_uncached()
+    _gpu_cache = (now, val)
+    return val
+
+
+def read_cpu_temp() -> Optional[float]:
+    global _cpu_cache
+    now = time.monotonic()
+    ts, val = _cpu_cache
+    if now - ts < _CACHE_TTL:
+        return val
+    val = _read_cpu_temp_uncached()
+    _cpu_cache = (now, val)
+    return val
+
+
+def read_cpu_temp_avg(samples: int = 3, max_seconds: float = 3.0) -> Optional[float]:
+    """Averaged CPU temperature for stable resume/cooldown decisions.
+
+    CPU sensors are noisy — two consecutive reads can swing ±10°C — so a single
+    sample is a poor basis for deciding the hardware has cooled down. Averaging a
+    few *uncached* samples gives a representative value. Only used for the resume
+    side; the pause decision deliberately keeps a single read so it reacts
+    immediately to a spike and never lets the CPU overheat.
+
+    Bounded to ``max_seconds`` total (default 3s): the samples are spread across
+    the budget (≈1s apart for 3 samples) so they actually capture the second-to-
+    second swing, and the loop stops at the deadline. This runs during the
+    cooldown wait — which already sleeps ``poll_seconds`` (e.g. 10s) between
+    checks — so spending up to 3s of that gathering samples adds no real delay.
+    """
+    global _cpu_cache
+    samples = max(1, samples)
+    # Spread reads evenly across the budget (e.g. 3 samples over 3s → ~1s apart).
+    gap = (max_seconds / samples) if samples > 1 else 0.0
+    deadline = time.monotonic() + max_seconds
+    vals = []
+    for i in range(samples):
+        v = _read_cpu_temp_uncached()
+        if v is not None:
+            vals.append(v)
+        if i >= samples - 1:
+            break
+        # Stop early rather than sleep past the 3s budget.
+        if time.monotonic() + gap >= deadline:
+            break
+        time.sleep(gap)
+    if not vals:
+        return None
+    avg = sum(vals) / len(vals)
+    # Refresh the cache so nearby cached reads see the averaged figure.
+    _cpu_cache = (time.monotonic(), avg)
+    return avg
+
+
+def _debug_enabled() -> bool:
+    """Thermal debug output is gated on the dedicated --debug-thermal flag
+    (not the global --debug, which is too noisy for sensor polling)."""
+    try:
+        from codai.api.state import get_global_args
+        ga = get_global_args()
+        return bool(getattr(ga, "debug_thermal", False))
+    except Exception:
+        return False
+
+
+def _dbg(msg: str) -> None:
+    if _debug_enabled():
+        print(f"[thermal][debug] {msg}")
+
+
+def _fmt(temp: Optional[float]) -> str:
+    return f"{temp:.1f}°C" if temp is not None else "n/a"
+
+
+# ---------------------------------------------------------------------------
+# Settings + guard
+# ---------------------------------------------------------------------------
+
+class ThermalSettings:
+    """Resolved thermal-protection settings (with sane defaults)."""
+
+    __slots__ = (
+        "cpu_enabled", "gpu_enabled",
+        "cpu_high", "cpu_resume", "gpu_high", "gpu_resume",
+        "poll_seconds",
+    )
+
+    def __init__(self, cpu_enabled=True, gpu_enabled=True,
+                 cpu_high=90.0, cpu_resume=87.0,
+                 gpu_high=90.0, gpu_resume=87.0,
+                 poll_seconds=5.0):
+        self.cpu_enabled = bool(cpu_enabled)
+        self.gpu_enabled = bool(gpu_enabled)
+        self.cpu_high = float(cpu_high)
+        self.cpu_resume = float(cpu_resume)
+        self.gpu_high = float(gpu_high)
+        self.gpu_resume = float(gpu_resume)
+        self.poll_seconds = max(1.0, float(poll_seconds))
+
+
+def _settings_from_global_args() -> ThermalSettings:
+    """Build settings from the live global_args, falling back to defaults."""
+    try:
+        from codai.api.state import get_global_args
+        ga = get_global_args()
+    except Exception:
+        ga = None
+    if ga is None:
+        return ThermalSettings()
+    g = lambda name, default: getattr(ga, name, default)
+    return ThermalSettings(
+        cpu_enabled=g("thermal_cpu_enabled", True),
+        gpu_enabled=g("thermal_gpu_enabled", True),
+        cpu_high=g("thermal_cpu_high", 90.0),
+        cpu_resume=g("thermal_cpu_resume", 87.0),
+        gpu_high=g("thermal_gpu_high", 90.0),
+        gpu_resume=g("thermal_gpu_resume", 87.0),
+        poll_seconds=g("thermal_poll_seconds", 5.0),
+    )
+
+
+_last_checkpoint: dict = {}
+
+
+def checkpoint(context: str = "", throttle_seconds: float = 0.0) -> None:
+    """Mid-generation thermal checkpoint — call between denoise steps / tokens.
+
+    Same semantics as ``wait_until_safe`` (pause while too hot, resume on
+    cooldown), but cheap to call in a hot loop: when ``throttle_seconds`` > 0 it
+    only re-checks once that much wall-time has elapsed for this ``context``, so
+    a token-by-token text loop doesn't read sensors on every token. Pass 0 for
+    low-frequency callers (e.g. per-step diffusion callbacks).
+    """
+    if throttle_seconds and throttle_seconds > 0:
+        now = time.monotonic()
+        last = _last_checkpoint.get(context, 0.0)
+        if (now - last) < throttle_seconds:
+            return
+        _last_checkpoint[context] = now
+    wait_until_safe(context=context)
+
+
+def wait_until_safe(settings: Optional[ThermalSettings] = None,
+                    debug: bool = False,
+                    context: str = "") -> None:
+    """Block until CPU and GPU temperatures are within safe limits.
+
+    Returns immediately when protection is disabled or temperatures are below
+    their trigger thresholds.  Designed to be called from a worker thread (it
+    uses a blocking ``time.sleep``); the heavy request paths already run
+    ``request_model`` inside ``asyncio.to_thread``.
+    """
+    if settings is None:
+        settings = _settings_from_global_args()
+    if not settings.cpu_enabled and not settings.gpu_enabled:
+        _dbg(f"protection disabled (cpu/gpu both off) — proceeding [{context}]")
+        return
+
+    desc0 = f" [{context}]" if context else ""
+
+    # Read current temps once (cached) and log the full picture in debug mode.
+    gpu_t = read_gpu_temp() if settings.gpu_enabled else None
+    cpu_t = read_cpu_temp() if settings.cpu_enabled else None
+    _dbg(
+        f"check{desc0}: "
+        f"GPU {_fmt(gpu_t)} (enabled={settings.gpu_enabled}, "
+        f"pause>={settings.gpu_high:.0f} resume<={settings.gpu_resume:.0f}) | "
+        f"CPU {_fmt(cpu_t)} (enabled={settings.cpu_enabled}, "
+        f"pause>={settings.cpu_high:.0f} resume<={settings.cpu_resume:.0f})"
+    )
+
+    hot = []
+    if settings.gpu_enabled and gpu_t is not None and gpu_t >= settings.gpu_high:
+        hot.append(("GPU", gpu_t, settings.gpu_resume))
+    if settings.cpu_enabled and cpu_t is not None and cpu_t >= settings.cpu_high:
+        hot.append(("CPU", cpu_t, settings.cpu_resume))
+    if not hot:
+        _dbg(f"within safe limits — serving immediately{desc0}")
+        return
+
+    # Enter cooldown: wait until *every* triggered sensor is at/below resume.
+    desc = f" ({context})" if context else ""
+    trig = ", ".join(f"{lbl} {t:.0f}°C>={settings.gpu_high if lbl=='GPU' else settings.cpu_high:.0f}°C"
+                     for lbl, t, _ in hot)
+    print(f"[thermal] Hardware too hot{desc}: {trig} — pausing requests "
+          f"until cooldown (GPU<={settings.gpu_resume:.0f}°C / "
+          f"CPU<={settings.cpu_resume:.0f}°C)")
+    waited = 0.0
+    while True:
+        # Re-evaluate against resume thresholds (lower than trigger → hysteresis).
+        # CPU temps are noisy, so average a few samples for the resume decision
+        # (the pause check above stays single-read to react fast to spikes).
+        gt = read_gpu_temp() if settings.gpu_enabled else None
+        ct = read_cpu_temp_avg() if settings.cpu_enabled else None
+        still = []
+        if gt is not None and gt > settings.gpu_resume:
+            still.append(("GPU", gt, settings.gpu_resume))
+        if ct is not None and ct > settings.cpu_resume:
+            still.append(("CPU", ct, settings.cpu_resume))
+        _dbg(f"cooldown{desc} {int(waited)}s: GPU {_fmt(gt)} CPU {_fmt(ct)} (avg-3) "
+             f"(still hot: {[s[0] for s in still] or 'none'})")
+        if not still:
+            break
+        msg = ", ".join(f"{lbl} {t:.0f}°C>{r:.0f}°C" for lbl, t, r in still)
+        print(f"[thermal] Cooling{desc}: {msg} — waiting "
+              f"({int(waited)}s elapsed)")
+        time.sleep(settings.poll_seconds)
+        waited += settings.poll_seconds
+    print(f"[thermal] Temperatures back within safe limits{desc} — resuming "
+          f"after {int(waited)}s")