Thermal soft-throttle, accel hot-swap + UI, township progress & concat fixes

coderai:
- Thermal: configurable proactive CPU soft-throttle (engage temp + max
  per-step sleep) that gently slows generation in a warm band so it rarely hits
  the hard pause; CPU-only, hard pause always takes precedence. Tasks page shows
  a soft-throttle banner + per-task badge (live, gated on a running task).
- Acceleration hot-swap: toggling/changing a model's acceleration now evicts the
  loaded model (manager.unload_model) so the next request reloads with the new
  setting — no restart. (acceleration is fused at load time.)
- Models UI: cascading distill-LoRA pickers — new /admin/api/accel-loras scans
  the cache for distill repos; pick the distill model, then its high/low (or
  single) LoRA from dropdowns. Presets now also fill the high/low fields.
- Tasks queue summary now reflects ALL model activity (derived from the unified
  task list), not just queue-manager requests — fixes the stuck "0 active".
- images.py: proactive eviction no longer skipped by a NameError (model_key).

township (tools/gen_township_fighters.py):
- Per-clip/outcome/keyframe progress now shows real diffusion-step progress
  (polls /v1/{images,video}/progress) on the CLI spinner and the web step bars,
  including "shot part N/total" for chained single shots.
- Chained-shot concat re-encodes (CFR) instead of stream-copy, fixing the
  "first half is a static image" freeze at the part boundary.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -1988,6 +1988,81 @@ async def api_accel_presets(username: str = Depends(require_admin)):
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.get("/admin/api/accel-loras", summary="List cached distill-LoRA files for acceleration")
+async def api_accel_loras(username: str = Depends(require_admin)):
+    """Scan the HF cache for distill / step-distillation LoRA repos and return each
+    repo's ``.safetensors`` files, pre-classified into high-noise / low-noise (for
+    Wan2.2's two experts) so the model-config UI can offer cascading dropdowns:
+    pick the distill model first, then the high/low (or single) LoRA from it.
+
+    A repo qualifies if its id matches a distill keyword (lightning/lightx2v/lcm/
+    hyper/turbo/distill/dmd) or is referenced by an ACCEL_PRESETS entry."""
+    import os
+    import re as _re
+    out: list = []
+    try:
+        from codai.models.acceleration import ACCEL_PRESETS
+        from codai.models.cache import get_all_cache_dirs
+        from huggingface_hub import scan_cache_dir
+
+        # Repos named by presets are always relevant (the `repo:file` head before ':').
+        preset_repos = set()
+        for p in (ACCEL_PRESETS or {}).values():
+            for k in ("lora", "lora_high", "lora_low"):
+                ref = p.get(k)
+                if ref and ":" in str(ref):
+                    preset_repos.add(str(ref).split(":", 1)[0])
+                elif ref:
+                    preset_repos.add(str(ref))
+
+        kw = _re.compile(r"light(ning|x2v)|lcm|hyper[-_ ]?sd|turbo|distill|dmd|seko",
+                         _re.IGNORECASE)
+        _hi = _re.compile(r"high[-_ ]?noise", _re.IGNORECASE)
+        _lo = _re.compile(r"low[-_ ]?noise", _re.IGNORECASE)
+        # A file is a distill LoRA (vs a full model's component weights like
+        # vae/transformer/text_encoder) if its path names a LoRA or noise level.
+        _loraname = _re.compile(r"lora|noise|light(ning|x2v)|lcm|hyper|distill|dmd",
+                                _re.IGNORECASE)
+
+        hf_dir = (get_all_cache_dirs() or {}).get("huggingface")
+        info = scan_cache_dir(hf_dir) if hf_dir else scan_cache_dir()
+        for repo in info.repos:
+            rid = repo.repo_id
+            in_preset = rid in preset_repos
+            if not (in_preset or kw.search(rid)):
+                continue
+            # Newest revision's snapshot → relative .safetensors paths. Keep only
+            # LoRA-looking files, unless the repo is a curated preset repo (then
+            # trust all its safetensors). This drops full-model component weights
+            # from repos that merely match a keyword (e.g. "*-Turbo" base models).
+            rev = max(repo.revisions, key=lambda r: (r.last_modified or 0), default=None)
+            if rev is None:
+                continue
+            snap = str(rev.snapshot_path)
+            files = []
+            for f in rev.files:
+                fp = str(f.file_path)
+                if not fp.endswith(".safetensors"):
+                    continue
+                rel = os.path.relpath(fp, snap).replace(os.sep, "/")
+                if in_preset or _loraname.search(rel):
+                    files.append(rel)
+            if not files:
+                continue
+            files.sort()
+            out.append({
+                "repo": rid,
+                "files": files,
+                "high": [f for f in files if _hi.search(f)],
+                "low":  [f for f in files if _lo.search(f)],
+            })
+        out.sort(key=lambda m: m["repo"].lower())
+    except Exception as e:
+        # Cache scan is best-effort; the UI falls back to the free-text fields.
+        return {"models": [], "error": str(e)}
+    return {"models": out}
+
+
 @router.get("/admin/api/turboquant-info", summary="TurboQuant backend availability")
 async def api_turboquant_info(username: str = Depends(require_admin)):
     """Report which TurboQuant embedding-quantization backends are available so
@@ -2109,7 +2184,41 @@ async def api_tasks(username: str = Depends(require_admin)):
     except Exception:
         pass
 
-    return {"tasks": tasks, "queue": queue_manager.get_metrics(), "thermal": cooling}
+    # Proactive CPU soft-throttle (distinct from the hard cooldown): generations
+    # are being gently slowed because the CPU is in its warm band. Computed live
+    # from the CPU temp, and only shown when something is actually running (idle
+    # warmth isn't being throttled). Suppressed during a hard cooldown.
+    soft = {"active": False}
+    try:
+        from codai.models import thermal as _therm
+        _any_running = any(t.get("status") == "running" for t in tasks)
+        ss = _therm.soft_throttle_status()
+        if ss.get("active") and _any_running and not cooling.get("active"):
+            _cpu = ss.get("cpu")
+            _slp = ss.get("sleep") or 0
+            label = "CPU soft-throttle"
+            if _cpu is not None:
+                label += f" — CPU {_cpu:.0f}°C"
+            if _slp:
+                label += f" (+{_slp:.1f}s/step)"
+            soft = {"active": True, "message": label, "cpu": _cpu, "sleep": _slp}
+            for t in tasks:
+                if (t.get("active") and t.get("status") == "running"
+                        and not t.get("cooling")):
+                    t["throttling"] = True
+                    t["throttle_message"] = label
+    except Exception:
+        pass
+
+    # The queue-summary header must reflect ALL model activity, not just requests
+    # that flow through queue_manager (text/pipelines/training). Image/video/audio
+    # generations run their own paths and live only in the task registry, so derive
+    # active/waiting from the unified `tasks` list; keep max_parallel from the
+    # queue manager.
+    queue = dict(queue_manager.get_metrics())
+    queue["active"] = sum(1 for t in tasks if t.get("status") == "running")
+    queue["waiting"] = sum(1 for t in tasks if t.get("status") == "queued")
+    return {"tasks": tasks, "queue": queue, "thermal": cooling, "soft_throttle": soft}
 
 
 def _read_vram_info() -> Optional[dict]:
@@ -2336,6 +2445,9 @@ async def api_get_settings(username: str = Depends(require_admin)):
             "gpu_high": c.thermal.gpu_high,
             "gpu_resume": c.thermal.gpu_resume,
             "poll_seconds": c.thermal.poll_seconds,
+            "soft_throttle_enabled": c.thermal.soft_throttle_enabled,
+            "soft_throttle_temp": c.thermal.soft_throttle_temp,
+            "soft_throttle_max_sleep": c.thermal.soft_throttle_max_sleep,
         },
         "jobs": {
             "resume_on_restart": c.jobs.resume_on_restart,
@@ -2457,6 +2569,9 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
         c.thermal.gpu_high = float(th.get("gpu_high", c.thermal.gpu_high))
         c.thermal.gpu_resume = float(th.get("gpu_resume", c.thermal.gpu_resume))
         c.thermal.poll_seconds = max(1.0, float(th.get("poll_seconds", c.thermal.poll_seconds)))
+        c.thermal.soft_throttle_enabled = bool(th.get("soft_throttle_enabled", c.thermal.soft_throttle_enabled))
+        c.thermal.soft_throttle_temp = float(th.get("soft_throttle_temp", c.thermal.soft_throttle_temp))
+        c.thermal.soft_throttle_max_sleep = max(0.0, float(th.get("soft_throttle_max_sleep", c.thermal.soft_throttle_max_sleep)))
         # Push to the live global_args so changes apply without a restart.
         try:
             from codai.api.state import get_global_args
@@ -2469,6 +2584,9 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
                 ga.thermal_gpu_high = c.thermal.gpu_high
                 ga.thermal_gpu_resume = c.thermal.gpu_resume
                 ga.thermal_poll_seconds = c.thermal.poll_seconds
+                ga.thermal_soft_throttle_enabled = c.thermal.soft_throttle_enabled
+                ga.thermal_soft_throttle_temp = c.thermal.soft_throttle_temp
+                ga.thermal_soft_throttle_max_sleep = c.thermal.soft_throttle_max_sleep
         except Exception:
             pass
 

codai/admin/templates/models.html

@@ -730,12 +730,22 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
             </select>
           </div>
           <div class="form-row" style="max-width:560px">
-            <label class="form-label">Distill LoRA <span class="muted">(path or HF repo, optionally repo:weight_name.safetensors; blank for turbo full-models)</span></label>
+            <label class="form-label">Distill model <span class="muted">(cached — pick to fill the LoRA fields below)</span></label>
+            <select id="cfg-accel-distill" class="form-input" onchange="onAccelDistill()">
+              <option value="">— manual / pick a preset —</option>
+            </select>
+            <span class="form-hint" id="cfg-accel-distill-hint">Lists distill-LoRA repos found in the local cache. Choose one, then pick its high/low (or single) LoRA below.</span>
+          </div>
+          <div class="form-row" style="max-width:560px">
+            <label class="form-label">Distill LoRA <span class="muted">(single — path or repo:weight_name.safetensors; blank for turbo full-models)</span></label>
+            <select id="cfg-accel-lora-sel" class="form-input" style="display:none;margin-bottom:.4rem" onchange="onAccelLoraPick('lora')"></select>
             <input type="text" id="cfg-accel-lora" class="form-input" placeholder="e.g. ByteDance/SDXL-Lightning:sdxl_lightning_4step_lora.safetensors">
           </div>
           <div class="form-row" style="max-width:560px">
             <label class="form-label">Distill LoRA — high/low noise <span class="muted">(Wan2.2 A14B two-expert only; overrides the single LoRA per expert)</span></label>
+            <select id="cfg-accel-lora-high-sel" class="form-input" style="display:none;margin-bottom:.3rem" onchange="onAccelLoraPick('high')"></select>
             <input type="text" id="cfg-accel-lora-high" class="form-input" placeholder="high-noise → transformer  (e.g. repo:..._high_noise_..._4step.safetensors)">
+            <select id="cfg-accel-lora-low-sel" class="form-input" style="display:none;margin:.4rem 0 .3rem" onchange="onAccelLoraPick('low')"></select>
             <input type="text" id="cfg-accel-lora-low" class="form-input" style="margin-top:.4rem" placeholder="low-noise → transformer_2  (e.g. repo:..._low_noise_..._4step.safetensors)">
             <span class="form-hint">Wan2.2 A14B has two experts; the distill LoRA must be fused into <b>both</b> or the clip collapses to a solid colour at 4 steps. Leave blank to apply the single LoRA above to both.</span>
           </div>
@@ -2797,15 +2807,103 @@ function onAccelPreset(){
   const p = (_accelPresets || {})[key];
   if (!p) return;  // "custom" — leave fields as-is
   document.getElementById('cfg-accel-lora').value = p.lora || '';
+  document.getElementById('cfg-accel-lora-high').value = p.lora_high || '';
+  document.getElementById('cfg-accel-lora-low').value = p.lora_low || '';
   document.getElementById('cfg-accel-weight').value = p.lora_weight != null ? p.lora_weight : '';
   document.getElementById('cfg-accel-steps').value = p.steps != null ? p.steps : '';
   document.getElementById('cfg-accel-guidance').value = p.guidance_scale != null ? p.guidance_scale : '';
   document.getElementById('cfg-accel-flowshift').value = p.flow_shift != null ? p.flow_shift : '';
   document.getElementById('cfg-accel-scheduler').value = p.scheduler || '';
+  _syncAccelDistillFromFields();
+}
+
+// ---- Cascading distill-LoRA pickers (cached files) ----
+let _accelLoras = null;
+async function _loadAccelLoras(){
+  if (_accelLoras) return _accelLoras;
+  try {
+    const r = await fetch(ROOT_PATH + '/admin/api/accel-loras');
+    const d = await r.json();
+    _accelLoras = d.models || [];
+  } catch(e){ _accelLoras = []; }
+  return _accelLoras;
+}
+function _refreshAccelDistill(){
+  const sel = document.getElementById('cfg-accel-distill');
+  if (!sel) return;
+  const cur = sel.value;
+  sel.innerHTML = '<option value="">— manual / pick a preset —</option>';
+  (_accelLoras || []).forEach(m=>{
+    const o = document.createElement('option');
+    o.value = m.repo;
+    o.textContent = m.repo + '  (' + m.files.length + ' file' + (m.files.length===1?'':'s') + ')';
+    sel.appendChild(o);
+  });
+  if ([...sel.options].some(o=>o.value===cur)) sel.value = cur;
+  const hint = document.getElementById('cfg-accel-distill-hint');
+  if (hint && !(_accelLoras || []).length){
+    hint.textContent = 'No distill-LoRA repos found in the local cache — use a preset or type the path/repo manually below.';
+  }
+}
+function _fillAccelSel(selId, repo, files, keep){
+  const sel = document.getElementById(selId);
+  if (!sel) return;
+  sel.innerHTML = '<option value="">— choose —</option>';
+  (files || []).forEach(f=>{
+    const o = document.createElement('option');
+    o.value = repo + ':' + f; o.textContent = f;
+    sel.appendChild(o);
+  });
+  sel.style.display = (files && files.length) ? '' : 'none';
+  if (keep && [...sel.options].some(o=>o.value===keep)) sel.value = keep;
+}
+function onAccelDistill(){
+  const repo = document.getElementById('cfg-accel-distill').value;
+  const m = (_accelLoras || []).find(x=>x.repo===repo);
+  if (!m){
+    ['cfg-accel-lora-sel','cfg-accel-lora-high-sel','cfg-accel-lora-low-sel'].forEach(id=>{
+      const s=document.getElementById(id); if(s){ s.innerHTML=''; s.style.display='none'; }
+    });
+    return;
+  }
+  // Single-LoRA dropdown = all files; high/low = classified subsets (fall back to
+  // all files when the repo doesn't name them by noise level).
+  _fillAccelSel('cfg-accel-lora-sel', repo, m.files);
+  _fillAccelSel('cfg-accel-lora-high-sel', repo, (m.high && m.high.length) ? m.high : m.files);
+  _fillAccelSel('cfg-accel-lora-low-sel', repo, (m.low && m.low.length) ? m.low : m.files);
+}
+function onAccelLoraPick(kind){
+  const tmap = {lora:'cfg-accel-lora', high:'cfg-accel-lora-high', low:'cfg-accel-lora-low'};
+  const smap = {lora:'cfg-accel-lora-sel', high:'cfg-accel-lora-high-sel', low:'cfg-accel-lora-low-sel'};
+  const v = document.getElementById(smap[kind]).value;
+  if (v) document.getElementById(tmap[kind]).value = v;
+}
+// On load (or after a preset fill), preselect the distill model + dropdowns from
+// whatever repo the current high/single LoRA field points at.
+function _syncAccelDistillFromFields(){
+  const sel = document.getElementById('cfg-accel-distill');
+  if (!sel) return;
+  const refHigh = (document.getElementById('cfg-accel-lora-high').value || '').trim();
+  const refLow  = (document.getElementById('cfg-accel-lora-low').value || '').trim();
+  const refOne  = (document.getElementById('cfg-accel-lora').value || '').trim();
+  const ref = refHigh || refOne || refLow;
+  const repo = ref.includes(':') ? ref.split(':')[0] : ref;
+  if (repo && [...sel.options].some(o=>o.value===repo)){
+    sel.value = repo;
+    onAccelDistill();
+    const m = (_accelLoras || []).find(x=>x.repo===repo);
+    if (m){
+      _fillAccelSel('cfg-accel-lora-sel', repo, m.files, refOne);
+      _fillAccelSel('cfg-accel-lora-high-sel', repo, (m.high&&m.high.length)?m.high:m.files, refHigh);
+      _fillAccelSel('cfg-accel-lora-low-sel', repo, (m.low&&m.low.length)?m.low:m.files, refLow);
+    }
+  }
 }
 async function _populateAccel(a){
   await _loadAccelPresets();
+  await _loadAccelLoras();
   _refreshAccelVisibility();
+  _refreshAccelDistill();
   a = a || {};
   document.getElementById('cfg-accel-enabled').checked = !!a.enabled;
   const sel = document.getElementById('cfg-accel-preset');
@@ -2818,6 +2916,7 @@ async function _populateAccel(a){
   document.getElementById('cfg-accel-guidance').value = a.guidance_scale != null ? a.guidance_scale : '';
   document.getElementById('cfg-accel-flowshift').value = a.flow_shift != null ? a.flow_shift : '';
   document.getElementById('cfg-accel-scheduler').value = a.scheduler || '';
+  _syncAccelDistillFromFields();
   onAccelToggle();
 }
 function _collectAccel(){

codai/admin/templates/settings.html

@@ -146,11 +146,33 @@
     </div>
   </div>
 
-  <div class="form-row" style="margin:0">
+  <div class="form-row" style="margin:0 0 .75rem">
     <label class="form-label">Re-check interval while cooling down (seconds)</label>
     <input type="number" id="s-therm-poll" class="form-input" style="max-width:200px" min="1" max="120" step="1" placeholder="5">
     <span class="form-hint">How often to re-read temperatures while waiting for cooldown.</span>
   </div>
+
+  <div class="form-row" style="margin:0;border-top:1px solid var(--border,#2a2a2a);padding-top:.75rem">
+    <label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
+      <input type="checkbox" id="s-therm-soft-enabled" onchange="toggleThermalFields()">
+      <span style="font-size:13px;font-weight:500">Enable CPU soft-throttle</span>
+    </label>
+    <span class="form-hint" style="display:block;margin-top:.35rem">
+      Before the hard pause, gently slow generation (short per-step sleeps) once the
+      CPU enters the warm band, so its temperature climbs slower and the full
+      cooldown is rarely hit. CPU only — GPU is unaffected.
+    </span>
+  </div>
+  <div id="therm-soft-fields" class="form-row" style="display:grid;grid-template-columns:1fr 1fr;gap:1rem;margin-top:.5rem">
+    <div>
+      <label class="form-label">Engage above CPU temp (°C)</label>
+      <input type="number" id="s-therm-soft-temp" class="form-input" min="40" max="120" step="1" placeholder="80">
+    </div>
+    <div>
+      <label class="form-label">Max slow-down per step (seconds)</label>
+      <input type="number" id="s-therm-soft-sleep" class="form-input" min="0" max="30" step="0.5" placeholder="3">
+    </div>
+  </div>
 </div>
 
 <!-- Background jobs -->
@@ -284,6 +306,8 @@ function toggleThermalFields(){
     document.getElementById('s-therm-gpu-enabled').checked ? 'grid' : 'none';
   document.getElementById('therm-cpu-fields').style.display =
     document.getElementById('s-therm-cpu-enabled').checked ? 'grid' : 'none';
+  document.getElementById('therm-soft-fields').style.display =
+    document.getElementById('s-therm-soft-enabled').checked ? 'grid' : 'none';
 }
 
 function showAlert(type, msg){
@@ -345,6 +369,9 @@ async function loadSettings(){
     document.getElementById('s-therm-cpu-high').value = therm.cpu_high ?? 90;
     document.getElementById('s-therm-cpu-resume').value = therm.cpu_resume ?? 87;
     document.getElementById('s-therm-poll').value = therm.poll_seconds ?? 5;
+    document.getElementById('s-therm-soft-enabled').checked = !!therm.soft_throttle_enabled;
+    document.getElementById('s-therm-soft-temp').value = therm.soft_throttle_temp ?? 80;
+    document.getElementById('s-therm-soft-sleep').value = therm.soft_throttle_max_sleep ?? 3;
     toggleThermalFields();
     // Background jobs
     const jobs = d.jobs || {};
@@ -383,6 +410,9 @@ async function saveSettings(){
       cpu_high:   parseFloat(document.getElementById('s-therm-cpu-high').value)   || 90,
       cpu_resume: parseFloat(document.getElementById('s-therm-cpu-resume').value) || 87,
       poll_seconds: parseFloat(document.getElementById('s-therm-poll').value) || 5,
+      soft_throttle_enabled: document.getElementById('s-therm-soft-enabled').checked,
+      soft_throttle_temp: parseFloat(document.getElementById('s-therm-soft-temp').value) || 80,
+      soft_throttle_max_sleep: parseFloat(document.getElementById('s-therm-soft-sleep').value) || 0,
     },
     jobs:{
       resume_on_restart: document.getElementById('s-jobs-resume').checked,

codai/admin/templates/tasks.html

@@ -19,6 +19,13 @@
   — running work is paused until the hardware cools.
 </div>
 
+<div id="throttle-banner" style="display:none;margin:0 0 1rem;padding:.6rem .85rem;border-radius:8px;
+     background:rgba(56,189,248,.10);border:1px solid rgba(56,189,248,.4);color:#38bdf8;font-size:13px">
+  <span style="font-weight:600">🐢 CPU soft-throttle</span>
+  <span id="throttle-banner-msg" class="mono"></span>
+  — generation is being gently slowed to keep the CPU below its pause limit.
+</div>
+
 <!-- Live hardware telemetry -->
 <div id="sys-stats" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));
      gap:.75rem;margin:0 0 1.25rem">
@@ -165,6 +172,16 @@ async function loadTasks() {
       banner.style.display = 'none';
     }
 
+    const soft = data.soft_throttle || {};
+    const sbanner = document.getElementById('throttle-banner');
+    // Hide the soft-throttle banner during a hard cooldown (the pause supersedes it).
+    if (soft.active && !therm.active) {
+      document.getElementById('throttle-banner-msg').textContent = ' ' + (soft.message || '');
+      sbanner.style.display = '';
+    } else {
+      sbanner.style.display = 'none';
+    }
+
     const tbody = document.getElementById('tasks-body');
     if (!tasks.length) {
       tbody.innerHTML = '<tr class="empty-row"><td colspan="6">No tasks yet</td></tr>';
@@ -177,6 +194,10 @@ async function loadTasks() {
       if (t.cooling) {
         statusCell = `<span class="badge badge-warn">❄ Cooling down</span>`
           + `<div class="dim small">${esc(t.cooling_message || 'paused for thermal cooldown')}</div>`;
+      } else if (t.throttling) {
+        statusCell = `<span class="badge ${badge}">${esc(t.status)}</span>`
+          + ` <span class="badge badge-user">🐢 throttling</span>`
+          + `<div class="dim small">${esc(t.throttle_message || 'CPU soft-throttle')}</div>`;
       } else if (t.paused) {
         statusCell = `<span class="badge badge-warn">⏸ Paused</span>`
           + `<div class="dim small">suspended — click Resume to continue</div>`;

codai/config.py

@@ -124,6 +124,13 @@ class ThermalConfig:
     gpu_high: float = 90.0      # pause when GPU reaches this temperature
     gpu_resume: float = 87.0    # resume once GPU drops back to/below this
     poll_seconds: float = 5.0   # how often to re-check while cooling down
+    # Proactive soft-throttle: before a hard pause, when a sensor enters the warm
+    # band [soft_throttle_temp, *_high) insert a short per-step sleep (scaled by
+    # how close to the pause threshold) so the temperature climbs slower and the
+    # hard cooldown is rarely hit. Caps the heat-rate of a single pegged core.
+    soft_throttle_enabled: bool = False
+    soft_throttle_temp: float = 80.0       # engage at/above this temperature (°C)
+    soft_throttle_max_sleep: float = 3.0   # max seconds to sleep/checkpoint at the limit
 
 
 @dataclass
@@ -423,6 +430,9 @@ class ConfigManager:
                 "gpu_high": self.config.thermal.gpu_high,
                 "gpu_resume": self.config.thermal.gpu_resume,
                 "poll_seconds": self.config.thermal.poll_seconds,
+                "soft_throttle_enabled": self.config.thermal.soft_throttle_enabled,
+                "soft_throttle_temp": self.config.thermal.soft_throttle_temp,
+                "soft_throttle_max_sleep": self.config.thermal.soft_throttle_max_sleep,
             },
             "jobs": {
                 "resume_on_restart": self.config.jobs.resume_on_restart,

codai/main.py

@@ -204,15 +204,41 @@ def apply_model_entry_live(entry, model_types) -> int:
     mid = entry.get("path") or entry.get("id") or ""
     if not mid:
         return 0
+    def _accel_sig(c):
+        """Stable signature of a config's acceleration so we can tell whether a
+        save toggled/changed it. Returns 'off' when acceleration is absent."""
+        try:
+            from codai.models.acceleration import resolve_acceleration
+            import json as _json
+            a = resolve_acceleration(c)
+            return _json.dumps(a, sort_keys=True, default=str) if a else "off"
+        except Exception:
+            return "off"
+
     updated = 0
     for cat in (model_types or []):
         tp = _CATEGORY_TYPE_PREFIX.get(cat)
         if not tp:
             continue
         type_str, prefix = tp
+        key = f"{prefix}{mid}"
+        old_cfg = multi_model_manager.config.get(key)
         cfg = build_runtime_model_cfg(entry, type_str)
-        multi_model_manager.config[f"{prefix}{mid}"] = cfg
+        multi_model_manager.config[key] = cfg
         updated += 1
+        # Acceleration (Lightning/Lightx2v/LCM distill LoRA + scheduler) is FUSED
+        # into the pipeline at load time, so it can't be toggled on an already
+        # loaded model. If the save changed it and the model is loaded, evict it
+        # so the NEXT request for this model reloads with the new acceleration —
+        # applied immediately, no server restart.
+        try:
+            if (key in multi_model_manager.models
+                    and _accel_sig(old_cfg) != _accel_sig(cfg)):
+                if multi_model_manager.unload_model(key):
+                    print(f"  [admin] acceleration changed for {key} — model "
+                          f"evicted; next request reloads with the new setting")
+        except Exception as _e:
+            print(f"  [admin] accel-evict skipped for {key}: {_e}")
         alias = entry.get("alias")
         if alias:
             try:
@@ -766,6 +792,9 @@ def main():
     global_args.thermal_gpu_high = config.thermal.gpu_high
     global_args.thermal_gpu_resume = config.thermal.gpu_resume
     global_args.thermal_poll_seconds = config.thermal.poll_seconds
+    global_args.thermal_soft_throttle_enabled = config.thermal.soft_throttle_enabled
+    global_args.thermal_soft_throttle_temp = config.thermal.soft_throttle_temp
+    global_args.thermal_soft_throttle_max_sleep = config.thermal.soft_throttle_max_sleep
     global_args.n_gpu_layers = config.vulkan.n_gpu_layers
     global_args.n_ctx = [config.vulkan.n_ctx]
     global_args.vulkan_device = config.vulkan.device_id

codai/models/manager.py

@@ -2464,6 +2464,52 @@ class MultiModelManager:
             _time.sleep(0.25)
         return True
 
+    def unload_model(self, key: str) -> bool:
+        """Fully unload ONE model by key: drop it from the cache + instance pool and
+        free its VRAM/host RAM. Used when a config change (e.g. acceleration, which
+        is fused at load time) needs the next request to reload the model fresh.
+
+        Waits briefly if the model is mid-request; returns True if a model was
+        actually unloaded."""
+        if key not in self.models and key not in self.model_pools:
+            return False
+        if self._is_key_busy(key):
+            if not self._wait_until_idle(key):
+                print(f"  unload_model: '{key}' still busy — not unloaded")
+                return False
+        model_obj = self.models.pop(key, None)
+        self.models_in_vram.discard(key)
+        if self.current_model_key == key:
+            self.current_model_key = None
+        if self.active_in_vram == key:
+            self.active_in_vram = None
+        pool = self.model_pools.pop(key, None)
+        if pool is not None:
+            try:
+                pool.cleanup_all()
+            except Exception as e:
+                print(f"  Warning cleaning pool for '{key}': {e}")
+        if model_obj is not None:
+            try:
+                if hasattr(model_obj, 'cleanup'):
+                    model_obj.cleanup()
+                elif hasattr(model_obj, 'to'):
+                    model_obj.to('cpu')
+            except Exception as e:
+                print(f"  Warning during unload of '{key}': {e}")
+        del model_obj
+        for _ in range(3):
+            gc.collect()
+        try:
+            import torch
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+                torch.cuda.empty_cache()
+        except Exception:
+            pass
+        _trim_cpu_ram()
+        return True
+
     def _evict_models_for_vram(self, needed_gb: float):
         """Unload loaded models (LRU first) until we have at least needed_gb free VRAM.
 

codai/models/thermal.py

@@ -63,6 +63,25 @@ def get_cooldown_state() -> dict:
         return dict(_cooldown_state)
 
 
+def soft_throttle_status() -> dict:
+    """Current proactive CPU soft-throttle status, computed LIVE from the CPU temp
+    and settings (so the Tasks page reflects reality regardless of when the last
+    per-step checkpoint fired — video steps can be tens of seconds apart).
+
+    ``active`` is True when soft-throttle is enabled and the CPU sits in the warm
+    band [soft_temp, cpu_high). The caller (Tasks API) additionally gates this on
+    there being a running generation, since throttling only happens during work.
+    CPU-only by design."""
+    s = _settings_from_global_args()
+    if not s.soft_enabled or not s.cpu_enabled or s.soft_max_sleep <= 0:
+        return {"active": False}
+    cpu_t = read_cpu_temp()
+    if cpu_t is None or cpu_t < s.soft_temp or cpu_t >= s.cpu_high:
+        return {"active": False}
+    return {"active": True, "cpu": cpu_t, "sleep": _soft_throttle_sleep(s, cpu_t),
+            "engage": s.soft_temp, "cpu_high": s.cpu_high}
+
+
 def _cooldown_active() -> bool:
     """True while at least one worker is in the cooldown wait loop. Used so that
     other parallel workers join the pause (cross-worker hysteresis) instead of
@@ -423,12 +442,14 @@ class ThermalSettings:
         "cpu_enabled", "gpu_enabled",
         "cpu_high", "cpu_resume", "gpu_high", "gpu_resume",
         "poll_seconds",
+        "soft_enabled", "soft_temp", "soft_max_sleep",
     )
 
     def __init__(self, cpu_enabled=True, gpu_enabled=True,
                  cpu_high=90.0, cpu_resume=87.0,
                  gpu_high=90.0, gpu_resume=87.0,
-                 poll_seconds=5.0):
+                 poll_seconds=5.0,
+                 soft_enabled=False, soft_temp=80.0, soft_max_sleep=3.0):
         self.cpu_enabled = bool(cpu_enabled)
         self.gpu_enabled = bool(gpu_enabled)
         self.cpu_high = float(cpu_high)
@@ -436,6 +457,9 @@ class ThermalSettings:
         self.gpu_high = float(gpu_high)
         self.gpu_resume = float(gpu_resume)
         self.poll_seconds = max(1.0, float(poll_seconds))
+        self.soft_enabled = bool(soft_enabled)
+        self.soft_temp = float(soft_temp)
+        self.soft_max_sleep = max(0.0, float(soft_max_sleep))
 
 
 def _settings_from_global_args() -> ThermalSettings:
@@ -456,9 +480,29 @@ def _settings_from_global_args() -> ThermalSettings:
         gpu_high=g("thermal_gpu_high", 90.0),
         gpu_resume=g("thermal_gpu_resume", 87.0),
         poll_seconds=g("thermal_poll_seconds", 5.0),
+        soft_enabled=g("thermal_soft_throttle_enabled", False),
+        soft_temp=g("thermal_soft_throttle_temp", 80.0),
+        soft_max_sleep=g("thermal_soft_throttle_max_sleep", 3.0),
     )
 
 
+def _soft_throttle_sleep(settings: "ThermalSettings", cpu_t) -> float:
+    """Seconds to sleep at this checkpoint for proactive CPU soft-throttling.
+
+    0 unless the CPU is in its warm band [soft_temp, cpu_high). Scales linearly
+    from 0 at soft_temp to soft_max_sleep just below the CPU pause threshold.
+    CPU-ONLY by design — GPU heat is left entirely to the hard cooldown."""
+    if (not settings.soft_enabled or settings.soft_max_sleep <= 0
+            or not settings.cpu_enabled or cpu_t is None):
+        return 0.0
+    t0 = settings.soft_temp
+    if cpu_t < t0:
+        return 0.0
+    span = max(1.0, settings.cpu_high - t0)
+    frac = min(1.0, max(0.0, (cpu_t - t0) / span))
+    return settings.soft_max_sleep * frac
+
+
 _last_checkpoint: dict = {}
 
 
@@ -527,7 +571,18 @@ def wait_until_safe(settings: Optional[ThermalSettings] = None,
            (settings.cpu_enabled and cpu_t is not None and cpu_t > settings.cpu_resume):
             joined = True
     if not hot and not joined:
-        _dbg(f"within safe limits — serving immediately{desc0}")
+        # Proactive soft-throttle (CPU only): not hot enough to hard-pause, but if
+        # the CPU is in the warm band [soft_temp, cpu_high) sleep a little (scaled
+        # by how close to the pause threshold) so its temperature climbs slower and
+        # we rarely hit the full cooldown. Caps the heat-rate of a single pegged
+        # core. GPU heat is handled solely by the hard cooldown.
+        _sleep = _soft_throttle_sleep(settings, cpu_t)
+        if _sleep > 0:
+            _dbg(f"soft-throttle{desc0}: sleeping {_sleep:.2f}s "
+                 f"(GPU {_fmt(gpu_t)} CPU {_fmt(cpu_t)}, engage>={settings.soft_temp:.0f})")
+            time.sleep(_sleep)
+        else:
+            _dbg(f"within safe limits — serving immediately{desc0}")
         return
 
     # Enter cooldown: wait until *every* triggered sensor is at/below resume.

tools/gen_township_fighters.py

@@ -32,11 +32,17 @@ def _log(*args, **kwargs):
     print(*args, **kwargs, flush=True)
 
 
-def _run_with_spinner(label: str, fn, *args, **kwargs):
+def _run_with_spinner(label: str, fn, *args, poll_fn=None, step_cb=None, **kwargs):
     """
     Run fn(*args, **kwargs) in a background thread while printing a live
     elapsed-time ticker on stdout so the user knows the script is alive.
     Returns the function's return value, or re-raises any exception it threw.
+
+    ``poll_fn`` (optional): a zero-arg callable returning the server's live
+    generation-progress dict (``current``/``total``/``pct``/``it_per_s``/``phase``)
+    — e.g. ``client.video_progress``. When given, the spinner shows the real
+    diffusion step ("step 12/25 (48%) 1.3it/s") and each sample is forwarded to
+    ``step_cb(progress_dict)`` so a web caller can drive a per-clip step bar.
     """
     result_box = [None]
     exc_box    = [None]
@@ -54,12 +60,33 @@ def _run_with_spinner(label: str, fn, *args, **kwargs):
     idx = 0
     while t.is_alive():
         elapsed = time.monotonic() - start
-        sys.stdout.write(f"\r    {spinner[idx % len(spinner)]} {label}  {elapsed:.0f}s elapsed…")
+        step_txt = ""
+        if poll_fn is not None:
+            try:
+                p = poll_fn() or {}
+            except Exception:
+                p = {}
+            if p.get("active") and (p.get("total") or 0) > 0:
+                cur, tot = int(p.get("current") or 0), int(p.get("total") or 0)
+                pct = int(p.get("pct") or 0)
+                its = p.get("it_per_s") or 0
+                phase = p.get("phase") or ""
+                step_txt = (f" · {phase} {cur}/{tot} ({pct}%)"
+                            + (f" {its:.2f}it/s" if its else ""))
+            elif p.get("phase") == "loading":
+                step_txt = " · loading model…"
+            if step_cb is not None:
+                try:
+                    step_cb(p)
+                except Exception:
+                    pass
+        sys.stdout.write(f"\r    {spinner[idx % len(spinner)]} {label}  "
+                         f"{elapsed:.0f}s{step_txt}…    ")
         sys.stdout.flush()
         idx += 1
         t.join(timeout=1.0)
     # Clear the spinner line
-    sys.stdout.write("\r" + " " * 72 + "\r")
+    sys.stdout.write("\r" + " " * 96 + "\r")
     sys.stdout.flush()
 
     if exc_box[0] is not None:
@@ -766,6 +793,21 @@ class CoderAIClient:
         except Exception:
             return {}
 
+    def image_progress(self) -> dict:
+        """Live diffusion-step progress of the in-flight image generation
+        (keyframes / character / environment refs). Best-effort; {} on error."""
+        try:
+            return self._get("/v1/images/progress") or {}
+        except Exception:
+            return {}
+
+    def video_progress(self) -> dict:
+        """Live diffusion-step progress of the in-flight video clip. {} on error."""
+        try:
+            return self._get("/v1/video/progress") or {}
+        except Exception:
+            return {}
+
     def chat_complete(self, model: str, system: str, user: str,
                       max_tokens: int = 400) -> str:
         import re as _re
@@ -1025,12 +1067,38 @@ class PromptGenerator:
 # Video utilities
 # ─────────────────────────────────────────────────────────────────────────────
 
-def concat_videos(clip_paths: list, out_path: str):
+def concat_videos(clip_paths: list, out_path: str, reencode: bool = False,
+                  fps: int = None):
+    """Concatenate clips with the ffmpeg concat demuxer.
+
+    ``reencode=False`` stream-copies (fast) — fine for the final short/long
+    assembly where each segment is a separate shot and a cut between them is
+    expected.
+
+    ``reencode=True`` RE-ENCODES with a constant frame rate. Use it to join the
+    sub-renders of ONE chained shot: stream-copying mp4s that carry B-frames /
+    an edit list / reset PTS makes players FREEZE on a segment's first frame for
+    its whole duration (the "first half is a static image" bug). Re-encoding
+    regenerates clean, monotonic, constant-rate timestamps so the join is truly
+    seamless. Falls back to stream copy if the encoder is unavailable."""
     with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as f:
         for p in clip_paths:
             f.write(f"file '{os.path.abspath(p)}'\n")
         list_path = f.name
     try:
+        if reencode:
+            cmd = ["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", list_path,
+                   "-c:v", "libx264", "-preset", "veryfast", "-crf", "18",
+                   "-pix_fmt", "yuv420p", "-vsync", "cfr"]
+            if fps:
+                cmd += ["-r", str(int(fps))]
+            cmd += ["-movflags", "+faststart", out_path]
+            try:
+                subprocess.run(cmd, check=True, capture_output=True)
+                return
+            except subprocess.CalledProcessError as e:
+                _log(f"    ⚠ seamless re-encode concat failed ({e.stderr.decode(errors='replace')[:120] if e.stderr else e}); "
+                     f"falling back to stream copy")
         subprocess.run(
             ["ffmpeg", "-y", "-f", "concat", "-safe", "0",
              "-i", list_path, "-c", "copy", out_path],
@@ -1203,6 +1271,7 @@ def stage_characters(client: CoderAIClient, image_model: str, out_dir: Path,
                 name=name, prompt=fighter["prompt"],
                 description=fighter["description"], model=image_model,
                 n=max(1, int(n_refs or 4)),
+                poll_fn=client.image_progress,
             )
             _log(f"    ✓ {d.get('image_count', '?')} reference images saved in CoderAI")
             # Fetch and save locally
@@ -1285,6 +1354,7 @@ def stage_environments(client: CoderAIClient, image_model: str, out_dir: Path,
                 name=name, prompt=env["prompt"],
                 description=env["description"], model=image_model,
                 n=max(1, int(n_refs or 3)), size="768x512",
+                poll_fn=client.image_progress,
             )
             _log(f"    ✓ {d.get('image_count', '?')} reference images saved in CoderAI")
             images = client.fetch_profile_images("environment", name)
@@ -1917,6 +1987,8 @@ def _generate_keyframes(client: CoderAIClient, image_model: str, keyframe_dir: P
                 character_profiles=profiles, loras=loras,
                 character_strength=char_strength, size=keyframe_size,
                 steps=keyframe_steps,
+                poll_fn=client.image_progress,
+                step_cb=(lambda prog, _s=stem: _kf(_s, "step", prog)),
             )
             out_png.write_bytes(img)
             made += 1
@@ -2298,6 +2370,15 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
             except Exception:
                 pass
 
+    def _step(prog):
+        """Forward a live diffusion-step progress dict for the CURRENT clip to the
+        web UI (per-clip step bar). No-op without a clip_cb."""
+        if clip_cb and prog:
+            try:
+                clip_cb(_gidx, "step", prog)
+            except Exception:
+                pass
+
     def _keyframe_bytes(stem: str):
         if not keyframe_dir:
             return None
@@ -2315,7 +2396,7 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
                             MODEL_MAX_FRAMES))
 
     def _render_once(label, prompt, profiles, env, nf, out_path,
-                     fighters=None, init_override=None):
+                     fighters=None, init_override=None, step_cb=None):
         """One model generation → out_path. `init_override` (PNG bytes) wins over
         any keyframe; pass it to chain a sub-render onto the previous one's last
         frame. Returns (ok, duration_or_None, fatal)."""
@@ -2343,6 +2424,7 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
                 num_frames=nf, fps=fps, seed=random.randint(0, 2**31),
                 width=_vw, height=_vh,
                 init_image=init_image, loras=loras,
+                poll_fn=client.video_progress, step_cb=step_cb,
             )
             Path(out_path).write_bytes(mp4)
             return True, (get_video_duration(out_path) or None), False
@@ -2357,7 +2439,8 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
             time.sleep(backoff)
             return False, None, False
 
-    def _render(label, prompt, profiles, env, nf, out_path, stem=None, fighters=None):
+    def _render(label, prompt, profiles, env, nf, out_path, stem=None, fighters=None,
+                step_cb=None):
         """Render one CLIP, splitting into chained sub-renders when the budget
         exceeds the single-render cap. The parts are concatenated into out_path as
         one continuous shot and discarded, so callers (and the Matches page) still
@@ -2366,7 +2449,8 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
         budget = _split_frame_budget(int(nf), _chunk_max)
         if len(budget) == 1:
             return _render_once(label, prompt, profiles, env, budget[0], out_path,
-                                fighters=fighters, init_override=keyframe)
+                                fighters=fighters, init_override=keyframe,
+                                step_cb=step_cb)
         # Chained multi-part shot: part 0 starts from the clip keyframe; each later
         # part is seeded by the previous part's last frame → seamless single take.
         # Parts live in a throwaway temp dir (NOT video_dir) so a crash can't leave
@@ -2381,10 +2465,15 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
             for pi, pn in enumerate(budget):
                 part_path = os.path.join(tmpd, f"part{pi:02d}.mp4")
                 seed_img = keyframe if pi == 0 else (prev_last or keyframe)
+                # Tag each part's step updates with part N/total so the UI can show
+                # "concatenating shot — part 2/3" alongside the diffusion step.
+                _pcb = ((lambda prog, _p=pi + 1, _n=len(budget):
+                         step_cb({**(prog or {}), "part": _p, "parts": _n}))
+                        if step_cb else None)
                 ok, _dur, is_fatal = _render_once(
                     f"{label} [part {pi+1}/{len(budget)}, {pn}f]",
                     prompt, profiles, env, pn, part_path,
-                    fighters=fighters, init_override=seed_img)
+                    fighters=fighters, init_override=seed_img, step_cb=_pcb)
                 if not ok:
                     return False, None, is_fatal
                 parts.append(part_path)
@@ -2392,7 +2481,10 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
                 if pi < len(budget) - 1 and prev_last is None:
                     _log("    ⚠ could not read part's last frame — next part falls "
                          "back to the clip keyframe (possible visible seam)")
-            concat_videos(parts, out_path)
+            # Re-encode the join: stream-copying the parts makes players freeze on
+            # each part's first frame for its duration (static-first-half bug). The
+            # parts are one continuous shot, so a clean CFR re-encode is correct.
+            concat_videos(parts, out_path, reencode=True, fps=fps)
             return True, (get_video_duration(out_path) or None), False
         finally:
             _sh.rmtree(tmpd, ignore_errors=True)
@@ -2433,7 +2525,7 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
             ok, dur, is_fatal = _render(
                 f"clip {c['idx']:02d} — {m['f1']} vs {m['f2']}",
                 c["prompt"], [m["f1"], m["f2"]], m["env"], _nf, str(clip_path),
-                stem=clip_stem, fighters=[m["f1"], m["f2"]])
+                stem=clip_stem, fighters=[m["f1"], m["f2"]], step_cb=_step)
             if is_fatal:
                 fatal = True
                 _clip("end", False)
@@ -2502,7 +2594,7 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
         ok, dur, is_fatal = _render(
             f"{clip_name} outcome clip",
             o["prompt"], [o["fighter"]], o["env"], _onf, out_path,
-            stem=clip_name, fighters=_ofighters)
+            stem=clip_name, fighters=_ofighters, step_cb=_step)
         if is_fatal:
             fatal = True
             _clip("end", False)
@@ -3103,11 +3195,29 @@ def launch_web_ui(default_args):
                     {"label": lbl, "status": "pending"} for lbl in labels]
 
         def _item(gidx, phase, ok=None):
-            st = "rendering" if phase == "start" else ("done" if ok else "failed")
             with _jobs_lock:
                 items = _state["jobs"][job_id].get("items") or []
-                if 0 <= gidx < len(items):
-                    items[gidx]["status"] = st
+                if not (0 <= gidx < len(items)):
+                    return
+                if phase == "step":
+                    # `ok` carries the live progress dict (current/total/pct/
+                    # it_per_s, plus part/parts when the clip is a chained shot).
+                    prog = ok or {}
+                    if (prog.get("total") or 0) > 0:
+                        items[gidx]["step"] = {
+                            "cur": int(prog.get("current") or 0),
+                            "tot": int(prog.get("total") or 0),
+                            "pct": int(prog.get("pct") or 0),
+                            "its": round(float(prog.get("it_per_s") or 0), 2),
+                            "phase": prog.get("phase") or "",
+                            "part": int(prog.get("part") or 0),
+                            "parts": int(prog.get("parts") or 0),
+                        }
+                    return
+                items[gidx]["status"] = (
+                    "rendering" if phase == "start" else ("done" if ok else "failed"))
+                if phase != "start":
+                    items[gidx].pop("step", None)  # clear step bar on completion
 
         def _load_map(fname):
             fp = out_dir / fname
@@ -4642,12 +4752,22 @@ function _renderMatchBars(wrap, d){
     h+='<div class=prg-items>';
     for(const it of items){
       const s=it.status||'pending';
-      let w='0%', cls='';
-      if(s==='rendering'){ w='100%'; cls=' striped'; }
+      let w='0%', cls='', sub=s;
+      if(s==='rendering'){
+        const st=it.step;
+        if(st && st.tot>0){
+          // Real diffusion-step progress (and part X/Y when this clip is a
+          // chained single shot made of several concatenated generations).
+          w=st.pct+'%';
+          const partTxt=(st.parts>1)?('shot part '+st.part+'/'+st.parts+' · '):'';
+          const itsTxt=st.its?(' · '+st.its+'it/s'):'';
+          sub=partTxt+(st.phase||'step')+' '+st.cur+'/'+st.tot+' ('+st.pct+'%)'+itsTxt;
+        } else { w='100%'; cls=' striped'; sub='starting…'; }
+      }
       else if(s==='done'){ w='100%'; cls=' ok'; }
       else if(s==='failed'){ w='100%'; cls=' fail'; }
       const icon=s==='done'?'✓':(s==='failed'?'✗':(s==='rendering'?'⏳':'·'));
-      h+='<div class=prg-item><div class=prg-ilabel>'+icon+' '+_esch(it.label)+' — '+s+'</div>'
+      h+='<div class=prg-item><div class=prg-ilabel>'+icon+' '+_esch(it.label)+' — '+_esch(sub)+'</div>'
         +'<div class=progress-bar><div class="progress-fill'+cls+'" style="width:'+w+'"></div></div></div>';
     }
     h+='</div>';