tasks page: engine indicator, live image-gen progress, per-task cooling

- Show which engine each task runs on (badge on active AND history rows; backend
  already tags t.engine) and an "● processing" badge on the engines card (new
  inflight/processing fields in /admin/api/engines).
- Image-gen advancement showed only "working…": poll() short-circuited to the
  synthesized task list whenever the primary had any in-flight request, dropping the
  engine's real step/total. Now it quick-polls the engine first (image/diffusers gen
  releases the GIL so it answers with real progress) and only falls back to the
  synthesized list when a GIL-bound text gen times out.
- Surface thermal cooldown on the task entry itself: _merge_engine_tasks marks tasks
  on a cooling engine with cooling/cooling_message (the row already renders it).
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RdMufYvtTbtGDWsiZVoXce
parent df8939fe
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
# Canonical product version for CoderAI — single source of truth. Both the API # Canonical product version for CoderAI — single source of truth. Both the API
# metadata and the admin web UI read from here. # metadata and the admin web UI read from here.
__version__ = "0.1.22" __version__ = "0.1.23"
# Configure the CUDA caching allocator BEFORE torch is imported anywhere. # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
# expandable_segments lets the allocator return freed pages to the driver even # expandable_segments lets the allocator return freed pages to the driver even
......
...@@ -218,7 +218,7 @@ function historyRow(t) { ...@@ -218,7 +218,7 @@ function historyRow(t) {
} }
return `<tr> return `<tr>
<td><span class="badge badge-user">${esc(KIND_LABEL[t.kind] || t.kind)}</span></td> <td><span class="badge badge-user">${esc(KIND_LABEL[t.kind] || t.kind)}</span></td>
<td><div class="td-name">${esc(title)}</div><div class="dim small mono">${esc(t.model || '')}</div></td> <td><div class="td-name">${esc(title)}${t.engine?` <span class="badge badge-user" style="font-size:9px;padding:.05rem .3rem;vertical-align:middle" title="Ran on engine">${esc(t.engine)}</span>`:''}</div><div class="dim small mono">${esc(t.model || '')}</div></td>
<td>${result}</td> <td>${result}</td>
<td>${thru}</td> <td>${thru}</td>
<td class="dim small">${fmtTime(t.ended_at)}</td> <td class="dim small">${fmtTime(t.ended_at)}</td>
...@@ -312,12 +312,13 @@ async function loadEngines(){ ...@@ -312,12 +312,13 @@ async function loadEngines(){
const vram = e.vram ? `${(e.vram.used ?? 0).toFixed ? e.vram.used.toFixed(1) : e.vram.used}/${e.vram.total} GB` : ''; const vram = e.vram ? `${(e.vram.used ?? 0).toFixed ? e.vram.used.toFixed(1) : e.vram.used}/${e.vram.total} GB` : '';
const cool = e.cooling ? ` <span class="badge badge-warn" style="font-size:9px">❄ cooling</span>` : ''; const cool = e.cooling ? ` <span class="badge badge-warn" style="font-size:9px">❄ cooling</span>` : '';
const prim = e.primary ? ` <span class="badge badge-user" style="font-size:9px">primary</span>` : ''; const prim = e.primary ? ` <span class="badge badge-user" style="font-size:9px">primary</span>` : '';
const proc = e.processing ? ` <span class="badge badge-admin" style="font-size:9px" title="actively processing a request">● processing</span>` : '';
const loaded = e.loaded_models||[]; const loaded = e.loaded_models||[];
const models = loaded.length; const models = loaded.length;
const mtip = models ? esc(loaded.join('\n')) : 'no models loaded'; const mtip = models ? esc(loaded.join('\n')) : 'no models loaded';
return `<div class="sys-tile"> return `<div class="sys-tile">
<div class="sys-head"> <div class="sys-head">
<span class="sys-name">${esc(e.name)} <span class="dim" style="text-transform:none">(${esc(e.backend)})</span>${prim}${cool}</span> <span class="sys-name">${esc(e.name)} <span class="dim" style="text-transform:none">(${esc(e.backend)})</span>${prim}${proc}${cool}</span>
<span style="width:9px;height:9px;border-radius:50%;background:${dot};display:inline-block" title="${state}"></span> <span style="width:9px;height:9px;border-radius:50%;background:${dot};display:inline-block" title="${state}"></span>
</div> </div>
<div class="sys-sub"><span>${esc(state)}${vram?' · '+esc(vram):''}</span><span title="${mtip}" style="cursor:${models?'help':'default'};${models?'text-decoration:underline dotted':''}">${models} model${models!==1?'s':''}</span></div> <div class="sys-sub"><span>${esc(state)}${vram?' · '+esc(vram):''}</span><span title="${mtip}" style="cursor:${models?'help':'default'};${models?'text-decoration:underline dotted':''}">${models} model${models!==1?'s':''}</span></div>
......
...@@ -779,6 +779,28 @@ class FrontProxy: ...@@ -779,6 +779,28 @@ class FrontProxy:
# the timeout). Serve the merged/synthesized task list from the front's # the timeout). Serve the merged/synthesized task list from the front's
# last-good cache + live in-flight tracking — no engine hit during work. # last-good cache + live in-flight tracking — no engine hit during work.
if is_tasks and int(getattr(prim, "inflight", 0) or 0) > 0: if is_tasks and int(getattr(prim, "inflight", 0) or 0) > 0:
# In-flight: try a SHORT poll FIRST. An image/diffusers generation
# releases the GIL, so the engine can still answer with the REAL
# step/total progress (otherwise the task page only shows "working…").
# Only fall back to the synthesized list when the engine is genuinely
# GIL-bound (text gen) and the quick poll times out — keeping the page
# responsive either way.
try:
headers = self._filter_headers(request.headers, _DROP_REQ)
r = await self._short.get(
prim.url + request.url.path, headers=headers,
params=request.query_params,
timeout=httpx.Timeout(connect=2.0, read=1.5, write=2.0, pool=2.0))
if r.status_code == 200:
data = r.json()
_ptasks = data.get("tasks") or []
self._primary_tasks_cache = _ptasks
self._primary_tasks_at = time.monotonic()
data["tasks"] = self._merge_engine_tasks(prim, _ptasks)
data["cooling_engines"] = self._cooling_engines()
return JSONResponse(data)
except Exception:
pass
ptasks = (self._primary_tasks_cache or []) \ ptasks = (self._primary_tasks_cache or []) \
if (time.monotonic() - self._primary_tasks_at) < 120 else [] if (time.monotonic() - self._primary_tasks_at) < 120 else []
return JSONResponse({"engine": "busy", "stale": True, return JSONResponse({"engine": "busy", "stale": True,
...@@ -1008,6 +1030,9 @@ class FrontProxy: ...@@ -1008,6 +1030,9 @@ class FrontProxy:
"thermal_paused": bool(getattr(e, "therm_paused", False)), "thermal_paused": bool(getattr(e, "therm_paused", False)),
"thermal_frozen": bool(getattr(e, "therm_sigstopped", False)), "thermal_frozen": bool(getattr(e, "therm_sigstopped", False)),
"loaded_models": self._canonical_loaded(e.loaded_models), "loaded_models": self._canonical_loaded(e.loaded_models),
"inflight": int(getattr(e, "inflight", 0) or 0),
"processing": (int(getattr(e, "inflight", 0) or 0) > 0
or bool(getattr(e, "loading", None))),
"pid": pid}) "pid": pid})
return out return out
...@@ -1385,10 +1410,18 @@ class FrontProxy: ...@@ -1385,10 +1410,18 @@ class FrontProxy:
merged = [] merged = []
seen = set() seen = set()
# Primary's tasks (from its authed response) — tag with the primary name. # Primary's tasks (from its authed response) — tag with the primary name.
def _mark_cooling(t, e):
"""Surface a thermal cooldown on the task entry itself (the row reads
t.cooling / t.cooling_message), so a paused-for-heat task shows why."""
if e is not None and getattr(e, "cooling", None) and t.get("status") == "running":
t["cooling"] = True
_cm = e.cooling.get("message") if isinstance(e.cooling, dict) else None
t.setdefault("cooling_message", _cm or "paused for thermal cooldown")
for t in primary_tasks: for t in primary_tasks:
if isinstance(t, dict): if isinstance(t, dict):
t = dict(t) t = dict(t)
t.setdefault("engine", primary.name if primary else None) t.setdefault("engine", primary.name if primary else None)
_mark_cooling(t, primary)
seen.add(t.get("id")) seen.add(t.get("id"))
merged.append(t) merged.append(t)
# Tasks the supervisor saw on the other engines. # Tasks the supervisor saw on the other engines.
...@@ -1400,6 +1433,7 @@ class FrontProxy: ...@@ -1400,6 +1433,7 @@ class FrontProxy:
continue continue
t = dict(t) t = dict(t)
t["engine"] = e.name t["engine"] = e.name
_mark_cooling(t, e)
merged.append(t) merged.append(t)
seen.add(t.get("id")) seen.add(t.get("id"))
# Synthetic "loading" tasks parsed from the log stream, for any engine that # Synthetic "loading" tasks parsed from the log stream, for any engine that
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment