thermal: front-driven central supervisor with cooperative pause + SIGSTOP escalation

The thermal guard was checkpoint-based and ran inside each engine, so it went
blind during a single long native call (llama.cpp prefill / image encode): the
engine can't reach a between-token checkpoint, and the [thermal][debug] heartbeat
stops exactly when the GPU is hottest.

Move the AUTHORITATIVE monitor to the front, which stays responsive regardless of
what an engine is doing:

* Front (engine_supervisor): a thermal thread reads per-card temps via gpu_stats()
  + CPU temp, maps each card to the owning engine by its CODERAI_ENGINE_GPUS
  selectors, and drives pause/resume with hysteresis (pause at *_high, resume only
  back at *_resume). A hot GPU pauses just its engine; a hot CPU pauses all.
* Engines stop cooperatively, as before, but triggered remotely: the front POSTs
  /internal/thermal-pause; thermal.set_external_pause() makes wait_until_safe()/
  checkpoint() block at the next safe point (publishing cooldown state so the Tasks
  page shows it), until /internal/thermal-resume.
* Escalation: if a paused engine keeps generating (inflight > 0) — stuck in a
  native call it can't interrupt — for stop_escalate_checks (default 3) consecutive
  checks, the front SIGSTOPs the engine's process group; SIGCONT on cooldown. Both
  signals target the session group so children freeze too.
* stop_all()/restart_engine() SIGCONT a frozen engine first (a stopped process
  ignores SIGTERM until continued); _spawn() resets the thermal flags.
* Config: thermal.supervisor_enabled (default on), thermal.stop_escalate_checks.
* UI: per-engine temp + pause/frozen state in engines_list and the Tasks cooldown
  banner (covers a SIGSTOPped engine that can't report its own cooldown).

Also: exclude coderai-runtime/*-runtime from the Docker build context (.dockerignore)
— a root-owned runtime temp file was breaking the image build.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RdMufYvtTbtGDWsiZVoXce

.dockerignore

@@ -15,6 +15,8 @@ __pycache__
 !.packaging-cache/build-manifest.json
 
 # Large runtime/generated data
+coderai-runtime
+*-runtime
 models
 offload
 township_output

codai/api/app.py

@@ -331,8 +331,45 @@ async def internal_engine_state():
                        "message": cs.get("message")}
     except Exception:
         cooling = None
+    # Whether the front's thermal supervisor currently holds this engine in a
+    # cooperative pause (so the front can confirm the engine acknowledged it).
+    paused = False
+    try:
+        from codai.models import thermal as _therm
+        paused = _therm.external_pause_active()
+    except Exception:
+        paused = False
     return {"ok": True, "pid": _os.getpid(), "loaded_models": loaded,
-            "vram": vram, "tasks": tasks, "cooling": cooling}
+            "vram": vram, "tasks": tasks, "cooling": cooling, "paused": paused}
+
+
+@app.post("/internal/thermal-pause", include_in_schema=False)
+async def internal_thermal_pause(request: Request):
+    """Front-driven thermal pause: the supervisor (which monitors temps centrally)
+    asks this engine to stop generating. Honoured cooperatively at the next safe
+    point — request entry / between tokens — via codai.models.thermal."""
+    import json as _json
+    try:
+        data = _json.loads((await request.body()) or b"{}") or {}
+    except Exception:
+        data = {}
+    try:
+        from codai.models import thermal as _therm
+        _therm.set_external_pause(True, reason=str(data.get("reason") or "thermal"))
+    except Exception as e:
+        return {"ok": False, "error": str(e)}
+    return {"ok": True, "paused": True}
+
+
+@app.post("/internal/thermal-resume", include_in_schema=False)
+async def internal_thermal_resume(request: Request):
+    """Lift the front-driven thermal pause once the hardware has cooled."""
+    try:
+        from codai.models import thermal as _therm
+        _therm.set_external_pause(False)
+    except Exception as e:
+        return {"ok": False, "error": str(e)}
+    return {"ok": True, "paused": False}
 
 
 @app.post("/internal/reload-config", include_in_schema=False)

codai/config.py

@@ -248,6 +248,14 @@ class ThermalConfig:
     soft_throttle_enabled: bool = False
     soft_throttle_temp: float = 80.0       # engage at/above this temperature (°C)
     soft_throttle_max_sleep: float = 3.0   # max seconds to sleep/checkpoint at the limit
+    # Front-driven thermal supervision. The front proxy monitors temperatures
+    # centrally (it stays responsive even while an engine is GIL-blocked in a long
+    # native call) and tells engines to pause/resume cooperatively. When an engine
+    # ignores a pause — stuck in a native call it can't interrupt — for
+    # `stop_escalate_checks` consecutive monitor checks, the front escalates to an
+    # OS-level SIGSTOP (and SIGCONT to resume after cooldown).
+    supervisor_enabled: bool = True
+    stop_escalate_checks: int = 3
 
 
 @dataclass
@@ -702,6 +710,8 @@ class ConfigManager:
                 "soft_throttle_enabled": self.config.thermal.soft_throttle_enabled,
                 "soft_throttle_temp": self.config.thermal.soft_throttle_temp,
                 "soft_throttle_max_sleep": self.config.thermal.soft_throttle_max_sleep,
+                "supervisor_enabled": self.config.thermal.supervisor_enabled,
+                "stop_escalate_checks": self.config.thermal.stop_escalate_checks,
             },
             "jobs": {
                 "resume_on_restart": self.config.jobs.resume_on_restart,

codai/frontproxy/app.py

@@ -957,6 +957,9 @@ class FrontProxy:
             out.append({"id": e.id, "name": e.name, "backend": e.backend,
                         "gpu": e.gpu, "healthy": e.healthy, "primary": e.primary,
                         "vram": e.vram, "cooling": bool(e.cooling),
+                        "temp": getattr(e, "therm_temp", None),
+                        "thermal_paused": bool(getattr(e, "therm_paused", False)),
+                        "thermal_frozen": bool(getattr(e, "therm_sigstopped", False)),
                         "loaded_models": self._canonical_loaded(e.loaded_models),
                         "pid": pid})
         return out
@@ -1107,13 +1110,24 @@ class FrontProxy:
         return resp
 
     def _cooling_engines(self) -> list:
-        """Which engines are in thermal cooldown right now (for the Tasks banner)."""
+        """Which engines are in thermal cooldown right now (for the Tasks banner).
+
+        Covers both a locally-reported cooldown (engine.cooling, from the engine's
+        own checkpoint loop) and a front-driven supervisor pause (engine.therm_paused)
+        — the latter matters when the engine is SIGSTOPped and can't report at all."""
         out = []
         for e in self.registry.all():
             if e.cooling:
                 out.append({"engine": e.name, "gpu": e.cooling.get("gpu"),
                             "cpu": e.cooling.get("cpu"),
                             "message": e.cooling.get("message")})
+            elif getattr(e, "therm_paused", False):
+                out.append({
+                    "engine": e.name, "gpu": getattr(e, "therm_temp", None),
+                    "cpu": None,
+                    "message": ("thermal pause (frozen)"
+                                if getattr(e, "therm_sigstopped", False)
+                                else "thermal pause")})
         return out
 
     @staticmethod

codai/frontproxy/registry.py

@@ -75,6 +75,13 @@ class Engine:
     draining: bool = False         # restart pending: stop routing NEW requests here
                                     # and let in-flight ones finish (drain grace period)
     inflight: int = 0              # proxied requests currently streaming through
+    # Front-side thermal supervision bookkeeping (written by the EngineSupervisor's
+    # thermal monitor; NOT reported by the engine). therm_temp is this engine's
+    # hottest owned card.
+    therm_temp: Optional[float] = None
+    therm_paused: bool = False     # front asked this engine to pause (cooperatively)
+    therm_sigstopped: bool = False  # front escalated to an OS-level SIGSTOP
+    therm_stop_checks: int = 0     # consecutive checks the engine stayed busy after a pause
     _inflight_lock: object = field(default_factory=threading.Lock, repr=False, compare=False)
     # In-flight request metadata {rid: {"model","kind","path","started_at"}} so the
     # front can synthesize Tasks-page entries for work it dispatched — visible even

codai/models/thermal.py

@@ -118,6 +118,67 @@ def _cooldown_exit() -> None:
             })
 
 
+# ---------------------------------------------------------------------------
+# External (front-driven) pause
+# ---------------------------------------------------------------------------
+# The front proxy runs the AUTHORITATIVE thermal monitor: it reads temperatures
+# centrally and stays responsive even while an engine is GIL-blocked inside a
+# long native call (a single llama.cpp prefill/image-encode that the engine's own
+# between-token checkpoint can't interrupt). When the front decides an engine must
+# stop, it POSTs /internal/thermal-pause, which sets this flag; the engine honours
+# it COOPERATIVELY at the next safe point (request entry / between tokens) — the
+# same mechanism as a local temperature cooldown, just triggered remotely. The
+# front clears it (/internal/thermal-resume) once the hardware has cooled. If an
+# engine ignores the pause (stuck in a native call) the front escalates to an
+# OS-level SIGSTOP, which doesn't need the engine to cooperate.
+_external_lock = threading.Lock()
+_external_pause = {"active": False, "reason": "", "since": 0.0}
+
+
+def set_external_pause(active: bool, reason: str = "") -> None:
+    """Set/clear the front-driven pause. Called from the /internal/thermal-* routes."""
+    with _external_lock:
+        active = bool(active)
+        if active and not _external_pause["active"]:
+            _external_pause["since"] = time.time()
+        if not active:
+            _external_pause["since"] = 0.0
+        _external_pause["active"] = active
+        _external_pause["reason"] = reason if active else ""
+
+
+def external_pause_active() -> bool:
+    with _external_lock:
+        return _external_pause["active"]
+
+
+def external_pause_state() -> dict:
+    with _external_lock:
+        return dict(_external_pause)
+
+
+def _wait_external_pause(settings: "ThermalSettings", context: str = "") -> None:
+    """Block while the front holds an external thermal pause, publishing cooldown
+    state so the Tasks page shows the engine is paused for cooldown."""
+    desc = f" ({context})" if context else ""
+    st = external_pause_state()
+    print(f"[thermal] Paused by supervisor{desc}: {st.get('reason') or 'cooldown'} "
+          f"— holding until resume")
+    waited = 0.0
+    poll = min(2.0, max(0.5, settings.poll_seconds))
+    _cooldown_enter()
+    try:
+        while external_pause_active():
+            _cooldown_update(read_gpu_temp(), read_cpu_temp(), waited,
+                             external_pause_state().get("reason")
+                             or "paused by thermal supervisor")
+            time.sleep(poll)
+            waited += poll
+    finally:
+        _cooldown_exit()
+    print(f"[thermal] Supervisor lifted pause{desc} after {int(waited)}s — resuming")
+
+
 # ---------------------------------------------------------------------------
 # Temperature readers
 # ---------------------------------------------------------------------------
@@ -601,6 +662,14 @@ def wait_until_safe(settings: Optional[ThermalSettings] = None,
     """
     if settings is None:
         settings = _settings_from_global_args()
+
+    # Front-driven pause takes precedence: the central supervisor decided this
+    # engine must stop. Block here — a safe point — until it lifts the pause after
+    # cooldown. Honoured even when local CPU/GPU protection is disabled, since the
+    # front owns this decision (it sees temps the GIL-blocked engine can't).
+    if external_pause_active():
+        _wait_external_pause(settings, context)
+
     if not settings.cpu_enabled and not settings.gpu_enabled:
         _dbg(f"protection disabled (cpu/gpu both off) — proceeding [{context}]")
         return