thermal: CPU-hot pauses ALL engines (incl colibri) — revert per-engine exemption

Per design: a hot CPU is a global hardware event, so every engine pauses — not just
the CPU-heat source. Reverts the per-engine CPU exemption from 0.1.75 (front
_thermal_loop) and 0.1.76 (engine-level wait_until_safe): the shared-CPU term again
applies to all engines. The original "radeon stuck while colibri ran free" bug is
addressed the right way — by making colibri ACTUALLY honour the pause (targeted
SIGSTOP that keeps the engine's HTTP alive, and colibri's cooperative PAUSE/RESUME
mux frame), both retained — so colibri stops, the CPU cools, and all engines resume
together instead of the innocent engine being stranded.

Removed the now-unused _scan_pgroup_cpu/_engine_cpu_relevant helpers and
_self_cpu_relevant.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LoSpEthysqmseCc6Geizty

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.76"
+__version__ = "0.1.77"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/frontproxy/engine_supervisor.py

@@ -643,84 +643,6 @@ class EngineSupervisor:
         return [c for c in all_cards
                 if c.get("vendor") in sels or (c.get("uuid") and c["uuid"] in sels)]
 
-    # An engine's process group using at least this many CPU cores is a real source
-    # of CPU heat; below it, pausing the engine can't meaningfully cool the CPU.
-    _CPU_RELEVANT_CORES = 1.5
-    _CPU_HIST_LEN = 6          # rolling window (~poll_seconds × 6) for CPU relevance
-
-    @staticmethod
-    def _scan_pgroup_cpu() -> dict:
-        """One pass over /proc → {pgid: total (utime+stime) ticks}. Used to attribute
-        CPU load to each engine's process group (its own PID + children like
-        colibri/ds4/whisper-server, which inherit the engine's setsid pgid)."""
-        out: dict = {}
-        try:
-            for name in os.listdir("/proc"):
-                if not name.isdigit():
-                    continue
-                try:
-                    with open("/proc/" + name + "/stat", "rb") as f:
-                        data = f.read()
-                    # comm (field 2) is parenthesised and may contain spaces/parens;
-                    # fields after the last ')' are stable and space-separated.
-                    fields = data[data.rindex(b")") + 2:].split()
-                    pgid = int(fields[2])                      # field 5: pgrp
-                    out[pgid] = out.get(pgid, 0) + int(fields[11]) + int(fields[12])
-                except Exception:
-                    continue
-        except Exception:
-            pass
-        return out
-
-    def _engine_cpu_cores(self, engine, pg_ticks: dict, now: float):
-        """CPU cores this engine's process group used since the last poll, or None
-        (first sample / unmeasurable)."""
-        proc = getattr(engine, "proc", None)
-        pid = getattr(proc, "pid", None) if proc is not None else None
-        if not pid:
-            return None
-        try:
-            pgid = os.getpgid(pid)
-        except Exception:
-            return None
-        ticks = pg_ticks.get(pgid)
-        if ticks is None:
-            return None
-        cache = getattr(self, "_cpu_sample", None)
-        if cache is None:
-            cache = self._cpu_sample = {}
-        prev = cache.get(engine.id)
-        cache[engine.id] = (ticks, now)
-        if not prev:
-            return None
-        dt = now - prev[1]
-        if dt <= 0:
-            return None
-        try:
-            clk = os.sysconf("SC_CLK_TCK")
-        except Exception:
-            clk = 100.0
-        return max(0.0, (ticks - prev[0]) / clk / dt)
-
-    def _engine_cpu_relevant(self, engine, pg_ticks: dict, now: float) -> bool:
-        """True when this engine is a meaningful CPU-heat source, so the shared-CPU
-        thermal term applies to it. Uses a rolling MAX so an engine that just got
-        SIGSTOP-ed/idled (e.g. colibri between throttle cycles) still counts as
-        CPU-heavy and keeps its normal hysteresis, while a persistently GPU-bound
-        engine (Vulkan embeddings, ~0 CPU) is exempt and can't be stranded by another
-        engine's CPU heat. Fails safe to True when it can't be measured."""
-        cores = self._engine_cpu_cores(engine, pg_ticks, now)
-        hist = getattr(self, "_cpu_hist", None)
-        if hist is None:
-            hist = self._cpu_hist = {}
-        lst = hist.setdefault(engine.id, [])
-        if cores is not None:
-            lst.append(cores)
-            del lst[:-self._CPU_HIST_LEN]
-        if not lst:
-            return True
-        return max(lst) >= self._CPU_RELEVANT_CORES
-
     def _thermal_signal(self, engine, sig) -> None:
         """Freeze/resume this engine's heavy native compute WITHOUT freezing its
         Python HTTP server, so it can still ack pause/resume and report health.
@@ -900,13 +822,6 @@ class EngineSupervisor:
                     cpu_warm = (cpu_t is not None and cpu_t > settings.cpu_resume)
                     escalate_n = int(getattr(self.config.thermal,
                                              "stop_escalate_checks", 3) or 3)
-                    # CPU is a SHARED resource, but only the engines actually loading it
-                    # can be cooled by pausing them. Sample per-engine CPU once per poll
-                    # so the shared-CPU thermal term applies only to real CPU-heat
-                    # sources — a GPU-bound engine (embeddings) is never paused/stranded
-                    # by another engine's (e.g. colibri's) CPU heat.
-                    pg_ticks = self._scan_pgroup_cpu() if cpu_t is not None else {}
-                    _now = time.monotonic()
                     for engine in self.registry.all():
                         if getattr(engine, "role", "engine") == "system":
                             continue   # the cache/downloads worker isn't on the GPU
@@ -941,19 +856,15 @@ class EngineSupervisor:
                             if t > resume:
                                 gpu_warm = True
                         engine.therm_temp = max(temps) if temps else None
-                        # Apply the (global) CPU term only to engines that actually heat
-                        # the CPU — pausing a GPU-bound engine can't cool it, and the
-                        # hysteresis would strand it in the resume<CPU<high dead-band a
-                        # CPU-heavy engine keeps the shared CPU parked in.
-                        cpu_rel = (self._engine_cpu_relevant(engine, pg_ticks, _now)
-                                   if cpu_t is not None else False)
-                        e_cpu_hot = cpu_hot and cpu_rel
-                        e_cpu_warm = cpu_warm and cpu_rel
-                        # Pause when this engine's GPU is over high OR (it's a CPU-heat
-                        # source AND) the CPU is over high. Resume only when BOTH clear.
-                        want_pause = (settings.gpu_enabled and gpu_hot) or e_cpu_hot
+                        # CPU is shared hardware: when it's over the limit, EVERY engine
+                        # pauses (a hot CPU is a global event). What matters is that the
+                        # engine actually heating it (colibri, dense/attention on CPU)
+                        # honours the pause — handled by the targeted SIGSTOP below and
+                        # colibri's cooperative PAUSE frame — so the CPU cools and all
+                        # engines resume together.
+                        want_pause = (settings.gpu_enabled and gpu_hot) or cpu_hot
                         want_resume_ok = (not (settings.gpu_enabled and gpu_warm)
-                                          and not e_cpu_warm)
+                                          and not cpu_warm)
                         self._thermal_apply(client, engine, settings, want_pause,
                                             want_resume_ok, engine.therm_temp, cpu_t,
                                             escalate_n)

codai/models/thermal.py

@@ -494,33 +494,6 @@ def read_process_tree_cpu() -> Optional[float]:
     return round(total, 1)
 
 
-# --- per-engine CPU relevance ---------------------------------------------------
-# The CPU is shared, but only an engine that actually LOADS the CPU can be cooled by
-# pausing it. A GPU-bound engine (e.g. the Vulkan embedding engine — its compute is on
-# the GPU, ~0 CPU) contributes no CPU heat, so making it wait on a hot CPU can't cool
-# anything; it would just be stranded while a CPU-heavy engine (colibri, dense-on-CPU)
-# keeps the shared CPU above the resume line. So the CPU thermal term below applies
-# only when THIS engine's own process tree is a real CPU-heat source.
-_CPU_RELEVANT_PCT = 150.0          # this engine's tree using >= 1.5 cores = a CPU source
-_self_cpu_hist: list = []
-_self_cpu_lock = threading.Lock()
-
-
-def _self_cpu_relevant() -> bool:
-    """True when this engine's process tree (itself + children like colibri) is a
-    meaningful CPU-heat source. Rolling MAX so a briefly-idle CPU-heavy engine still
-    counts (keeps its normal hysteresis); a persistently GPU-bound engine is exempt.
-    Fails safe to True when CPU can't be measured (no psutil)."""
-    pct = read_process_tree_cpu()
-    with _self_cpu_lock:
-        if pct is not None:
-            _self_cpu_hist.append(pct)
-            del _self_cpu_hist[:-6]
-        if not _self_cpu_hist:
-            return True
-        return max(_self_cpu_hist) >= _CPU_RELEVANT_PCT
-
-
 def read_cpu_temp_avg(samples: int = 3, max_seconds: float = 3.0) -> Optional[float]:
     """Averaged CPU temperature for stable resume/cooldown decisions.
 
@@ -751,18 +724,18 @@ def wait_until_safe(settings: Optional[ThermalSettings] = None,
         gpu_eval(settings) if settings.gpu_enabled else (False, False, None))
     gpu_t = gpu_worst["temp"] if gpu_worst else None
     cpu_t = read_cpu_temp() if settings.cpu_enabled else None
-    # Only honour the CPU term if THIS engine is actually heating the CPU (see
-    # _self_cpu_relevant). A GPU-bound engine ignores CPU heat it didn't cause.
-    cpu_rel = _self_cpu_relevant() if settings.cpu_enabled else False
-    cpu_active = settings.cpu_enabled and cpu_rel
     _dbg(
         f"check{desc0}: "
         f"GPU {_fmt(gpu_t)} (enabled={settings.gpu_enabled}, "
         f"over_high={gpu_over} over_resume={gpu_over_resume}) | "
-        f"CPU {_fmt(cpu_t)} (enabled={settings.cpu_enabled}, cpu_source={cpu_rel}, "
+        f"CPU {_fmt(cpu_t)} (enabled={settings.cpu_enabled}, "
         f"pause>={settings.cpu_high:.0f} resume<={settings.cpu_resume:.0f})"
     )
 
+    # CPU is shared hardware: when it's over the limit EVERY engine pauses (a global
+    # event), so the engine actually heating it (colibri) stops too and it can cool.
+    cpu_active = settings.cpu_enabled
+
     hot = []
     if settings.gpu_enabled and gpu_over:
         hot.append(("GPU", gpu_worst["temp"], gpu_worst["resume"]))