thermal: ignore physically impossible GPU temperature readings

After a GPU reset the amdgpu SMU can return a constant garbage reading
(observed: 511°C = 0x1FF invalid-ADC, with nonsense voltage/fan values).
The thermal supervisor believed it, paused the radeon engine and
escalated to SIGSTOP — freezing a healthy card forever. Both readers
(engine-side thermal.py and the front's supervisor loop) now discard
readings ≥150°C with a one-time warning naming the broken sensor;
thermal protection for that card resumes when it reads sane values.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014S8VtAvG499SsCbeESRK7V

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.49"
+__version__ = "0.1.50"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/frontproxy/engine_supervisor.py

@@ -757,6 +757,22 @@ class EngineSupervisor:
                             t = c.get("temp")
                             if t is None:
                                 continue
+                            # Broken-sensor guard: no real GPU reports ≥150°C
+                            # (observed: amdgpu SMU stuck at a constant 511°C =
+                            # 0x1FF invalid-ADC after a GPU reset). Believing it
+                            # would SIGSTOP-freeze the engine forever on a
+                            # healthy card — ignore the reading instead.
+                            if t >= 150:
+                                if not getattr(self, "_warned_bad_sensor", False):
+                                    self._warned_bad_sensor = True
+                                    print(f"[front] thermal: IGNORING impossible "
+                                          f"GPU temp {t:.0f}°C from "
+                                          f"'{c.get('name') or c.get('vendor')}' — "
+                                          f"broken/stuck sensor; power-cycle to "
+                                          f"restore it. Thermal protection for "
+                                          f"that card is disabled until it reads "
+                                          f"sane values.", flush=True)
+                                continue
                             temps.append(t)
                             high, resume = settings.gpu_thresholds(c.get("vendor"))
                             if t >= high:

codai/models/thermal.py

@@ -204,6 +204,31 @@ def _run(cmd, timeout=4.0) -> Optional[str]:
     return None
 
 
+# Sensor sanity ceiling: no GPU reports a REAL temperature this high — silicon
+# dies long before. Readings at/above it are a broken/stuck sensor (observed:
+# amdgpu SMU returning a constant 511°C = 0x1FF invalid-ADC after a GPU reset),
+# and believing one would freeze the engine forever on a healthy card.
+_TEMP_SANE_MAX = 150.0
+
+
+def _sane_temps(temps):
+    good = [t for t in temps if t is not None and 0 < t < _TEMP_SANE_MAX]
+    bad = [t for t in temps if t is not None and t >= _TEMP_SANE_MAX]
+    if bad:
+        global _WARNED_BAD_SENSOR
+        if not _WARNED_BAD_SENSOR:
+            _WARNED_BAD_SENSOR = True
+            print(f"[thermal] IGNORING physically impossible GPU temperature "
+                  f"reading(s) {bad} °C — broken/stuck sensor (e.g. amdgpu SMU "
+                  f"after a GPU reset). Power-cycle the machine to restore the "
+                  f"sensor; thermal protection for that card is DISABLED until "
+                  f"it reads sane values again.")
+    return good
+
+
+_WARNED_BAD_SENSOR = False
+
+
 def _read_gpu_temp_uncached() -> Optional[float]:
     """Hottest GPU temperature in °C across ALL installed cards, or None.
 
@@ -214,7 +239,8 @@ def _read_gpu_temp_uncached() -> Optional[float]:
     below if the unified reader fails."""
     try:
         from codai.frontproxy.gpu_detect import engine_gpu_stats
-        temps = [c["temp"] for c in engine_gpu_stats() if c.get("temp") is not None]
+        temps = _sane_temps(
+            [c["temp"] for c in engine_gpu_stats() if c.get("temp") is not None])
         if temps:
             return max(temps)
     except Exception: