thermal: filter impossible GPU temps at the source (gpu_detect + read chokepoint)

The 0.1.50 guard only covered _read_gpu_temp_uncached's FIRST probe path;
its rocm/psutil fallbacks and gpu_eval() (which reads engine_gpu_stats
directly) still saw the raw 511°C and cooling-waited a healthy card
forever. Filter at the real source — the amdgpu sysfs temp1_input read
in gpu_detect — so temp is None for any ≥150°C reading and NO downstream
reader can act on it; plus a final chokepoint in read_gpu_temp().
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014S8VtAvG499SsCbeESRK7V

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.50"
+__version__ = "0.1.51"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/frontproxy/gpu_detect.py

@@ -339,8 +339,15 @@ def _amd_stats() -> list:
             except OSError:
                 t = None
             if t is not None:
-                temp = t / 1000.0
-                break
+                _c = t / 1000.0
+                # Broken-sensor guard at the SOURCE: amdgpu's SMU can return a
+                # stuck invalid reading (observed 511°C = 0x1FF invalid-ADC
+                # after a GPU reset). No real GPU is ≥150°C; report the card as
+                # temp-unknown (None) rather than let a bogus value reach the
+                # thermal supervisor, which would pause/SIGSTOP a healthy card.
+                if _c < 150.0:
+                    temp = _c
+                    break
         cards.append({
             "vendor": "amd", "index": int(base[4:]),
             "name": _amd_gpu_name(dev, base),

codai/models/thermal.py

@@ -363,6 +363,21 @@ def read_gpu_temp() -> Optional[float]:
     if now - ts < _CACHE_TTL:
         return val
     val = _read_gpu_temp_uncached()
+    # FINAL broken-sensor chokepoint: _read_gpu_temp_uncached has several probe
+    # fallbacks (rocm-smi, psutil amdgpu) that _sane_temps does NOT cover — an
+    # invalid reading (e.g. amdgpu SMU stuck at 511°C = 0x1FF after a GPU reset)
+    # slips through the fallback path. Drop any physically impossible value HERE
+    # so no reader can act on it, then treat the card as temp-unknown (None).
+    if val is not None and val >= _TEMP_SANE_MAX:
+        global _WARNED_BAD_SENSOR
+        if not _WARNED_BAD_SENSOR:
+            _WARNED_BAD_SENSOR = True
+            print(f"[thermal] IGNORING physically impossible GPU temperature "
+                  f"{val:.0f}°C — broken/stuck sensor (e.g. amdgpu SMU after a "
+                  f"GPU reset; power-cycle to restore it). Thermal protection "
+                  f"for that card is DISABLED until it reads sane values.",
+                  flush=True)
+        val = None
     _gpu_cache = (now, val)
     return val