read pod telemetry from the endpoint pods actually serve (v0.4.68)

v0.4.67 asked each pod for /admin/api/gpu-stats. Pods answer 404: a pod runs
the SLIM app — /healthz, /v1/models, /internal/engine-state — and no admin API
at all. Built on an assumption; the live check caught it. Probe the pod before
choosing its endpoint.

/internal/engine-state already carries a `vram` block: used, free and total per
card with the GPU's name, built context-free and cached 4s there. It is what
the front health-polls, so every pod has it and it costs the pod nothing extra.
Verified against a live A40: 43.8/48 GB with qwen38 resident.

The pod reports FREE, not used — that is what a context-free query returns — so
used is the difference. Inverting that would show an idle pod as full.

Utilisation and temperature added AT THE SOURCE: nvidia-smi now asks for
utilization.gpu and temperature.gpu in the same call as the memory query rather
than paying for a second one (a saturated card answers slowly enough that one
call matters), with the pynvml equivalent, and both ride into
engine-state's vram.devices. Absent or "[N/A]" is left out, never zeroed: a
zero draws a bar that reads as "idle" when the truth is "not known".

So VRAM works on every pod already running; utilisation needs the next pod
image, and until then that column shows a dash — which is the degradation the
page was built for.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S8LAP3ueNzQJtwzhgVbq66

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.4.67"
+__version__ = "0.4.68"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/app.py

@@ -442,8 +442,15 @@ def _cached_engine_vram():
                 used += (t - f); free += f; total += t
                 if i not in _names:
                     _names[i] = d["name"]
-                devs.append({"index": i, "name": _names[i],
-                             "free": round(f / 1e9, 2), "total": round(t / 1e9, 2)})
+                dev = {"index": i, "name": _names[i],
+                       "free": round(f / 1e9, 2), "total": round(t / 1e9, 2)}
+                # Utilisation and temperature when the collector could read
+                # them. Reported per card so a caller can see WHICH card is
+                # busy; absent rather than zero when unknown.
+                for key in ("util", "temp"):
+                    if isinstance(d.get(key), (int, float)):
+                        dev[key] = round(float(d[key]), 1)
+                devs.append(dev)
             n = len(devs)
             label = (devs[0]["name"] if n == 1 else f"{n}× GPU")
             vram = {"used": round(used / 1e9, 2), "free": round(free / 1e9, 2),

codai/api/pod_telemetry.py

@@ -8,10 +8,11 @@
 
 """Live GPU/VRAM telemetry for rented pods, read FROM THE PODS.
 
-A pod runs coderai, so it already serves `/admin/api/gpu-stats` — per-card
-utilisation, VRAM used and total, and temperature, collected by the front with
-nvidia-smi off its event loop and cached there for 1.5s. Asking the pod is
-therefore both cheaper and fresher than the alternatives:
+A pod serves `/internal/engine-state`, which already carries a `vram` block —
+used, free and total, per card, with the GPU's name — built context-free and
+cached there for 4s. (It is the endpoint the front health-polls, so it exists on
+every pod and costs the pod nothing extra.) Asking the pod is cheaper and
+fresher than the alternatives:
 
   * RunPod's own API reports utilisation too, but every read spends quota from
     the same budget renting and boot-polling spend (a 429 there stops the fleet
@@ -100,16 +101,51 @@ def summarise(cards: Iterable[dict]) -> dict:
     return out
 
 
+def cards_from_engine_state(state: dict) -> list:
+    """The `vram` block of `/internal/engine-state`, as :func:`summarise` cards.
+
+    The pod reports FREE and total per card (that is what a context-free query
+    returns); used is the difference. `util`/`temp` are present only on a pod
+    new enough to collect them — older pods report VRAM alone, and the page
+    shows a dash for what it was not told rather than inventing a zero.
+    """
+    vram = (state or {}).get("vram") or {}
+    devs = vram.get("devices")
+    if not isinstance(devs, list) or not devs:
+        # A single-card pod that reports only the totals is still worth showing.
+        if isinstance(vram.get("total"), (int, float)):
+            devs = [{"free": vram.get("free"), "total": vram.get("total"),
+                     "util": vram.get("util"), "temp": vram.get("temp")}]
+        else:
+            return []
+    out = []
+    for d in devs:
+        if not isinstance(d, dict):
+            continue
+        total, free = d.get("total"), d.get("free")
+        card = {}
+        if isinstance(total, (int, float)):
+            card["mem_total"] = float(total)
+            if isinstance(free, (int, float)):
+                card["mem_used"] = max(0.0, float(total) - float(free))
+        for key in ("util", "temp"):
+            if isinstance(d.get(key), (int, float)):
+                card[key] = float(d[key])
+        if card:
+            out.append(card)
+    return out
+
+
 def _fetch(pod_id: str, url: str, key: str) -> dict:
     """Ask ONE pod. Blocking; always called off the event loop."""
     from codai.api import pod_http
     headers = {"Authorization": f"Bearer {key}"} if key else {}
     try:
-        r = pod_http.get(_root(url) + "/admin/api/gpu-stats",
+        r = pod_http.get(_root(url) + "/internal/engine-state",
                          headers=headers, timeout=TIMEOUT_S)
         if r.status_code != 200:
             return {}
-        return summarise((r.json() or {}).get("cards") or [])
+        return summarise(cards_from_engine_state(r.json() or {}))
     except Exception:
         # A pod that cannot be reached has no telemetry, which is not an error
         # worth surfacing: the pod's own health is reported beside it already.

codai/models/gpu_query.py

@@ -77,8 +77,18 @@ def _via_pynvml() -> Optional[List[Dict[str, Any]]]:
                     uuid = uuid.decode("utf-8", "replace")
             except Exception:
                 uuid = None
-            out.append({"index": i, "name": str(name), "uuid": uuid,
-                        "free": int(m.free), "total": int(m.total)})
+            row = {"index": i, "name": str(name), "uuid": uuid,
+                   "free": int(m.free), "total": int(m.total)}
+            try:
+                row["util"] = float(p.nvmlDeviceGetUtilizationRates(h).gpu)
+            except Exception:
+                pass
+            try:
+                row["temp"] = float(p.nvmlDeviceGetTemperature(
+                    h, p.NVML_TEMPERATURE_GPU))
+            except Exception:
+                pass
+            out.append(row)
         return out or None
     except Exception:
         return None
@@ -90,7 +100,11 @@ def _via_nvidia_smi() -> Optional[List[Dict[str, Any]]]:
         return None
     try:
         r = subprocess.run(
-            [exe, "--query-gpu=index,name,uuid,memory.free,memory.total",
+            # util/temp ride along with the memory query rather than costing a
+            # second nvidia-smi: the page that shows VRAM wants both, and a
+            # saturated card answers slowly enough that one call matters.
+            [exe, "--query-gpu=index,name,uuid,memory.free,memory.total,"
+                  "utilization.gpu,temperature.gpu",
              "--format=csv,noheader,nounits"],
             capture_output=True, text=True, timeout=5)
         if r.returncode != 0:
@@ -101,12 +115,21 @@ def _via_nvidia_smi() -> Optional[List[Dict[str, Any]]]:
             if len(parts) < 5:
                 continue
             try:
-                out.append({"index": int(parts[0]), "name": parts[1],
-                            "uuid": parts[2] or None,
-                            "free": int(float(parts[3])) * 1024 * 1024,
-                            "total": int(float(parts[4])) * 1024 * 1024})
+                row = {"index": int(parts[0]), "name": parts[1],
+                       "uuid": parts[2] or None,
+                       "free": int(float(parts[3])) * 1024 * 1024,
+                       "total": int(float(parts[4])) * 1024 * 1024}
             except (TypeError, ValueError):
                 continue
+            # Absent or "[N/A]" on some cards/drivers: left out rather than
+            # zeroed, because a zero reads as "idle".
+            for key, idx in (("util", 5), ("temp", 6)):
+                if len(parts) > idx:
+                    try:
+                        row[key] = float(parts[idx])
+                    except (TypeError, ValueError):
+                        pass
+            out.append(row)
         return out or None
     except Exception:
         return None

tests/test_pod_gpu_telemetry.py

 """Live GPU/VRAM for rented pods on the RunPod page, read FROM THE PODS.
 
-A pod runs coderai, so it already serves /admin/api/gpu-stats (nvidia-smi off
-its own loop, cached 1.5s there). Asking the pod is cheaper and fresher than
-asking RunPod, whose utilisation lags by tens of seconds and whose every read
-spends the same quota that renting and boot-polling need -- telemetry must
-never be able to cost a fleet its ability to rent.
+A pod serves /internal/engine-state, which already carries a `vram` block
+(used/free/total per card, cached 4s there). I first built this against
+/admin/api/gpu-stats and the pods answered 404: a pod runs the SLIM app, which
+serves /healthz, /v1/models and /internal/engine-state and no admin API at all.
+Probe the pod before choosing its endpoint.
+
+Asking the pod is cheaper and fresher than asking RunPod, whose utilisation
+lags by tens of seconds and whose every read spends the same quota that renting
+and boot-polling need -- telemetry must never be able to cost a fleet its
+ability to rent.
 """
 import asyncio
 import types
@@ -69,13 +74,25 @@ def test_the_admin_path_hangs_off_the_root(given, want):
 
 # ------------------------------------------------------------- the fetching #
 
-def _answer(cards, status=200):
+def _state(devices, status=200):
+    """A pod's /internal/engine-state answer."""
     r = types.SimpleNamespace()
     r.status_code = status
-    r.json = lambda: {"cards": cards}
+    r.json = lambda: {"ok": True, "vram": {"devices": devices,
+                                           "device_count": len(devices)}}
     return r
 
 
+def _answer(cards, status=200):
+    """Cards in summarise() shape, expressed as a pod would report them:
+    the pod reports FREE, not used."""
+    devs = [{"total": c.get("mem_total", 48),
+             "free": c.get("mem_total", 48) - c.get("mem_used", 0),
+             **{k: c[k] for k in ("util", "temp") if k in c}}
+            for c in cards]
+    return _state(devs, status)
+
+
 def test_a_pod_is_asked_with_its_own_bearer(monkeypatch):
     seen = {}
 
@@ -85,7 +102,7 @@ def test_a_pod_is_asked_with_its_own_bearer(monkeypatch):
     monkeypatch.setattr("codai.api.pod_http.get", _get)
     out = asyncio.run(pt.collect([("p1", "https://pod:1/v1", "its-own-key")]))
     assert out["p1"]["gpu_util"] == 55.0
-    assert seen["url"] == "https://pod:1/admin/api/gpu-stats"
+    assert seen["url"] == "https://pod:1/internal/engine-state"
     assert seen["headers"]["Authorization"] == "Bearer its-own-key"
     assert seen["timeout"] == pt.TIMEOUT_S
 
@@ -204,3 +221,53 @@ def test_endpoints_carry_each_pods_own_key():
         rw._pools.clear()
         rw._pools.update(saved)
     assert got == {"a": "own-key", "b": "pool-key"}
+
+
+# --------------------------- the pod reports FREE, so used is the difference #
+
+def test_used_is_derived_from_free_because_that_is_what_the_pod_reports():
+    cards = pt.cards_from_engine_state(
+        {"vram": {"devices": [{"index": 0, "name": "NVIDIA A40",
+                               "free": 4.5, "total": 48.31}]}})
+    assert cards == [{"mem_total": 48.31, "mem_used": pytest.approx(43.81)}]
+
+
+def test_util_and_temp_ride_along_when_the_pod_reports_them():
+    cards = pt.cards_from_engine_state(
+        {"vram": {"devices": [{"free": 4.5, "total": 48.31,
+                               "util": 97, "temp": 74}]}})
+    assert cards[0]["util"] == 97.0 and cards[0]["temp"] == 74.0
+
+
+def test_an_older_pod_reports_vram_only():
+    """It serves engine-state without util/temp. VRAM must still show, and the
+    page shows a dash for utilisation rather than a zeroed bar."""
+    out = pt.summarise(pt.cards_from_engine_state(
+        {"vram": {"devices": [{"free": 4.5, "total": 48.31}]}}))
+    assert out["vram_used_gb"] == pytest.approx(43.81)
+    assert "gpu_util" not in out and "gpu_temp_c" not in out
+
+
+def test_a_pod_with_no_card_reports_nothing():
+    assert pt.cards_from_engine_state({"vram": None}) == []
+    assert pt.cards_from_engine_state({}) == []
+    assert pt.cards_from_engine_state({"vram": {"devices": []}}) == []
+
+
+def test_totals_without_a_device_list_are_still_used():
+    cards = pt.cards_from_engine_state({"vram": {"used": 43.8, "free": 4.5,
+                                                 "total": 48.31}})
+    assert cards[0]["mem_total"] == 48.31
+
+
+def test_the_real_shape_a_live_pod_returned():
+    """Copied from pod 997ags1r9l6im3, 2026-10-22."""
+    out = pt.summarise(pt.cards_from_engine_state({
+        "ok": True, "vram": {"used": 43.8, "free": 4.5, "total": 48.31,
+                             "gpu": "NVIDIA A40",
+                             "devices": [{"index": 0, "name": "NVIDIA A40",
+                                          "free": 4.5, "total": 48.31}],
+                             "device_count": 1}}))
+    assert out["vram_used_gb"] == pytest.approx(43.81)
+    assert out["vram_total_gb"] == 48.31
+    assert out["gpu_cards"] == 1