count what each pod has actually served (v0.4.71)

The RunPod page showed only the instant: in-flight, KV occupancy, queue depth.
Nothing said how much work a pod had done, so a healthy pod billed by the hour
that had served nothing looked exactly like one carrying the fleet.

Each pod now reports the requests it has finished and the prompt/generation
tokens that went through it, read from the serving engine's own counters and
shown as a "Served" column, with the live fleet's total beside the pod count.

Two things had to be true for the number to be worth printing:

* prometheus_client renames a Counter. vLLM DECLARES "vllm:num_preemptions"
  and serves "vllm:num_preemptions_total", alongside a "_created" sample whose
  value is a unix timestamp. Keying on the declared name alone silently dropped
  every counter -- so the preemption and prefix figures added in 0.4.69 never
  matched anything, and the page drew a dash that read as "not supported".
  Both spellings are accepted now, and "_created" is never read as a count.
* vLLM's counters restart at zero with the engine process (model swap, crash,
  OOM kill), so a pod that had served four thousand requests would report
  twelve. Each drop is treated as a restart and the previous reading carried
  forward, which makes the column monotonic for the pod's lifetime.

Also drops the prefix hit rate rather than fix its name: in vLLM 0.27.1 the
prefix-cache hit counter is registered from the multimodal-cache counter, so a
rate computed from it is not the prefix hit rate. A wrong number on an
operator's page is worse than a dash.

Verified against the shipped image's own prometheus_client, not a hand-written
fixture: the exposition names and the "_created" timestamp in the tests are
what ghcr.io/nextime/coderai-vllm:latest actually renders.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S8LAP3ueNzQJtwzhgVbq66

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.4.70"
+__version__ = "0.4.71"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/admin/templates/runpod.html

@@ -20,7 +20,7 @@
 
 <!-- Cost tiles -->
 <div id="rp-tiles" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(200px,1fr));gap:.75rem;margin-bottom:1rem">
-  <div class="card"><div class="dim small">Running pods</div><div id="rp-podcount" style="font-size:1.6rem;font-weight:700">—</div></div>
+  <div class="card"><div class="dim small">Running pods</div><div id="rp-podcount" style="font-size:1.6rem;font-weight:700">—</div><div id="rp-served" class="dim small"></div></div>
   <div class="card"><div class="dim small">Live rate ($/hr)</div><div id="rp-rate" style="font-size:1.6rem;font-weight:700">—</div><div id="rp-rate-cap" class="dim small"></div></div>
   <div class="card"><div class="dim small">Uncommitted cost</div><div id="rp-uncommitted" style="font-size:1.6rem;font-weight:700">—</div><div class="dim small">pods running, not yet billed to the ledger</div></div>
   <div class="card"><div class="dim small">Global spend (period)</div><div id="rp-global-spend" style="font-size:1.6rem;font-weight:700">—</div><div id="rp-global-cap" class="dim small"></div></div>
@@ -32,9 +32,9 @@
   <div style="overflow-x:auto">
     <table style="width:100%;border-collapse:collapse;font-size:13px">
       <thead><tr style="text-align:left;color:var(--text-3)">
-        <th>Model</th><th>GPU</th><th>Load</th><th>Memory</th><th>Pool</th><th>In-flight</th><th>$/hr</th><th>Uptime</th><th>Cost so far</th><th>Health</th><th>Logs</th>
+        <th>Model</th><th>GPU</th><th>Load</th><th>Memory</th><th>Pool</th><th>In-flight</th><th>Served</th><th>$/hr</th><th>Uptime</th><th>Cost so far</th><th>Health</th><th>Logs</th>
       </tr></thead>
-      <tbody id="rp-pods"><tr><td colspan="11" class="dim">No pods running.</td></tr></tbody>
+      <tbody id="rp-pods"><tr><td colspan="12" class="dim">No pods running.</td></tr></tbody>
     </table>
   </div>
 </div>
@@ -101,6 +101,30 @@ function _memCell(p){
   const pct = 100 * p.ram_used_gb / p.ram_total_gb;
   return _meter(pct, `${p.ram_used_gb.toFixed(1)}/${Math.round(p.ram_total_gb)} GB`) + vr;
 }
+/* Counts, not rates: how many requests this pod has finished and how many
+   tokens went through it since it booted. In-flight next to it is the instant;
+   this is the pod's whole working life, and it is what says whether a pod
+   billed by the hour has earned it -- a healthy pod at 0 served is pure cost.
+   vLLM's counters rewind when the engine process is replaced, so the pod
+   carries them forward (codai.api.serving_load._accumulate) and this column
+   never goes backwards while the pod lives. */
+function _fmtCount(n){
+  if (n == null) return null;
+  if (n >= 1e9) return (n/1e9).toFixed(1) + 'B';
+  if (n >= 1e6) return (n/1e6).toFixed(1) + 'M';
+  if (n >= 1e3) return (n/1e3).toFixed(1) + 'k';
+  return String(n);
+}
+function _servedCell(p){
+  if (p.requests_done == null && p.generation_tokens == null) return '<span class="dim">—</span>';
+  const req = p.requests_done != null
+    ? `<div>${_fmtCount(p.requests_done)} <span style="color:var(--text-3)">req</span></div>` : '';
+  const tok = [];
+  if (p.prompt_tokens != null) tok.push(`${_fmtCount(p.prompt_tokens)} in`);
+  if (p.generation_tokens != null) tok.push(`${_fmtCount(p.generation_tokens)} out`);
+  return req + (tok.length
+    ? `<div style="font-size:10px;color:var(--text-3)">${tok.join(' · ')} tok</div>` : '');
+}
 
 async function rpRefresh(){
   let d;
@@ -117,6 +141,14 @@ async function rpRefresh(){
   const pods = d.pods || [];
   const running = pods.filter(p=>p.healthy);
   document.getElementById('rp-podcount').textContent = running.length + (pods.length>running.length ? ` (+${pods.length-running.length} unhealthy)` : '');
+  // What the pods now running have served between them. Pods that have already
+  // been released are not in this listing, so this is the live fleet's work,
+  // not the account's history -- the ledger below is the history.
+  const served = pods.reduce((a,p)=> a + (p.requests_done||0), 0);
+  const servedTok = pods.reduce((a,p)=> a + (p.generation_tokens||0), 0);
+  document.getElementById('rp-served').textContent = served
+    ? `${_fmtCount(served)} requests served · ${_fmtCount(servedTok)} tokens out`
+    : (pods.length ? 'nothing served yet' : '');
   document.getElementById('rp-rate').textContent = _usd(d.live_hourly_usd);
   // The rate now counts BOOTING pods too, because RunPod bills from creation.
   // Say how much of it is boot: a fleet that is mostly boot is a ramp that is
@@ -133,7 +165,7 @@ async function rpRefresh(){
 
   // pods table
   const tb = document.getElementById('rp-pods');
-  if(!pods.length){ tb.innerHTML = '<tr><td colspan="11" class="dim">No pods running.</td></tr>'; }
+  if(!pods.length){ tb.innerHTML = '<tr><td colspan="12" class="dim">No pods running.</td></tr>'; }
   else {
     tb.innerHTML = pods.map(p => `<tr>
       <td>${p.model||''}</td>
@@ -142,6 +174,7 @@ async function rpRefresh(){
       <td>${_memCell(p)}</td>
       <td>${p.is_spot?'spot':'on-demand'}</td>
       <td>${p.inflight||0}</td>
+      <td>${_servedCell(p)}</td>
       <td>${_usd(p.hourly_usd)}</td>
       <td>${_fmtDur(p.uptime_s)}</td>
       <td>${_usd(p.live_cost_usd)}</td>

codai/api/pod_telemetry.py

@@ -147,13 +147,14 @@ def _fetch(pod_id: str, url: str, key: str) -> dict:
             return {}
         state = r.json() or {}
         out = summarise(cards_from_engine_state(state))
-        # Host memory and serving load: the two limits that actually bind. See
-        # codai.api.serving_load for why reserved VRAM is not one of them.
+        # Host memory, serving load, and what the pod has served since it
+        # booted. See codai.api.serving_load for why reserved VRAM is not one
+        # of the limits, and why the served counters survive an engine restart.
         load = state.get("load")
         if isinstance(load, dict):
             for k in ("ram_used_gb", "ram_total_gb", "ram_free_gb",
                       "kv_cache_pct", "running", "waiting", "preemptions",
-                      "prefix_hit_pct"):
+                      "requests_done", "prompt_tokens", "generation_tokens"):
                 if isinstance(load.get(k), (int, float)):
                     out[k] = load[k]
         return out
@@ -192,7 +193,7 @@ def forget(pod_id: Optional[str] = None) -> None:
 
 async def collect(endpoints: List[Tuple[str, str, str]],
                   overall_timeout: float = 3.0) -> Dict[str, dict]:
-    """``{pod_id: {gpu_util, vram_used_gb, vram_total_gb, gpu_temp_c}}``.
+    """``{pod_id: {gpu_util, vram_*, gpu_temp_c, ram_*, requests_done, ...}}``.
 
     ``endpoints`` is ``[(pod_id, url, bearer)]``. Fresh cache entries are
     answered without a request; the rest are fetched concurrently off the loop.

codai/api/serving_load.py

@@ -28,6 +28,17 @@ What does move with load:
   is, and it is invisible in utilisation or VRAM.
 * **waiting** — requests admitted but not yet running: queueing, i.e. latency
   the client is already paying.
+* **requests served, and the tokens in and out** — the work this engine has
+  actually done since it started. Not a limit, but the figure that says whether
+  a pod being paid for by the hour is earning it: a rented pod at 0 served
+  requests is pure cost, and that was invisible on a page showing only
+  instantaneous load. Carried across an engine restart by :func:`_accumulate`,
+  since vLLM's own counters rewind to zero with the process.
+
+``vllm:prefix_cache_hits`` is deliberately absent: in vLLM 0.27.1 the gauge is
+registered from the multimodal-cache counter (`counter_prefix_cache_hits`
+assigned `counter_mm_cache_hits`), so a hit rate computed from it is not the
+prefix hit rate. A number that is wrong is worse than a dash.
 
 Read from the serving engine's own Prometheus endpoint and from cgroup memory
 accounting, both cheap, both cached briefly, and every failure degrades to
@@ -40,20 +51,31 @@ import threading
 import time
 from typing import Dict, Optional
 
-#: Gauges worth carrying, mapped to the names used in the API.
+#: Metrics worth carrying, mapped to the names used in the API. Names are the
+#: ones vLLM DECLARES; a counter is exposed by prometheus_client with a
+#: ``_total`` suffix, which :func:`parse_prometheus` accepts for either form.
 _WANTED = {
     "vllm:kv_cache_usage_perc": "kv_cache_pct",
     "vllm:num_requests_running": "running",
     "vllm:num_requests_waiting": "waiting",
     "vllm:num_preemptions": "preemptions",
-    "vllm:prefix_cache_hits": "prefix_hits",
-    "vllm:prefix_cache_queries": "prefix_queries",
+    "vllm:request_success": "requests_done",
+    "vllm:prompt_tokens": "prompt_tokens",
+    "vllm:generation_tokens": "generation_tokens",
 }
 
+#: Figures that only ever go up, and so can be carried across an engine
+#: restart. Everything else describes this instant and must not be.
+_CUMULATIVE = ("requests_done", "prompt_tokens", "generation_tokens",
+               "preemptions")
+
 TTL_S = 2.0
 TIMEOUT_S = 1.5
 
 _CACHE: Dict[str, tuple] = {}
+
+#: ``{base_url: {key: (last_raw, carried)}}`` -- see :func:`_accumulate`.
+_CARRY: Dict[str, Dict[str, tuple]] = {}
 _LOCK = threading.Lock()
 
 
@@ -76,7 +98,14 @@ def parse_prometheus(text: str) -> dict:
             _, _, value = rest.rpartition("}")
         else:
             metric, _, value = line.partition(" ")
-        key = _WANTED.get(metric.strip())
+        metric = metric.strip()
+        key = _WANTED.get(metric)
+        if key is None and metric.endswith("_total"):
+            # prometheus_client renders a Counter named "vllm:prompt_tokens" as
+            # the sample "vllm:prompt_tokens_total" (plus a "_created"
+            # timestamp, which must never be read as a count). Matching only
+            # the declared name silently dropped every counter on this page.
+            key = _WANTED.get(metric[:-len("_total")])
         if not key:
             continue
         try:
@@ -94,12 +123,39 @@ def parse_prometheus(text: str) -> dict:
         out["kv_cache_pct"] = round(maxes["kv_cache_pct"] * 100.0, 1)
     for k, v in sums.items():
         out[k] = int(v) if float(v).is_integer() else round(v, 2)
-    if out.get("prefix_queries"):
-        out["prefix_hit_pct"] = round(
-            100.0 * out.get("prefix_hits", 0) / out["prefix_queries"], 1)
     return out
 
 
+def _accumulate(base: str, data: dict) -> dict:
+    """Keep the cumulative figures cumulative across an engine restart.
+
+    vLLM's counters start again from zero every time the engine process is
+    replaced -- a model swap, a crash, an OOM kill -- so a pod that has served
+    four thousand requests reports twelve shortly after one. An operator reads
+    the column as "what this pod has done", and a counter that silently rewinds
+    is worse than no counter: it hides exactly the restart worth noticing.
+
+    So each drop is treated as a restart and the previous reading is carried
+    forward. The result is monotonic for as long as this process lives, which
+    is the pod's own lifetime.
+    """
+    if not data:
+        return data
+    with _LOCK:
+        seen = _CARRY.setdefault(base, {})
+        for key in _CUMULATIVE:
+            raw = data.get(key)
+            if not isinstance(raw, (int, float)):
+                continue
+            last, carried = seen.get(key, (0.0, 0.0))
+            if raw < last:                       # the engine was replaced
+                carried += last
+            seen[key] = (float(raw), carried)
+            total = carried + raw
+            data[key] = int(total) if float(total).is_integer() else round(total, 2)
+    return data
+
+
 def engine_load(base_url: str) -> dict:
     """Serving load from one engine's ``/metrics``, or {} if unavailable."""
     base = str(base_url or "").strip().rstrip("/")
@@ -113,6 +169,7 @@ def engine_load(base_url: str) -> dict:
         import requests
         r = requests.get(base + "/metrics", timeout=TIMEOUT_S)
         data = parse_prometheus(r.text) if r.status_code == 200 else {}
+        data = _accumulate(base, data)
     except Exception:
         data = {}
     with _LOCK:
@@ -184,5 +241,7 @@ def snapshot(base_url: str = "") -> dict:
 
 
 def forget() -> None:
+    """Drop the cached readings AND the restart carry-over."""
     with _LOCK:
         _CACHE.clear()
+        _CARRY.clear()

tests/test_pod_gpu_telemetry.py

@@ -271,3 +271,35 @@ def test_the_real_shape_a_live_pod_returned():
     assert out["vram_used_gb"] == pytest.approx(43.81)
     assert out["vram_total_gb"] == 48.31
     assert out["gpu_cards"] == 1
+
+
+def test_what_the_pod_has_served_comes_through(monkeypatch):
+    """The counter an operator reads as "is this rented pod earning its hour?".
+    It travels in the engine-state `load` block beside the limits, and only the
+    numeric keys the page draws are copied -- a pod on an older image simply
+    does not send them, and the page shows a dash rather than a zero."""
+    r = types.SimpleNamespace()
+    r.status_code = 200
+    r.json = lambda: {"ok": True,
+                      "vram": {"devices": [{"total": 48, "free": 8}]},
+                      "load": {"requests_done": 4012, "prompt_tokens": 1250000,
+                               "generation_tokens": 412000, "running": 3,
+                               "ram_used_gb": 19.4, "ram_total_gb": 60.0,
+                               "kv_cache_pct": 26.5, "preemptions": 4,
+                               "ram_source": "cgroup"}}
+    monkeypatch.setattr("codai.api.pod_http.get",
+                        lambda *a, **k: r)
+    out = asyncio.run(pt.collect([("p1", "https://pod:1", "k")]))["p1"]
+    assert out["requests_done"] == 4012
+    assert out["prompt_tokens"] == 1250000
+    assert out["generation_tokens"] == 412000
+    assert out["kv_cache_pct"] == 26.5
+    assert "ram_source" not in out          # a string, not a figure to draw
+
+
+def test_an_older_pod_sends_no_served_counts(monkeypatch):
+    monkeypatch.setattr("codai.api.pod_http.get",
+                        lambda *a, **k: _answer([{"mem_used": 40, "mem_total": 48}]))
+    out = asyncio.run(pt.collect([("p9", "https://pod:1", "k")]))["p9"]
+    assert "requests_done" not in out
+    assert out["vram_used_gb"] == 40.0

tests/test_serving_load.py

@@ -21,15 +21,26 @@ def setup_function():
     sl.forget()
 
 
+#: Exactly as vLLM 0.27.1 exposes it. The counter names carry the ``_total``
+#: suffix prometheus_client appends -- vLLM DECLARES "vllm:num_preemptions" and
+#: serves "vllm:num_preemptions_total" -- and each one brings a "_created"
+#: sample whose value is a unix TIMESTAMP, which must never be read as a count.
 BODY = """
 # HELP vllm:kv_cache_usage_perc GPU KV-cache usage
 # TYPE vllm:kv_cache_usage_perc gauge
 vllm:kv_cache_usage_perc{engine="0",model_name="m"} 0.265
 vllm:num_requests_running{engine="0",model_name="m"} 75
 vllm:num_requests_waiting{engine="0",model_name="m"} 12
-vllm:num_preemptions{engine="0",model_name="m"} 4
-vllm:prefix_cache_hits{engine="0",model_name="m"} 250
-vllm:prefix_cache_queries{engine="0",model_name="m"} 1000
+# TYPE vllm:num_preemptions counter
+vllm:num_preemptions_total{engine="0",model_name="m"} 4.0
+vllm:num_preemptions_created{engine="0",model_name="m"} 1.76e+09
+# TYPE vllm:request_success counter
+vllm:request_success_total{engine="0",model_name="m",finished_reason="stop"} 3900.0
+vllm:request_success_total{engine="0",model_name="m",finished_reason="length"} 100.0
+vllm:request_success_created{engine="0",model_name="m",finished_reason="stop"} 1.76e+09
+# TYPE vllm:prompt_tokens counter
+vllm:prompt_tokens_total{engine="0",model_name="m"} 1250000.0
+vllm:generation_tokens_total{engine="0",model_name="m"} 412000.0
 vllm:something_else{engine="0"} 999
 """
 
@@ -41,7 +52,26 @@ def test_the_ladders_numbers_come_back():
     assert out["running"] == 75
     assert out["waiting"] == 12
     assert out["preemptions"] == 4
-    assert out["prefix_hit_pct"] == 25.0
+
+
+def test_the_counters_are_matched_despite_the_total_suffix():
+    """THE TRAP: prometheus_client renames a Counter. vLLM declares
+    "vllm:num_preemptions" and serves "vllm:num_preemptions_total", so keying on
+    the declared name alone silently dropped every counter -- the page drew a
+    dash and looked merely unsupported."""
+    out = sl.parse_prometheus(BODY)
+    assert out["requests_done"] == 4000          # summed over finished_reason
+    assert out["prompt_tokens"] == 1250000
+    assert out["generation_tokens"] == 412000
+
+
+def test_a_created_timestamp_is_never_read_as_a_count():
+    """"_created" carries a unix time; counting it would report 1.76 billion
+    preemptions on an idle pod."""
+    out = sl.parse_prometheus(
+        'vllm:num_preemptions_total{e="0"} 2.0\n'
+        'vllm:num_preemptions_created{e="0"} 1760000000.0\n')
+    assert out["preemptions"] == 2
 
 
 def test_unrelated_metrics_are_ignored():
@@ -69,10 +99,14 @@ def test_junk_is_survivable():
     assert sl.parse_prometheus("vllm:num_requests_running notanumber") == {}
 
 
-def test_no_queries_means_no_hit_rate():
-    """0/0 is not 0%."""
-    out = sl.parse_prometheus('vllm:prefix_cache_queries{e="0"} 0')
+def test_the_broken_prefix_hit_rate_is_not_reported():
+    """vLLM 0.27.1 registers its prefix-cache HIT counter from the MULTIMODAL
+    cache counter, so a rate computed from it is not the prefix hit rate. A
+    wrong number on an operator's page is worse than a dash."""
+    out = sl.parse_prometheus('vllm:prefix_cache_hits_total{e="0"} 250\n'
+                              'vllm:prefix_cache_queries_total{e="0"} 1000\n')
     assert "prefix_hit_pct" not in out
+    assert "prefix_hits" not in out
 
 
 # --------------------------------------------------------------- host memory #
@@ -150,6 +184,45 @@ def test_a_fresh_read_is_reused(monkeypatch):
     assert len(calls) == 1
 
 
+def test_a_restarted_engine_does_not_rewind_the_served_count(monkeypatch):
+    """vLLM's counters start at zero with every new engine process -- a model
+    swap, a crash, an OOM kill. An operator reads the column as "what this pod
+    has done", so a rewind would hide the restart AND lose the work."""
+    bodies = iter([
+        'vllm:request_success_total{e="0"} 4000\nvllm:generation_tokens_total{e="0"} 90\n',
+        'vllm:request_success_total{e="0"} 12\nvllm:generation_tokens_total{e="0"} 5\n',
+        'vllm:request_success_total{e="0"} 30\nvllm:generation_tokens_total{e="0"} 11\n',
+    ])
+
+    class _R:
+        status_code = 200
+        def __init__(self):
+            self.text = next(bodies)
+    monkeypatch.setattr("requests.get", lambda *a, **k: _R())
+    assert sl.engine_load("http://e:1")["requests_done"] == 4000
+    sl._CACHE.clear()                       # the TTL would answer from cache
+    assert sl.engine_load("http://e:1")["requests_done"] == 4012
+    sl._CACHE.clear()
+    out = sl.engine_load("http://e:1")
+    assert out["requests_done"] == 4030
+    assert out["generation_tokens"] == 101
+
+
+def test_the_instantaneous_figures_are_never_carried(monkeypatch):
+    """Running/waiting describe this moment; adding them up would be nonsense."""
+    bodies = iter(['vllm:num_requests_running{e="0"} 9\n',
+                   'vllm:num_requests_running{e="0"} 2\n'])
+
+    class _R:
+        status_code = 200
+        def __init__(self):
+            self.text = next(bodies)
+    monkeypatch.setattr("requests.get", lambda *a, **k: _R())
+    sl.engine_load("http://e:1")
+    sl._CACHE.clear()
+    assert sl.engine_load("http://e:1")["running"] == 2
+
+
 def test_an_engine_that_does_not_answer_is_not_a_failure(monkeypatch):
     def _boom(*a, **k):
         raise OSError("refused")