runpod: fleet status for an external monitor (v0.2.86)

Digesta needs to see what the fleet is doing, not only what it costs: how
many pods are running, where, and which model. The spend endpoint already
carried the pod rows, but a pod did not report its placement and there was
no roll-up to read at a glance.

A pod now carries and reports its data_center, cloud_type and gpu_count,
plus the pool and engine it belongs to. GET /v1/runpod/status (an alias of
/v1/runpod/spend, one implementation so the two can never disagree) adds a
`summary`: pods_total/ready/booting/unhealthy, gpus_in_use, inflight,
spot_pods, hourly_usd, and tallies by model, region, GPU and cloud type.

Same gate as the spend figures — an admin-scoped API key, 403 otherwise.

Three deliberate distinctions, documented: summary.hourly_usd counts only
pods actually serving (a booting pod bills but answers nothing, so counting
it would overstate capacity); pods_unhealthy is the alert-worthy number, a
pod past boot that is costing money and serving nothing; and gpus_in_use
counts cards, so a gpu_count:2 pod counts twice. An unknown region or GPU is
labelled "unknown" rather than dropped, so the tallies always sum to
pods_total.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.85"
+__version__ = "0.2.86"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/app.py

@@ -831,11 +831,26 @@ async def runpod_spend(request: Request):
                     "enabled": cfg.enabled}
     except Exception:
         caps = {}
-    return {"pods": pods, "ledger": ledger, "caps": caps, "schedules": schedules,
+    try:
+        summary = runpod_worker.fleet_summary(pods)
+    except Exception:
+        summary = {}
+    return {"summary": summary, "pods": pods, "ledger": ledger, "caps": caps,
+            "schedules": schedules,
             "live_hourly_usd": round(live_hourly, 4),
             "live_uncommitted_usd": round(live_cost, 4)}
 
 
+@app.get("/v1/runpod/status", summary="RunPod fleet status", tags=["Core"])
+async def runpod_status(request: Request):
+    """How many pods are running, where, and which model — for an admin key.
+
+    The same document as /v1/runpod/spend; this is the name to poll when the
+    question is "what is the fleet doing" rather than "what is it costing".
+    """
+    return await runpod_spend(request)
+
+
 @app.get("/coderai/capabilities", summary="Server capability document", tags=["Core"])
 async def get_broker_capabilities():
     """Return broker capability metadata."""

codai/api/runpod_worker.py

@@ -1697,6 +1697,12 @@ class PodHandle:
     inflight: int = 0
     last_used: float = 0.0
     console_url: str = ""
+    #: Where this pod actually is and what it is. Reported to monitoring: the
+    #: region matters for latency and for where the data is allowed to live, and
+    #: "which model" is the first question anyone asks of a running fleet.
+    data_center: str = ""
+    cloud_type: str = ""
+    gpu_count: int = 1
 
 
 def _deploy_tag(account_cfg) -> str:
@@ -2261,7 +2267,10 @@ class RunpodPodPool:
             h = PodHandle(pod_id=pod_id, url=url, hourly_usd=sel["price"],
                           started_at=time.time(), gpu=sel["display_name"],
                           is_spot=sel["is_spot"], healthy=True, last_used=time.time(),
-                          console_url=console)
+                          console_url=console,
+                          data_center=(getattr(self, "_volume_dc", "") or dc or ""),
+                          cloud_type=str(sel.get("cloud_type") or ""),
+                          gpu_count=int(sel.get("gpu_count") or 1))
             with self._cv:
                 self.pods.append(h)
                 self._cv.notify_all()
@@ -2757,6 +2766,41 @@ def _all_pools_live_cost() -> float:
         return sum(p.live_cost_usd() for p in _pools.values())
 
 
+def fleet_summary(pods: list = None) -> dict:
+    """"How many, where, which model" — the questions asked of a running fleet.
+
+    Derived from pods_status() so there is one source of truth. Counts only
+    pods this install rented: one created by hand in the RunPod console is not
+    in the pools and is deliberately not counted here.
+    """
+    rows = pods_status() if pods is None else pods
+    ready = [p for p in rows if p.get("state") == "ready" and p.get("healthy")]
+    booting = [p for p in rows if p.get("state") != "ready"]
+
+    def _tally(key, src):
+        out = {}
+        for r in src:
+            k = str(r.get(key) or "") or "unknown"
+            out[k] = out.get(k, 0) + 1
+        return out
+
+    return {
+        "pods_total": len(rows),
+        "pods_ready": len(ready),
+        "pods_booting": len(booting),
+        "pods_unhealthy": len([p for p in rows
+                               if p.get("state") == "ready" and not p.get("healthy")]),
+        "gpus_in_use": sum(int(p.get("gpu_count") or 1) for p in ready),
+        "inflight": sum(int(p.get("inflight") or 0) for p in rows),
+        "by_model": _tally("model", rows),
+        "by_region": _tally("data_center", rows),
+        "by_gpu": _tally("gpu", rows),
+        "by_cloud_type": _tally("cloud_type", rows),
+        "spot_pods": len([p for p in rows if p.get("is_spot")]),
+        "hourly_usd": round(sum((p.get("hourly_usd") or 0.0) for p in ready), 4),
+    }
+
+
 def pools_schedule_status() -> list:
     """Per-pool warm-pod schedule: where each model stands right now.
 
@@ -2798,10 +2842,15 @@ def pods_status() -> list:
             for p in pool.pods:
                 out.append({
                     "model": key, "pod_id": p.pod_id, "gpu": p.gpu,
+                    "gpu_count": getattr(p, "gpu_count", 1),
                     "is_spot": p.is_spot, "healthy": p.healthy, "inflight": p.inflight,
                     "hourly_usd": p.hourly_usd, "uptime_s": int(now - p.started_at),
                     "live_cost_usd": round((now - p.started_at) / 3600.0 * p.hourly_usd, 4),
                     "console_url": p.console_url, "state": "ready",
+                    "data_center": getattr(p, "data_center", "") or "",
+                    "cloud_type": getattr(p, "cloud_type", "") or "",
+                    "pool": (pool.mcfg.pool or "") if getattr(pool, "mcfg", None) else "",
+                    "engine": (pool.mcfg.engine or "") if getattr(pool, "mcfg", None) else "",
                 })
     # Booting pods (created, not yet serving) — visible so a slow/failing boot is
     # obvious and its vLLM log is one click away.

docs/orchestrator-runpod-guide.md

@@ -661,8 +661,10 @@ provisioning stops rather than quietly continuing.
 uncommitted cost, per-model spend from the ledger, and — for scheduled models —
 whether each is inside its window and when that next flips.
 
-**The API, for your own monitoring.** `GET /v1/runpod/spend` returns the same
-figures as JSON:
+**The API, for your own monitoring.** `GET /v1/runpod/status` (or
+`/v1/runpod/spend` — the same document under two names, so pick the one that
+reads right where you call it) answers both "what is the fleet doing" and "what
+is it costing":
 
 ```bash
 curl -fsS -H "Authorization: Bearer sk-coderai-…" \
@@ -671,8 +673,19 @@ curl -fsS -H "Authorization: Bearer sk-coderai-…" \
 
 ```json
 {
-  "pods":   [{"model":"…","pod_id":"…","hourly_usd":1.09,"uptime_s":940,
-              "live_cost_usd":0.28,"healthy":true,"inflight":2}],
+  "summary": {
+    "pods_total": 3, "pods_ready": 2, "pods_booting": 1, "pods_unhealthy": 0,
+    "gpus_in_use": 2, "inflight": 2, "spot_pods": 1, "hourly_usd": 2.18,
+    "by_model":  {"datalab-to/surya-ocr-2": 2, "Qwen/Qwen3-8B-Instruct": 1},
+    "by_region": {"EU-SE-1": 2, "EU-RO-1": 1},
+    "by_gpu":    {"A40": 2, "A100": 1},
+    "by_cloud_type": {"SECURE": 3}
+  },
+  "pods": [{"model":"datalab-to/surya-ocr-2","pod_id":"x0nr…","state":"ready",
+            "healthy":true,"inflight":2,"gpu":"A40","gpu_count":1,
+            "data_center":"EU-SE-1","cloud_type":"SECURE","is_spot":false,
+            "pool":"","engine":"vllm","hourly_usd":1.09,"uptime_s":940,
+            "live_cost_usd":0.28,"console_url":"https://…"}],
   "ledger": {"per_model": {"…": {"day": 12.40}}, "global": {"day": 31.05}},
   "caps":   {"global_max_hourly_usd":20.0,"global_cost_limit_usd":40.0,
              "global_cost_period":"day","enabled":true},
@@ -683,6 +696,17 @@ curl -fsS -H "Authorization: Bearer sk-coderai-…" \
 }
 ```
 
+`summary` is derived from `pods`, so the two can never disagree. Three
+distinctions worth knowing when you build a dashboard on this:
+
+- **`pods_ready` vs `pods_total`** — a booting pod bills but serves nothing, so
+  `summary.hourly_usd` counts only pods that are actually serving. For real
+  spend use the ledger plus `live_uncommitted_usd`.
+- **`pods_unhealthy`** is a pod past boot that is failing its probe: it is
+  costing money and answering nothing. A non-zero value here is the thing to
+  alert on.
+- **`gpus_in_use`** counts cards, not pods, so a `gpu_count: 2` pod counts twice.
+
 This endpoint needs an **admin-scoped** API key — Admin → **Tokens**, tick
 *Admin scope* when generating it (or toggle it on an existing one). An ordinary
 key gets `403`: a key that may ask a model a question has no business reading the

tests/test_runpod_fleet_status.py

+"""Fleet status for an external monitor: how many pods, where, which model.
+
+Admin-scoped only — the same gate as the spend figures, because this is the
+same document.
+"""
+import pathlib
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+
+
+def _rows():
+    return [
+        {"state": "ready", "healthy": True, "model": "surya", "data_center": "EU-SE-1",
+         "gpu": "A40", "gpu_count": 1, "inflight": 2, "hourly_usd": 1.09,
+         "cloud_type": "SECURE", "is_spot": False},
+        {"state": "ready", "healthy": True, "model": "surya", "data_center": "EU-SE-1",
+         "gpu": "A40", "gpu_count": 1, "inflight": 0, "hourly_usd": 1.09,
+         "cloud_type": "SECURE", "is_spot": False},
+        {"state": "booting", "healthy": False, "model": "qwen", "data_center": "EU-RO-1",
+         "gpu": "A100", "gpu_count": 2, "inflight": 0, "hourly_usd": 2.40,
+         "cloud_type": "SECURE", "is_spot": True},
+        {"state": "ready", "healthy": False, "model": "qwen", "data_center": "EU-RO-1",
+         "gpu": "A100", "gpu_count": 2, "inflight": 0, "hourly_usd": 2.40,
+         "cloud_type": "SECURE", "is_spot": False},
+    ]
+
+
+def test_the_summary_answers_how_many_where_and_which_model():
+    from codai.api.runpod_worker import fleet_summary
+    s = fleet_summary(_rows())
+    assert s["pods_total"] == 4
+    assert s["pods_ready"] == 2          # healthy and serving
+    assert s["pods_booting"] == 1
+    assert s["pods_unhealthy"] == 1      # ready-state but failed its probe
+    assert s["by_model"] == {"surya": 2, "qwen": 2}
+    assert s["by_region"] == {"EU-SE-1": 2, "EU-RO-1": 2}
+    assert s["by_gpu"] == {"A40": 2, "A100": 2}
+    assert s["spot_pods"] == 1
+
+
+def test_the_hourly_rate_counts_only_what_is_actually_serving():
+    """A booting or unhealthy pod still bills, but reporting it as capacity
+    would overstate what the fleet can do; the ledger covers real spend."""
+    from codai.api.runpod_worker import fleet_summary
+    s = fleet_summary(_rows())
+    assert s["hourly_usd"] == round(1.09 * 2, 4)
+    assert s["gpus_in_use"] == 2         # multi-GPU pods count their cards
+    assert s["inflight"] == 2
+
+
+def test_an_unknown_region_is_labelled_not_dropped():
+    from codai.api.runpod_worker import fleet_summary
+    s = fleet_summary([{"state": "ready", "healthy": True, "model": "m",
+                        "gpu": "", "gpu_count": 1, "hourly_usd": 0.5}])
+    assert s["by_region"] == {"unknown": 1}
+    assert s["by_gpu"] == {"unknown": 1}
+
+
+def test_each_pod_reports_where_it_is_and_what_it_serves():
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    blk = src.split("def pods_status")[1].split("def ")[0]
+    for field in ('"data_center"', '"cloud_type"', '"pool"', '"engine"',
+                  '"gpu_count"', '"model"'):
+        assert field in blk, field
+    # and the placement has to be carried on the pod to be reportable
+    assert "data_center: str = \"\"" in src.split("class PodHandle")[1][:600]
+    assert 'data_center=(getattr(self, "_volume_dc", "") or dc or "")' in src
+
+
+def test_status_and_spend_are_the_same_document_behind_the_same_gate():
+    src = (ROOT / "codai/api/app.py").read_text()
+    assert '@app.get("/v1/runpod/status"' in src
+    blk = src.split('async def runpod_status')[1].split("@app.get")[0]
+    assert "return await runpod_spend(request)" in blk, \
+        "one implementation, so the two can never disagree"
+    spend = src.split('async def runpod_spend')[1].split("@app.get")[0]
+    assert "_admin_scoped_token(request)" in spend and "status_code=403" in spend
+
+
+def test_the_summary_is_in_the_payload():
+    src = (ROOT / "codai/api/app.py").read_text()
+    blk = src.split('async def runpod_spend')[1].split("@app.get")[0]
+    assert "fleet_summary(pods)" in blk
+    assert '"summary": summary' in blk