runpod: fix pods /v1 path; add stale-pod reaper, multi-GPU fallback, pod logs

Live-tested against a real RunPod account (Secure RTX 4090, Qwen2.5-0.5B):
provision → boot → vLLM /v1/models health → idle auto-teardown + ledger ($0.021)
all confirmed. Serverless + spillover proxy paths confirmed (reach RunPod, correct
URL/auth). Fixes/additions from that run:

- FIX: pods mode posted to <proxy>/chat/completions (missing /v1) → 404. The pod's
  vLLM serves under /v1 (health check hit /v1/models=200), so RunpodBackend now
  appends /v1 to the pod proxy URL (serverless base already ends in /openai/v1).

- Stale-pod REAPER (safety): runpod_worker.reap_orphans() lists account pods tagged
  coderai-<deployment_id>-* and terminates any NOT tracked by a live pool (orphans
  from a crash/restart) — never touches in-flight provisions or another deployment's
  pods. Runs every ~30s in the scaler loop AND once at startup; started on the
  primary engine (where pods are hosted) via start_runpod_maintenance(). New
  config.runpod.deployment_id (default "default"); pods named with the tag.

- Multi-GPU capacity FALLBACK: _rank_gpus() returns ranked candidates and
  _create_with_fallback() tries the next on a RunPod "no instances available"
  capacity error instead of failing (hit a real A5000 community capacity miss).

- Pod LOGS: RunpodClient.get_pod_logs() (best-effort REST) + pod_console_url deep
  link; admin GET /admin/api/runpod/pod-logs; RunPod stats page gets a per-pod
  "view logs" modal + console link. PodHandle/pods_status carry console_url.

- Settings: deployment_id field.

Built coderai:base=5cffe23d1d20. Relaunch to activate (running image had the /v1
bug + no reaper). After relaunch, a pods chat returns a real completion.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R6iugoNeEgt9PkajKrFyq4

codai/admin/routes.py

@@ -3616,6 +3616,7 @@ def build_settings_dict(c, gpu_cards):
             "serverless_base": c.runpod.serverless_base,
             "default_gpu_type": c.runpod.default_gpu_type,
             "data_center": c.runpod.data_center,
+            "deployment_id": c.runpod.deployment_id,
             "global_max_hourly_usd": c.runpod.global_max_hourly_usd,
             "global_cost_limit_usd": c.runpod.global_cost_limit_usd,
             "global_cost_period": c.runpod.global_cost_period,
@@ -4157,6 +4158,8 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
             rp.default_gpu_type = (d.get("default_gpu_type") or "").strip()
         if "data_center" in d:
             rp.data_center = (d.get("data_center") or "").strip()
+        if "deployment_id" in d:
+            rp.deployment_id = (d.get("deployment_id") or "default").strip() or "default"
         if "global_max_hourly_usd" in d:
             try: rp.global_max_hourly_usd = max(0.0, float(d.get("global_max_hourly_usd") or 0))
             except (TypeError, ValueError): pass
@@ -4354,6 +4357,21 @@ async def api_runpod_stats(username: str = Depends(require_admin)):
             "live_uncommitted_usd": round(live_cost, 4)}
 
 
+@router.get("/admin/api/runpod/pod-logs", summary="Fetch a RunPod pod's container/vLLM logs")
+async def api_runpod_pod_logs(pod_id: str, tail: int = 200,
+                             username: str = Depends(require_admin)):
+    """Best-effort pod + vLLM logs via the RunPod API, plus a console deep-link
+    (the always-available fallback where the container/vLLM output lives)."""
+    from codai.api.runpod_client import RunpodError, pod_console_url
+    if not pod_id:
+        raise HTTPException(status_code=400, detail="pod_id required")
+    client = _runpod_client_or_400()
+    try:
+        return {"success": True, **client.get_pod_logs(pod_id, tail=tail)}
+    except RunpodError as exc:
+        return {"success": False, "error": str(exc), "console_url": pod_console_url(pod_id)}
+
+
 # =============================================================================
 # Archive management
 # =============================================================================

codai/admin/templates/runpod.html

@@ -32,9 +32,9 @@
   <div style="overflow-x:auto">
     <table style="width:100%;border-collapse:collapse;font-size:13px">
       <thead><tr style="text-align:left;color:var(--text-3)">
-        <th>Model</th><th>GPU</th><th>Pool</th><th>In-flight</th><th>$/hr</th><th>Uptime</th><th>Cost so far</th><th>Health</th>
+        <th>Model</th><th>GPU</th><th>Pool</th><th>In-flight</th><th>$/hr</th><th>Uptime</th><th>Cost so far</th><th>Health</th><th>Logs</th>
       </tr></thead>
-      <tbody id="rp-pods"><tr><td colspan="8" class="dim">No pods running.</td></tr></tbody>
+      <tbody id="rp-pods"><tr><td colspan="9" class="dim">No pods running.</td></tr></tbody>
     </table>
   </div>
 </div>
@@ -83,7 +83,7 @@ async function rpRefresh(){
 
   // pods table
   const tb = document.getElementById('rp-pods');
-  if(!pods.length){ tb.innerHTML = '<tr><td colspan="8" class="dim">No pods running.</td></tr>'; }
+  if(!pods.length){ tb.innerHTML = '<tr><td colspan="9" class="dim">No pods running.</td></tr>'; }
   else {
     tb.innerHTML = pods.map(p => `<tr>
       <td>${p.model||''}</td>
@@ -94,6 +94,7 @@ async function rpRefresh(){
       <td>${_fmtDur(p.uptime_s)}</td>
       <td>${_usd(p.live_cost_usd)}</td>
       <td>${p.healthy?'<span style="color:#22c55e">healthy</span>':'<span style="color:#ef4444">down</span>'}</td>
+      <td><a href="#" onclick="rpLogs('${p.pod_id}');return false">view</a>${p.console_url?` · <a href="${p.console_url}" target="_blank" rel="noopener">console</a>`:''}</td>
     </tr>`).join('');
   }
 
@@ -111,7 +112,41 @@ async function rpRefresh(){
   const now = new Date();
   document.getElementById('rp-updated').textContent = 'updated ' + now.toLocaleTimeString();
 }
+async function rpLogs(podId){
+  const modal = document.getElementById('rp-log-modal');
+  const pre = document.getElementById('rp-log-pre');
+  const title = document.getElementById('rp-log-title');
+  title.textContent = 'Logs — pod ' + podId;
+  pre.textContent = 'loading…';
+  modal.style.display = 'flex';
+  try {
+    const r = await fetch(ROOT_PATH + '/admin/api/runpod/pod-logs?pod_id=' + encodeURIComponent(podId) + '&tail=300');
+    const d = await r.json();
+    let txt = d.logs || '';
+    if (!txt && d.error) txt = '(' + d.error + ')';
+    if (!txt) txt = '(no logs returned by the API)';
+    pre.textContent = txt;
+    const link = document.getElementById('rp-log-console');
+    if (d.console_url) { link.href = d.console_url; link.style.display = ''; }
+    else link.style.display = 'none';
+  } catch(e){ pre.textContent = 'error: ' + e.message; }
+}
+function rpCloseLogs(){ document.getElementById('rp-log-modal').style.display = 'none'; }
 rpRefresh();
 setInterval(rpRefresh, 10000);
 </script>
+
+<div id="rp-log-modal" style="display:none;position:fixed;inset:0;background:rgba(0,0,0,.5);
+     z-index:1000;align-items:center;justify-content:center" onclick="if(event.target===this)rpCloseLogs()">
+  <div style="background:var(--bg-1,#1a1a1a);border:1px solid var(--border,#333);border-radius:10px;
+       width:min(900px,92vw);max-height:80vh;display:flex;flex-direction:column;padding:1rem">
+    <div style="display:flex;align-items:center;gap:.5rem;margin-bottom:.5rem">
+      <b id="rp-log-title">Logs</b>
+      <a id="rp-log-console" href="#" target="_blank" rel="noopener" class="dim small" style="display:none">open in RunPod console →</a>
+      <button class="btn btn-secondary" style="margin-left:auto" onclick="rpCloseLogs()">Close</button>
+    </div>
+    <pre id="rp-log-pre" style="overflow:auto;flex:1;font-size:12px;white-space:pre-wrap;
+         background:var(--bg-0,#111);padding:.6rem;border-radius:6px;margin:0"></pre>
+  </div>
+</div>
 {% endblock %}

codai/admin/templates/settings.html

@@ -866,9 +866,16 @@
         </select>
       </div>
     </div>
-    <div class="form-group">
-      <label class="form-label">Data center <span class="muted">(optional filter)</span></label>
-      <input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
+    <div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
+      <div class="form-group" style="flex:1;min-width:150px">
+        <label class="form-label">Data center <span class="muted">(optional filter)</span></label>
+        <input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
+      </div>
+      <div class="form-group" style="flex:1;min-width:150px">
+        <label class="form-label">Deployment id</label>
+        <input type="text" id="s-runpod-deploy" class="form-input" placeholder="default">
+        <span class="form-hint" style="font-size:11px">Tags pods as yours (<code>coderai-&lt;id&gt;-…</code>) so the stale-pod reaper only ever stops your own pods. Use distinct ids for separate coderai deployments on one account.</span>
+      </div>
     </div>
     <details style="margin:.4rem 0">
       <summary style="cursor:pointer;font-size:12px" class="muted">Advanced API endpoints</summary>
@@ -1784,6 +1791,7 @@ async function loadSettings(){
     document.getElementById('s-runpod-cost-limit').value = rp.global_cost_limit_usd ?? 0;
     document.getElementById('s-runpod-cost-period').value = rp.global_cost_period || 'unlimited';
     document.getElementById('s-runpod-dc').value = rp.data_center ?? '';
+    document.getElementById('s-runpod-deploy').value = rp.deployment_id ?? 'default';
     document.getElementById('s-runpod-api-base').value = rp.api_base ?? '';
     document.getElementById('s-runpod-rest-base').value = rp.rest_base ?? '';
     document.getElementById('s-runpod-serverless-base').value = rp.serverless_base ?? '';
@@ -1977,6 +1985,7 @@ async function saveSettings(silent){
       global_cost_limit_usd: parseFloat(document.getElementById('s-runpod-cost-limit').value) || 0,
       global_cost_period: document.getElementById('s-runpod-cost-period').value || 'unlimited',
       data_center: document.getElementById('s-runpod-dc').value.trim(),
+      deployment_id: document.getElementById('s-runpod-deploy').value.trim() || 'default',
       api_base: document.getElementById('s-runpod-api-base').value.trim(),
       rest_base: document.getElementById('s-runpod-rest-base').value.trim(),
       serverless_base: document.getElementById('s-runpod-serverless-base').value.trim(),

codai/api/runpod_client.py

@@ -36,6 +36,11 @@ def pod_proxy_url(pod_id: str, port: int) -> str:
     return f"https://{pod_id}-{int(port)}.proxy.runpod.net"
 
 
+# Deep link to the pod in the RunPod web console (container + vLLM logs live there).
+def pod_console_url(pod_id: str) -> str:
+    return f"https://www.runpod.io/console/pods/{pod_id}"
+
+
 class RunpodClient:
     def __init__(self, cfg):
         self._cfg = cfg
@@ -269,6 +274,32 @@ class RunpodClient:
             "image": p.get("imageName"),
         } for p in (me.get("pods") or [])]
 
+    def get_pod_logs(self, pod_id: str, tail: int = 200) -> dict:
+        """Best-effort container/vLLM logs for a pod via the REST API.
+
+        RunPod's log surface is the web console; this tries the REST logs route and
+        always returns a console URL to fall back to. Returns
+        {console_url, logs?: str, error?: str}."""
+        import requests
+        out = {"console_url": pod_console_url(pod_id)}
+        rest = (getattr(self._cfg, "rest_base", "") or "https://rest.runpod.io/v1").rstrip("/")
+        try:
+            r = requests.get(f"{rest}/pods/{pod_id}/logs",
+                             headers={"Authorization": f"Bearer {self._api_key}"},
+                             params={"tail": tail}, timeout=15)
+            if r.status_code == 200:
+                try:
+                    body = r.json()
+                    out["logs"] = body if isinstance(body, str) else \
+                        (body.get("logs") or body.get("data") or str(body))
+                except ValueError:
+                    out["logs"] = r.text
+            else:
+                out["error"] = f"logs API HTTP {r.status_code} — use the console link"
+        except requests.RequestException as exc:
+            out["error"] = f"logs API unreachable ({exc}) — use the console link"
+        return out
+
     def wait_ready(self, pod_id: str, port: int, ready_timeout: float = 900.0,
                    poll_every: float = 5.0) -> str:
         """Poll until the pod exposes its proxy port; return the proxy base URL.

codai/backends/runpod.py

@@ -141,7 +141,9 @@ class RunpodBackend(ModelBackend):
         if not self._pool:
             raise RuntimeError("RunPod pod pool not initialised")
         pod, url = self._pool.acquire()
-        return url, (lambda: self._pool.release(pod))
+        # The pod runs a vLLM OpenAI server at /v1; its proxy URL has no path, so
+        # add /v1 to match the serverless base (which already ends in /openai/v1).
+        return url.rstrip("/") + "/v1", (lambda: self._pool.release(pod))
 
     def _store_usage(self, usage: dict) -> None:
         if usage:

codai/config.py

@@ -577,6 +577,9 @@ class RunpodConfig:
     serverless_base: str = "https://api.runpod.ai/v2"  # serverless invoke base
     default_gpu_type: str = ""            # fallback RunPod gpuTypeId when a model gives none
     data_center: str = ""                 # optional data-center id filter (blank = any)
+    # Stable tag baked into every pod name (coderai-<deployment_id>-<model>-<rand>) so the
+    # reaper can identify OUR pods across restarts and never touch another deployment's.
+    deployment_id: str = "default"
     # Account-wide safety net: the scaler will not let the SUM of all running RunPod
     # pods' hourly cost exceed this. 0 = no global cap (per-model $/hr still applies).
     global_max_hourly_usd: float = 0.0
@@ -1138,6 +1141,7 @@ class ConfigManager:
                 "serverless_base": self.config.runpod.serverless_base,
                 "default_gpu_type": self.config.runpod.default_gpu_type,
                 "data_center": self.config.runpod.data_center,
+                "deployment_id": self.config.runpod.deployment_id,
                 "global_max_hourly_usd": self.config.runpod.global_max_hourly_usd,
                 "global_cost_limit_usd": self.config.runpod.global_cost_limit_usd,
                 "global_cost_period": self.config.runpod.global_cost_period,

codai/main.py

@@ -1375,6 +1375,20 @@ def main():
         except ValueError:
             pass
 
+    # RunPod: on the PRIMARY engine (which hosts runpod pods), start the scaler +
+    # the independent stale-pod reaper as soon as RunPod is enabled — so pods
+    # orphaned by a previous crash/restart are terminated even before the first
+    # request. (The reaper only touches pods tagged with our deployment_id and not
+    # tracked by a live pool, and runs in the same process as the pools.)
+    try:
+        if (os.environ.get("CODERAI_ENGINE_PRIMARY") == "1"
+                and getattr(config, "runpod", None) and config.runpod.enabled):
+            from codai.api import runpod_worker
+            runpod_worker.start_runpod_maintenance()
+            print("[runpod] maintenance started (pod scaler + stale-pod reaper)", flush=True)
+    except Exception as _e:
+        print(f"[runpod] failed to start maintenance: {_e}", flush=True)
+
     # Configure Python logging so broker/API log calls reach the terminal.
     # uvicorn is started with log_config=None to keep our config in place.
     _log_level = logging.DEBUG if global_debug else logging.INFO