runpod: fix pods /v1 path; add stale-pod reaper, multi-GPU fallback, pod logs

Live-tested against a real RunPod account (Secure RTX 4090, Qwen2.5-0.5B):
provision → boot → vLLM /v1/models health → idle auto-teardown + ledger ($0.021)
all confirmed. Serverless + spillover proxy paths confirmed (reach RunPod, correct
URL/auth). Fixes/additions from that run:

- FIX: pods mode posted to <proxy>/chat/completions (missing /v1) → 404. The pod's
  vLLM serves under /v1 (health check hit /v1/models=200), so RunpodBackend now
  appends /v1 to the pod proxy URL (serverless base already ends in /openai/v1).

- Stale-pod REAPER (safety): runpod_worker.reap_orphans() lists account pods tagged
  coderai-<deployment_id>-* and terminates any NOT tracked by a live pool (orphans
  from a crash/restart) — never touches in-flight provisions or another deployment's
  pods. Runs every ~30s in the scaler loop AND once at startup; started on the
  primary engine (where pods are hosted) via start_runpod_maintenance(). New
  config.runpod.deployment_id (default "default"); pods named with the tag.

- Multi-GPU capacity FALLBACK: _rank_gpus() returns ranked candidates and
  _create_with_fallback() tries the next on a RunPod "no instances available"
  capacity error instead of failing (hit a real A5000 community capacity miss).

- Pod LOGS: RunpodClient.get_pod_logs() (best-effort REST) + pod_console_url deep
  link; admin GET /admin/api/runpod/pod-logs; RunPod stats page gets a per-pod
  "view logs" modal + console link. PodHandle/pods_status carry console_url.

- Settings: deployment_id field.

Built coderai:base=5cffe23d1d20. Relaunch to activate (running image had the /v1
bug + no reaper). After relaunch, a pods chat returns a real completion.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R6iugoNeEgt9PkajKrFyq4

codai/admin/routes.py

@@ -3616,6 +3616,7 @@ def build_settings_dict(c, gpu_cards):
             "serverless_base": c.runpod.serverless_base,
             "default_gpu_type": c.runpod.default_gpu_type,
             "data_center": c.runpod.data_center,
+            "deployment_id": c.runpod.deployment_id,
             "global_max_hourly_usd": c.runpod.global_max_hourly_usd,
             "global_cost_limit_usd": c.runpod.global_cost_limit_usd,
             "global_cost_period": c.runpod.global_cost_period,
@@ -4157,6 +4158,8 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
             rp.default_gpu_type = (d.get("default_gpu_type") or "").strip()
         if "data_center" in d:
             rp.data_center = (d.get("data_center") or "").strip()
+        if "deployment_id" in d:
+            rp.deployment_id = (d.get("deployment_id") or "default").strip() or "default"
         if "global_max_hourly_usd" in d:
             try: rp.global_max_hourly_usd = max(0.0, float(d.get("global_max_hourly_usd") or 0))
             except (TypeError, ValueError): pass
@@ -4354,6 +4357,21 @@ async def api_runpod_stats(username: str = Depends(require_admin)):
             "live_uncommitted_usd": round(live_cost, 4)}
 
 
+@router.get("/admin/api/runpod/pod-logs", summary="Fetch a RunPod pod's container/vLLM logs")
+async def api_runpod_pod_logs(pod_id: str, tail: int = 200,
+                             username: str = Depends(require_admin)):
+    """Best-effort pod + vLLM logs via the RunPod API, plus a console deep-link
+    (the always-available fallback where the container/vLLM output lives)."""
+    from codai.api.runpod_client import RunpodError, pod_console_url
+    if not pod_id:
+        raise HTTPException(status_code=400, detail="pod_id required")
+    client = _runpod_client_or_400()
+    try:
+        return {"success": True, **client.get_pod_logs(pod_id, tail=tail)}
+    except RunpodError as exc:
+        return {"success": False, "error": str(exc), "console_url": pod_console_url(pod_id)}
+
+
 # =============================================================================
 # Archive management
 # =============================================================================

codai/admin/templates/runpod.html

@@ -32,9 +32,9 @@
   <div style="overflow-x:auto">
     <table style="width:100%;border-collapse:collapse;font-size:13px">
       <thead><tr style="text-align:left;color:var(--text-3)">
-        <th>Model</th><th>GPU</th><th>Pool</th><th>In-flight</th><th>$/hr</th><th>Uptime</th><th>Cost so far</th><th>Health</th>
+        <th>Model</th><th>GPU</th><th>Pool</th><th>In-flight</th><th>$/hr</th><th>Uptime</th><th>Cost so far</th><th>Health</th><th>Logs</th>
       </tr></thead>
-      <tbody id="rp-pods"><tr><td colspan="8" class="dim">No pods running.</td></tr></tbody>
+      <tbody id="rp-pods"><tr><td colspan="9" class="dim">No pods running.</td></tr></tbody>
     </table>
   </div>
 </div>
@@ -83,7 +83,7 @@ async function rpRefresh(){
 
   // pods table
   const tb = document.getElementById('rp-pods');
-  if(!pods.length){ tb.innerHTML = '<tr><td colspan="8" class="dim">No pods running.</td></tr>'; }
+  if(!pods.length){ tb.innerHTML = '<tr><td colspan="9" class="dim">No pods running.</td></tr>'; }
   else {
     tb.innerHTML = pods.map(p => `<tr>
       <td>${p.model||''}</td>
@@ -94,6 +94,7 @@ async function rpRefresh(){
       <td>${_fmtDur(p.uptime_s)}</td>
       <td>${_usd(p.live_cost_usd)}</td>
       <td>${p.healthy?'<span style="color:#22c55e">healthy</span>':'<span style="color:#ef4444">down</span>'}</td>
+      <td><a href="#" onclick="rpLogs('${p.pod_id}');return false">view</a>${p.console_url?` · <a href="${p.console_url}" target="_blank" rel="noopener">console</a>`:''}</td>
     </tr>`).join('');
   }
 
@@ -111,7 +112,41 @@ async function rpRefresh(){
   const now = new Date();
   document.getElementById('rp-updated').textContent = 'updated ' + now.toLocaleTimeString();
 }
+async function rpLogs(podId){
+  const modal = document.getElementById('rp-log-modal');
+  const pre = document.getElementById('rp-log-pre');
+  const title = document.getElementById('rp-log-title');
+  title.textContent = 'Logs — pod ' + podId;
+  pre.textContent = 'loading…';
+  modal.style.display = 'flex';
+  try {
+    const r = await fetch(ROOT_PATH + '/admin/api/runpod/pod-logs?pod_id=' + encodeURIComponent(podId) + '&tail=300');
+    const d = await r.json();
+    let txt = d.logs || '';
+    if (!txt && d.error) txt = '(' + d.error + ')';
+    if (!txt) txt = '(no logs returned by the API)';
+    pre.textContent = txt;
+    const link = document.getElementById('rp-log-console');
+    if (d.console_url) { link.href = d.console_url; link.style.display = ''; }
+    else link.style.display = 'none';
+  } catch(e){ pre.textContent = 'error: ' + e.message; }
+}
+function rpCloseLogs(){ document.getElementById('rp-log-modal').style.display = 'none'; }
 rpRefresh();
 setInterval(rpRefresh, 10000);
 </script>
+
+<div id="rp-log-modal" style="display:none;position:fixed;inset:0;background:rgba(0,0,0,.5);
+     z-index:1000;align-items:center;justify-content:center" onclick="if(event.target===this)rpCloseLogs()">
+  <div style="background:var(--bg-1,#1a1a1a);border:1px solid var(--border,#333);border-radius:10px;
+       width:min(900px,92vw);max-height:80vh;display:flex;flex-direction:column;padding:1rem">
+    <div style="display:flex;align-items:center;gap:.5rem;margin-bottom:.5rem">
+      <b id="rp-log-title">Logs</b>
+      <a id="rp-log-console" href="#" target="_blank" rel="noopener" class="dim small" style="display:none">open in RunPod console →</a>
+      <button class="btn btn-secondary" style="margin-left:auto" onclick="rpCloseLogs()">Close</button>
+    </div>
+    <pre id="rp-log-pre" style="overflow:auto;flex:1;font-size:12px;white-space:pre-wrap;
+         background:var(--bg-0,#111);padding:.6rem;border-radius:6px;margin:0"></pre>
+  </div>
+</div>
 {% endblock %}

codai/admin/templates/settings.html

@@ -866,9 +866,16 @@
         </select>
       </div>
     </div>
-    <div class="form-group">
-      <label class="form-label">Data center <span class="muted">(optional filter)</span></label>
-      <input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
+    <div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
+      <div class="form-group" style="flex:1;min-width:150px">
+        <label class="form-label">Data center <span class="muted">(optional filter)</span></label>
+        <input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
+      </div>
+      <div class="form-group" style="flex:1;min-width:150px">
+        <label class="form-label">Deployment id</label>
+        <input type="text" id="s-runpod-deploy" class="form-input" placeholder="default">
+        <span class="form-hint" style="font-size:11px">Tags pods as yours (<code>coderai-&lt;id&gt;-…</code>) so the stale-pod reaper only ever stops your own pods. Use distinct ids for separate coderai deployments on one account.</span>
+      </div>
     </div>
     <details style="margin:.4rem 0">
       <summary style="cursor:pointer;font-size:12px" class="muted">Advanced API endpoints</summary>
@@ -1784,6 +1791,7 @@ async function loadSettings(){
     document.getElementById('s-runpod-cost-limit').value = rp.global_cost_limit_usd ?? 0;
     document.getElementById('s-runpod-cost-period').value = rp.global_cost_period || 'unlimited';
     document.getElementById('s-runpod-dc').value = rp.data_center ?? '';
+    document.getElementById('s-runpod-deploy').value = rp.deployment_id ?? 'default';
     document.getElementById('s-runpod-api-base').value = rp.api_base ?? '';
     document.getElementById('s-runpod-rest-base').value = rp.rest_base ?? '';
     document.getElementById('s-runpod-serverless-base').value = rp.serverless_base ?? '';
@@ -1977,6 +1985,7 @@ async function saveSettings(silent){
       global_cost_limit_usd: parseFloat(document.getElementById('s-runpod-cost-limit').value) || 0,
       global_cost_period: document.getElementById('s-runpod-cost-period').value || 'unlimited',
       data_center: document.getElementById('s-runpod-dc').value.trim(),
+      deployment_id: document.getElementById('s-runpod-deploy').value.trim() || 'default',
       api_base: document.getElementById('s-runpod-api-base').value.trim(),
       rest_base: document.getElementById('s-runpod-rest-base').value.trim(),
       serverless_base: document.getElementById('s-runpod-serverless-base').value.trim(),

codai/api/runpod_client.py

@@ -36,6 +36,11 @@ def pod_proxy_url(pod_id: str, port: int) -> str:
     return f"https://{pod_id}-{int(port)}.proxy.runpod.net"
 
 
+# Deep link to the pod in the RunPod web console (container + vLLM logs live there).
+def pod_console_url(pod_id: str) -> str:
+    return f"https://www.runpod.io/console/pods/{pod_id}"
+
+
 class RunpodClient:
     def __init__(self, cfg):
         self._cfg = cfg
@@ -269,6 +274,32 @@ class RunpodClient:
             "image": p.get("imageName"),
         } for p in (me.get("pods") or [])]
 
+    def get_pod_logs(self, pod_id: str, tail: int = 200) -> dict:
+        """Best-effort container/vLLM logs for a pod via the REST API.
+
+        RunPod's log surface is the web console; this tries the REST logs route and
+        always returns a console URL to fall back to. Returns
+        {console_url, logs?: str, error?: str}."""
+        import requests
+        out = {"console_url": pod_console_url(pod_id)}
+        rest = (getattr(self._cfg, "rest_base", "") or "https://rest.runpod.io/v1").rstrip("/")
+        try:
+            r = requests.get(f"{rest}/pods/{pod_id}/logs",
+                             headers={"Authorization": f"Bearer {self._api_key}"},
+                             params={"tail": tail}, timeout=15)
+            if r.status_code == 200:
+                try:
+                    body = r.json()
+                    out["logs"] = body if isinstance(body, str) else \
+                        (body.get("logs") or body.get("data") or str(body))
+                except ValueError:
+                    out["logs"] = r.text
+            else:
+                out["error"] = f"logs API HTTP {r.status_code} — use the console link"
+        except requests.RequestException as exc:
+            out["error"] = f"logs API unreachable ({exc}) — use the console link"
+        return out
+
     def wait_ready(self, pod_id: str, port: int, ready_timeout: float = 900.0,
                    poll_every: float = 5.0) -> str:
         """Poll until the pod exposes its proxy port; return the proxy base URL.

codai/api/runpod_worker.py

@@ -190,13 +190,12 @@ def _pod_health_ok(url: str, timeout: float = 4.0) -> bool:
         return False
 
 
-def _select_gpu(client, mcfg: "RunpodModelConfig", account_cfg) -> dict:
-    """Choose a GPU across the model's allowed pools per its selection criteria.
-
-    Returns {gpu_type_id, display_name, memory_gb, cloud_type, price, is_spot,
-    bid} where ``price`` is the effective hourly cost we'll be billed and ``bid``
-    is the max bid to place for a spot pod. Raises RunpodError if nothing fits.
-    """
+def _rank_gpus(client, mcfg: "RunpodModelConfig", account_cfg) -> list:
+    """Ranked list of GPU options across the model's allowed pools per its selection
+    criteria (best first). Each item: {gpu_type_id, display_name, memory_gb,
+    cloud_type, price, is_spot, bid}. Raises RunpodError if nothing fits the
+    constraints. _provision_one tries them in order so a capacity miss on the top
+    pick falls through to the next."""
     from codai.api.runpod_client import RunpodError
     catalog = client.list_gpu_types()
     allowed = [c.upper() for c in (mcfg.cloud_types or ["SECURE"])] or ["SECURE"]
@@ -236,10 +235,15 @@ def _select_gpu(client, mcfg: "RunpodModelConfig", account_cfg) -> dict:
                                   -(c[3].get("memory_gb") or 0), c[0]))
     else:  # cheaper
         cands.sort(key=lambda c: (c[0], c[1] is False))
-    price, is_spot, ct, g, on_demand = cands[0]
-    return {"gpu_type_id": g["id"], "display_name": g["display_name"],
-            "memory_gb": g.get("memory_gb"), "cloud_type": ct, "price": price,
-            "is_spot": is_spot, "bid": on_demand if is_spot else 0.0}
+    return [{"gpu_type_id": g["id"], "display_name": g["display_name"],
+             "memory_gb": g.get("memory_gb"), "cloud_type": ct, "price": price,
+             "is_spot": is_spot, "bid": on_demand if is_spot else 0.0}
+            for (price, is_spot, ct, g, on_demand) in cands]
+
+
+def _select_gpu(client, mcfg: "RunpodModelConfig", account_cfg) -> dict:
+    """The single best GPU option (top of _rank_gpus)."""
+    return _rank_gpus(client, mcfg, account_cfg)[0]
 
 
 @_dc_pod
@@ -253,6 +257,18 @@ class PodHandle:
     healthy: bool = True
     inflight: int = 0
     last_used: float = 0.0
+    console_url: str = ""
+
+
+def _deploy_tag(account_cfg) -> str:
+    """Stable, filesystem-safe deployment tag baked into pod names."""
+    t = (getattr(account_cfg, "deployment_id", "") or "default").strip() or "default"
+    return "".join(c if (c.isalnum() or c in "-_") else "_" for c in t)[:24]
+
+
+# Pod ids currently mid-provision (created on RunPod but not yet in a pool). The
+# reaper must NOT kill these. Guarded by _pools_lock.
+_provisioning_ids: set = set()
 
 
 def _vllm_docker_args(mcfg: "RunpodModelConfig", served: str) -> str:
@@ -323,30 +339,54 @@ class RunpodPodPool:
         return ""
 
     # -- provisioning ----------------------------------------------------- #
+    def _create_with_fallback(self, client, ranked):
+        """Try ranked GPU candidates in order; skip capacity misses. Returns
+        (pod_id, sel). Raises if all fail."""
+        from codai.api.runpod_client import RunpodError
+        port = self.mcfg.port or 8000
+        image = self.mcfg.image or DEFAULT_POD_IMAGE
+        env = dict(self.mcfg.env or {})
+        args = _vllm_docker_args(self.mcfg, self.served)
+        dc = getattr(self.account, "data_center", "") or ""
+        last = None
+        for sel in ranked:
+            block = self._budget_blocks(sel["price"])
+            if block:
+                raise RunpodError(f"RunPod budget cap hit — not provisioning ({block}).")
+            name = (f"coderai-{_deploy_tag(self.account)}-"
+                    + str(self.model_key)[:20].replace("/", "_").replace(" ", "_")
+                    + "-" + uuid.uuid4().hex[:6])
+            print(f"[runpod] provisioning pod for {self.model_key!r}: {sel['display_name']} "
+                  f"({sel['cloud_type']}{'/spot' if sel['is_spot'] else ''}) ${sel['price']}/hr",
+                  flush=True)
+            try:
+                pod_id = client.create_pod(
+                    name=name, image=image, gpu_type_id=sel["gpu_type_id"], port=port,
+                    cloud_type=sel["cloud_type"], container_disk_gb=self.mcfg.container_disk_gb,
+                    volume_gb=self.mcfg.volume_gb, env=env, docker_args=args,
+                    is_spot=sel["is_spot"], bid_per_gpu=sel["bid"], data_center_id=dc)
+                return pod_id, sel
+            except RunpodError as exc:
+                msg = str(exc).lower()
+                if "no longer any instances" in msg or "no instances" in msg or "capacity" in msg:
+                    print(f"[runpod] {sel['display_name']} ({sel['cloud_type']}) unavailable — "
+                          "trying next candidate", flush=True)
+                    last = exc
+                    continue
+                raise
+        raise last or RunpodError("RunPod: no candidate GPU could be deployed (capacity).")
+
     def _provision_one(self):
         """Create + boot one pod; append it healthy. Blocking (minutes)."""
-        from codai.api.runpod_client import RunpodClient, RunpodError
+        from codai.api.runpod_client import RunpodClient, RunpodError, pod_console_url
         client = RunpodClient(self.account)
-        sel = _select_gpu(client, self.mcfg, self.account)
-        block = self._budget_blocks(sel["price"])
-        if block:
-            raise RunpodError(f"RunPod budget cap hit — not provisioning ({block}).")
-        name = ("coderai-" + str(self.model_key)[:24].replace("/", "_").replace(" ", "_")
-                + "-" + uuid.uuid4().hex[:6])
-        env = dict(self.mcfg.env or {})
-        image = self.mcfg.image or DEFAULT_POD_IMAGE
-        print(f"[runpod] provisioning pod for {self.model_key!r}: {sel['display_name']} "
-              f"({sel['cloud_type']}{'/spot' if sel['is_spot'] else ''}) ${sel['price']}/hr",
-              flush=True)
-        pod_id = client.create_pod(
-            name=name, image=image, gpu_type_id=sel["gpu_type_id"], port=self.mcfg.port or 8000,
-            cloud_type=sel["cloud_type"], container_disk_gb=self.mcfg.container_disk_gb,
-            volume_gb=self.mcfg.volume_gb, env=env,
-            docker_args=_vllm_docker_args(self.mcfg, self.served),
-            is_spot=sel["is_spot"], bid_per_gpu=sel["bid"],
-            data_center_id=getattr(self.account, "data_center", "") or "")
+        ranked = _rank_gpus(client, self.mcfg, self.account)
+        port = self.mcfg.port or 8000
+        pod_id, sel = self._create_with_fallback(client, ranked)
+        with _pools_lock:
+            _provisioning_ids.add(pod_id)   # shield from the reaper during boot
         try:
-            url = client.wait_ready(pod_id, self.mcfg.port or 8000, ready_timeout=900.0)
+            url = client.wait_ready(pod_id, port, ready_timeout=900.0)
             # Now wait for the OpenAI server inside the pod (image pull + model load).
             deadline = time.time() + 900.0
             while time.time() < deadline:
@@ -360,13 +400,17 @@ class RunpodPodPool:
                 client.terminate_pod(pod_id)
             except Exception:
                 pass
+            with _pools_lock:
+                _provisioning_ids.discard(pod_id)
             raise
         h = PodHandle(pod_id=pod_id, url=url, hourly_usd=sel["price"], started_at=time.time(),
                       gpu=sel["display_name"], is_spot=sel["is_spot"], healthy=True,
-                      last_used=time.time())
+                      last_used=time.time(), console_url=pod_console_url(pod_id))
         with self._cv:
             self.pods.append(h)
             self._cv.notify_all()
+        with _pools_lock:
+            _provisioning_ids.discard(pod_id)
         print(f"[runpod] pod {pod_id} ready for {self.model_key!r} at {url}", flush=True)
         return h
 
@@ -518,13 +562,73 @@ def pods_status() -> list:
                     "is_spot": p.is_spot, "healthy": p.healthy, "inflight": p.inflight,
                     "hourly_usd": p.hourly_usd, "uptime_s": int(now - p.started_at),
                     "live_cost_usd": round((now - p.started_at) / 3600.0 * p.hourly_usd, 4),
+                    "console_url": p.console_url,
                 })
     return out
 
 
+def _known_pod_ids() -> set:
+    """All pod ids this process is responsible for: live pool pods + mid-provision."""
+    ids = set()
+    with _pools_lock:
+        for pool in _pools.values():
+            with pool._cv:
+                ids.update(p.pod_id for p in pool.pods)
+        ids.update(_provisioning_ids)
+    return ids
+
+
+def reap_orphans() -> int:
+    """Terminate RunPod pods tagged as OURS that no live pool is tracking — stale
+    pods from a crash/restart that would otherwise bill silently. Only touches pods
+    named ``coderai-<our deployment_id>-*`` (never another deployment's), and never
+    a pod currently mid-provision. Returns the number reaped."""
+    try:
+        from codai.models.manager import get_active_runpod_config
+        acct = get_active_runpod_config()
+    except Exception:
+        acct = None
+    if acct is None or not getattr(acct, "enabled", False) or not getattr(acct, "api_key", ""):
+        return 0
+    from codai.api.runpod_client import RunpodClient, RunpodError
+    prefix = f"coderai-{_deploy_tag(acct)}-"
+    try:
+        client = RunpodClient(acct)
+        pods = client.list_pods()
+    except RunpodError as exc:
+        print(f"[runpod-reaper] list failed: {exc}", flush=True)
+        return 0
+    known = _known_pod_ids()
+    reaped = 0
+    for p in pods:
+        pid, name = p.get("id"), (p.get("name") or "")
+        status = str(p.get("status") or "").upper()
+        if not pid or not name.startswith(prefix):
+            continue                      # not ours (or another deployment's)
+        if pid in known:
+            continue                      # tracked by a live pool / provisioning
+        if status in ("TERMINATED", "EXITED"):
+            continue
+        try:
+            client.terminate_pod(pid)
+            reaped += 1
+            print(f"[runpod-reaper] terminated STALE pod {pid} ({name}) — "
+                  "not tracked by any pool", flush=True)
+        except Exception as exc:
+            print(f"[runpod-reaper] failed to terminate {pid}: {exc}", flush=True)
+    return reaped
+
+
 def _scaler_loop():
+    tick = 0
+    # Reap once promptly on start to catch orphans left by a crash/restart.
+    try:
+        reap_orphans()
+    except Exception as exc:
+        print(f"[runpod-reaper] startup: {exc}", flush=True)
     while True:
         time.sleep(15)
+        tick += 1
         try:
             with _pools_lock:
                 pools = list(_pools.values())
@@ -535,6 +639,12 @@ def _scaler_loop():
                     print(f"[runpod] scaler: {exc}", flush=True)
         except Exception:
             pass
+        # Independent stale-pod safety sweep every ~30s, even with no local pools.
+        if tick % 2 == 0:
+            try:
+                reap_orphans()
+            except Exception as exc:
+                print(f"[runpod-reaper] {exc}", flush=True)
 
 
 def _ensure_scaler():
@@ -546,6 +656,13 @@ def _ensure_scaler():
     threading.Thread(target=_scaler_loop, daemon=True, name="runpod-scaler").start()
 
 
+def start_runpod_maintenance():
+    """Start the scaler + stale-pod reaper independently of any loaded model — call
+    at engine startup when RunPod is enabled so orphaned pods from a previous run are
+    reaped even before the first request."""
+    _ensure_scaler()
+
+
 def stop_all_pods():
     with _pools_lock:
         pools = list(_pools.values())

codai/backends/runpod.py

@@ -141,7 +141,9 @@ class RunpodBackend(ModelBackend):
         if not self._pool:
             raise RuntimeError("RunPod pod pool not initialised")
         pod, url = self._pool.acquire()
-        return url, (lambda: self._pool.release(pod))
+        # The pod runs a vLLM OpenAI server at /v1; its proxy URL has no path, so
+        # add /v1 to match the serverless base (which already ends in /openai/v1).
+        return url.rstrip("/") + "/v1", (lambda: self._pool.release(pod))
 
     def _store_usage(self, usage: dict) -> None:
         if usage:

codai/config.py

@@ -577,6 +577,9 @@ class RunpodConfig:
     serverless_base: str = "https://api.runpod.ai/v2"  # serverless invoke base
     default_gpu_type: str = ""            # fallback RunPod gpuTypeId when a model gives none
     data_center: str = ""                 # optional data-center id filter (blank = any)
+    # Stable tag baked into every pod name (coderai-<deployment_id>-<model>-<rand>) so the
+    # reaper can identify OUR pods across restarts and never touch another deployment's.
+    deployment_id: str = "default"
     # Account-wide safety net: the scaler will not let the SUM of all running RunPod
     # pods' hourly cost exceed this. 0 = no global cap (per-model $/hr still applies).
     global_max_hourly_usd: float = 0.0
@@ -1138,6 +1141,7 @@ class ConfigManager:
                 "serverless_base": self.config.runpod.serverless_base,
                 "default_gpu_type": self.config.runpod.default_gpu_type,
                 "data_center": self.config.runpod.data_center,
+                "deployment_id": self.config.runpod.deployment_id,
                 "global_max_hourly_usd": self.config.runpod.global_max_hourly_usd,
                 "global_cost_limit_usd": self.config.runpod.global_cost_limit_usd,
                 "global_cost_period": self.config.runpod.global_cost_period,

codai/main.py

@@ -1375,6 +1375,20 @@ def main():
         except ValueError:
             pass
 
+    # RunPod: on the PRIMARY engine (which hosts runpod pods), start the scaler +
+    # the independent stale-pod reaper as soon as RunPod is enabled — so pods
+    # orphaned by a previous crash/restart are terminated even before the first
+    # request. (The reaper only touches pods tagged with our deployment_id and not
+    # tracked by a live pool, and runs in the same process as the pools.)
+    try:
+        if (os.environ.get("CODERAI_ENGINE_PRIMARY") == "1"
+                and getattr(config, "runpod", None) and config.runpod.enabled):
+            from codai.api import runpod_worker
+            runpod_worker.start_runpod_maintenance()
+            print("[runpod] maintenance started (pod scaler + stale-pod reaper)", flush=True)
+    except Exception as _e:
+        print(f"[runpod] failed to start maintenance: {_e}", flush=True)
+
     # Configure Python logging so broker/API log calls reach the terminal.
     # uvicorn is started with log_config=None to keep our config in place.
     _log_level = logging.DEBUG if global_debug else logging.INFO