Skip to content

C
coderai
Commit 7c057af2 authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

runpod(phase1): account config + GraphQL client + admin key/GPU-catalog plumbing 

First slice of the RunPod remote-GPU backend (rent cloud GPUs, proxy inference).
Dormant unless enabled — no request path yet.

- config.py: RunpodConfig (enabled/api_key/cloud_type/api_base/rest_base/
  serverless_base/default_gpu_type/data_center/global_max_hourly_usd); registered,
  loaded, serialized. get_active_runpod_config() in manager.py.
- codai/api/runpod_client.py: GraphQL client — ping, list_gpu_types (+prices),
  pick_gpu (cheapest under a $/hr ceiling, VRAM filter, spot-aware), create_pod
  (on-demand + interruptible/spot), get_pod, stop/terminate, list_pods, wait_ready.
  API key via ?api_key=, masked in all error strings.
- admin: settings GET/save runpod block (key masked, overwritten only on a real
  new value); GET /admin/api/runpod/gpu-types (key check + catalog) and
  /admin/api/runpod/status (pods + live $/hr).
- settings.html: RunPod card (key/cloud/default-gpu/global $/hr cap/DC/advanced
  bases) + "Test key & list GPUs" button.

Decisions locked: host in primary engine (Layer-2, zero local VRAM); default
vLLM-OpenAI pod image; per-model $/hr + global budget; per-model allow_spot;
mode pods|serverless per model. TODO (request-path phase): RunPod requests must
bypass the GLOBAL concurrency gate only (QueueManager.max_parallel_requests).

Client GraphQL field names written from docs — pending live verification with a key.
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R6iugoNeEgt9PkajKrFyq4
parent 717ab1ad
Show whitespace changes
Inline Side-by-side
Showing with 548 additions and 4 deletions
codai/admin/routes.py
View file @ 7c057af2
......@@ -3525,6 +3525,20 @@ def build_settings_dict(c, gpu_cards):
"extra_env": c.vllm.extra_env,
"auto_build": c.vllm.auto_build,
},
"runpod": {
"enabled": c.runpod.enabled,
# API key is never returned in the clear — only whether it's set + a mask.
"api_key_set": bool(c.runpod.api_key),
"api_key_mask": (c.runpod.api_key[:4] + "…" + c.runpod.api_key[-4:])
if len(c.runpod.api_key) > 10 else ("****" if c.runpod.api_key else ""),
"cloud_type": c.runpod.cloud_type,
"api_base": c.runpod.api_base,
"rest_base": c.runpod.rest_base,
"serverless_base": c.runpod.serverless_base,
"default_gpu_type": c.runpod.default_gpu_type,
"data_center": c.runpod.data_center,
"global_max_hourly_usd": c.runpod.global_max_hourly_usd,
},
"ocr": {
"enabled": c.ocr.enabled,
"default_engine": c.ocr.default_engine,
......@@ -4040,6 +4054,32 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
if "extra_env" in d: v.extra_env = (d.get("extra_env") or "").strip()
if "auto_build" in d: v.auto_build = bool(d["auto_build"])
if "runpod" in data:
d = data["runpod"]
rp = c.runpod
rp.enabled = bool(d.get("enabled", rp.enabled))
if "api_key" in d:
newk = (d.get("api_key") or "").strip()
# Only overwrite when a real new key is supplied — the UI sends back the
# masked preview ("…") for an unchanged key, which must NOT clobber it.
if newk and "…" not in newk and newk != "****":
rp.api_key = newk
if "cloud_type" in d:
rp.cloud_type = (d.get("cloud_type") or "SECURE").strip().upper()
if "api_base" in d and (d.get("api_base") or "").strip():
rp.api_base = d["api_base"].strip()
if "rest_base" in d and (d.get("rest_base") or "").strip():
rp.rest_base = d["rest_base"].strip()
if "serverless_base" in d and (d.get("serverless_base") or "").strip():
rp.serverless_base = d["serverless_base"].strip()
if "default_gpu_type" in d:
rp.default_gpu_type = (d.get("default_gpu_type") or "").strip()
if "data_center" in d:
rp.data_center = (d.get("data_center") or "").strip()
if "global_max_hourly_usd" in d:
try: rp.global_max_hourly_usd = max(0.0, float(d.get("global_max_hourly_usd") or 0))
except (TypeError, ValueError): pass
if "ocr" in data:
d = data["ocr"]
o = c.ocr
......@@ -4151,6 +4191,54 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
return {"success": True, "warnings": _settings_warnings}
# =============================================================================
# RunPod remote-GPU backend
# =============================================================================
def _runpod_client_or_400():
"""Build a RunpodClient from the live config, or raise HTTP 400 with why."""
from codai.api.runpod_client import RunpodClient, RunpodError
cfg = config_manager.config.runpod if config_manager and config_manager.config else None
if cfg is None or not getattr(cfg, "api_key", ""):
raise HTTPException(status_code=400,
detail="RunPod API key not set (Settings → RunPod).")
try:
return RunpodClient(cfg)
except RunpodError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/admin/api/runpod/gpu-types", summary="List RunPod GPU types + prices")
async def api_runpod_gpu_types(username: str = Depends(require_admin)):
"""Probe the RunPod key and return the GPU catalog with USD/hr pricing.
This is the Phase-1 key-verification path — a 200 with a non-empty list proves
the stored API key works."""
from codai.api.runpod_client import RunpodError
client = _runpod_client_or_400()
try:
gpus = client.list_gpu_types()
except RunpodError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
return {"success": True, "gpu_types": gpus, "count": len(gpus)}
@router.get("/admin/api/runpod/status", summary="RunPod account pods + live cost")
async def api_runpod_status(username: str = Depends(require_admin)):
"""List the account's RunPod pods and the aggregate live $/hr (also a key check)."""
from codai.api.runpod_client import RunpodError
client = _runpod_client_or_400()
try:
me = client.ping()
pods = client.list_pods()
except RunpodError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
total = sum((p.get("cost_per_hr") or 0.0) for p in pods
if str(p.get("status")).upper() == "RUNNING")
return {"success": True, "account_id": me.get("id"), "pods": pods,
"total_hourly_usd": round(total, 4)}
# =============================================================================
# Archive management
# =============================================================================
......
codai/admin/templates/settings.html
View file @ 7c057af2
......@@ -813,6 +813,62 @@
</div>
</div>
<!-- RunPod (remote cloud GPUs) -->
<div class="card">
<div class="card-title">RunPod (remote GPUs)</div>
<p class="form-hint" style="margin-bottom:.6rem">Burst inference to rented GPUs on <a href="https://runpod.io" target="_blank" rel="noopener">RunPod</a> — as <b>Pods</b> (coderai provisions, load-balances and autoscales them) or <b>Serverless</b> endpoints (RunPod autoscales). Uses no local VRAM: coderai proxies OpenAI requests to the remote GPU. Set the account key here, then tag a model in the <b>Models</b> list with <code>backend: runpod</code> and its per-model pod/serverless block. These are the account-wide settings shared by every RunPod model.</p>
<div class="form-row">
<label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
<input type="checkbox" id="s-runpod-enabled" onchange="toggleRunpodFields()">
<span style="font-size:13px;font-weight:500">Enable RunPod</span>
</label>
</div>
<div id="runpod-fields" style="display:none">
<div class="form-group">
<label class="form-label">API key</label>
<input type="password" id="s-runpod-api-key" class="form-input" autocomplete="off" placeholder="(unchanged)">
<span class="form-hint" id="s-runpod-key-hint">Your RunPod API key. Stored server-side, never logged. Leave blank to keep the current key.</span>
</div>
<div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
<div class="form-group" style="flex:1;min-width:150px">
<label class="form-label">Cloud type</label>
<select id="s-runpod-cloud" class="form-input">
<option value="SECURE">Secure</option>
<option value="COMMUNITY">Community (cheaper)</option>
</select>
</div>
<div class="form-group" style="flex:1;min-width:180px">
<label class="form-label">Default GPU type <span class="muted">(optional)</span></label>
<input type="text" id="s-runpod-default-gpu" class="form-input" placeholder="e.g. NVIDIA GeForce RTX 4090">
</div>
<div class="form-group" style="flex:1;min-width:140px">
<label class="form-label">Global $/hr cap</label>
<input type="number" id="s-runpod-budget" class="form-input" min="0" step="0.1" placeholder="0 = no cap">
</div>
</div>
<div class="form-group">
<label class="form-label">Data center <span class="muted">(optional filter)</span></label>
<input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
</div>
<details style="margin:.4rem 0">
<summary style="cursor:pointer;font-size:12px" class="muted">Advanced API endpoints</summary>
<div class="form-group" style="margin-top:.5rem">
<label class="form-label">GraphQL base</label>
<input type="text" id="s-runpod-api-base" class="form-input" placeholder="https://api.runpod.io/graphql">
</div>
<div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
<div class="form-group" style="flex:1;min-width:200px"><label class="form-label">REST base</label><input type="text" id="s-runpod-rest-base" class="form-input" placeholder="https://rest.runpod.io/v1"></div>
<div class="form-group" style="flex:1;min-width:200px"><label class="form-label">Serverless base</label><input type="text" id="s-runpod-serverless-base" class="form-input" placeholder="https://api.runpod.ai/v2"></div>
</div>
</details>
<div class="form-row" style="display:flex;gap:.6rem;align-items:center;flex-wrap:wrap">
<button type="button" class="btn btn-secondary" onclick="runpodTest()">Test key &amp; list GPUs</button>
<span id="s-runpod-test-status" class="form-hint"></span>
</div>
<div id="s-runpod-gpu-list" style="margin-top:.6rem;font-size:12px;max-height:220px;overflow:auto"></div>
</div>
</div>
<!-- OCR (dedicated document OCR engines) -->
<div class="card">
<div class="card-title">OCR (document → text)</div>
......@@ -1134,6 +1190,39 @@ function toggleVllmFields(){
document.getElementById('vllm-fields').style.display =
document.getElementById('s-vllm-enabled').checked ? 'block' : 'none';
}
function toggleRunpodFields(){
document.getElementById('runpod-fields').style.display =
document.getElementById('s-runpod-enabled').checked ? 'block' : 'none';
}
async function runpodTest(){
const st = document.getElementById('s-runpod-test-status');
const box = document.getElementById('s-runpod-gpu-list');
st.textContent = 'saving settings + querying RunPod…'; st.style.color = '';
box.innerHTML = '';
try{
// Persist current settings first so the key just typed is used by the probe.
await saveSettings(true);
const r = await fetch(ROOT_PATH + '/admin/api/runpod/gpu-types');
const d = await r.json();
if(!r.ok){ throw new Error(d.detail || ('HTTP ' + r.status)); }
const gpus = (d.gpu_types || []).filter(g => g.on_demand_price != null)
.sort((a,b) => (a.on_demand_price||1e9) - (b.on_demand_price||1e9));
st.textContent = '✓ key OK — ' + d.count + ' GPU types';
st.style.color = '#2e7d32';
if(gpus.length){
let html = '<table style="width:100%;border-collapse:collapse"><tr style="text-align:left">'
+ '<th>GPU</th><th>VRAM</th><th>On-demand $/hr</th><th>Spot $/hr</th></tr>';
for(const g of gpus){
html += '<tr><td>' + (g.display_name||g.id) + '</td><td>' + (g.memory_gb ?? '?') + 'GB</td>'
+ '<td>' + (g.on_demand_price!=null ? '$'+g.on_demand_price : '—') + '</td>'
+ '<td>' + (g.spot_price!=null ? '$'+g.spot_price : '—') + '</td></tr>';
}
box.innerHTML = html + '</table>';
}
}catch(e){
st.textContent = '✗ ' + e.message; st.style.color = '#c62828';
}
}
// ---- OCR isolated-venv build (paddle/surya) ----
const _ocrBuildPollers = {};
......@@ -1662,6 +1751,22 @@ async function loadSettings(){
document.getElementById('s-vllm-auto-build').checked = !!vll.auto_build;
toggleVllmFields();
// RunPod remote GPUs
const rp = d.runpod || {};
document.getElementById('s-runpod-enabled').checked = !!rp.enabled;
document.getElementById('s-runpod-api-key').value = '';
document.getElementById('s-runpod-key-hint').textContent = rp.api_key_set
? ('Current key: ' + (rp.api_key_mask || 'set') + ' — leave blank to keep it.')
: 'Your RunPod API key. Stored server-side, never logged.';
document.getElementById('s-runpod-cloud').value = rp.cloud_type || 'SECURE';
document.getElementById('s-runpod-default-gpu').value = rp.default_gpu_type ?? '';
document.getElementById('s-runpod-budget').value = rp.global_max_hourly_usd ?? 0;
document.getElementById('s-runpod-dc').value = rp.data_center ?? '';
document.getElementById('s-runpod-api-base').value = rp.api_base ?? '';
document.getElementById('s-runpod-rest-base').value = rp.rest_base ?? '';
document.getElementById('s-runpod-serverless-base').value = rp.serverless_base ?? '';
toggleRunpodFields();
// OCR subsystem
const ocr = d.ocr || {};
document.getElementById('s-ocr-enabled').checked = !!ocr.enabled;
......@@ -1702,7 +1807,7 @@ async function loadSettings(){
}catch(e){ showAlert('error','Failed to load settings: '+e.message); }
}
async function saveSettings(){
async function saveSettings(silent){
const strOrNull = id => document.getElementById(id).value.trim() || null;
const data = {
server:{
......@@ -1840,6 +1945,18 @@ async function saveSettings(){
extra_env: document.getElementById('s-vllm-extra-env').value.trim(),
auto_build: document.getElementById('s-vllm-auto-build').checked,
},
runpod:{
enabled: document.getElementById('s-runpod-enabled').checked,
// Blank = keep the current key (backend ignores empty / masked values).
api_key: document.getElementById('s-runpod-api-key').value.trim(),
cloud_type: document.getElementById('s-runpod-cloud').value || 'SECURE',
default_gpu_type: document.getElementById('s-runpod-default-gpu').value.trim(),
global_max_hourly_usd: parseFloat(document.getElementById('s-runpod-budget').value) || 0,
data_center: document.getElementById('s-runpod-dc').value.trim(),
api_base: document.getElementById('s-runpod-api-base').value.trim(),
rest_base: document.getElementById('s-runpod-rest-base').value.trim(),
serverless_base: document.getElementById('s-runpod-serverless-base').value.trim(),
},
ocr:{
enabled: document.getElementById('s-ocr-enabled').checked,
default_engine: document.getElementById('s-ocr-default-engine').value || 'paddle',
......@@ -1907,14 +2024,15 @@ async function saveSettings(){
});
if(r.ok){
const d = await r.json().catch(()=>({}));
if(d.warnings && d.warnings.length){
if(silent){ /* caller (e.g. runpodTest) handles its own status */ }
else if(d.warnings && d.warnings.length){
showAlert('error', 'Saved, but: ' + d.warnings.join(' '));
} else {
showAlert('info','Settings saved. Archive and thermal-protection changes take effect immediately; restart CoderAI for other changes.');
}
}
else{ const e=await r.json(); showAlert('error', e.detail||'Save failed'); }
}catch(e){ showAlert('error','Error: '+e.message); }
else{ const e=await r.json(); if(!silent) showAlert('error', e.detail||'Save failed'); else throw new Error(e.detail||'Save failed'); }
}catch(e){ if(!silent) showAlert('error','Error: '+e.message); else throw e; }
}
loadSettings();
......
codai/api/runpod_client.py 0 → 100644
View file @ 7c057af2
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
"""Thin RunPod API client (GPU catalog + Pod lifecycle).
coderai talks to RunPod over its GraphQL API (https://graphql-spec.runpod.io) — the most
stable surface for GPU pricing and pod create/query/stop/terminate. Serverless invocation
uses the separate ``/v2/{endpoint}`` REST base and is handled in the worker, not here.
This module is pure transport: no scaling/policy logic (that lives in
:mod:`codai.api.runpod_worker`). The API key comes from the caller (RunpodConfig) and is
NEVER logged — only masked in error strings.
"""
import time
from typing import Optional
class RunpodError(RuntimeError):
"""A RunPod API call failed (network, auth, or GraphQL error)."""
def _mask(key: str) -> str:
if not key:
return "<none>"
return (key[:4] + "…" + key[-4:]) if len(key) > 10 else "****"
# A pod's public HTTP proxy hostname is derived from its id + the container port.
def pod_proxy_url(pod_id: str, port: int) -> str:
return f"https://{pod_id}-{int(port)}.proxy.runpod.net"
class RunpodClient:
def __init__(self, cfg):
self._cfg = cfg
self._api_key = (getattr(cfg, "api_key", "") or "").strip()
self._gql_base = (getattr(cfg, "api_base", "") or "https://api.runpod.io/graphql").strip()
if not self._api_key:
raise RunpodError("RunPod API key not configured (config.runpod.api_key).")
# ------------------------------------------------------------------ #
# transport
# ------------------------------------------------------------------ #
def _gql(self, query: str, variables: Optional[dict] = None,
timeout: float = 30.0) -> dict:
import requests
# RunPod authenticates GraphQL via the ?api_key= query param.
url = self._gql_base + ("&" if "?" in self._gql_base else "?") + "api_key=" + self._api_key
try:
r = requests.post(url, json={"query": query, "variables": variables or {}},
timeout=timeout,
headers={"Content-Type": "application/json"})
except requests.RequestException as exc:
raise RunpodError(f"RunPod API unreachable ({_mask(self._api_key)}): {exc}")
if r.status_code == 401:
raise RunpodError(f"RunPod rejected the API key ({_mask(self._api_key)}) — 401.")
if r.status_code >= 400:
raise RunpodError(f"RunPod API HTTP {r.status_code}: {r.text[:300]}")
try:
body = r.json()
except ValueError:
raise RunpodError(f"RunPod API returned non-JSON: {r.text[:200]}")
if body.get("errors"):
msg = "; ".join(e.get("message", str(e)) for e in body["errors"])
raise RunpodError(f"RunPod GraphQL error: {msg}")
return body.get("data") or {}
def ping(self) -> dict:
"""Cheap auth check — returns {"id","email"?} for the account or raises."""
data = self._gql("query { myself { id } }", timeout=15.0)
return data.get("myself") or {}
# ------------------------------------------------------------------ #
# GPU catalog + selection
# ------------------------------------------------------------------ #
def list_gpu_types(self) -> list:
"""Return the RunPod GPU catalog with pricing, normalized:
[{id, display_name, memory_gb, secure_price, community_price,
on_demand_price, spot_price}] (prices are USD/hr for 1 GPU; None if N/A)
"""
q = """
query GpuTypes {
gpuTypes {
id
displayName
memoryInGb
securePrice
communityPrice
secureSpotPrice
communitySpotPrice
}
}"""
data = self._gql(q)
out = []
for g in (data.get("gpuTypes") or []):
secure = g.get("securePrice")
community = g.get("communityPrice")
on_demand = secure if secure is not None else community
spot = g.get("secureSpotPrice")
if spot is None:
spot = g.get("communitySpotPrice")
out.append({
"id": g.get("id"),
"display_name": g.get("displayName") or g.get("id"),
"memory_gb": g.get("memoryInGb"),
"secure_price": secure,
"community_price": community,
"on_demand_price": on_demand,
"spot_price": spot,
})
return out
def pick_gpu(self, *, min_vram_gb: float = 0.0, max_hourly_usd: float = 0.0,
allow_spot: bool = False, cloud_type: str = "SECURE",
explicit_gpu_type: str = "") -> dict:
"""Choose the cheapest GPU that meets the constraints.
Returns {gpu_type_id, display_name, memory_gb, price, is_spot} or raises
RunpodError if nothing qualifies under the price ceiling.
"""
catalog = self.list_gpu_types()
secure = (cloud_type or "SECURE").upper() == "SECURE"
def _price(g, spot):
if spot:
return g.get("spot_price")
return g.get("secure_price") if secure else g.get("community_price")
# Explicit GPU type: honor it, still enforce the price ceiling.
if explicit_gpu_type:
for g in catalog:
if g["id"] == explicit_gpu_type or g["display_name"] == explicit_gpu_type:
for spot in ((True, False) if allow_spot else (False,)):
p = _price(g, spot)
if p is not None and (max_hourly_usd <= 0 or p <= max_hourly_usd):
return {"gpu_type_id": g["id"], "display_name": g["display_name"],
"memory_gb": g["memory_gb"], "price": p, "is_spot": spot}
raise RunpodError(
f"GPU {explicit_gpu_type!r} exceeds the ${max_hourly_usd}/hr ceiling "
f"(cheapest {cloud_type} price ${_price(g, allow_spot)}).")
raise RunpodError(f"GPU type {explicit_gpu_type!r} not found in the RunPod catalog.")
# Otherwise pick cheapest qualifying by VRAM + price. Prefer spot when allowed.
candidates = []
for g in catalog:
mem = g.get("memory_gb") or 0
if min_vram_gb and mem < min_vram_gb:
continue
for spot in ((True, False) if allow_spot else (False,)):
p = _price(g, spot)
if p is None or p <= 0:
continue
if max_hourly_usd > 0 and p > max_hourly_usd:
continue
candidates.append((p, spot, g))
if not candidates:
raise RunpodError(
f"No {cloud_type} GPU with ≥{min_vram_gb}GB VRAM under "
f"${max_hourly_usd}/hr (spot={'on' if allow_spot else 'off'}).")
candidates.sort(key=lambda c: (c[0], not c[1])) # cheapest, spot-first on tie
price, spot, g = candidates[0]
return {"gpu_type_id": g["id"], "display_name": g["display_name"],
"memory_gb": g["memory_gb"], "price": price, "is_spot": spot}
# ------------------------------------------------------------------ #
# Pod lifecycle
# ------------------------------------------------------------------ #
def create_pod(self, *, name: str, image: str, gpu_type_id: str, port: int,
cloud_type: str = "SECURE", gpu_count: int = 1,
container_disk_gb: int = 40, volume_gb: int = 0,
volume_mount_path: str = "/workspace", env: Optional[dict] = None,
docker_args: str = "", is_spot: bool = False,
bid_per_gpu: float = 0.0, data_center_id: str = "") -> str:
"""Provision a pod (on-demand or interruptible/spot). Returns the pod id."""
env_list = [{"key": str(k), "value": str(v)} for k, v in (env or {}).items()]
ports = f"{int(port)}/http"
common = {
"cloudType": (cloud_type or "SECURE").upper(),
"gpuCount": int(gpu_count),
"gpuTypeId": gpu_type_id,
"name": name,
"imageName": image,
"containerDiskInGb": int(container_disk_gb),
"volumeInGb": int(volume_gb),
"volumeMountPath": volume_mount_path,
"ports": ports,
"dockerArgs": docker_args or "",
"env": env_list,
}
if data_center_id:
common["dataCenterId"] = data_center_id
if is_spot:
common["bidPerGpu"] = float(bid_per_gpu)
mutation = "podRentInterruptable"
var_type = "PodRentInterruptableInput"
else:
mutation = "podFindAndDeployOnDemand"
var_type = "PodFindAndDeployOnDemandInput"
q = f"""
mutation Deploy($input: {var_type}!) {{
{mutation}(input: $input) {{ id imageName machineId }}
}}"""
data = self._gql(q, {"input": common}, timeout=90.0)
node = data.get(mutation) or {}
pod_id = node.get("id")
if not pod_id:
raise RunpodError(f"RunPod did not return a pod id for {name!r} (capacity? {data}).")
return pod_id
def get_pod(self, pod_id: str) -> dict:
"""Return {id, status, uptime_s, cost_per_hr, ports:[...], ready:bool, url?}."""
q = """
query Pod($input: PodFilter!) {
pod(input: $input) {
id
desiredStatus
costPerHr
runtime {
uptimeInSeconds
ports { ip isIpPublic privatePort publicPort type }
}
}
}"""
data = self._gql(q, {"input": {"podId": pod_id}}, timeout=20.0)
p = data.get("pod") or {}
rt = p.get("runtime") or {}
ports = rt.get("ports") or []
return {
"id": p.get("id") or pod_id,
"status": p.get("desiredStatus"),
"uptime_s": rt.get("uptimeInSeconds") or 0,
"cost_per_hr": p.get("costPerHr"),
"ports": ports,
# A pod is "ready" for us once its http proxy port is exposed.
"ready": bool(ports),
}
def stop_pod(self, pod_id: str) -> None:
q = "mutation($input: PodStopInput!) { podStop(input: $input) { id } }"
try:
self._gql(q, {"input": {"podId": pod_id}}, timeout=30.0)
except RunpodError:
raise
def terminate_pod(self, pod_id: str) -> None:
q = "mutation($input: PodTerminateInput!) { podTerminate(input: $input) }"
self._gql(q, {"input": {"podId": pod_id}}, timeout=30.0)
def list_pods(self) -> list:
"""List the account's pods: [{id, name, status, cost_per_hr, image}]."""
q = """
query Pods {
myself {
pods { id name desiredStatus costPerHr imageName }
}
}"""
data = self._gql(q)
me = data.get("myself") or {}
return [{
"id": p.get("id"), "name": p.get("name"),
"status": p.get("desiredStatus"), "cost_per_hr": p.get("costPerHr"),
"image": p.get("imageName"),
} for p in (me.get("pods") or [])]
def wait_ready(self, pod_id: str, port: int, ready_timeout: float = 900.0,
poll_every: float = 5.0) -> str:
"""Poll until the pod exposes its proxy port; return the proxy base URL.
Only waits for the port to be exposed — the HTTP service inside (vLLM) is
health-checked separately by the worker."""
deadline = time.time() + ready_timeout
while time.time() < deadline:
info = self.get_pod(pod_id)
if info.get("status") in ("TERMINATED", "FAILED"):
raise RunpodError(f"Pod {pod_id} entered status {info.get('status')} before ready.")
if info.get("ready"):
return pod_proxy_url(pod_id, port)
time.sleep(poll_every)
raise RunpodError(f"Pod {pod_id} did not expose port {port} within {ready_timeout}s.")
codai/config.py
View file @ 7c057af2
......@@ -554,6 +554,34 @@ class VllmConfig:
auto_build: bool = False # create the isolated venv + pip install vllm if missing
@dataclass
class RunpodConfig:
"""RunPod remote-GPU backend — account-level settings.
RunPod (https://runpod.io) rents GPUs by the second, either as **Pods** (full GPU
containers coderai provisions, load-balances and scales itself) or **Serverless**
endpoints (RunPod autoscales; coderai just proxies). A model is sent to RunPod by
tagging its models.json entry with ``backend: "runpod"`` plus a per-model ``runpod``
block (mode/gpu/price/scaling — see codai.api.runpod_worker). This object holds only
the account-wide settings shared by every RunPod model.
The RunPod backend uses NO local VRAM: it runs as a thin HTTP proxy inside the
primary engine (like ds4/kt) and forwards OpenAI requests to the remote pod/endpoint
(:mod:`codai.backends.runpod`, :mod:`codai.api.runpod_worker`).
"""
enabled: bool = False
api_key: str = "" # RunPod API key (masked in the UI, never logged)
cloud_type: str = "SECURE" # SECURE | COMMUNITY (default; per-model can override)
api_base: str = "https://api.runpod.io/graphql" # GraphQL endpoint (override for testing)
rest_base: str = "https://rest.runpod.io/v1" # REST endpoint (pod lifecycle)
serverless_base: str = "https://api.runpod.ai/v2" # serverless invoke base
default_gpu_type: str = "" # fallback RunPod gpuTypeId when a model gives none
data_center: str = "" # optional data-center id filter (blank = any)
# Account-wide safety net: the scaler will not let the SUM of all running RunPod
# pods' hourly cost exceed this. 0 = no global cap (per-model $/hr still applies).
global_max_hourly_usd: float = 0.0
@dataclass
class OcrConfig:
"""Dedicated OCR subsystem configuration.
......@@ -656,6 +684,7 @@ class Config:
k3: K3Config = field(default_factory=K3Config)
ktransformers: KtransformersConfig = field(default_factory=KtransformersConfig)
vllm: VllmConfig = field(default_factory=VllmConfig)
runpod: RunpodConfig = field(default_factory=RunpodConfig)
ocr: OcrConfig = field(default_factory=OcrConfig)
compaction: CompactionConfig = field(default_factory=CompactionConfig)
broker: BrokerConfig = field(default_factory=BrokerConfig)
......@@ -846,6 +875,7 @@ class ConfigManager:
k3=_dc(K3Config, config_data.get("k3", {})),
ktransformers=_dc(KtransformersConfig, config_data.get("ktransformers", {})),
vllm=_dc(VllmConfig, config_data.get("vllm", {})),
runpod=_dc(RunpodConfig, config_data.get("runpod", {})),
ocr=_dc(OcrConfig, config_data.get("ocr", {})),
compaction=_dc(CompactionConfig, config_data.get("compaction", {})),
broker=_dc(BrokerConfig, config_data.get("broker", {})),
......@@ -1094,6 +1124,17 @@ class ConfigManager:
"extra_env": self.config.vllm.extra_env,
"auto_build": self.config.vllm.auto_build,
},
"runpod": {
"enabled": self.config.runpod.enabled,
"api_key": self.config.runpod.api_key,
"cloud_type": self.config.runpod.cloud_type,
"api_base": self.config.runpod.api_base,
"rest_base": self.config.runpod.rest_base,
"serverless_base": self.config.runpod.serverless_base,
"default_gpu_type": self.config.runpod.default_gpu_type,
"data_center": self.config.runpod.data_center,
"global_max_hourly_usd": self.config.runpod.global_max_hourly_usd,
},
"ocr": {
"enabled": self.config.ocr.enabled,
"default_engine": self.config.ocr.default_engine,
......
codai/models/manager.py
View file @ 7c057af2
......@@ -91,6 +91,17 @@ def get_active_vllm_config():
return None
def get_active_runpod_config():
"""Return the active RunpodConfig from the server config, or None."""
try:
from codai.admin.routes import config_manager
if config_manager is not None and config_manager.config is not None:
return config_manager.config.runpod
except Exception:
pass
return None
_GGUF_ARCH_CACHE: Dict[tuple, str] = {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment