Skip to content

C
coderai
Commit 7c057af2 authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

runpod(phase1): account config + GraphQL client + admin key/GPU-catalog plumbing 

First slice of the RunPod remote-GPU backend (rent cloud GPUs, proxy inference).
Dormant unless enabled — no request path yet.

- config.py: RunpodConfig (enabled/api_key/cloud_type/api_base/rest_base/
  serverless_base/default_gpu_type/data_center/global_max_hourly_usd); registered,
  loaded, serialized. get_active_runpod_config() in manager.py.
- codai/api/runpod_client.py: GraphQL client — ping, list_gpu_types (+prices),
  pick_gpu (cheapest under a $/hr ceiling, VRAM filter, spot-aware), create_pod
  (on-demand + interruptible/spot), get_pod, stop/terminate, list_pods, wait_ready.
  API key via ?api_key=, masked in all error strings.
- admin: settings GET/save runpod block (key masked, overwritten only on a real
  new value); GET /admin/api/runpod/gpu-types (key check + catalog) and
  /admin/api/runpod/status (pods + live $/hr).
- settings.html: RunPod card (key/cloud/default-gpu/global $/hr cap/DC/advanced
  bases) + "Test key & list GPUs" button.

Decisions locked: host in primary engine (Layer-2, zero local VRAM); default
vLLM-OpenAI pod image; per-model $/hr + global budget; per-model allow_spot;
mode pods|serverless per model. TODO (request-path phase): RunPod requests must
bypass the GLOBAL concurrency gate only (QueueManager.max_parallel_requests).

Client GraphQL field names written from docs — pending live verification with a key.
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R6iugoNeEgt9PkajKrFyq4
parent 717ab1ad
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 548 additions and 4 deletions
codai/admin/routes.py
View file @ 7c057af2
......@@ -3525,6 +3525,20 @@ def build_settings_dict(c, gpu_cards):
"extra_env": c.vllm.extra_env,
"auto_build": c.vllm.auto_build,
},
"runpod": {
"enabled": c.runpod.enabled,
# API key is never returned in the clear — only whether it's set + a mask.
"api_key_set": bool(c.runpod.api_key),
"api_key_mask": (c.runpod.api_key[:4] + "…" + c.runpod.api_key[-4:])
if len(c.runpod.api_key) > 10 else ("****" if c.runpod.api_key else ""),
"cloud_type": c.runpod.cloud_type,
"api_base": c.runpod.api_base,
"rest_base": c.runpod.rest_base,
"serverless_base": c.runpod.serverless_base,
"default_gpu_type": c.runpod.default_gpu_type,
"data_center": c.runpod.data_center,
"global_max_hourly_usd": c.runpod.global_max_hourly_usd,
},
"ocr": {
"enabled": c.ocr.enabled,
"default_engine": c.ocr.default_engine,
......@@ -4040,6 +4054,32 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
if "extra_env" in d: v.extra_env = (d.get("extra_env") or "").strip()
if "auto_build" in d: v.auto_build = bool(d["auto_build"])
if "runpod" in data:
d = data["runpod"]
rp = c.runpod
rp.enabled = bool(d.get("enabled", rp.enabled))
if "api_key" in d:
newk = (d.get("api_key") or "").strip()
# Only overwrite when a real new key is supplied — the UI sends back the
# masked preview ("…") for an unchanged key, which must NOT clobber it.
if newk and "…" not in newk and newk != "****":
rp.api_key = newk
if "cloud_type" in d:
rp.cloud_type = (d.get("cloud_type") or "SECURE").strip().upper()
if "api_base" in d and (d.get("api_base") or "").strip():
rp.api_base = d["api_base"].strip()
if "rest_base" in d and (d.get("rest_base") or "").strip():
rp.rest_base = d["rest_base"].strip()
if "serverless_base" in d and (d.get("serverless_base") or "").strip():
rp.serverless_base = d["serverless_base"].strip()
if "default_gpu_type" in d:
rp.default_gpu_type = (d.get("default_gpu_type") or "").strip()
if "data_center" in d:
rp.data_center = (d.get("data_center") or "").strip()
if "global_max_hourly_usd" in d:
try: rp.global_max_hourly_usd = max(0.0, float(d.get("global_max_hourly_usd") or 0))
except (TypeError, ValueError): pass
if "ocr" in data:
d = data["ocr"]
o = c.ocr
......@@ -4151,6 +4191,54 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
return {"success": True, "warnings": _settings_warnings}
# =============================================================================
# RunPod remote-GPU backend
# =============================================================================
def _runpod_client_or_400():
"""Build a RunpodClient from the live config, or raise HTTP 400 with why."""
from codai.api.runpod_client import RunpodClient, RunpodError
cfg = config_manager.config.runpod if config_manager and config_manager.config else None
if cfg is None or not getattr(cfg, "api_key", ""):
raise HTTPException(status_code=400,
detail="RunPod API key not set (Settings → RunPod).")
try:
return RunpodClient(cfg)
except RunpodError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/admin/api/runpod/gpu-types", summary="List RunPod GPU types + prices")
async def api_runpod_gpu_types(username: str = Depends(require_admin)):
"""Probe the RunPod key and return the GPU catalog with USD/hr pricing.
This is the Phase-1 key-verification path — a 200 with a non-empty list proves
the stored API key works."""
from codai.api.runpod_client import RunpodError
client = _runpod_client_or_400()
try:
gpus = client.list_gpu_types()
except RunpodError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
return {"success": True, "gpu_types": gpus, "count": len(gpus)}
@router.get("/admin/api/runpod/status", summary="RunPod account pods + live cost")
async def api_runpod_status(username: str = Depends(require_admin)):
"""List the account's RunPod pods and the aggregate live $/hr (also a key check)."""
from codai.api.runpod_client import RunpodError
client = _runpod_client_or_400()
try:
me = client.ping()
pods = client.list_pods()
except RunpodError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
total = sum((p.get("cost_per_hr") or 0.0) for p in pods
if str(p.get("status")).upper() == "RUNNING")
return {"success": True, "account_id": me.get("id"), "pods": pods,
"total_hourly_usd": round(total, 4)}
# =============================================================================
# Archive management
# =============================================================================
......
codai/admin/templates/settings.html
View file @ 7c057af2
......@@ -813,6 +813,62 @@
</div>
</div>
<!-- RunPod (remote cloud GPUs) -->
<div class="card">
<div class="card-title">RunPod (remote GPUs)</div>
<p class="form-hint" style="margin-bottom:.6rem">Burst inference to rented GPUs on <a href="https://runpod.io" target="_blank" rel="noopener">RunPod</a> — as <b>Pods</b> (coderai provisions, load-balances and autoscales them) or <b>Serverless</b> endpoints (RunPod autoscales). Uses no local VRAM: coderai proxies OpenAI requests to the remote GPU. Set the account key here, then tag a model in the <b>Models</b> list with <code>backend: runpod</code> and its per-model pod/serverless block. These are the account-wide settings shared by every RunPod model.</p>
<div class="form-row">
<label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
<input type="checkbox" id="s-runpod-enabled" onchange="toggleRunpodFields()">
<span style="font-size:13px;font-weight:500">Enable RunPod</span>
</label>
</div>
<div id="runpod-fields" style="display:none">
<div class="form-group">
<label class="form-label">API key</label>
<input type="password" id="s-runpod-api-key" class="form-input" autocomplete="off" placeholder="(unchanged)">
<span class="form-hint" id="s-runpod-key-hint">Your RunPod API key. Stored server-side, never logged. Leave blank to keep the current key.</span>
</div>
<div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
<div class="form-group" style="flex:1;min-width:150px">
<label class="form-label">Cloud type</label>
<select id="s-runpod-cloud" class="form-input">
<option value="SECURE">Secure</option>
<option value="COMMUNITY">Community (cheaper)</option>
</select>
</div>
<div class="form-group" style="flex:1;min-width:180px">
<label class="form-label">Default GPU type <span class="muted">(optional)</span></label>
<input type="text" id="s-runpod-default-gpu" class="form-input" placeholder="e.g. NVIDIA GeForce RTX 4090">
</div>
<div class="form-group" style="flex:1;min-width:140px">
<label class="form-label">Global $/hr cap</label>
<input type="number" id="s-runpod-budget" class="form-input" min="0" step="0.1" placeholder="0 = no cap">
</div>
</div>
<div class="form-group">
<label class="form-label">Data center <span class="muted">(optional filter)</span></label>
<input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
</div>
<details style="margin:.4rem 0">
<summary style="cursor:pointer;font-size:12px" class="muted">Advanced API endpoints</summary>
<div class="form-group" style="margin-top:.5rem">
<label class="form-label">GraphQL base</label>
<input type="text" id="s-runpod-api-base" class="form-input" placeholder="https://api.runpod.io/graphql">
</div>
<div class="form-row" style="display:flex;gap:1rem;flex-wrap:wrap">
<div class="form-group" style="flex:1;min-width:200px"><label class="form-label">REST base</label><input type="text" id="s-runpod-rest-base" class="form-input" placeholder="https://rest.runpod.io/v1"></div>
<div class="form-group" style="flex:1;min-width:200px"><label class="form-label">Serverless base</label><input type="text" id="s-runpod-serverless-base" class="form-input" placeholder="https://api.runpod.ai/v2"></div>
</div>
</details>
<div class="form-row" style="display:flex;gap:.6rem;align-items:center;flex-wrap:wrap">
<button type="button" class="btn btn-secondary" onclick="runpodTest()">Test key &amp; list GPUs</button>
<span id="s-runpod-test-status" class="form-hint"></span>
</div>
<div id="s-runpod-gpu-list" style="margin-top:.6rem;font-size:12px;max-height:220px;overflow:auto"></div>
</div>
</div>
<!-- OCR (dedicated document OCR engines) -->
<div class="card">
<div class="card-title">OCR (document → text)</div>
......@@ -1134,6 +1190,39 @@ function toggleVllmFields(){
document.getElementById('vllm-fields').style.display =
document.getElementById('s-vllm-enabled').checked ? 'block' : 'none';
}
function toggleRunpodFields(){
document.getElementById('runpod-fields').style.display =
document.getElementById('s-runpod-enabled').checked ? 'block' : 'none';
}
async function runpodTest(){
const st = document.getElementById('s-runpod-test-status');
const box = document.getElementById('s-runpod-gpu-list');
st.textContent = 'saving settings + querying RunPod…'; st.style.color = '';
box.innerHTML = '';
try{
// Persist current settings first so the key just typed is used by the probe.
await saveSettings(true);
const r = await fetch(ROOT_PATH + '/admin/api/runpod/gpu-types');
const d = await r.json();
if(!r.ok){ throw new Error(d.detail || ('HTTP ' + r.status)); }
const gpus = (d.gpu_types || []).filter(g => g.on_demand_price != null)
.sort((a,b) => (a.on_demand_price||1e9) - (b.on_demand_price||1e9));
st.textContent = '✓ key OK — ' + d.count + ' GPU types';
st.style.color = '#2e7d32';
if(gpus.length){
let html = '<table style="width:100%;border-collapse:collapse"><tr style="text-align:left">'
+ '<th>GPU</th><th>VRAM</th><th>On-demand $/hr</th><th>Spot $/hr</th></tr>';
for(const g of gpus){
html += '<tr><td>' + (g.display_name||g.id) + '</td><td>' + (g.memory_gb ?? '?') + 'GB</td>'
+ '<td>' + (g.on_demand_price!=null ? '$'+g.on_demand_price : '—') + '</td>'
+ '<td>' + (g.spot_price!=null ? '$'+g.spot_price : '—') + '</td></tr>';
}
box.innerHTML = html + '</table>';
}
}catch(e){
st.textContent = '✗ ' + e.message; st.style.color = '#c62828';
}
}
// ---- OCR isolated-venv build (paddle/surya) ----
const _ocrBuildPollers = {};
......@@ -1662,6 +1751,22 @@ async function loadSettings(){
document.getElementById('s-vllm-auto-build').checked = !!vll.auto_build;
toggleVllmFields();
// RunPod remote GPUs
const rp = d.runpod || {};
document.getElementById('s-runpod-enabled').checked = !!rp.enabled;
document.getElementById('s-runpod-api-key').value = '';
document.getElementById('s-runpod-key-hint').textContent = rp.api_key_set
? ('Current key: ' + (rp.api_key_mask || 'set') + ' — leave blank to keep it.')
: 'Your RunPod API key. Stored server-side, never logged.';
document.getElementById('s-runpod-cloud').value = rp.cloud_type || 'SECURE';
document.getElementById('s-runpod-default-gpu').value = rp.default_gpu_type ?? '';
document.getElementById('s-runpod-budget').value = rp.global_max_hourly_usd ?? 0;
document.getElementById('s-runpod-dc').value = rp.data_center ?? '';
document.getElementById('s-runpod-api-base').value = rp.api_base ?? '';
document.getElementById('s-runpod-rest-base').value = rp.rest_base ?? '';
document.getElementById('s-runpod-serverless-base').value = rp.serverless_base ?? '';
toggleRunpodFields();
// OCR subsystem
const ocr = d.ocr || {};
document.getElementById('s-ocr-enabled').checked = !!ocr.enabled;
......@@ -1702,7 +1807,7 @@ async function loadSettings(){
}catch(e){ showAlert('error','Failed to load settings: '+e.message); }
}
async function saveSettings(){
async function saveSettings(silent){
const strOrNull = id => document.getElementById(id).value.trim() || null;
const data = {
server:{
......@@ -1840,6 +1945,18 @@ async function saveSettings(){
extra_env: document.getElementById('s-vllm-extra-env').value.trim(),
auto_build: document.getElementById('s-vllm-auto-build').checked,
},
runpod:{
enabled: document.getElementById('s-runpod-enabled').checked,
// Blank = keep the current key (backend ignores empty / masked values).
api_key: document.getElementById('s-runpod-api-key').value.trim(),
cloud_type: document.getElementById('s-runpod-cloud').value || 'SECURE',
default_gpu_type: document.getElementById('s-runpod-default-gpu').value.trim(),
global_max_hourly_usd: parseFloat(document.getElementById('s-runpod-budget').value) || 0,
data_center: document.getElementById('s-runpod-dc').value.trim(),
api_base: document.getElementById('s-runpod-api-base').value.trim(),
rest_base: document.getElementById('s-runpod-rest-base').value.trim(),
serverless_base: document.getElementById('s-runpod-serverless-base').value.trim(),
},
ocr:{
enabled: document.getElementById('s-ocr-enabled').checked,
default_engine: document.getElementById('s-ocr-default-engine').value || 'paddle',
......@@ -1907,14 +2024,15 @@ async function saveSettings(){
});
if(r.ok){
const d = await r.json().catch(()=>({}));
if(d.warnings && d.warnings.length){
if(silent){ /* caller (e.g. runpodTest) handles its own status */ }
else if(d.warnings && d.warnings.length){
showAlert('error', 'Saved, but: ' + d.warnings.join(' '));
} else {
showAlert('info','Settings saved. Archive and thermal-protection changes take effect immediately; restart CoderAI for other changes.');
}
}
else{ const e=await r.json(); showAlert('error', e.detail||'Save failed'); }
}catch(e){ showAlert('error','Error: '+e.message); }
else{ const e=await r.json(); if(!silent) showAlert('error', e.detail||'Save failed'); else throw new Error(e.detail||'Save failed'); }
}catch(e){ if(!silent) showAlert('error','Error: '+e.message); else throw e; }
}
loadSettings();
......
codai/api/runpod_client.py 0 → 100644
View file @ 7c057af2
This diff is collapsed.
codai/config.py
View file @ 7c057af2
......@@ -554,6 +554,34 @@ class VllmConfig:
auto_build: bool = False # create the isolated venv + pip install vllm if missing
@dataclass
class RunpodConfig:
"""RunPod remote-GPU backend — account-level settings.
RunPod (https://runpod.io) rents GPUs by the second, either as **Pods** (full GPU
containers coderai provisions, load-balances and scales itself) or **Serverless**
endpoints (RunPod autoscales; coderai just proxies). A model is sent to RunPod by
tagging its models.json entry with ``backend: "runpod"`` plus a per-model ``runpod``
block (mode/gpu/price/scaling — see codai.api.runpod_worker). This object holds only
the account-wide settings shared by every RunPod model.
The RunPod backend uses NO local VRAM: it runs as a thin HTTP proxy inside the
primary engine (like ds4/kt) and forwards OpenAI requests to the remote pod/endpoint
(:mod:`codai.backends.runpod`, :mod:`codai.api.runpod_worker`).
"""
enabled: bool = False
api_key: str = "" # RunPod API key (masked in the UI, never logged)
cloud_type: str = "SECURE" # SECURE | COMMUNITY (default; per-model can override)
api_base: str = "https://api.runpod.io/graphql" # GraphQL endpoint (override for testing)
rest_base: str = "https://rest.runpod.io/v1" # REST endpoint (pod lifecycle)
serverless_base: str = "https://api.runpod.ai/v2" # serverless invoke base
default_gpu_type: str = "" # fallback RunPod gpuTypeId when a model gives none
data_center: str = "" # optional data-center id filter (blank = any)
# Account-wide safety net: the scaler will not let the SUM of all running RunPod
# pods' hourly cost exceed this. 0 = no global cap (per-model $/hr still applies).
global_max_hourly_usd: float = 0.0
@dataclass
class OcrConfig:
"""Dedicated OCR subsystem configuration.
......@@ -656,6 +684,7 @@ class Config:
k3: K3Config = field(default_factory=K3Config)
ktransformers: KtransformersConfig = field(default_factory=KtransformersConfig)
vllm: VllmConfig = field(default_factory=VllmConfig)
runpod: RunpodConfig = field(default_factory=RunpodConfig)
ocr: OcrConfig = field(default_factory=OcrConfig)
compaction: CompactionConfig = field(default_factory=CompactionConfig)
broker: BrokerConfig = field(default_factory=BrokerConfig)
......@@ -846,6 +875,7 @@ class ConfigManager:
k3=_dc(K3Config, config_data.get("k3", {})),
ktransformers=_dc(KtransformersConfig, config_data.get("ktransformers", {})),
vllm=_dc(VllmConfig, config_data.get("vllm", {})),
runpod=_dc(RunpodConfig, config_data.get("runpod", {})),
ocr=_dc(OcrConfig, config_data.get("ocr", {})),
compaction=_dc(CompactionConfig, config_data.get("compaction", {})),
broker=_dc(BrokerConfig, config_data.get("broker", {})),
......@@ -1094,6 +1124,17 @@ class ConfigManager:
"extra_env": self.config.vllm.extra_env,
"auto_build": self.config.vllm.auto_build,
},
"runpod": {
"enabled": self.config.runpod.enabled,
"api_key": self.config.runpod.api_key,
"cloud_type": self.config.runpod.cloud_type,
"api_base": self.config.runpod.api_base,
"rest_base": self.config.runpod.rest_base,
"serverless_base": self.config.runpod.serverless_base,
"default_gpu_type": self.config.runpod.default_gpu_type,
"data_center": self.config.runpod.data_center,
"global_max_hourly_usd": self.config.runpod.global_max_hourly_usd,
},
"ocr": {
"enabled": self.config.ocr.enabled,
"default_engine": self.config.ocr.default_engine,
......
codai/models/manager.py
View file @ 7c057af2
......@@ -91,6 +91,17 @@ def get_active_vllm_config():
return None
def get_active_runpod_config():
"""Return the active RunpodConfig from the server config, or None."""
try:
from codai.admin.routes import config_manager
if config_manager is not None and config_manager.config is not None:
return config_manager.config.runpod
except Exception:
pass
return None
_GGUF_ARCH_CACHE: Dict[tuple, str] = {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment