Skip to content

C
coderai
Commit 17604ab4 authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

runpod(phase2): serverless request path + per-model config + no-local-download 

End-to-end serverless: a model tagged backend:runpod (mode:serverless) proxies
OpenAI chat/completions to a RunPod serverless endpoint. Runs on a remote GPU,
holds NO local weights.

- codai/api/runpod_worker.py: RunpodModelConfig + parse_model_runpod (mode/pools/
  criteria/gpu/price/scaling/cost/serverless), serverless_base_url + auth_headers.
- codai/backends/runpod.py: RunpodBackend — thin OpenAI proxy (chat/stream, Bearer
  auth for serverless), uses_local_vram=False; pods/auto raise a clear "next phase"
  error for now.
- manager.py: runpod in _ENGINE_BACKENDS + _runpod_name_claims (pin-only) +
  runpod_should_handle + load_model dispatch branch. _model_is_runpod() gate:
  set_default_model skips local download/cache for runpod-served models.
- router.py + registry.py: runpod is an explicit-pin capability; added to every
  engine's default caps (no local VRAM → any node hosts the proxy; prefers primary).
- admin routes: whitelist per-model `runpod` + `runpod_spillover` blocks; skip
  local-VRAM estimate for runpod models.
- models.html: RunPod backend option + full per-model sub-form (mode w/ explainer,
  serverless endpoint, served model, pools/criteria/gpu/price/spot, min/max pods,
  idle timeout, image/disk/port/env, spend limit+period) + prefill/collect.

Serverless verified wired in-image (ef1fd7a3802c). Pods pool/scaler, cost ledger,
spillover routing, and stats UI are later phases.
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R6iugoNeEgt9PkajKrFyq4
parent 7c057af2
Hide whitespace changes
Inline Side-by-side
Showing with 772 additions and 9 deletions
codai/admin/routes.py
View file @ 17604ab4
......@@ -2673,8 +2673,11 @@ async def api_model_configure(request: Request, username: str = Depends(require_
# Auto-estimate used_vram_gb from file size if not provided. For a not-yet-
# downloaded HF model this returns None (nothing on disk to measure); the
# download-completion backfill fills it in once the weights land.
# RunPod-served models hold NO local weights (the remote pod pulls them), so
# never estimate local VRAM for them.
_is_runpod = (str(data.get("backend") or "").strip().lower() == "runpod")
used_vram_gb = data.get("used_vram_gb")
if used_vram_gb is None:
if used_vram_gb is None and not _is_runpod:
used_vram_gb = _estimate_used_vram_gb(path)
# Build settings entry
......@@ -2754,6 +2757,82 @@ async def api_model_configure(request: Request, username: str = Depends(require_
else:
entry.pop("colibri", None)
# Per-model runpod block (mode/pools/criteria/gpu/price/scaling/cost/serverless).
# Stored as a normalized dict on the entry; the backend reads it via
# codai.api.runpod_worker.parse_model_runpod. Only meaningful for backend:runpod
# (pods|serverless|auto) or a local model's spillover target.
if "runpod" in data:
src = data.get("runpod") if isinstance(data.get("runpod"), dict) else {}
rpo = {}
# strings
for k in ("mode", "selection_criteria", "gpu_type", "image", "served_model",
"endpoint_id", "cost_period"):
v = src.get(k)
if isinstance(v, str) and v.strip():
rpo[k] = v.strip()
# cloud_types: list or comma string
ct = src.get("cloud_types")
if isinstance(ct, str):
ct = [c.strip().upper() for c in ct.split(",") if c.strip()]
elif isinstance(ct, list):
ct = [str(c).strip().upper() for c in ct if str(c).strip()]
else:
ct = None
if ct:
rpo["cloud_types"] = ct
# bools
for k in ("allow_spot",):
if k in src and src.get(k) is not None and src.get(k) != "":
sv = src.get(k)
rpo[k] = (sv.lower() in ("1", "true", "on", "yes")
if isinstance(sv, str) else bool(sv))
# ints
for k in ("container_disk_gb", "volume_gb", "port", "ctx", "min_pods",
"max_pods", "scale_up_inflight_per_pod", "idle_timeout_s",
"min_workers", "max_workers"):
v = src.get(k)
if v not in (None, ""):
try:
rpo[k] = int(v)
except (TypeError, ValueError):
pass
# floats
for k in ("min_vram_gb", "max_hourly_usd", "cost_limit_usd"):
v = src.get(k)
if v not in (None, ""):
try:
rpo[k] = float(v)
except (TypeError, ValueError):
pass
# env dict (extra pod env)
env = src.get("env")
if isinstance(env, dict) and env:
rpo["env"] = {str(k): str(v) for k, v in env.items()}
if rpo:
entry["runpod"] = rpo
else:
entry.pop("runpod", None)
# Per-model spillover-to-RunPod block for a LOCAL model (local is primary; burst
# to RunPod on the configured triggers). Kept separate from the `runpod` block so
# a model can be local-primary with a runpod overflow target.
if "runpod_spillover" in data:
src = data.get("runpod_spillover") if isinstance(data.get("runpod_spillover"), dict) else {}
sp = {}
for k in ("enabled", "on_concurrency_full", "on_no_gpu", "on_local_error"):
if k in src and src.get(k) is not None and src.get(k) != "":
sv = src.get(k)
sp[k] = (sv.lower() in ("1", "true", "on", "yes")
if isinstance(sv, str) else bool(sv))
# the runpod target spec reuses the same shape as the `runpod` block
tgt = src.get("target")
if isinstance(tgt, dict) and tgt:
sp["target"] = tgt
if sp.get("enabled"):
entry["runpod_spillover"] = sp
else:
entry.pop("runpod_spillover", None)
# A GGUF LLM is served by llama.cpp. Its multimodal projector (mmproj) gives
# it VISION INPUT, which is the `image_to_text` capability served through
# llama.cpp — NOT the diffusers `vision_models`/`image_models` categories
......
codai/admin/templates/models.html
View file @ 17604ab4
......@@ -545,7 +545,7 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
<div style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem">
<div class="form-row" style="margin:0">
<label class="form-label">Compute backend</label>
<select id="cfg-backend" class="form-input" onchange="_updateSttVisibility()">
<select id="cfg-backend" class="form-input" onchange="_updateSttVisibility();_updateRunpodVisibility()">
<option value="auto">Auto-detect</option>
<option value="nvidia">NVIDIA (CUDA)</option>
<option value="vulkan">Vulkan</option>
......@@ -556,6 +556,7 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
<option value="k3">k3 (Kimi-K3 / kimi-k3-in-c)</option>
<option value="kt">ktransformers (SGLang, CPU+GPU)</option>
<option value="vllm">vLLM (high-concurrency, CUDA)</option>
<option value="runpod">RunPod (remote cloud GPU — not stored locally)</option>
<option value="wav2vec2">wav2vec2 / HF-ASR (speech-to-text)</option>
<option value="whisper-hf">whisper-hf / CrisperWhisper (HF transformers STT)</option>
<option value="vosk">Vosk (offline STT, CPU, per-language dir)</option>
......@@ -675,6 +676,100 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
<span class="form-hint" style="font-size:11px"><code>KEY=VALUE</code> pairs for colibri's env-only tuning. Blank = inherit global.</span>
</div>
</div>
<!-- Per-model RunPod (remote GPU) config: shown when backend = runpod -->
<div id="cfg-runpod-row" style="margin-top:.75rem;display:none;border:1px solid var(--border,#333);border-radius:8px;padding:.6rem">
<div style="font-weight:600;font-size:13px;margin-bottom:.3rem">RunPod — remote GPU</div>
<div class="form-hint" style="font-size:11px;margin-bottom:.5rem">This model runs on a rented RunPod GPU and is <b>not downloaded locally</b>. Set the model <b>Path/ID</b> (top) to the served model id (e.g. an HF repo) — the remote worker pulls it. Requires RunPod enabled in Settings.</div>
<div class="form-row" style="margin:0">
<label class="form-label">Mode</label>
<select id="cfg-rp-mode" class="form-input" onchange="_updateRunpodModeHint()">
<option value="serverless">Serverless — RunPod autoscales workers; coderai proxies (simplest)</option>
<option value="pods">Pods — coderai provisions, load-balances &amp; autoscales GPU pods itself</option>
<option value="auto">Auto — pick pods vs serverless per request by the criteria below</option>
</select>
<span class="form-hint" id="cfg-rp-mode-hint" style="font-size:11px"></span>
</div>
<div class="form-row" style="margin:.5rem 0 0">
<label class="form-label">Served model <span class="muted">(HF id the remote worker loads)</span></label>
<input type="text" id="cfg-rp-served" class="form-input" placeholder="(blank = use the model Path/ID above)">
</div>
<div class="form-row" style="margin:.5rem 0 0">
<label class="form-label">Max context <span class="muted">(tokens, optional)</span></label>
<input type="number" id="cfg-rp-ctx" class="form-input" min="0" placeholder="0 = default">
</div>
<!-- Serverless-only -->
<div id="cfg-rp-serverless" style="margin-top:.5rem">
<div class="form-row" style="margin:0">
<label class="form-label">Serverless endpoint id</label>
<input type="text" id="cfg-rp-endpoint" class="form-input" placeholder="from your RunPod serverless (vLLM) endpoint">
</div>
<div style="display:grid;grid-template-columns:1fr 1fr;gap:.5rem;margin-top:.5rem">
<div class="form-row" style="margin:0"><label class="form-label">Min workers</label><input type="number" id="cfg-rp-min-workers" class="form-input" min="0" placeholder="0"></div>
<div class="form-row" style="margin:0"><label class="form-label">Max workers</label><input type="number" id="cfg-rp-max-workers" class="form-input" min="1" placeholder="1"></div>
</div>
</div>
<!-- Pods / auto -->
<div id="cfg-rp-pods" style="margin-top:.5rem;display:none">
<div class="form-row" style="margin:0">
<label class="form-label">Allowed pools</label>
<div style="display:flex;gap:1rem;flex-wrap:wrap;font-size:12px">
<label style="display:flex;align-items:center;gap:.3rem"><input type="checkbox" id="cfg-rp-pool-secure" style="width:auto" checked> Secure</label>
<label style="display:flex;align-items:center;gap:.3rem"><input type="checkbox" id="cfg-rp-pool-community" style="width:auto"> Community (cheaper)</label>
</div>
</div>
<div style="display:grid;grid-template-columns:1fr 1fr;gap:.5rem;margin-top:.5rem">
<div class="form-row" style="margin:0">
<label class="form-label">Selection</label>
<select id="cfg-rp-criteria" class="form-input">
<option value="cheaper">Cheaper — lowest $/hr</option>
<option value="faster">Faster — reliability + throughput</option>
</select>
</div>
<div class="form-row" style="margin:0">
<label class="form-label">GPU type <span class="muted">(optional)</span></label>
<input type="text" id="cfg-rp-gpu-type" class="form-input" placeholder="auto — e.g. NVIDIA GeForce RTX 4090">
</div>
</div>
<div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:.5rem;margin-top:.5rem">
<div class="form-row" style="margin:0"><label class="form-label">Min VRAM (GB)</label><input type="number" id="cfg-rp-min-vram" class="form-input" min="0" placeholder="0 = any"></div>
<div class="form-row" style="margin:0"><label class="form-label">Max $/hr</label><input type="number" id="cfg-rp-max-hourly" class="form-input" min="0" step="0.01" placeholder="0 = no cap"></div>
<div class="form-row" style="margin:0;align-self:end"><label style="display:flex;align-items:center;gap:.3rem;font-size:12px"><input type="checkbox" id="cfg-rp-spot" style="width:auto"> Allow spot</label></div>
</div>
<div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:.5rem;margin-top:.5rem">
<div class="form-row" style="margin:0"><label class="form-label">Min pods</label><input type="number" id="cfg-rp-min-pods" class="form-input" min="0" placeholder="0"></div>
<div class="form-row" style="margin:0"><label class="form-label">Max pods</label><input type="number" id="cfg-rp-max-pods" class="form-input" min="1" placeholder="1"></div>
<div class="form-row" style="margin:0"><label class="form-label">Idle timeout (s)</label><input type="number" id="cfg-rp-idle" class="form-input" min="0" placeholder="300"></div>
</div>
<div style="display:grid;grid-template-columns:2fr 1fr 1fr;gap:.5rem;margin-top:.5rem">
<div class="form-row" style="margin:0"><label class="form-label">Pod image <span class="muted">(blank = default vLLM)</span></label><input type="text" id="cfg-rp-image" class="form-input" placeholder="vllm/vllm-openai:latest"></div>
<div class="form-row" style="margin:0"><label class="form-label">Disk (GB)</label><input type="number" id="cfg-rp-disk" class="form-input" min="1" placeholder="40"></div>
<div class="form-row" style="margin:0"><label class="form-label">Port</label><input type="number" id="cfg-rp-port" class="form-input" min="1" placeholder="8000"></div>
</div>
<div class="form-row" style="margin:.5rem 0 0">
<label class="form-label">Pod env <span class="muted">(KEY=VAL, space-separated — e.g. HF_TOKEN=…)</span></label>
<input type="text" id="cfg-rp-env" class="form-input" placeholder="HF_TOKEN=hf_xxx">
</div>
</div>
<!-- Cost budget (cumulative spend cap, distinct from the $/hr rate cap) -->
<div style="display:grid;grid-template-columns:1fr 1fr;gap:.5rem;margin-top:.5rem">
<div class="form-row" style="margin:0"><label class="form-label">Spend limit ($)</label><input type="number" id="cfg-rp-cost-limit" class="form-input" min="0" step="0.01" placeholder="0 = unlimited"></div>
<div class="form-row" style="margin:0">
<label class="form-label">Per period</label>
<select id="cfg-rp-cost-period" class="form-input">
<option value="unlimited">Unlimited</option>
<option value="hour">Hour</option>
<option value="day">Day</option>
<option value="week">Week</option>
<option value="month">Month</option>
</select>
</div>
</div>
</div>
<div style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem;margin-top:.75rem">
<div class="form-row" style="margin:0">
<label class="form-label">Used VRAM <span class="muted">(GB)</span></label>
......@@ -3451,6 +3546,7 @@ function openCfgModal(idx, cfgIdx){
(s.split_secondary_cap_gb != null ? s.split_secondary_cap_gb : '');
_populateDs4(m, s);
_populateColibri(m, s);
_populateRunpod(s);
document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
document.getElementById('cfg-tools').checked = !!s.tools_closer_prompt;
......@@ -3854,6 +3950,96 @@ function _updateSttVisibility(){
if (row) row.style.display = (be === 'wav2vec2' || be === 'vosk' || be === 'nemo' || be === 'whisper-hf') ? '' : 'none';
}
// Show the RunPod block only when the backend is runpod; toggle mode sub-groups.
function _updateRunpodVisibility(){
const be = (document.getElementById('cfg-backend').value || '');
const row = document.getElementById('cfg-runpod-row');
if (row) row.style.display = (be === 'runpod') ? '' : 'none';
_updateRunpodModeHint();
}
function _updateRunpodModeHint(){
const mode = (document.getElementById('cfg-rp-mode') || {}).value || 'serverless';
const sl = document.getElementById('cfg-rp-serverless');
const pd = document.getElementById('cfg-rp-pods');
if (sl) sl.style.display = (mode === 'serverless') ? '' : 'none';
if (pd) pd.style.display = (mode === 'serverless') ? 'none' : '';
const hint = document.getElementById('cfg-rp-mode-hint');
if (hint) hint.textContent = mode === 'serverless'
? 'RunPod runs an autoscaling endpoint (min/max workers); coderai only proxies. Paste an existing serverless endpoint id below.'
: (mode === 'pods'
? 'coderai provisions GPU pods within your price/size limits, load-balances requests, and destroys idle pods.'
: 'coderai picks pods vs serverless per request using the selection criteria (e.g. serverless for a cold single request, a pod for sustained load).');
}
function _populateRunpod(s){
const rp = (s && s.runpod) || {};
const set = (id,v)=>{ const el=document.getElementById(id); if(el!=null) el.value=(v==null?'':v); };
const chk = (id,v)=>{ const el=document.getElementById(id); if(el) el.checked=!!v; };
set('cfg-rp-mode', rp.mode || 'serverless');
set('cfg-rp-served', rp.served_model || '');
set('cfg-rp-ctx', rp.ctx || '');
set('cfg-rp-endpoint', rp.endpoint_id || '');
set('cfg-rp-min-workers', rp.min_workers != null ? rp.min_workers : '');
set('cfg-rp-max-workers', rp.max_workers != null ? rp.max_workers : '');
const pools = (rp.cloud_types || ['SECURE']).map(x=>String(x).toUpperCase());
chk('cfg-rp-pool-secure', pools.includes('SECURE'));
chk('cfg-rp-pool-community', pools.includes('COMMUNITY'));
set('cfg-rp-criteria', rp.selection_criteria || 'cheaper');
set('cfg-rp-gpu-type', rp.gpu_type || '');
set('cfg-rp-min-vram', rp.min_vram_gb != null ? rp.min_vram_gb : '');
set('cfg-rp-max-hourly', rp.max_hourly_usd != null ? rp.max_hourly_usd : '');
chk('cfg-rp-spot', rp.allow_spot);
set('cfg-rp-min-pods', rp.min_pods != null ? rp.min_pods : '');
set('cfg-rp-max-pods', rp.max_pods != null ? rp.max_pods : '');
set('cfg-rp-idle', rp.idle_timeout_s != null ? rp.idle_timeout_s : '');
set('cfg-rp-image', rp.image || '');
set('cfg-rp-disk', rp.container_disk_gb != null ? rp.container_disk_gb : '');
set('cfg-rp-port', rp.port != null ? rp.port : '');
const env = rp.env || {};
set('cfg-rp-env', Object.keys(env).map(k=>k+'='+env[k]).join(' '));
set('cfg-rp-cost-limit', rp.cost_limit_usd != null ? rp.cost_limit_usd : '');
set('cfg-rp-cost-period', rp.cost_period || 'unlimited');
_updateRunpodVisibility();
}
// Build the per-model runpod block. Only meaningful when backend=runpod; the
// backend normalizes/drops empty fields.
function _collectRunpod(){
if ((document.getElementById('cfg-backend').value || '') !== 'runpod') return undefined;
const val = id => (document.getElementById(id) || {}).value;
const numOrU = id => { const v = parseFloat(val(id)); return isNaN(v) ? undefined : v; };
const intOrU = id => { const v = parseInt(val(id)); return isNaN(v) ? undefined : v; };
const pools = [];
if (document.getElementById('cfg-rp-pool-secure').checked) pools.push('SECURE');
if (document.getElementById('cfg-rp-pool-community').checked) pools.push('COMMUNITY');
const env = {};
(val('cfg-rp-env') || '').trim().split(/\s+/).forEach(tok => {
const i = tok.indexOf('='); if (i > 0) env[tok.slice(0,i)] = tok.slice(i+1);
});
const o = {
mode: val('cfg-rp-mode') || 'serverless',
served_model: (val('cfg-rp-served') || '').trim(),
ctx: intOrU('cfg-rp-ctx'),
endpoint_id: (val('cfg-rp-endpoint') || '').trim(),
min_workers: intOrU('cfg-rp-min-workers'),
max_workers: intOrU('cfg-rp-max-workers'),
cloud_types: pools,
selection_criteria: val('cfg-rp-criteria') || 'cheaper',
gpu_type: (val('cfg-rp-gpu-type') || '').trim(),
min_vram_gb: numOrU('cfg-rp-min-vram'),
max_hourly_usd: numOrU('cfg-rp-max-hourly'),
allow_spot: document.getElementById('cfg-rp-spot').checked,
min_pods: intOrU('cfg-rp-min-pods'),
max_pods: intOrU('cfg-rp-max-pods'),
idle_timeout_s: intOrU('cfg-rp-idle'),
image: (val('cfg-rp-image') || '').trim(),
container_disk_gb: intOrU('cfg-rp-disk'),
port: intOrU('cfg-rp-port'),
env: Object.keys(env).length ? env : undefined,
cost_limit_usd: numOrU('cfg-rp-cost-limit'),
cost_period: val('cfg-rp-cost-period') || 'unlimited',
};
return o;
}
async function saveModelConfig(){
const path = document.getElementById('cfg-path').value;
const maxGpu = parseFloat(document.getElementById('cfg-max-gpu').value);
......@@ -3872,6 +4058,7 @@ async function saveModelConfig(){
alias: document.getElementById('cfg-alias').value.trim() || null,
config_name: document.getElementById('cfg-config-name').value.trim() || null,
backend: document.getElementById('cfg-backend').value,
runpod: _collectRunpod(),
load_mode: document.getElementById('cfg-load-mode').value,
languages: (function(){ const v=document.getElementById('cfg-stt-languages').value.trim();
return v ? v.split(',').map(s=>s.trim()).filter(Boolean) : []; })(),
......
codai/api/runpod_worker.py 0 → 100644
View file @ 17604ab4
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
"""RunPod worker — per-model config parsing + endpoint resolution.
This module owns the RunPod-specific model config (parsed from the model's
models.json ``runpod`` block) and turns it into a live OpenAI base URL that
:mod:`codai.backends.runpod` proxies to.
Two modes (per-model ``mode``):
* ``serverless`` — RunPod autoscales an endpoint; coderai just proxies to
``<serverless_base>/<endpoint_id>/openai/v1`` with the account API key.
RunPod owns the worker lifecycle; there is nothing for coderai to provision.
* ``pods`` — coderai provisions, load-balances and scales a POOL of GPU pods
itself (lifecycle in this module; see RunpodPodPool). [built in a later phase]
* ``auto`` — pick pods-vs-serverless by the selection criteria. [later phase]
The account settings (API key, endpoints, global caps) live in
:class:`codai.config.RunpodConfig`; this module reads the per-model block.
"""
import os
from dataclasses import dataclass, field
from typing import Optional
# --------------------------------------------------------------------------- #
# Per-model config
# --------------------------------------------------------------------------- #
@dataclass
class RunpodModelConfig:
"""Parsed per-model ``runpod`` block. All fields optional with sane defaults."""
mode: str = "pods" # pods | serverless | auto
# --- selection (pods) ---
cloud_types: list = field(default_factory=lambda: ["SECURE"]) # allowed pools
selection_criteria: str = "cheaper" # cheaper | faster
gpu_type: str = "" # explicit RunPod gpuTypeId (optional)
min_vram_gb: float = 0.0
max_hourly_usd: float = 0.0 # per-model $/hr GPU ceiling (0 = none)
allow_spot: bool = False
# --- serving (pods) ---
image: str = "" # blank = default vLLM-OpenAI image
served_model: str = "" # HF id the pod/endpoint serves
container_disk_gb: int = 40
volume_gb: int = 0
port: int = 8000
ctx: int = 0
env: dict = field(default_factory=dict) # extra pod env (HF_TOKEN, etc.)
# --- scaling (pods) ---
min_pods: int = 0
max_pods: int = 1
scale_up_inflight_per_pod: int = 4
idle_timeout_s: int = 300 # destroy a pod this long after its last request
# --- serverless ---
endpoint_id: str = "" # reference an existing serverless endpoint
min_workers: int = 0
max_workers: int = 1
# --- cost budget (cumulative spend, distinct from the $/hr rate cap) ---
cost_limit_usd: float = 0.0 # 0 = unlimited
cost_period: str = "unlimited" # hour | day | week | month | unlimited
@property
def is_serverless(self) -> bool:
return (self.mode or "").lower() == "serverless"
# Default OpenAI-compatible pod image (vLLM's official OpenAI server). The pod
# pulls the model named by ``served_model`` (needs HF_TOKEN in env for gated repos).
DEFAULT_POD_IMAGE = "vllm/vllm-openai:latest"
def _as_bool(v, default=False):
if isinstance(v, bool):
return v
if isinstance(v, str):
return v.strip().lower() in ("1", "true", "yes", "on")
return default
def _as_float(v, default=0.0):
try:
return float(v)
except (TypeError, ValueError):
return default
def _as_int(v, default=0):
try:
return int(v)
except (TypeError, ValueError):
return default
def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
"""Build a RunpodModelConfig from a model entry's ``runpod`` dict (or {})."""
b = block or {}
cloud = b.get("cloud_types")
if isinstance(cloud, str):
cloud = [c.strip().upper() for c in cloud.split(",") if c.strip()]
elif isinstance(cloud, list):
cloud = [str(c).strip().upper() for c in cloud if str(c).strip()]
else:
cloud = None
env = b.get("env")
if not isinstance(env, dict):
env = {}
cfg = RunpodModelConfig()
cfg.mode = (b.get("mode") or cfg.mode).strip().lower()
if cloud:
cfg.cloud_types = cloud
cfg.selection_criteria = (b.get("selection_criteria") or cfg.selection_criteria).strip().lower()
cfg.gpu_type = (b.get("gpu_type") or "").strip()
cfg.min_vram_gb = _as_float(b.get("min_vram_gb"), cfg.min_vram_gb)
cfg.max_hourly_usd = _as_float(b.get("max_hourly_usd"), cfg.max_hourly_usd)
cfg.allow_spot = _as_bool(b.get("allow_spot"), cfg.allow_spot)
cfg.image = (b.get("image") or "").strip()
cfg.served_model = (b.get("served_model") or "").strip()
cfg.container_disk_gb = _as_int(b.get("container_disk_gb"), cfg.container_disk_gb)
cfg.volume_gb = _as_int(b.get("volume_gb"), cfg.volume_gb)
cfg.port = _as_int(b.get("port"), cfg.port) or 8000
cfg.ctx = _as_int(b.get("ctx"), cfg.ctx)
cfg.env = {str(k): str(v) for k, v in env.items()}
cfg.min_pods = max(0, _as_int(b.get("min_pods"), cfg.min_pods))
cfg.max_pods = max(1, _as_int(b.get("max_pods"), cfg.max_pods))
cfg.scale_up_inflight_per_pod = max(1, _as_int(b.get("scale_up_inflight_per_pod"),
cfg.scale_up_inflight_per_pod))
cfg.idle_timeout_s = max(0, _as_int(b.get("idle_timeout_s"), cfg.idle_timeout_s))
cfg.endpoint_id = (b.get("endpoint_id") or "").strip()
cfg.min_workers = max(0, _as_int(b.get("min_workers"), cfg.min_workers))
cfg.max_workers = max(1, _as_int(b.get("max_workers"), cfg.max_workers))
cfg.cost_limit_usd = _as_float(b.get("cost_limit_usd"), cfg.cost_limit_usd)
cfg.cost_period = (b.get("cost_period") or cfg.cost_period).strip().lower()
return cfg
def model_runpod_block(model_name: str) -> dict:
"""Fetch the ``runpod`` block from the model's models.json entry, or {}."""
try:
from codai.models.manager import _model_entry_for
entry = _model_entry_for(model_name)
if entry and isinstance(entry.get("runpod"), dict):
return entry["runpod"]
except Exception:
pass
return {}
# --------------------------------------------------------------------------- #
# Serverless resolution (RunPod owns the lifecycle — we only build the URL)
# --------------------------------------------------------------------------- #
def serverless_base_url(account_cfg, mcfg: RunpodModelConfig) -> str:
"""Return the OpenAI base URL for a serverless endpoint, e.g.
``https://api.runpod.ai/v2/<endpoint_id>/openai/v1``."""
eid = (mcfg.endpoint_id or "").strip()
if not eid:
raise RuntimeError(
"RunPod serverless: no endpoint_id set on the model's runpod config. "
"Create a serverless endpoint in RunPod (a vLLM worker) and paste its id.")
base = (getattr(account_cfg, "serverless_base", "") or "https://api.runpod.ai/v2").rstrip("/")
return f"{base}/{eid}/openai/v1"
def auth_headers(account_cfg) -> dict:
"""Authorization header for RunPod serverless (Bearer API key)."""
key = (getattr(account_cfg, "api_key", "") or "").strip()
return {"Authorization": f"Bearer {key}"} if key else {}
codai/backends/runpod.py 0 → 100644
View file @ 17604ab4
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
"""RunPod remote-GPU proxy backend.
A thin OpenAI proxy (mirroring :class:`codai.backends.vllm.VllmBackend`) that
forwards chat/completion requests to a model running on RunPod — either a
serverless endpoint (RunPod autoscales) or a coderai-managed pod pool. It uses
NO local VRAM and downloads NOTHING: the remote worker holds the weights.
Lifecycle + URL resolution live in :mod:`codai.api.runpod_worker`; account
settings (API key, endpoints, caps) in :class:`codai.config.RunpodConfig`.
Phase 2: the serverless path is live. Pods mode raises a clear error until the
pod pool lands in the next phase.
"""
import asyncio
import threading
from typing import AsyncGenerator, Dict, List, Optional
from codai.backends.base import ModelBackend
class RunpodBackend(ModelBackend):
"""Thin OpenAI proxy to a model hosted on RunPod."""
# RunPod requests run on REMOTE GPUs, so they must not count against the local
# global concurrency gate. This class-level inflight is informational only.
_inflight = 0
_inflight_lock = threading.Lock()
@classmethod
def _enter_request(cls):
with cls._inflight_lock:
cls._inflight += 1
@classmethod
def _exit_request(cls):
with cls._inflight_lock:
cls._inflight = max(0, cls._inflight - 1)
@classmethod
def any_request_active(cls) -> bool:
with cls._inflight_lock:
return cls._inflight > 0
# RunPod models never occupy local VRAM.
uses_local_vram = False
def __init__(self, cfg=None):
if cfg is None:
from codai.config import RunpodConfig
cfg = RunpodConfig()
self._acct = cfg
self._model_id = "runpod"
self._served_name = "runpod"
self._url: Optional[str] = None
self._headers: Dict[str, str] = {}
self._ctx = 32768
self._mcfg = None
self._last_usage: Dict = {}
# ------------------------------------------------------------------ #
# lifecycle
# ------------------------------------------------------------------ #
def load_model(self, model_name: str, **kwargs) -> None:
"""Resolve the remote endpoint for ``model_name``. Downloads nothing."""
from codai.api import runpod_worker
if model_name:
self._model_id = model_name
block = runpod_worker.model_runpod_block(model_name)
self._mcfg = runpod_worker.parse_model_runpod(block)
# served name: explicit served_model wins, else the model's own name.
self._served_name = self._mcfg.served_model or model_name or "runpod"
if self._mcfg.ctx:
self._ctx = self._mcfg.ctx
_ctx = kwargs.get("n_ctx", kwargs.get("ctx"))
if isinstance(_ctx, (list, tuple)):
_ctx = _ctx[0] if _ctx else None
try:
_ctx = int(_ctx) if _ctx else 0
except (TypeError, ValueError):
_ctx = 0
if _ctx > 0:
self._ctx = _ctx
mode = (self._mcfg.mode or "pods").lower()
if mode == "serverless":
self._url = runpod_worker.serverless_base_url(self._acct, self._mcfg)
self._headers = runpod_worker.auth_headers(self._acct)
print(f"[runpod] '{model_name}' -> serverless endpoint "
f"{self._mcfg.endpoint_id} ({self._url})", flush=True)
else:
# pods / auto: provisioned pool — lands in the next phase.
raise RuntimeError(
f"RunPod mode '{mode}' is not available yet — the pod pool is "
"built in a later phase. Set the model's runpod mode to 'serverless' "
"for now.")
def get_model_name(self) -> str:
return self._model_id
def get_context_size(self) -> int:
return self._ctx
def get_last_usage(self) -> dict:
return dict(self._last_usage)
def cleanup(self) -> None:
# Serverless: nothing to tear down (RunPod owns it). Pods: the pool
# reaper handles teardown (next phase).
self._url = None
# ------------------------------------------------------------------ #
# helpers
# ------------------------------------------------------------------ #
def _base(self) -> str:
if not self._url:
raise RuntimeError("RunPod endpoint not resolved")
return self._url
def _store_usage(self, usage: dict) -> None:
if usage:
self._last_usage = {
"prompt_tokens": usage.get("prompt_tokens", 0),
"completion_tokens": usage.get("completion_tokens", 0),
"total_tokens": usage.get("total_tokens", 0),
}
def format_messages(self, messages) -> str:
parts = []
for m in messages:
role = m.get("role") if isinstance(m, dict) else getattr(m, "role", "")
content = m.get("content") if isinstance(m, dict) else getattr(m, "content", "")
parts.append(f"{role}: {content}")
return "\n".join(parts)
def _chat_payload(self, messages, max_tokens, temperature, top_p, stop, stream):
payload = {
"model": self._served_name,
"messages": messages,
"temperature": temperature,
"top_p": top_p,
"stream": stream,
}
if max_tokens:
payload["max_tokens"] = max_tokens
if stop:
payload["stop"] = stop
return payload
# ------------------------------------------------------------------ #
# chat-level generation
# ------------------------------------------------------------------ #
def generate_chat(self, messages: List[Dict], max_tokens=None, temperature=0.7,
top_p=1.0, stop=None, tools=None, response_format=None):
import requests
self._enter_request()
try:
payload = self._chat_payload(messages, max_tokens, temperature, top_p, stop, False)
if response_format and response_format.get("type") == "json_object":
payload["response_format"] = {"type": "json_object"}
if tools:
payload["tools"] = tools
r = requests.post(self._base() + "/chat/completions", json=payload,
headers=self._headers, timeout=3600)
r.raise_for_status()
data = r.json()
self._store_usage(data.get("usage", {}))
return data["choices"][0]["message"].get("content") or ""
finally:
self._exit_request()
async def generate_chat_stream(self, messages: List[Dict], max_tokens=None,
temperature=0.7, top_p=1.0, stop=None, tools=None,
response_format=None) -> AsyncGenerator[str, None]:
self._enter_request()
try:
payload = self._chat_payload(messages, max_tokens, temperature, top_p, stop, True)
if tools:
payload["tools"] = tools
async for chunk in self._stream(self._base() + "/chat/completions", payload,
delta_key="delta"):
yield chunk
finally:
self._exit_request()
# ------------------------------------------------------------------ #
# plain completion (fallback path)
# ------------------------------------------------------------------ #
def generate(self, prompt: str, max_tokens=None, temperature: float = 0.7,
top_p: float = 1.0, stop=None, repeat_penalty: float = 1.0,
presence_penalty: float = 0.0, frequency_penalty: float = 0.0) -> str:
return self.generate_chat([{"role": "user", "content": prompt}],
max_tokens, temperature, top_p, stop)
async def generate_stream(self, prompt: str, max_tokens=None, temperature: float = 0.7,
top_p: float = 1.0, stop=None, repeat_penalty: float = 1.0,
presence_penalty: float = 0.0,
frequency_penalty: float = 0.0) -> AsyncGenerator[str, None]:
async for chunk in self.generate_chat_stream(
[{"role": "user", "content": prompt}], max_tokens, temperature, top_p, stop):
yield chunk
# ------------------------------------------------------------------ #
# SSE streaming
# ------------------------------------------------------------------ #
async def _stream(self, url: str, payload: dict, delta_key: str
) -> AsyncGenerator[str, None]:
import json
loop = asyncio.get_event_loop()
queue: asyncio.Queue = asyncio.Queue()
_SENTINEL = object()
headers = self._headers
def _handle_line(line: str) -> bool:
if not line or not line.startswith("data:"):
return False
data = line[len("data:"):].strip()
if data == "[DONE]":
return True
try:
obj = json.loads(data)
except ValueError:
return False
choice = (obj.get("choices") or [{}])[0]
text = (choice.get(delta_key) or {}).get("content") or ""
if text:
loop.call_soon_threadsafe(queue.put_nowait, text)
if obj.get("usage"):
self._store_usage(obj["usage"])
return bool(choice.get("finish_reason"))
def _worker():
import requests
try:
with requests.post(url, json=payload, headers=headers,
stream=True, timeout=3600) as r:
r.raise_for_status()
buf = b""
done = False
for bchunk in r.iter_content(chunk_size=8192):
if not bchunk:
continue
buf += bchunk
while b"\n" in buf:
raw, buf = buf.split(b"\n", 1)
if _handle_line(raw.decode("utf-8", "replace").strip()):
done = True
break
if done:
break
if not done and buf.strip():
_handle_line(buf.decode("utf-8", "replace").strip())
except Exception as exc:
loop.call_soon_threadsafe(queue.put_nowait, exc)
finally:
loop.call_soon_threadsafe(queue.put_nowait, _SENTINEL)
threading.Thread(target=_worker, daemon=True).start()
while True:
item = await queue.get()
if item is _SENTINEL:
break
if isinstance(item, Exception):
raise item
yield item
codai/frontproxy/registry.py
View file @ 17604ab4
......@@ -43,12 +43,15 @@ def _short_stem(key: str) -> str:
# whisper, but not transformers, ds4 or colibri.
# k3 (kimi-k3-in-c) is a CPU engine → any node can host it; kt (ktransformers/SGLang)
# is CPU+GPU → the GPU-capable nodes.
# RunPod ("runpod") proxies to a remote GPU and uses NO local VRAM, so every node
# can host it — it's added to all cap sets. The front prefers the primary/default
# engine to host it (routing falls to default_engine before least_loaded).
_DEFAULT_CAPS = {
"nvidia": {"transformers", "gguf", "whisper", "ds4", "colibri", "k3", "kt"},
"cuda": {"transformers", "gguf", "whisper", "ds4", "colibri", "k3", "kt"},
"vulkan": {"gguf", "whisper", "k3"},
"opencl": {"gguf", "whisper", "k3"},
"auto": {"transformers", "gguf", "whisper", "ds4", "colibri", "k3", "kt"},
"nvidia": {"transformers", "gguf", "whisper", "ds4", "colibri", "k3", "kt", "runpod"},
"cuda": {"transformers", "gguf", "whisper", "ds4", "colibri", "k3", "kt", "runpod"},
"vulkan": {"gguf", "whisper", "k3", "runpod"},
"opencl": {"gguf", "whisper", "k3", "runpod"},
"auto": {"transformers", "gguf", "whisper", "ds4", "colibri", "k3", "kt", "runpod"},
}
......
codai/frontproxy/router.py
View file @ 17604ab4
......@@ -112,7 +112,7 @@ def required_capability(model: Optional[str], path: Optional[str] = None,
return "whisper"
# 1. Explicit engine-backend pin wins (authoritative), like the manager resolver.
b = (backend or "").lower()
if b in ("colibri", "ds4", "k3", "kt", "vllm"):
if b in ("colibri", "ds4", "k3", "kt", "vllm", "runpod"):
return b
m = (model or "").lower()
......
codai/models/manager.py
View file @ 17604ab4
......@@ -234,7 +234,7 @@ def _gguf_architecture(path: str):
# --------------------------------------------------------------------------- #
# Ordered by arbitration preference for ambiguous names. New engines append here.
_ENGINE_BACKENDS = ("ds4", "colibri", "k3", "kt", "vllm")
_ENGINE_BACKENDS = ("ds4", "colibri", "k3", "kt", "vllm", "runpod")
def _engine_config(engine: str):
......@@ -245,6 +245,7 @@ def _engine_config(engine: str):
"k3": get_active_k3_config,
"kt": get_active_ktransformers_config,
"vllm": get_active_vllm_config,
"runpod": get_active_runpod_config,
}.get(engine)
return getter() if getter else None
......@@ -357,12 +358,19 @@ def _vllm_name_claims(model_name: str) -> bool:
return False
def _runpod_name_claims(model_name: str) -> bool:
"""RunPod can serve any model, so it must NEVER auto-claim by a name marker.
Routes ONLY via an explicit ``backend: "runpod"`` pin on the model entry."""
return False
_ENGINE_NAME_CLAIMS = {
"ds4": _ds4_name_claims,
"colibri": _colibri_name_claims,
"k3": _k3_name_claims,
"kt": _kt_name_claims,
"vllm": _vllm_name_claims,
"runpod": _runpod_name_claims,
}
......@@ -427,6 +435,29 @@ def vllm_should_handle(model_name: str) -> bool:
return resolve_engine_backend(model_name) == "vllm"
def runpod_should_handle(model_name: str) -> bool:
"""True when runpod is the resolved engine backend for ``model_name``."""
return resolve_engine_backend(model_name) == "runpod"
def _model_is_runpod(model_name: str, config=None) -> bool:
"""True for a RunPod-served model — from the runtime config's backend pin (or its
embedded _raw_cfg), else the resolver. Such models hold NO local weights, so the
manager must never download/cache them or estimate local VRAM for them."""
if isinstance(config, dict):
b = (config.get("backend") or "").strip().lower()
if not b:
raw = config.get("_raw_cfg")
if isinstance(raw, dict):
b = (raw.get("backend") or "").strip().lower()
if b == "runpod":
return True
try:
return resolve_engine_backend(model_name) == "runpod"
except Exception:
return False
def _trim_cpu_ram() -> None:
"""Return freed CPU heap memory to the OS (and let the kernel reclaim swap).
......@@ -610,6 +641,18 @@ class ModelManager:
self.tool_parser = ModelParserAdapter(model_name=model_name)
return
# RunPod: proxy matching models to a remote GPU on RunPod (serverless
# endpoint or a managed pod pool). Uses NO local VRAM and downloads
# nothing — the remote worker holds the weights. Pin-selected only.
if runpod_should_handle(model_name):
from codai.backends.runpod import RunpodBackend
print(f"Routing '{model_name}' to RunPod (remote GPU) backend")
self.backend_type = "runpod"
self.backend = RunpodBackend(get_active_runpod_config())
self.backend.load_model(model_name, **kwargs)
self.tool_parser = ModelParserAdapter(model_name=model_name)
return
available = detect_available_backends()
# Check if model is a GGUF file. The name alone isn't reliable: a gguf's
......@@ -1475,6 +1518,13 @@ class MultiModelManager:
self.model_backend_types[model_name] = backend_type
self._remember_registered_type(model_name, "text")
# RunPod-served models hold no local weights — the remote pod/endpoint pulls
# them. Never download/cache locally; the backend resolves the remote URL on
# first request.
if _model_is_runpod(model_name, self.config.get(model_name)):
print(f"Model '{model_name}' is RunPod-served — skipping local download/cache")
return
# Download/cache the model at startup if it's a URL or HF ID
resolved_model = self.load_model(model_name)
if resolved_model != model_name:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment