runpod: coderai-vllm and coderai-llama — the LLM servers as coderai pods

coderai-text was transformers only, so a GGUF or a vLLM-served repo still
went to an upstream image: no TLS on the direct path, no /boot, no local
LoRAs, nothing on the Tasks page. coderai-llama compiles llama-cpp-python
with CUDA for every pod GPU on the full core; coderai-vllm is vLLM in its
own uv-resolved venv on the light core, found by the worker through
CODERAI_VLLM_VENV, with the local vLLM settings forwarded. The upstream
engines stay selectable as vllm / llamacpp.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/templates/models.html

@@ -783,8 +783,10 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               <label class="form-label">Pod engine</label>
               <select id="cfg-rp-engine" class="form-input" onchange="_updateRunpodEngineHint()">
                 <option value="auto">Auto — vLLM, or llama.cpp for a GGUF</option>
-                <option value="vllm">vLLM (HF repo)</option>
-                <option value="llamacpp">llama.cpp (GGUF)</option>
+                <option value="vllm">vLLM — upstream vllm/vllm-openai image (HF repo)</option>
+                <option value="llamacpp">llama.cpp — upstream ggml-org image (GGUF)</option>
+                <option value="coderai-vllm">vLLM as a coderai pod (coderai-vllm image)</option>
+                <option value="coderai-llama">llama.cpp as a coderai pod (coderai-llama image, GGUF)</option>
                 <option value="coderai">coderai (a capability image)</option>
                 <option value="ds4">ds4 — DeepSeek-V4 (engines image)</option>
                 <option value="colibri">colibri — GLM-5.2 / DeepSeek-V4 / Kimi-K3 (engines image)</option>
@@ -4400,6 +4402,8 @@ function _updateRunpodEngineHint(){
     auto: 'A GGUF goes to llama.cpp, anything else to vLLM.',
     vllm: 'Serves an HF repo. vLLM\'s GGUF support is experimental — use llama.cpp for those.',
     llamacpp: 'Serves the GGUF named on the right; the pod downloads it itself.',
+    'coderai-vllm': 'vLLM inside a coderai pod: the same server as upstream, plus TLS on the direct path, /boot, seeding and the Tasks page. Your local vLLM settings (dtype, quantization, memory utilization…) travel to the pod.',
+    'coderai-llama': 'llama-cpp-python (CUDA, every pod GPU) inside a coderai pod for a GGUF: same kernels as upstream, plus what a coderai pod gives — including local LoRAs.',
     coderai: 'A whole coderai serving one capability. Needs a pod image you have pushed.',
     ds4: 'ds4-server on the engines image, built for every pod GPU. Downloads the configured variant unless the weights are on the volume.',
     colibri: 'colibri on the engines image (GLM-5.2 on CUDA; DeepSeek-V4 and Kimi-K3 on CPU). Never downloads: the container dir must be on the volume.',

codai/api/runpod_worker.py

@@ -218,14 +218,25 @@ PUBLISHED_CAPABILITY_IMAGES = (
     # kt-kernel dispatches at runtime (AMX → AVX-512 → AVX2 llamafile → AMD
     # BLIS), so one image serves whatever CPU the pod host has.
     "engines-kt",
+    # The two LLM servers as coderai pods rather than upstream images: the
+    # same kernels, plus what a coderai pod gives — TLS on the direct path,
+    # /boot, seeding, local LoRAs, the Tasks page. `llama` is llama-cpp-python
+    # built with CUDA for every pod GPU on the full core (the GGUF backend
+    # asks torch about the card); `vllm` is vLLM in its own venv on the light
+    # core, driven by coderai's vLLM proxy backend.
+    "llama",
+    "vllm",
 )
 
 #: Model backends served by an engine image, and which image. ds4, colibri
 #: and k3 are C binaries and share one image. ktransformers is a Python stack
 #: (kt-kernel + SGLang, pinning their own torch) that takes a venv of its own,
 #: so it has an image of its own rather than doubling the first.
-_ENGINE_POD_BACKENDS = ("ds4", "colibri", "k3", "kt")
-_ENGINE_POD_IMAGE = {"kt": "engines-kt"}
+_ENGINE_POD_BACKENDS = ("ds4", "colibri", "k3", "kt", "vllm")
+_ENGINE_POD_IMAGE = {"kt": "engines-kt", "vllm": "vllm"}
+#: runpod-block `engine` values that mean "a coderai pod running THIS server",
+#: as opposed to `vllm`/`llamacpp`, which mean the upstream images.
+_CODERAI_SERVER_ENGINES = {"coderai-vllm": "vllm", "coderai-llama": "llama"}
 
 #: Capabilities served by a published image other than their own name.
 _CAPABILITY_IMAGE_ALIASES = {
@@ -377,6 +388,8 @@ def resolve_pod_engine(mcfg: "RunpodModelConfig", model_key: str = "",
     vLLM.
     """
     want = (mcfg.engine or "auto").strip().lower()
+    if want in _CODERAI_SERVER_ENGINES:
+        return "coderai"
     if want in ("vllm", "llamacpp", "coderai", "custom"):
         return want
     # A non-text model is served by a whole coderai carrying that capability.
@@ -422,6 +435,8 @@ def engine_pod_backend(entry: dict, mcfg: "RunpodModelConfig" = None) -> str:
     want = str(getattr(mcfg, "engine", "") or "").strip().lower() if mcfg else ""
     if want in _ENGINE_POD_BACKENDS:
         return want
+    if want == "coderai-vllm":
+        return "vllm"
     b = str((entry or {}).get("backend") or "").strip().lower()
     return b if b in _ENGINE_POD_BACKENDS else ""
 
@@ -455,6 +470,9 @@ def pod_plan(mcfg: "RunpodModelConfig", served: str, model_key: str = "",
         image = mcfg.image
         capability_hint = model_capability(entry or {}, include_text=True)
         _eng = engine_pod_backend(entry or {}, mcfg)
+        _want = str(mcfg.engine or "").strip().lower()
+        if not image and _want in _CODERAI_SERVER_ENGINES:
+            image = default_capability_image(_CODERAI_SERVER_ENGINES[_want])
         if not image and _eng:
             image = default_capability_image(_ENGINE_POD_IMAGE.get(_eng, "engines"))
         if not image:
@@ -752,6 +770,8 @@ _ENGINE_FORWARDED_FIELDS = {
     "k3": ("ctx", "preset", "trunk_gb", "cache_gb", "extra_args", "extra_env",
            "model_id"),
     "kt": ("ctx", "extra_args", "extra_env", "model_id"),
+    "vllm": ("ctx", "gpu_memory_utilization", "tensor_parallel_size", "max_num_seqs",
+             "dtype", "quantization", "extra_args", "extra_env"),
 }
 #: The config attribute behind each engine name (ktransformers is the odd one).
 _ENGINE_CONFIG_ATTR = {"kt": "ktransformers"}
@@ -965,7 +985,15 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
             # SGLang + kt-kernel live in the image's own venv; the worker must
             # launch from there, not from coderai's interpreter.
             env.setdefault("CODERAI_KT_VENV", "/opt/coderai/venvs/sglang")
+        if eng == "vllm":
+            env.setdefault("CODERAI_VLLM_VENV", "/opt/coderai/venvs/vllm")
         forwarded = engine_config_for_pod(eng)
+        if eng == "vllm" and "tensor_parallel_size" in forwarded:
+            # The pod was rented with gpu_count cards; the local setting
+            # describes this machine's cards. The pod's count wins.
+            forwarded.pop("tensor_parallel_size", None)
+        if eng == "vllm" and int(getattr(mcfg, "gpu_count", 1) or 1) > 1:
+            forwarded["tensor_parallel_size"] = int(mcfg.gpu_count)
         if forwarded:
             env[f"CODERAI_{eng.upper()}_CONFIG"] = _json.dumps(forwarded)
         warn = weight_transfer_warning(entry, mcfg, _account_hint())

codai/api/vllm_worker.py

@@ -37,7 +37,8 @@ _REQ = os.path.join(os.path.dirname(__file__), "..", "..", "requirements-vllm.tx
 
 def resolve_venv_dir(cfg) -> str:
     """Isolated vLLM venv: config > baked /opt/coderai/vllm_venv > /cache mount > ~/.coderai."""
-    configured = (getattr(cfg, "venv", "") or "").strip()
+    configured = (getattr(cfg, "venv", "") or "").strip() \
+        or (os.environ.get("CODERAI_VLLM_VENV") or "").strip()
     if configured:
         return configured
     baked = "/opt/coderai/vllm_venv"

codai/config.py

@@ -1368,7 +1368,7 @@ class ConfigManager:
         # anything. Where the binary lives is the image's business
         # (CODERAI_<ENGINE>_DIR, read by the worker itself), not this block's.
         for name, attr in (("ds4", "ds4"), ("colibri", "colibri"), ("k3", "k3"),
-                           ("kt", "ktransformers")):
+                           ("kt", "ktransformers"), ("vllm", "vllm")):
             if not _flag(f"CODERAI_{name.upper()}_ENABLED"):
                 continue
             eng = getattr(self.config, attr, None)

docs/runpod.md

@@ -205,6 +205,22 @@ It stages the engine sources from `~/.coderai/{ds4,colibri,kimi-k3-in-c}` (or
 `CODERAI_DS4_DIR` etc.), so the pod runs the same engine code the local install
 does.
 
+### The LLM servers as coderai pods
+
+`engine: vllm` and `engine: llamacpp` run the **upstream** images
+(`vllm/vllm-openai`, `ghcr.io/ggml-org/llama.cpp:server-cuda`): well-tested,
+but not coderai — no TLS on the direct-TCP path, no `/boot`, no seeding, no
+local LoRAs, nothing on the Tasks page. Two more images run the same servers
+*inside* a coderai pod:
+
+| `engine` | Image | What it is |
+|---|---|---|
+| `coderai-vllm` | `coderai-vllm` (light core + vLLM venv, ~10 GB) | vLLM driven by coderai's vLLM backend. Your local **vLLM settings travel** (`dtype`, `quantization`, `gpu_memory_utilization`, `max_num_seqs`, `extra_args`); tensor parallelism follows the pod's `gpu_count`. |
+| `coderai-llama` | `coderai-llama` (full core + llama-cpp-python CUDA, ~9 GB) | A GGUF on coderai's own CUDA backend — llama-cpp-python compiled for every pod GPU generation. Local LoRAs work. |
+
+`coderai-text` remains the transformers pod (HF safetensors, 4/8-bit, PEFT
+adapters). Throughput under concurrency is vLLM's game on either image.
+
 **Naming the engine.** A RunPod-only model has `backend: runpod` — that slot is
 taken — so the engine goes on the runpod block, beside `vllm` and `llamacpp`:
 

packaging/runpod/Dockerfile.capability-llama

+# CoderAI llama pod — llama-cpp-python with CUDA, for every pod GPU generation.
+#
+# The upstream llama.cpp server image serves a GGUF perfectly well, but it is
+# not a coderai: no TLS on the direct-TCP path, no /boot record, no seeding,
+# no local LoRA upload, nothing on the Tasks page. This image is coderai's
+# text path with llama-cpp-python compiled for CUDA in a toolchain stage —
+# SASS for Ampere through Blackwell plus PTX for what comes next — on the
+# full core, because the CUDA GGUF backend asks torch about the card.
+
+ARG CORE_BASE=coderai-capability-base:latest
+ARG CUDA_DEVEL=nvidia/cuda:12.8.1-devel-ubuntu24.04
+ARG LLAMA_CPP_PYTHON=0.3.30
+
+# ---------------------------------------------------------------- build stage
+FROM ${CUDA_DEVEL} AS wheel
+ARG LLAMA_CPP_PYTHON
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential cmake git python3 python3-pip python3-venv ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+# Ubuntu 24.04's python3 is 3.12, the same as the core: the wheel is cp312 either way.
+RUN python3 -m venv /bw && /bw/bin/pip install --no-cache-dir -U pip build scikit-build-core
+ENV CMAKE_ARGS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=80;86;89;90;120 -DGGML_NATIVE=off -DGGML_AVX2=on"
+RUN --mount=type=cache,target=/root/.cache/pip set -eux; \
+    /bw/bin/pip wheel --no-cache-dir --no-deps --no-binary llama-cpp-python \
+        "llama-cpp-python==${LLAMA_CPP_PYTHON}" -w /wheels; \
+    ls -la /wheels
+
+# -------------------------------------------------------------- runtime stage
+FROM ${CORE_BASE}
+
+ARG PROFILE=llama
+ARG TORCH_INDEX=https://download.pytorch.org/whl/cu128
+ENV PROFILE=${PROFILE}
+ARG IMAGE_TAG=unknown
+ENV CODERAI_IMAGE_TAG=${IMAGE_TAG}
+
+WORKDIR /opt/coderai/app
+COPY packaging/runpod/profiles/ /tmp/profiles/
+COPY --from=wheel /wheels/ /tmp/wheels/
+RUN --mount=type=cache,target=/root/.cache/pip set -eux; \
+    python -m pip install --extra-index-url ${TORCH_INDEX} -r /tmp/profiles/${PROFILE}.txt; \
+    python -m pip install --no-cache-dir /tmp/wheels/llama_cpp_python-*.whl; \
+    python -c "import torch; print('torch', torch.__version__, 'cuda', torch.version.cuda)"; \
+    rm -rf /tmp/profiles /tmp/wheels
+
+# The CUDA runtime the wheel links, from torch's own wheels; fail the BUILD if
+# libllama cannot find a library, rather than the pod at its first load.
+ENV LD_LIBRARY_PATH=/usr/local/lib/python3.12/site-packages/nvidia/cuda_runtime/lib:/usr/local/lib/python3.12/site-packages/nvidia/cublas/lib:${LD_LIBRARY_PATH}
+COPY packaging/runpod/check_llama_wheel.py /tmp/check_llama_wheel.py
+RUN python /tmp/check_llama_wheel.py && rm /tmp/check_llama_wheel.py
+
+COPY . /opt/coderai/app
+RUN python -c "import codai.api.app; print('coderai imports OK for profile ${PROFILE}')"
+
+EXPOSE 8000
+HEALTHCHECK --interval=30s --timeout=5s --start-period=120s \
+    CMD curl -fsS http://127.0.0.1:8000/healthz || curl -fsSk https://127.0.0.1:8000/healthz || exit 1
+
+COPY packaging/runpod/boot.sh /opt/coderai/boot.sh
+RUN chmod +x /opt/coderai/boot.sh
+CMD ["/opt/coderai/boot.sh"]

packaging/runpod/check_llama_wheel.py

+# Build-time check for the llama pod image: the CUDA wheel is installed, its
+# shared libraries resolve, and the CUDA backend is actually in it.
+import glob
+import subprocess
+
+libs = glob.glob("/usr/local/lib/python3.12/site-packages/llama_cpp/lib/*.so*")
+assert libs, "no llama_cpp shared libraries installed"
+bad = [l for l in libs
+       if "not found" in subprocess.run(["ldd", l], capture_output=True, text=True).stdout]
+assert not bad, f"unresolved libraries in {bad}"
+import llama_cpp  # noqa: E402
+print("llama-cpp-python", llama_cpp.__version__, "with", len(libs), "libs, all resolved")
+assert any("ggml-cuda" in l for l in libs), "the wheel was built WITHOUT CUDA"

packaging/runpod/profiles/llama.dockerfile

+Dockerfile.capability-llama

packaging/runpod/profiles/llama.txt

+# llama.cpp as a coderai pod: llama-cpp-python built with CUDA for every pod
+# GPU generation, driving GGUF models through coderai's own CUDA backend —
+# the same kernels as the upstream ghcr.io/ggml-org/llama.cpp image, plus what
+# a coderai pod gives (TLS on the direct path, /boot, seeding, local LoRAs,
+# the Tasks page). FULL core: the GGUF backend asks torch about the card.
+#
+# The wheel itself is compiled in Dockerfile.capability-llama's toolchain
+# stage (see the `llama.dockerfile` marker); this file lists what else the
+# text path needs.
+sentencepiece>=0.1.99
+protobuf>=3.20.0

packaging/runpod/profiles/vllm.txt

+# vLLM as a coderai pod: vLLM in its own venv (vllm.venv-vllm.txt, resolved
+# with uv) on the LIGHT core, driven by coderai's vLLM proxy backend — the
+# same server as the upstream vllm/vllm-openai image, plus what a coderai pod
+# gives (TLS on the direct path, /boot, seeding, the Tasks page). The main
+# venv only proxies; nothing here touches torch.

packaging/runpod/profiles/vllm.venv-vllm.check

+# Deep check for the vllm venv: the OpenAI server module must import.
+import vllm.entrypoints.openai.api_server                 # noqa
+import torch, vllm
+print("vllm", vllm.__version__, "torch", torch.__version__, "cuda", torch.version.cuda)

packaging/runpod/profiles/vllm.venv-vllm.txt

+# The version the local vLLM node runs (requirements-vllm.txt). vLLM pins its
+# own torch and CUDA libraries, which is why it cannot share the main venv.
+vllm==0.27.1

tests/test_burst_placement.py

@@ -1122,3 +1122,65 @@ def test_a_direct_tcp_pod_speaks_tls_the_renting_coderai_can_verify(tmp_path, mo
     from pathlib import Path
     boot = Path("packaging/runpod/boot.sh").read_text()
     assert "CODERAI_TLS_CERT" in boot and "--ssl-certfile" in boot and "unset CODERAI_TLS_KEY" in boot
+
+
+def test_the_llm_servers_can_run_as_coderai_pods_instead_of_upstream_images(monkeypatch):
+    """coderai-text was transformers only, so a GGUF or a vLLM-served repo
+    still went to an upstream image — no TLS on the direct path, no /boot,
+    no local LoRAs. coderai-llama and coderai-vllm are those servers as
+    coderai pods; the upstream engines stay selectable."""
+    import json
+    from codai.api.runpod_worker import (pod_plan, parse_model_runpod,
+                                         PUBLISHED_CAPABILITY_IMAGES, resolve_pod_engine)
+    assert "llama" in PUBLISHED_CAPABILITY_IMAGES and "vllm" in PUBLISHED_CAPABILITY_IMAGES
+
+    class _V:
+        ctx = 16384; gpu_memory_utilization = 0.85; tensor_parallel_size = 2
+        max_num_seqs = 0; dtype = "bfloat16"; quantization = ""; extra_args = ""; extra_env = ""
+        venv = "/local/vllm_venv"; port = 9
+    class _Root:
+        vllm = _V()
+    class _CM:
+        config = _Root()
+    monkeypatch.setattr("codai.admin.routes.config_manager", _CM(), raising=False)
+
+    entry = {"path": "org/m", "model_type": "text_models", "backend": "runpod"}
+    # Upstream stays upstream.
+    assert resolve_pod_engine(parse_model_runpod({"engine": "vllm"}), "m", entry=entry) == "vllm"
+    up = pod_plan(parse_model_runpod({"engine": "vllm"}), "org/m", entry=entry)
+    assert up["image"].startswith("vllm/") and "CODERAI_VLLM_ENABLED" not in up["env"]
+
+    # vLLM as a coderai pod: its image, the engine on, local settings forwarded
+    # — except tensor parallelism, which follows the pod's own card count.
+    plan = pod_plan(parse_model_runpod({"engine": "coderai-vllm", "gpu_count": 2}),
+                    "org/m", entry=entry)
+    assert plan["engine"] == "coderai" and plan["image"].endswith("coderai-vllm:latest")
+    env = plan["env"]
+    assert env["CODERAI_VLLM_ENABLED"] == "1" and env["CODERAI_VLLM_VENV"] == "/opt/coderai/venvs/vllm"
+    fwd = json.loads(env["CODERAI_VLLM_CONFIG"])
+    assert fwd["dtype"] == "bfloat16" and fwd["ctx"] == 16384 and fwd["tensor_parallel_size"] == 2
+    assert "venv" not in fwd and "port" not in fwd
+    assert json.loads(env["CODERAI_SEED_MODELS"])[0]["backend"] == "vllm"
+    one = pod_plan(parse_model_runpod({"engine": "coderai-vllm"}), "org/m", entry=entry)
+    assert "tensor_parallel_size" not in json.loads(one["env"]["CODERAI_VLLM_CONFIG"])
+
+    # llama.cpp as a coderai pod: the GGUF goes to coderai's own CUDA backend.
+    gg = {"path": "org/m-GGUF", "model_type": "text_models", "backend": "runpod"}
+    plan = pod_plan(parse_model_runpod({"engine": "coderai-llama", "hf_gguf": "org/m-GGUF:Q4_K_M"}),
+                    "org/m-GGUF", entry=gg)
+    assert plan["engine"] == "coderai" and plan["image"].endswith("coderai-llama:latest")
+    assert "backend" not in json.loads(plan["env"]["CODERAI_SEED_MODELS"])[0]
+
+    # The worker finds the baked venv from env, ahead of the home-dir default.
+    from codai.api import vllm_worker as vw
+    class _Cfg: venv = ""
+    monkeypatch.setenv("CODERAI_VLLM_VENV", "/opt/coderai/venvs/vllm")
+    assert vw.resolve_venv_dir(_Cfg()) == "/opt/coderai/venvs/vllm"
+
+    from pathlib import Path
+    prof = Path("packaging/runpod/profiles")
+    assert (prof / "llama.dockerfile").read_text().strip() == "Dockerfile.capability-llama"
+    df = Path("packaging/runpod/Dockerfile.capability-llama").read_text()
+    assert "GGML_CUDA=on" in df and "80;86;89;90;120" in df and "check_llama_wheel" in df
+    assert (prof / "vllm.light").exists() and (prof / "vllm.venv-vllm.uv").exists()
+    assert "vllm==" in (prof / "vllm.venv-vllm.txt").read_text()