llm: make every text model remotizable, GGUF included

Two additions, one goal: no text model is stuck on this box.

1. codai/backends/remote_openai.py — set `service_url` on a model entry
   and coderai proxies that model to any OpenAI-compatible endpoint
   (another coderai, llama-server, vLLM, SGLang, a rented pod) instead of
   loading it. `served_model` renames it for the remote, `api_key` adds a
   bearer token. It reuses RunPod's request/SSE machinery and only
   replaces endpoint resolution, so streaming and usage accounting are
   the same code path.

   This is what makes the GGUF/llama.cpp catalogue remotizable: those
   models load in-process through llama-cpp-python, so there is no
   service to redirect — pointing the *model* at a URL is the only
   redirect available. The manager checks it first, ahead of every engine
   route, and `_model_is_runpod` grew into `_model_is_remote` so such
   models are never downloaded, cached, or charged local VRAM (old name
   kept as an alias).

2. RunPod pods pick their server per model. A GGUF cannot simply go to
   vLLM: its GGUF loader is experimental — single file only, a limited
   architecture list, and it still wants the original repo's tokenizer.
   So `engine: auto` sends an HF repo to vLLM and a GGUF to llama.cpp's
   own server image (ghcr.io/ggml-org/llama.cpp:server-cuda), which
   downloads the weights itself from `hf_gguf` ("user/repo:Q4_K_M") —
   a local /AI/….gguf path means nothing on a rented machine, and we
   don't upload multi-GB weights. `engine: vllm|llamacpp` overrides.
   Both servers answer /v1/models, so the readiness probe is unchanged.

Tested against fake servers: routing, /v1 normalisation, auth header,
chat, SSE streaming, usage, and each engine-selection branch.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/routes.py

@@ -2675,7 +2675,10 @@ async def api_model_configure(request: Request, username: str = Depends(require_
     # download-completion backfill fills it in once the weights land.
     # RunPod-served models hold NO local weights (the remote pod pulls them), so
     # never estimate local VRAM for them.
-    _is_runpod = (str(data.get("backend") or "").strip().lower() == "runpod")
+    # Same for a model pointed at an explicit `service_url` — the endpoint holds
+    # the weights, so there is nothing local to measure.
+    _is_runpod = (str(data.get("backend") or "").strip().lower() == "runpod"
+                  or bool(str(data.get("service_url") or "").strip()))
     used_vram_gb = data.get("used_vram_gb")
     if used_vram_gb is None and not _is_runpod:
         used_vram_gb = _estimate_used_vram_gb(path)
@@ -2706,7 +2709,11 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                 "auto_compact", "auto_compact_pct", "auto_compact_strategy",
                 "auto_compact_model", "suppress_reasoning",
                 # Speech-to-text (wav2vec2 / vosk / NeMo-Canary) options.
-                "languages", "supports_translation", "keep_resident"):
+                "languages", "supports_translation", "keep_resident",
+                # Serve this model from somewhere else instead of loading it here:
+                # any OpenAI-compatible endpoint (another coderai, llama-server,
+                # vLLM, a rented pod). See codai/backends/remote_openai.py.
+                "service_url", "served_model", "api_key"):
         if key in data:
             entry[key] = data[key]
 

codai/api/runpod_worker.py

@@ -44,8 +44,17 @@ class RunpodModelConfig:
     max_hourly_usd: float = 0.0              # per-model $/hr GPU ceiling (0 = none)
     allow_spot: bool = False
     # --- serving (pods) ---
-    image: str = ""                          # blank = default vLLM-OpenAI image
+    image: str = ""                          # blank = image picked from `engine`
+    # Which server the pod runs. "auto" reads the model: an HF repo goes to vLLM,
+    # a GGUF goes to llama.cpp (vLLM's GGUF support is experimental — single-file
+    # only, a limited architecture list, and it still wants the original repo's
+    # tokenizer — so it is not a drop-in for the llama.cpp catalogue).
+    engine: str = "auto"                     # auto | vllm | llamacpp
     served_model: str = ""                   # HF id the pod/endpoint serves
+    # GGUF to pull on a llama.cpp pod, in llama.cpp's own `-hf` form:
+    # "user/repo:Q4_K_M" or "user/repo:file.gguf". Needed because a local
+    # /AI/…/foo.gguf path means nothing on a rented machine.
+    hf_gguf: str = ""
     container_disk_gb: int = 40
     volume_gb: int = 0
     port: int = 8000
@@ -76,6 +85,60 @@ class RunpodModelConfig:
 # pulls the model named by ``served_model`` (needs HF_TOKEN in env for gated repos).
 DEFAULT_POD_IMAGE = "vllm/vllm-openai:latest"
 
+# llama.cpp's own server image. Its OpenAI surface (/v1/models, /v1/chat/completions)
+# matches vLLM's closely enough that the readiness probe and the proxy are unchanged.
+# This is what serves the GGUF catalogue remotely: those models have no safetensors
+# repo to hand vLLM, and vLLM's GGUF loader is experimental (single-file only, a
+# limited architecture list, and it needs the original repo for the tokenizer).
+LLAMACPP_POD_IMAGE = "ghcr.io/ggml-org/llama.cpp:server-cuda"
+
+
+def _looks_like_gguf(name: str) -> bool:
+    return (name or "").strip().lower().endswith(".gguf")
+
+
+def resolve_pod_engine(mcfg: "RunpodModelConfig", model_key: str = "",
+                       model_path: str = "") -> str:
+    """Decide which server a pod runs for this model: "vllm" or "llamacpp".
+
+    An explicit ``engine`` wins. Otherwise: a GGUF (by `hf_gguf`, by the model's
+    own path, or by the served name) goes to llama.cpp; anything else — an HF
+    repo id — goes to vLLM.
+    """
+    want = (mcfg.engine or "auto").strip().lower()
+    if want in ("vllm", "llamacpp"):
+        return want
+    if mcfg.hf_gguf:
+        return "llamacpp"
+    if _looks_like_gguf(mcfg.served_model) or _looks_like_gguf(model_path) \
+            or _looks_like_gguf(model_key):
+        return "llamacpp"
+    return "vllm"
+
+
+def _llamacpp_docker_args(mcfg: "RunpodModelConfig", served: str) -> str:
+    """Args for the llama.cpp server image so it downloads and serves the GGUF.
+
+    llama.cpp pulls weights itself with ``-hf user/repo:QUANT``; a local path is
+    meaningless on a rented machine, so ``hf_gguf`` (or an HF-shaped
+    ``served_model``) is required.
+    """
+    src = (mcfg.hf_gguf or "").strip()
+    if not src:
+        cand = (served or "").strip()
+        if cand and "/" in cand and not cand.startswith("/"):
+            src = cand
+    if not src:
+        raise RuntimeError(
+            "RunPod llama.cpp pod: set `hf_gguf` on the model's runpod block to the "
+            "GGUF to serve (\"user/repo:Q4_K_M\"). A local .gguf path does not exist "
+            "on a rented pod, and coderai does not upload multi-GB weights.")
+    args = ["--host", "0.0.0.0", "--port", str(mcfg.port or 8000),
+            "-hf", src, "--alias", served or src, "-ngl", "999"]
+    if mcfg.ctx and mcfg.ctx > 0:
+        args += ["-c", str(mcfg.ctx)]
+    return " ".join(args)
+
 
 def _as_bool(v, default=False):
     if isinstance(v, bool):
@@ -122,7 +185,9 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
     cfg.max_hourly_usd = _as_float(b.get("max_hourly_usd"), cfg.max_hourly_usd)
     cfg.allow_spot = _as_bool(b.get("allow_spot"), cfg.allow_spot)
     cfg.image = (b.get("image") or "").strip()
+    cfg.engine = (b.get("engine") or cfg.engine).strip().lower() or "auto"
     cfg.served_model = (b.get("served_model") or "").strip()
+    cfg.hf_gguf = (b.get("hf_gguf") or "").strip()
     cfg.container_disk_gb = _as_int(b.get("container_disk_gb"), cfg.container_disk_gb)
     cfg.volume_gb = _as_int(b.get("volume_gb"), cfg.volume_gb)
     cfg.port = _as_int(b.get("port"), cfg.port) or 8000
@@ -143,6 +208,16 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
     return cfg
 
 
+def _model_path_for(model_name: str) -> str:
+    """The model's local path from models.json (used to spot a GGUF), or ''."""
+    try:
+        from codai.models.manager import _model_entry_for
+        entry = _model_entry_for(model_name) or {}
+        return str(entry.get("path") or "")
+    except Exception:
+        return ""
+
+
 def model_runpod_block(model_name: str) -> dict:
     """Fetch the ``runpod`` block from the model's models.json entry, or {}."""
     try:
@@ -351,9 +426,15 @@ class RunpodPodPool:
         (pod_id, sel, next_index). Raises if all fail."""
         from codai.api.runpod_client import RunpodError
         port = self.mcfg.port or 8000
-        image = self.mcfg.image or DEFAULT_POD_IMAGE
+        engine = resolve_pod_engine(self.mcfg, str(self.model_key),
+                                    _model_path_for(self.model_key))
+        if engine == "llamacpp":
+            image = self.mcfg.image or LLAMACPP_POD_IMAGE
+            args = _llamacpp_docker_args(self.mcfg, self.served)
+        else:
+            image = self.mcfg.image or DEFAULT_POD_IMAGE
+            args = _vllm_docker_args(self.mcfg, self.served)
         env = dict(self.mcfg.env or {})
-        args = _vllm_docker_args(self.mcfg, self.served)
         dc = getattr(self.account, "data_center", "") or ""
         last = None
         for i in range(start, len(ranked)):
@@ -445,7 +526,7 @@ class RunpodPodPool:
                         break
                     time.sleep(5)
                 else:
-                    raise RunpodError(f"pod {pod_id} OpenAI server (vLLM) did not answer "
+                    raise RunpodError(f"pod {pod_id} OpenAI server did not answer "
                                       f"/v1/models within {int(load_to)}s")
             except Exception as exc:
                 # Self-diagnose: pull the vLLM/container log before tearing down, so

codai/backends/remote_openai.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+"""Remote OpenAI backend — serve any text model from somewhere else.
+
+Set ``service_url`` on a model entry and coderai stops loading it locally and
+proxies to that endpoint instead: another coderai, a llama.cpp ``llama-server``,
+a vLLM/SGLang/TGI server, a rented pod, or any hosted OpenAI-compatible API.
+Nothing is downloaded and no VRAM is used here.
+
+This is the piece that makes the GGUF/llama.cpp catalogue remotizable. Those
+models load in-process through llama-cpp-python, so there was no service to
+redirect; pointing the *model* at a URL sidesteps that entirely — the far side
+runs llama.cpp (or anything else) and coderai is a client.
+
+The request machinery is RunPod's — same OpenAI wire protocol, same SSE
+streaming — so this subclasses it and only replaces endpoint resolution.
+
+Model entry fields:
+  service_url     base URL. With or without a trailing /v1 (it is added if missing).
+  served_model    the name the remote knows the model by (default: the model's own).
+  api_key         bearer token for the remote, when it wants one.
+"""
+
+import os
+from typing import Dict, Optional
+
+from codai.backends.runpod import RunpodBackend
+
+
+def model_remote_url(model_name: str) -> str:
+    """The ``service_url`` configured on this model's entry, or ''."""
+    if not model_name:
+        return ""
+    try:
+        from codai.models.manager import _model_entry_for
+        entry = _model_entry_for(model_name) or {}
+    except Exception:
+        return ""
+    return str(entry.get("service_url") or "").strip()
+
+
+def remote_should_handle(model_name: str) -> bool:
+    """True when this model is configured to be served from somewhere else."""
+    return bool(model_remote_url(model_name))
+
+
+class RemoteOpenAIBackend(RunpodBackend):
+    """Proxy a model to any OpenAI-compatible endpoint."""
+
+    # Its own in-flight counter: these requests run on someone else's hardware,
+    # so like RunPod they must not count against the local concurrency gate —
+    # but they also must not be mistaken for RunPod spend.
+    _inflight = 0
+    import threading as _threading
+    _inflight_lock = _threading.Lock()
+    del _threading
+
+    def __init__(self, cfg=None):
+        super().__init__(cfg)
+        self._mode = "remote"
+        self._pool = None
+
+    def load_model(self, model_name: str, **kwargs) -> None:
+        """Resolve the remote endpoint for ``model_name``. Downloads nothing."""
+        try:
+            from codai.models.manager import _model_entry_for
+            entry = _model_entry_for(model_name) or {}
+        except Exception:
+            entry = {}
+        url = (str(entry.get("service_url") or "").strip()
+               or os.environ.get("CODERAI_REMOTE_SERVICE_URL", "").strip())
+        if not url:
+            raise RuntimeError(f"no service_url configured for model {model_name!r}")
+
+        self._model_id = model_name or "remote"
+        self._served_name = (str(entry.get("served_model") or "").strip()
+                             or model_name or "remote")
+        url = url.rstrip("/")
+        # Accept both "http://host:8000" and "http://host:8000/v1".
+        self._url = url if url.endswith("/v1") else url + "/v1"
+
+        key = str(entry.get("api_key") or "").strip() or os.environ.get("CODERAI_REMOTE_API_KEY", "")
+        self._headers: Dict[str, str] = {"Authorization": f"Bearer {key}"} if key else {}
+
+        ctx = kwargs.get("n_ctx", kwargs.get("ctx")) or entry.get("n_ctx")
+        if isinstance(ctx, (list, tuple)):
+            ctx = ctx[0] if ctx else None
+        try:
+            ctx = int(ctx or 0)
+        except (TypeError, ValueError):
+            ctx = 0
+        if ctx > 0:
+            self._ctx = ctx
+
+        print(f"[remote] '{model_name}' -> {self._url} (served as {self._served_name!r})",
+              flush=True)
+
+    def _acquire_endpoint(self):
+        """One fixed endpoint — nothing to provision or release."""
+        if not self._url:
+            raise RuntimeError("remote endpoint not resolved")
+        return self._url, (lambda: None)
+
+    def health(self) -> Optional[dict]:
+        import requests
+        try:
+            r = requests.get(self._base() + "/models", headers=self._headers, timeout=10)
+            return r.json() if r.ok else None
+        except Exception:
+            return None
+
+    def cleanup(self) -> None:
+        self._url = None

codai/models/manager.py

@@ -440,10 +440,22 @@ def runpod_should_handle(model_name: str) -> bool:
     return resolve_engine_backend(model_name) == "runpod"
 
 
-def _model_is_runpod(model_name: str, config=None) -> bool:
-    """True for a RunPod-served model — from the runtime config's backend pin (or its
-    embedded _raw_cfg), else the resolver. Such models hold NO local weights, so the
-    manager must never download/cache them or estimate local VRAM for them."""
+def _model_is_remote(model_name: str, config=None) -> bool:
+    """True for a model served from somewhere else — RunPod, or any endpoint named
+    by ``service_url``.
+
+    From the runtime config's backend pin (or its embedded ``_raw_cfg``), else the
+    resolver. Such models hold NO local weights, so the manager must never
+    download/cache them or estimate local VRAM for them.
+    """
+    if isinstance(config, dict) and str(config.get("service_url") or "").strip():
+        return True
+    try:
+        from codai.backends.remote_openai import remote_should_handle
+        if remote_should_handle(model_name):
+            return True
+    except Exception:
+        pass
     if isinstance(config, dict):
         b = (config.get("backend") or "").strip().lower()
         if not b:
@@ -458,6 +470,10 @@ def _model_is_runpod(model_name: str, config=None) -> bool:
         return False
 
 
+#: Back-compat alias — the predicate grew past RunPod to cover any remote endpoint.
+_model_is_runpod = _model_is_remote
+
+
 def _trim_cpu_ram() -> None:
     """Return freed CPU heap memory to the OS (and let the kernel reclaim swap).
 
@@ -597,6 +613,22 @@ class ModelManager:
             self.tool_parser = ModelParserAdapter(model_name=model_name)
             return
 
+        # Remote: a model entry carrying `service_url` is served from somewhere
+        # else entirely (another coderai, a llama.cpp/vLLM/SGLang server, a
+        # rented pod). This is checked FIRST because it overrides every local
+        # decision — no weights, no VRAM, no engine here. It is also what makes
+        # the GGUF/llama.cpp catalogue remotizable: those models load in-process,
+        # so pointing the model itself at a URL is the only redirect there is.
+        from codai.backends.remote_openai import remote_should_handle
+        if remote_should_handle(model_name):
+            from codai.backends.remote_openai import RemoteOpenAIBackend
+            print(f"Routing '{model_name}' to a remote OpenAI endpoint")
+            self.backend_type = "remote"
+            self.backend = RemoteOpenAIBackend()
+            self.backend.load_model(model_name, **kwargs)
+            self.tool_parser = ModelParserAdapter(model_name=model_name)
+            return
+
         # GLM-5.2 via colibri: when enabled, drive the managed colibri C engine
         # (in-process mux protocol) instead of the normal nvidia/vulkan backends.
         if colibri_should_handle(model_name):
@@ -1521,8 +1553,8 @@ class MultiModelManager:
         # RunPod-served models hold no local weights — the remote pod/endpoint pulls
         # them. Never download/cache locally; the backend resolves the remote URL on
         # first request.
-        if _model_is_runpod(model_name, self.config.get(model_name)):
-            print(f"Model '{model_name}' is RunPod-served — skipping local download/cache")
+        if _model_is_remote(model_name, self.config.get(model_name)):
+            print(f"Model '{model_name}' is served remotely — skipping local download/cache")
             return
 
         # Download/cache the model at startup if it's a URL or HF ID