runpod: send local text adapters to the pod that can receive them

A text LoRA that exists only on this machine had nowhere to go: vLLM and
llama.cpp resolve adapters at launch and have no endpoint to upload one
to, so the previous commit could only report and skip it.

Such a model is now routed to a coderai TEXT pod — the one pod image that
can be sent an adapter — and the transfer is arranged around the fact
that a pod is created before anything can be sent to it:

  1. the pod is told the adapter's sha256 id, computed HERE, so both
     sides agree on the name before the pod exists;
  2. the bytes go to the pod's content-addressed blob store before the
     first request, once per pod (a repeat costs one existence check);
  3. the pod resolves sha256:<hex> back to a file — text_loras.local_path
     now understands the content-addressed forms — and applies it.

A published adapter still goes to vLLM, which fetches it itself: cheaper
and faster, so publishing an adapter you use often remains the better
path, and the log says so. A PEFT adapter DIRECTORY is still refused with
a reason — the blob store holds single files.

packaging/runpod/profiles/text.txt adds the text profile (transformers,
peft, bitsandbytes for QLoRA bases). It has to be built and pushed before
use; the other nine are published already.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/api/remote_gateway.py

@@ -283,6 +283,9 @@ class Target:
         #: The model entry whose weights must be pushed before serving, when the
         #: remote has no way to fetch them (`source: "upload"`).
         self.upload_entry = None
+        #: The model entry whose LOCAL text adapters must be pushed before the
+        #: pod loads the model. Only a coderai pod can receive them.
+        self.lora_entry = None
 
     def acquire(self):
         """Return (base_url, release). A pool provisions on demand and bills."""
@@ -324,9 +327,22 @@ def resolve_target(path: str, method: str, query: str, body: bytes,
                                              resolve_model_source)
         target = Target(pool=get_model_pod_pool(model, entry, block),
                         reason=f"model {model!r} on its own RunPod pod")
-        kind, _ = resolve_model_source(entry, parse_model_runpod(block))
+        mcfg = parse_model_runpod(block)
+        kind, _ = resolve_model_source(entry, mcfg)
         if kind == "upload":
             target.upload_entry = entry
+        # Local adapters only reach a coderai pod: vLLM and llama.cpp images have
+        # no endpoint to receive an upload.
+        try:
+            from codai.api.runpod_worker import resolve_pod_engine
+            from codai.models.text_loras import configured_specs, portable_specs
+            if resolve_pod_engine(mcfg, model, str(entry.get("path") or ""), entry) \
+                    in ("coderai", "custom"):
+                _, needs_sending = portable_specs(configured_specs(entry))
+                if needs_sending:
+                    target.lora_entry = entry
+        except Exception:
+            pass
         return target
     if not cap:
         return None
@@ -412,6 +428,12 @@ class RemoteGatewayMiddleware:
             if target.upload_entry:
                 await asyncio.to_thread(ensure_model_uploaded, base, target.api_key,
                                         target.upload_entry)
+            # A text LoRA is applied when the model LOADS, so the adapter has to
+            # be on the pod before the first request — not carried by it.
+            if target.lora_entry is not None:
+                await asyncio.to_thread(ensure_text_loras, base, target.api_key,
+                                        target.lora_entry)
+                target.lora_entry = None        # sent once per pod
             # Ship any LoRA the request names, so the remote can actually load it.
             if b'"loras"' in (body or b"") and "json" in headers.get("content-type", ""):
                 body = await asyncio.to_thread(
@@ -582,6 +604,92 @@ def sync_loras(base: str, api_key: str, body: bytes) -> bytes:
 _UPLOADED: set = set()
 
 
+def push_lora_file(base: str, api_key: str, path: str) -> str:
+    """Put one adapter file in a remote's blob store; return its ``sha256:`` id.
+
+    Content-addressed, so a pod that already has it is told nothing and a repeat
+    costs one HEAD-shaped check. Returns '' when the transfer fails, which the
+    caller reports rather than silently serving a model without its adapter.
+    """
+    import hashlib
+    import requests
+
+    if not path or not os.path.isfile(path):
+        return ""
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+    try:
+        with open(path, "rb") as fh:
+            data = fh.read()
+        digest = hashlib.sha256(data).hexdigest()
+        have = requests.get(f"{base}/v1/loras/blob/{digest}", headers=headers, timeout=30)
+        if have.status_code != 200:
+            up = requests.post(f"{base}/v1/loras/upload", data=data,
+                               headers={**headers,
+                                        "Content-Type": "application/octet-stream"},
+                               timeout=1800)
+            up.raise_for_status()
+            print(f"[remote-gateway] sent adapter {os.path.basename(path)} "
+                  f"({len(data) / 1e6:.0f} MB) to {base}", flush=True)
+        return f"sha256:{digest}"
+    except Exception as exc:
+        print(f"[remote-gateway] could not send adapter {path} to {base}: {exc}",
+              flush=True)
+        return ""
+
+
+def ensure_text_loras(base: str, api_key: str, entry: dict) -> dict:
+    """Send a text model's LOCAL adapters to a coderai pod.
+
+    A text LoRA is part of how the model loads, not something the request
+    carries, so it has to be on the pod before the model is loaded there. Only a
+    coderai pod can receive one — a vLLM or llama.cpp pod has no endpoint to
+    upload to, which is why those need a HuggingFace repo id instead.
+
+    Returns the config the REMOTE should use: the same entry with local adapter
+    paths replaced by the content ids it now holds. An adapter that could not be
+    sent is dropped from that config, so the pod loads the model without it
+    rather than failing on a path it cannot see.
+    """
+    from codai.models.text_loras import configured_specs, local_path, is_portable
+
+    specs = configured_specs(entry or {})
+    if not specs:
+        return dict(entry or {})
+
+    rewritten, dropped = [], []
+    for spec in specs:
+        source = str(spec.get("source") or "")
+        if is_portable(source):
+            rewritten.append({**spec, "source": source})
+            continue
+        path = local_path(source)
+        if not path:
+            dropped.append(source)
+            continue
+        if os.path.isdir(path):
+            # A PEFT adapter directory is several files; the blob store holds
+            # single files. Publishing it to HuggingFace is the way to move it.
+            dropped.append(source)
+            print(f"[remote-gateway] {source} is a directory — publish it to "
+                  "HuggingFace to use it on a pod", flush=True)
+            continue
+        blob = push_lora_file(base, api_key, path)
+        if blob:
+            rewritten.append({**spec, "source": blob})
+        else:
+            dropped.append(source)
+
+    out = {k: v for k, v in (entry or {}).items()
+           if k not in ("lora_path", "lora_model_dir", "loras", "lora_scale")}
+    if rewritten:
+        out["loras"] = [{"path": s["source"], "weight": s.get("weight", 1.0),
+                         "name": s.get("name")} for s in rewritten]
+    if dropped:
+        print(f"[remote-gateway] adapters not available remotely: {', '.join(dropped)}",
+              flush=True)
+    return out
+
+
 def ensure_model_uploaded(base: str, api_key: str, entry: dict) -> None:
     """Push a model's weights to a remote that cannot fetch them itself.
 

codai/api/runpod_worker.py

@@ -166,6 +166,10 @@ CAPABILITY_IMAGE_TAG = os.environ.get("CODERAI_CAPABILITY_IMAGE_TAG", "latest")
 PUBLISHED_CAPABILITY_IMAGES = (
     "images", "video", "embeddings", "ocr", "tts", "stt", "voice", "audio",
     "faceswap",
+    # Text: a coderai pod that serves LLMs. Unlike a vLLM/llama.cpp pod it can be
+    # SENT a local LoRA adapter, which is the only way a local text adapter runs
+    # on RunPod. Build and push it with packaging/runpod before using it.
+    "text",
 )
 
 #: Capabilities served by a published image other than their own name.
@@ -182,6 +186,9 @@ _CAPABILITY_IMAGE_ALIASES = {
 #: lets a NON-TEXT model be sent to RunPod individually: the pod image follows
 #: from what the model is, not from whether its path ends in .gguf.
 MODEL_TYPE_CAPABILITY = {
+    "text_models": "text",
+    "gguf_models": "text",
+    "vision_models": "text",
     "image_models": "images",
     "video_models": "video",
     "audio_models": "stt",
@@ -195,13 +202,19 @@ MODEL_TYPE_CAPABILITY = {
 _LLM_MODEL_TYPES = ("text_models", "gguf_models", "vision_models")
 
 
-def model_capability(entry: dict) -> str:
-    """The capability a model entry belongs to, or '' for a text/LLM model."""
+def model_capability(entry: dict, include_text: bool = False) -> str:
+    """The capability a model entry belongs to.
+
+    Text models return '' by default: they normally go to vLLM or llama.cpp, not
+    to a coderai pod. ``include_text`` asks for the capability anyway, which is
+    what picks the image once something HAS decided a text model needs a coderai
+    pod (a local LoRA adapter, today).
+    """
     if not isinstance(entry, dict):
         return ""
     types = entry.get("model_types") or [entry.get("model_type") or ""]
     for mt in types:
-        if mt in _LLM_MODEL_TYPES:
+        if mt in _LLM_MODEL_TYPES and not include_text:
             return ""
         cap = MODEL_TYPE_CAPABILITY.get(mt)
         if cap:
@@ -299,6 +312,19 @@ def resolve_pod_engine(mcfg: "RunpodModelConfig", model_key: str = "",
         entry = _model_entry(model_key)
     if model_capability(entry or {}):
         return "coderai"
+    # A text model whose LoRA lives only on this disk has one option: a coderai
+    # pod, which can be sent the adapter. vLLM and llama.cpp resolve adapters
+    # themselves at launch and have no endpoint to receive one.
+    try:
+        from codai.models.text_loras import configured_specs, portable_specs
+        _portable, _local = portable_specs(configured_specs(entry or {}))
+        if _local:
+            print(f"[lora] {model_key!r} has adapters that exist only here "
+                  f"({', '.join(str(s.get('source')) for s in _local)}) — using a "
+                  "coderai pod, which can be sent them", flush=True)
+            return "coderai"
+    except Exception:
+        pass
     if mcfg.hf_gguf:
         return "llamacpp"
     if _looks_like_gguf(mcfg.served_model) or _looks_like_gguf(model_path) \
@@ -328,7 +354,7 @@ def pod_plan(mcfg: "RunpodModelConfig", served: str, model_key: str = "",
         image = mcfg.image or LLAMACPP_POD_IMAGE
         args = mcfg.docker_args or _llamacpp_docker_args(mcfg, served, entry)
     elif engine in ("coderai", "custom"):
-        cap = model_capability(entry or {})
+        cap = model_capability(entry or {}, include_text=True)
         if not mcfg.image and cap:
             # A model of a known kind gets that capability's published image.
             image = default_capability_image(cap)
@@ -413,6 +439,35 @@ def seed_model_env(entry: dict, served: str = "", source: str = "") -> str:
             "lora_path", "lora_model_dir", "lora_scale", "loras")
     out = {k: entry[k] for k in keep if k in entry and entry[k] is not None}
     out["path"] = path
+    # Adapters the pod cannot resolve by name are named by CONTENT id instead.
+    # The hash is computed here, so the pod is told the id when it is created and
+    # sent the bytes before the first request — the two agree without the pod
+    # having to exist yet. codai/api/remote_gateway.ensure_text_loras does the
+    # sending.
+    try:
+        from codai.models.text_loras import configured_specs, is_portable, blob_id, local_path
+        specs = configured_specs(entry)
+        if specs:
+            rewritten = []
+            for spec in specs:
+                src = str(spec.get("source") or "")
+                if is_portable(src) or src.startswith("sha256:"):
+                    rewritten.append({"path": src, "weight": spec.get("weight", 1.0),
+                                      "name": spec.get("name")})
+                    continue
+                local = local_path(src)
+                bid = blob_id(local) if local and os.path.isfile(local) else ""
+                if bid:
+                    rewritten.append({"path": bid, "weight": spec.get("weight", 1.0),
+                                      "name": spec.get("name")})
+            for key in ("lora_path", "lora_model_dir", "lora_scale"):
+                out.pop(key, None)
+            if rewritten:
+                out["loras"] = rewritten
+            else:
+                out.pop("loras", None)
+    except Exception as exc:
+        print(f"[seed] could not prepare adapters: {exc}", flush=True)
     if served and served != path:
         out["alias"] = served
     return _json.dumps([out])

codai/models/text_loras.py

@@ -110,11 +110,64 @@ def _default_name(source: str) -> str:
 
 
 def local_path(source: str) -> str:
-    """The adapter as a local path, or '' when it is a repo id / not present."""
-    p = os.path.expanduser(str(source or ""))
+    """The adapter as a path on THIS machine, or '' when it is not here.
+
+    Understands the content-addressed forms as well as plain paths —
+    ``sha256:<hex>`` and ``name:<registered>`` — which is how an adapter that was
+    uploaded rather than downloaded is referenced. That matters on a pod: the
+    file arrives in the blob store, and the model config names it by hash.
+    """
+    src = str(source or "").strip()
+    if not src:
+        return ""
+    if src.startswith(("sha256:", "name:")) or (len(src) == 64 and all(
+            c in "0123456789abcdef" for c in src.lower())):
+        try:
+            from codai.api.loras import resolve_lora_ref
+            resolved = resolve_lora_ref({"id": src})
+        except Exception:
+            resolved = None
+        return resolved if resolved and os.path.exists(resolved) else ""
+    p = os.path.expanduser(src)
     return p if p and os.path.exists(p) else ""
 
 
+def is_portable(source: str) -> bool:
+    """True when a REMOTE could resolve this adapter on its own.
+
+    A HuggingFace repo id can be downloaded anywhere. A local path cannot: it
+    has to be sent, and only a coderai pod has anywhere to put it.
+    """
+    src = str(source or "").strip()
+    return bool(src) and "/" in src and not src.startswith(("/", "~", ".")) \
+        and not src.startswith(("sha256:", "name:", "http://", "https://")) \
+        and src.count("/") == 1
+
+
+def blob_id(path: str) -> str:
+    """The ``sha256:<hex>`` id an adapter file will have in any blob store.
+
+    Computed here, so a pod can be TOLD the id when it is created and be SENT the
+    bytes afterwards — the two agree without the pod having to exist yet.
+    """
+    import hashlib
+    if not path or not os.path.isfile(path):
+        return ""
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return f"sha256:{h.hexdigest()}"
+
+
+def portable_specs(specs: List[dict]) -> tuple:
+    """Split adapters into (remote-resolvable, must-be-sent)."""
+    portable, local = [], []
+    for spec in specs:
+        (portable if is_portable(spec.get("source", "")) else local).append(spec)
+    return portable, local
+
+
 def gguf_adapter(spec: dict) -> str:
     """The GGUF adapter file for llama.cpp, or ''.
 

docs/remote-execution.md

@@ -430,13 +430,32 @@ Each runtime applies them its own way, and what reaches a pod differs:
 | vLLM | `--enable-lora --lora-modules` at launch | **HuggingFace repo ids only** |
 | diffusers (image/video) | `load_lora_weights` | uploaded to the remote automatically, content-addressed |
 
-The asymmetry is worth knowing: a **diffusion** LoRA is sent to the remote with
-the request (hashed, uploaded once, referenced by content id), because that API
-carries adapters per request. A **text** LoRA is part of how the model is
-*loaded*, so it has to be resolvable when the pod starts — which means a
-HuggingFace repo id for a vLLM or llama.cpp pod. A local adapter path is reported
-and skipped rather than baked into a launch command that would fail minutes later
-inside a pod. Publish it to HuggingFace, or serve that model on a coderai pod.
+The asymmetry is worth knowing: a **diffusion** LoRA is sent with the request
+(hashed, uploaded once, referenced by content id), because that API carries
+adapters per request. A **text** LoRA is part of how the model is *loaded*, so it
+has to be on the pod before the model loads there.
+
+**A local text adapter still works on RunPod** — it just decides which pod. A
+model whose adapter exists only on this machine is routed to a **coderai text
+pod** (`ghcr.io/<you>/coderai-text`), because that is the only pod image with an
+endpoint that can receive one. The sequence:
+
+1. the pod is created and told the adapter's `sha256:` id — computed here, so
+   the two agree before the pod exists;
+2. the adapter is sent to the pod's content-addressed blob store before the
+   first request, once per pod;
+3. the pod resolves `sha256:<hex>` back to a file and applies it with PEFT.
+
+A **published** adapter (a HuggingFace repo id) skips all of that and goes to a
+vLLM pod, which fetches it itself — cheaper and faster, so publish adapters you
+use often. A PEFT adapter **directory** cannot be sent (the blob store holds
+single files); publish that one.
+
+Build and push the text image before using it:
+
+```bash
+PROFILES=text ./packaging/runpod/publish_capability_images.sh
+```
 
 ### Keeping a pod warm
 

packaging/runpod/profiles/text.txt

+# Text generation (HuggingFace safetensors) — a coderai pod that serves LLMs.
+#
+# This is the profile that makes a LOCAL text LoRA usable on RunPod: vLLM and
+# llama.cpp pods can only load adapters they can resolve themselves (a HF repo
+# id), while a coderai pod is sent the adapter and applies it with PEFT.
+transformers>=4.35.0
+accelerate>=0.24.0
+peft>=0.10.0
+# QLoRA: the adapter is ordinary, but the BASE must load 4-bit the way it was
+# trained, which is what bitsandbytes does.
+bitsandbytes>=0.43.0
+sentencepiece>=0.1.99
+protobuf>=3.20.0

tests/test_remote_execution.py

@@ -925,6 +925,113 @@ def test_a_coderai_pod_is_told_about_the_models_adapters():
     seeded = json.loads(seed_model_env(
         {"path": "org/model", "model_type": "text_models",
          "lora_path": "org/adapter", "lora_scale": 0.6, "load_in_4bit": True}))[0]
-    assert seeded["lora_path"] == "org/adapter"
-    assert seeded["lora_scale"] == 0.6
+    # Normalised to one shape the pod can act on, weight preserved.
+    assert seeded["loras"] == [{"path": "org/adapter", "weight": 0.6,
+                                "name": "adapter"}]
+    # load_in_4bit is what makes a QLoRA adapter load against its own base.
     assert seeded["load_in_4bit"] is True
+
+
+# --------------------------------------------------------------------------- #
+# a LOCAL text adapter on RunPod
+# --------------------------------------------------------------------------- #
+def test_a_local_text_adapter_routes_to_a_coderai_pod():
+    """vLLM and llama.cpp resolve adapters themselves at launch and have no
+    endpoint to receive one, so an adapter that exists only here forces the pod
+    that CAN be sent it."""
+    from codai.api.runpod_worker import resolve_pod_engine, pod_plan, parse_model_runpod
+
+    plain = {"path": "Qwen/Qwen3.5-9B", "model_type": "text_models"}
+    assert resolve_pod_engine(parse_model_runpod({}), "q", plain["path"], plain) == "vllm"
+
+    local = dict(plain, lora_path="/AI/loras/mine.safetensors")
+    assert resolve_pod_engine(parse_model_runpod({}), "q", plain["path"], local) == "coderai"
+    assert pod_plan(parse_model_runpod({}), "q", "q",
+                    entry=local)["image"].endswith("coderai-text:latest")
+
+    # A published adapter needs none of that: vLLM can fetch it itself.
+    published = dict(plain, lora_path="org/published-lora")
+    assert resolve_pod_engine(parse_model_runpod({}), "q", plain["path"],
+                              published) == "vllm"
+
+
+def test_the_pod_is_told_the_adapters_content_id(tmp_path, monkeypatch):
+    """The pod is created before the adapter can be sent, so it is told the hash
+    the file WILL have — computed here — and sent the bytes afterwards."""
+    import hashlib
+    from codai.api.runpod_worker import seed_model_env
+
+    adapter = tmp_path / "mine.safetensors"
+    adapter.write_bytes(b"adapter-bytes" * 50)
+    digest = hashlib.sha256(adapter.read_bytes()).hexdigest()
+
+    seeded = json.loads(seed_model_env(
+        {"path": "Qwen/Qwen3.5-9B", "model_type": "text_models",
+         "lora_path": str(adapter), "lora_scale": 0.7}))[0]
+    assert seeded["loras"] == [{"path": f"sha256:{digest}", "weight": 0.7,
+                                "name": "mine"}]
+    # The local path must not survive: it means nothing on the pod.
+    assert "lora_path" not in seeded
+
+
+def test_local_adapters_are_sent_to_the_pod_once(tmp_path, monkeypatch):
+    from codai.api import remote_gateway as gw
+
+    adapter = tmp_path / "mine.safetensors"
+    adapter.write_bytes(b"weights" * 100)
+
+    calls = {"get": 0, "post": 0}
+    have = {"blob": False}
+
+    class _R:
+        def __init__(self, code): self.status_code = code
+        def raise_for_status(self): pass
+
+    monkeypatch.setattr("requests.get",
+                        lambda url, **kw: (calls.__setitem__("get", calls["get"] + 1),
+                                           _R(200 if have["blob"] else 404))[1])
+
+    def _post(url, **kw):
+        calls["post"] += 1
+        have["blob"] = True
+        return _R(200)
+
+    monkeypatch.setattr("requests.post", _post)
+
+    entry = {"path": "Qwen/Qwen3.5-9B", "model_type": "text_models",
+             "lora_path": str(adapter), "lora_scale": 0.5}
+    out = gw.ensure_text_loras("http://pod:8000", "tok", entry)
+    assert calls["post"] == 1
+    # The config handed to the remote names the adapter by content, not by path.
+    assert out["loras"][0]["path"].startswith("sha256:")
+    assert out["loras"][0]["weight"] == 0.5
+    assert "lora_path" not in out
+
+    gw.ensure_text_loras("http://pod:8000", "tok", entry)
+    assert calls["post"] == 1          # already there — nothing re-sent
+
+
+def test_a_published_adapter_is_not_uploaded(monkeypatch):
+    from codai.api import remote_gateway as gw
+
+    def _boom(*a, **k):
+        raise AssertionError("an HF repo id must never be uploaded")
+
+    monkeypatch.setattr("requests.post", _boom)
+    out = gw.ensure_text_loras("http://pod:8000", "tok",
+                               {"path": "org/m", "lora_path": "org/adapter"})
+    assert out["loras"][0]["path"] == "org/adapter"
+
+
+def test_a_content_addressed_adapter_resolves_through_the_blob_store(monkeypatch):
+    """On the pod the adapter arrives in the blob store and the config names it
+    by hash — which has to resolve back to a file."""
+    from codai.models import text_loras
+
+    monkeypatch.setattr("codai.api.loras.resolve_lora_ref",
+                        lambda spec: "/blobs/abc" if spec.get("id") else None)
+    monkeypatch.setattr(text_loras.os.path, "exists", lambda p: p == "/blobs/abc")
+    assert text_loras.local_path("sha256:" + "a" * 64) == "/blobs/abc"
+    # And it is not mistaken for something a remote could fetch by name.
+    assert text_loras.is_portable("sha256:" + "a" * 64) is False
+    assert text_loras.is_portable("org/adapter") is True