runpod: let a pod download before its server starts

Your idea, and it removes the last dead end. vLLM and llama.cpp pods
could not be given a local model or adapter: no endpoint to upload to,
and their server needs the file at launch. But a pod CAN fetch it before
the server runs — put the weights on any HTTP host it can reach and stage
them at boot.

The obstacle was that those images run their server as the ENTRYPOINT, so
everything we passed became an argument to the server. RunPod's REST pod
API accepts dockerEntrypoint/dockerStartCmd (GraphQL does not), so
runpod_client now creates a pod through REST when an override is needed
and stays on GraphQL otherwise.

The staged start command fetches with curl, wget or python — whichever
the image actually has rather than whichever we assumed — extracts a tar
for a multi-file model, and then execs the server pointed at what it
staged. One definition of the staged path (staged_model_dest) is used by
both the download and the server args: computing it twice is exactly how
a pod downloads to one path and serves another, which the first cut did,
along with emitting two --model flags.

Adapters given as a URL ride the same path: --lora-modules for vLLM,
--lora/--lora-scaled for llama.cpp. So a local adapter reaches a vLLM pod
after all — publish it to a URL instead of uploading it.

`source: upload` against a vLLM pod now says what to do instead, rather
than only that it cannot.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/routes.py

@@ -2809,7 +2809,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
         if ct:
             rpo["cloud_types"] = ct
         # bools
-        for k in ("allow_spot", "allow_open_pod", "sticky_sessions", "keep_warm"):
+        for k in ("allow_spot", "allow_open_pod", "sticky_sessions", "keep_warm",
+                  "model_url_is_tar"):
             if k in src and src.get(k) is not None and src.get(k) != "":
                 sv = src.get(k)
                 rpo[k] = (sv.lower() in ("1", "true", "on", "yes")

codai/admin/templates/models.html

@@ -795,6 +795,9 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               <label class="form-label" id="cfg-rp-source-label">Repo / URL</label>
               <input type="text" id="cfg-rp-hf-repo" class="form-input" placeholder="Qwen/Qwen3.5-9B">
               <input type="text" id="cfg-rp-model-url" class="form-input" style="display:none" placeholder="https://host/model-Q4_K_M.gguf">
+              <label style="display:none;align-items:center;gap:.3rem;font-size:11px;margin-top:.3rem" id="cfg-rp-tar-row">
+                <input type="checkbox" id="cfg-rp-url-tar" style="width:auto"> The URL is a tar of a multi-file model
+              </label>
             </div>
           </div>
           <div class="form-row" style="margin:.5rem 0 0">
@@ -4198,6 +4201,7 @@ function _populateRunpod(s){
   set('cfg-rp-source', rp.source || 'auto');
   set('cfg-rp-hf-repo', rp.hf_repo || '');
   set('cfg-rp-model-url', rp.model_url || '');
+  chk('cfg-rp-url-tar', rp.model_url_is_tar);
   chk('cfg-rp-keep-warm', rp.keep_warm);
   _updateRunpodSource();
   set('cfg-rp-api-key', rp.api_key || '');
@@ -4264,12 +4268,14 @@ function _updateRunpodSource(){
   const hint = document.getElementById('cfg-rp-source-hint');
   if (repo) repo.style.display = (v === 'url' || v === 'upload') ? 'none' : '';
   if (url) url.style.display = (v === 'url') ? '' : 'none';
+  const tarRow = document.getElementById('cfg-rp-tar-row');
+  if (tarRow) tarRow.style.display = (v === 'url') ? 'flex' : 'none';
   if (label) label.textContent = (v === 'url') ? 'Download URL'
     : (v === 'upload') ? 'Uploaded from this machine' : 'HuggingFace repo id';
   const HINTS = {
     auto: 'Uses the repo id if the model has one — including one recovered from the HuggingFace cache path.',
     hf: 'The pod downloads it from HuggingFace at datacenter speed. Costs nothing here.',
-    url: 'The pod downloads it directly — how most one-off GGUFs are had.',
+    url: 'The pod downloads it before its server starts — works for vLLM and llama.cpp pods too, so weights on any HTTP host you control can be served.',
     upload: 'Sent from this machine. Needed only when the weights exist nowhere else; every NEW pod pays the upload again.',
   };
   if (hint) hint.textContent = HINTS[v] || '';
@@ -4363,6 +4369,7 @@ function _collectRunpod(){
     source:             val('cfg-rp-source') || 'auto',
     hf_repo:            (val('cfg-rp-hf-repo') || '').trim(),
     model_url:          (val('cfg-rp-model-url') || '').trim(),
+    model_url_is_tar:   document.getElementById('cfg-rp-url-tar').checked,
     keep_warm:          document.getElementById('cfg-rp-keep-warm').checked,
     api_key:            (val('cfg-rp-api-key') || '').trim(),
     allow_open_pod:     document.getElementById('cfg-rp-open').checked,

codai/api/runpod_client.py

@@ -183,8 +183,26 @@ class RunpodClient:
                    volume_mount_path: str = "/workspace", env: Optional[dict] = None,
                    docker_args: str = "", is_spot: bool = False,
                    bid_per_gpu: float = 0.0, data_center_id: str = "",
-                   registry_auth_id: str = "") -> str:
-        """Provision a pod (on-demand or interruptible/spot). Returns the pod id."""
+                   registry_auth_id: str = "",
+                   entrypoint: Optional[list] = None,
+                   start_cmd: Optional[list] = None) -> str:
+        """Provision a pod (on-demand or interruptible/spot). Returns the pod id.
+
+        ``entrypoint``/``start_cmd`` override the image's ENTRYPOINT/CMD, which is
+        what lets a pod fetch something before its server starts — an image whose
+        ENTRYPOINT is the server itself (vllm/vllm-openai) otherwise treats every
+        argument as the server's own. Only the REST API accepts those, so the
+        request goes there when they are given and stays on GraphQL otherwise.
+        """
+        if entrypoint or start_cmd:
+            return self._create_pod_rest(
+                name=name, image=image, gpu_type_id=gpu_type_id, port=port,
+                cloud_type=cloud_type, gpu_count=gpu_count,
+                container_disk_gb=container_disk_gb, volume_gb=volume_gb,
+                volume_mount_path=volume_mount_path, env=env,
+                is_spot=is_spot, bid_per_gpu=bid_per_gpu,
+                data_center_id=data_center_id, registry_auth_id=registry_auth_id,
+                entrypoint=entrypoint, start_cmd=start_cmd)
         env_list = [{"key": str(k), "value": str(v)} for k, v in (env or {}).items()]
         ports = f"{int(port)}/http"
         common = {
@@ -225,6 +243,50 @@ class RunpodClient:
             raise RunpodError(f"RunPod did not return a pod id for {name!r} (capacity? {data}).")
         return pod_id
 
+    def _create_pod_rest(self, *, name, image, gpu_type_id, port, cloud_type,
+                         gpu_count, container_disk_gb, volume_gb, volume_mount_path,
+                         env, is_spot, bid_per_gpu, data_center_id,
+                         registry_auth_id, entrypoint, start_cmd) -> str:
+        """Create a pod through the REST API, which accepts entrypoint overrides."""
+        import requests
+        rest = (getattr(self._cfg, "rest_base", "") or "https://rest.runpod.io/v1").rstrip("/")
+        body = {
+            "name": name,
+            "imageName": image,
+            "gpuTypeIds": [gpu_type_id],
+            "gpuCount": int(gpu_count),
+            "cloudType": (cloud_type or "SECURE").upper(),
+            "containerDiskInGb": int(container_disk_gb),
+            "volumeInGb": int(volume_gb),
+            "volumeMountPath": volume_mount_path,
+            "ports": [f"{int(port)}/http"],
+            "env": {str(k): str(v) for k, v in (env or {}).items()},
+        }
+        if entrypoint:
+            body["dockerEntrypoint"] = list(entrypoint)
+        if start_cmd:
+            body["dockerStartCmd"] = list(start_cmd)
+        if data_center_id:
+            body["dataCenterIds"] = [data_center_id]
+        if registry_auth_id:
+            body["containerRegistryAuthId"] = registry_auth_id
+        if is_spot:
+            body["interruptible"] = True
+            body["bidPerGpu"] = float(bid_per_gpu)
+        r = requests.post(f"{rest}/pods", json=body,
+                          headers={"Authorization": f"Bearer {self._api_key}",
+                                   "Content-Type": "application/json"},
+                          timeout=90)
+        if r.status_code >= 400:
+            raise RunpodError(f"RunPod REST create failed ({r.status_code}): "
+                              f"{(r.text or '')[:400]}")
+        data = r.json() if r.content else {}
+        pod_id = (data.get("id") or (data.get("pod") or {}).get("id")
+                  or (data.get("data") or {}).get("id"))
+        if not pod_id:
+            raise RunpodError(f"RunPod REST create returned no pod id: {str(data)[:300]}")
+        return pod_id
+
     def get_pod(self, pod_id: str) -> dict:
         """Return {id, status, uptime_s, cost_per_hr, ports:[...], ready:bool, url?}."""
         q = """

docs/remote-execution.md

@@ -398,14 +398,30 @@ these has to resolve (`source` on the model's runpod block):
 downloaded through coderai live in the HuggingFace cache, whose directory names
 still say which repo they came from.
 
-Not every server can take every source, and the mismatch is refused at
-configuration time rather than after a pod has booted for five minutes:
-
-* **vLLM** is launched `--model <id>` and can only take a HuggingFace repo id.
-* **llama.cpp** takes `-hf user/repo:QUANT` or downloads a URL with `-mu`.
+How each server gets there differs, and a combination that cannot work is
+refused at configuration time rather than after a pod has booted for five
+minutes:
+
+* **llama.cpp** takes `-hf user/repo:QUANT`, or downloads a URL itself (`-mu`).
+* **vLLM** is launched `--model <id>` and downloads nothing but a HuggingFace
+  repo — *so the pod downloads it first*: with `source: url`, coderai overrides
+  the image's entrypoint with a small script that fetches the file (curl, wget or
+  python — whichever the image has), extracts it when `model_url_is_tar` is set,
+  and then execs vLLM pointed at what it staged. This is the "external bounce"
+  route: put the weights on any HTTP host the pod can reach — an object store,
+  your own HTTPS server — and a vLLM pod can serve them.
 * **Upload works only on a coderai pod.** vLLM and llama.cpp need the file to
-  exist before their container starts; a coderai pod loads models on demand, so
-  it can be given them afterwards.
+  exist before their container starts and have no endpoint to receive one; a
+  coderai pod loads models on demand, so it can be given them afterwards.
+
+The same staging carries **adapters given as a URL**: a vLLM pod gets
+`--lora-modules name=<staged path>`, a llama.cpp pod `--lora`/`--lora-scaled`.
+So a local adapter can reach a vLLM pod after all — put it on a URL rather than
+uploading it.
+
+Staging uses RunPod's REST pod API, which is the only one that accepts
+`dockerEntrypoint`/`dockerStartCmd`; pods that need no staging are created
+through the GraphQL path exactly as before.
 
 Uploading is a deliberate choice, not a fallback: it is the cold cost of every
 NEW pod, since pod storage is disposable. On a fast symmetric link that may be

tests/test_remote_execution.py

@@ -817,11 +817,11 @@ def test_each_pod_server_gets_a_source_it_can_actually_use():
                                  "m", gguf)
     assert "-mu https://h/m.gguf" in args
 
-    # vLLM is launched with --model and can only take a repo id; say so rather
-    # than boot a pod for minutes and fail.
-    with pytest.raises(RuntimeError, match="cannot fetch"):
-        _vllm_docker_args(parse_model_runpod({"model_url": "https://h/m.bin"}),
-                          "/AI/local/m", {"path": "/AI/local/m"})
+    # vLLM cannot download a URL itself, but the POD can before vLLM starts, so
+    # the args name the staged path rather than refusing the URL.
+    staged = _vllm_docker_args(parse_model_runpod({"model_url": "https://h/m.bin"}),
+                               "/AI/local/m", {"path": "/AI/local/m"})
+    assert "--model /runpod-volume/staged/m.bin" in staged
     # Upload cannot work for a server that needs the file before it starts.
     with pytest.raises(RuntimeError, match="cannot work here"):
         _llamacpp_docker_args(parse_model_runpod({"source": "upload"}), "m", gguf)
@@ -1035,3 +1035,70 @@ def test_a_content_addressed_adapter_resolves_through_the_blob_store(monkeypatch
     # And it is not mistaken for something a remote could fetch by name.
     assert text_loras.is_portable("sha256:" + "a" * 64) is False
     assert text_loras.is_portable("org/adapter") is True
+
+
+# --------------------------------------------------------------------------- #
+# staging: the pod downloads before its server starts
+# --------------------------------------------------------------------------- #
+def test_a_vllm_pod_can_download_a_model_before_starting():
+    """The third way onto a vLLM pod: not on HuggingFace, not uploadable (no
+    receiver) — but downloadable from any host the pod can reach, BEFORE vLLM
+    starts. That needs the image's ENTRYPOINT overridden, since vLLM is it."""
+    from codai.api.runpod_worker import pod_plan, parse_model_runpod, STAGE_DIR
+
+    entry = {"path": "/AI/models/merged", "model_type": "text_models"}
+    plan = pod_plan(parse_model_runpod(
+        {"engine": "vllm", "model_url": "https://host/merged.tar",
+         "model_url_is_tar": True}), "m", "m", entry=entry)
+
+    assert plan["entrypoint"] == ["/bin/sh", "-c"]
+    script = plan["start_cmd"][0]
+    # Downloads the tar, extracts to a directory of the same name without the
+    # suffix, and serves THAT — one definition of the path, not two.
+    assert f"{STAGE_DIR}/merged.tar" in script          # the archive
+    assert f"-C '{STAGE_DIR}/merged'" in script          # extracted here
+    assert script.rstrip().endswith(f"--served-model-name {STAGE_DIR}/merged")
+    assert script.count("--model ") == 1                 # never two --model flags
+
+
+def test_llamacpp_downloads_its_own_model_but_stages_an_adapter():
+    """llama-server fetches a model URL itself (-mu), so only the adapter needs
+    staging — and it takes one, with its scale."""
+    from codai.api.runpod_worker import pod_plan, parse_model_runpod, STAGE_DIR
+
+    entry = {"path": "Qwen/Qwen3.5-9B", "model_type": "text_models",
+             "loras": [{"path": "https://host/a.safetensors", "weight": 0.7,
+                        "name": "mine"}]}
+    plan = pod_plan(parse_model_runpod({"engine": "llamacpp", "hf_gguf": "u/r:Q4"}),
+                    "q", "q", entry=entry)
+    script = plan["start_cmd"][0]
+    assert f"--lora-scaled {STAGE_DIR}/loras/a.safetensors 0.7" in script
+    assert "-hf u/r:Q4" in script          # the model still comes from HF
+
+
+def test_upload_to_a_vllm_pod_says_what_to_do_instead():
+    from codai.api.runpod_worker import _vllm_docker_args, parse_model_runpod
+
+    with pytest.raises(RuntimeError, match="model_url"):
+        _vllm_docker_args(parse_model_runpod({"source": "upload"}),
+                          "/AI/local/m", {"path": "/AI/local/m"})
+
+
+def test_staging_quotes_what_it_downloads():
+    """The URL is configuration, but it still ends up in a shell command."""
+    from codai.api.runpod_worker import stage_script
+
+    script = stage_script(
+        [{"url": "https://host/a'; rm -rf /; echo '", "dest": "/tmp/x"}], "server")
+    # The quote is escaped, so the injected text stays one argument.
+    assert "rm -rf /" in script and "'\\''" in script
+    assert script.rstrip().endswith("exec server")
+
+
+def test_no_staging_when_nothing_needs_downloading():
+    """An HF repo id needs no entrypoint override — leave the image alone."""
+    from codai.api.runpod_worker import pod_plan, parse_model_runpod
+
+    plan = pod_plan(parse_model_runpod({}), "Qwen/Qwen3.5-9B", "q",
+                    entry={"path": "Qwen/Qwen3.5-9B", "model_type": "text_models"})
+    assert "entrypoint" not in plan and "start_cmd" not in plan