runpod: optional network volumes for weights, uploads and staging

A pod's disk dies with the pod, so every cold pod re-downloads its
weights and every upload has to be repeated. A network volume — created
by hand in the console, coderai never creates one — makes that a one-off.

Attaching it is the easy half; the useful half is pointing HF_HOME, the
diffusers/transformers caches, CODERAI_MODELS_DIR and the staging
directory at the mount. A volume attached without redirecting those
changes nothing, which is the mistake this avoids.

That also answers the bounce-host question: an upload lands on the volume
and every later pod sees it, so a QLoRA or a local model can be sent once
and used from any pod — no third-party storage, no public IP.

Two RunPod constraints are enforced rather than discovered the hard way:
volumes are Secure Cloud only (cloud_types is forced), and a pod must run
in the volume's own data center (looked up once and pinned, instead of
letting price ranking pick a region where the attach fails as a confusing
capacity error).

Settable account-wide in Settings or per pool. Container images still
come from a registry — a volume holds weights, not the image.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/routes.py

@@ -2794,7 +2794,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                   "engine", "hf_gguf", "health_path", "docker_args",
                   "registry_auth_id", "api_key", "pool",
                   # where the POD gets the weights
-                  "source", "hf_repo", "model_url", "quantization"):
+                  "source", "hf_repo", "model_url", "quantization",
+                  "network_volume_id", "volume_mount_path"):
             v = src.get(k)
             if isinstance(v, str) and v.strip():
                 rpo[k] = v.strip()
@@ -3662,6 +3663,8 @@ def build_settings_dict(c, gpu_cards):
             "default_gpu_type": c.runpod.default_gpu_type,
             "data_center": c.runpod.data_center,
             "registry_auth_id": c.runpod.registry_auth_id,
+            "network_volume_id": c.runpod.network_volume_id,
+            "volume_mount_path": c.runpod.volume_mount_path,
             "deployment_id": c.runpod.deployment_id,
             "global_max_hourly_usd": c.runpod.global_max_hourly_usd,
             "global_cost_limit_usd": c.runpod.global_cost_limit_usd,
@@ -4238,6 +4241,11 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
             rp.data_center = (d.get("data_center") or "").strip()
         if "registry_auth_id" in d:
             rp.registry_auth_id = (d.get("registry_auth_id") or "").strip()
+        if "network_volume_id" in d:
+            rp.network_volume_id = (d.get("network_volume_id") or "").strip()
+        if "volume_mount_path" in d:
+            rp.volume_mount_path = ((d.get("volume_mount_path") or "").strip()
+                                    or "/workspace")
         if "deployment_id" in d:
             rp.deployment_id = (d.get("deployment_id") or "default").strip() or "default"
         if "global_max_hourly_usd" in d:

codai/admin/templates/settings.html

@@ -896,6 +896,15 @@
         <label class="form-label">Data center <span class="muted">(optional filter)</span></label>
         <input type="text" id="s-runpod-dc" class="form-input" placeholder="(any)">
       </div>
+      <div class="form-group" style="flex:1;min-width:180px">
+        <label class="form-label">Network volume id <span class="muted">(optional)</span></label>
+        <input type="text" id="s-runpod-volume" class="form-input" placeholder="blank = none (create it in the RunPod console)">
+        <span class="form-hint" style="font-size:11px">Pods attach it at launch, so downloaded weights, uploaded models and LoRA adapters survive the pod — the second pod does not re-download. Secure Cloud only, and pods are pinned to the volume's data center.</span>
+      </div>
+      <div class="form-group" style="flex:1;min-width:140px">
+        <label class="form-label">Volume mount path</label>
+        <input type="text" id="s-runpod-volume-mount" class="form-input" placeholder="/workspace">
+      </div>
       <div class="form-group" style="flex:1;min-width:180px">
         <label class="form-label">Registry credential id <span class="muted">(private images)</span></label>
         <input type="text" id="s-runpod-registry-auth" class="form-input" placeholder="blank = public images only">
@@ -1852,6 +1861,8 @@ async function loadSettings(){
     document.getElementById('s-runpod-default-gpu').value = rp.default_gpu_type ?? '';
     document.getElementById('s-runpod-budget').value = rp.global_max_hourly_usd ?? 0;
     document.getElementById('s-runpod-registry-auth').value = rp.registry_auth_id ?? '';
+    document.getElementById('s-runpod-volume').value = rp.network_volume_id ?? '';
+    document.getElementById('s-runpod-volume-mount').value = rp.volume_mount_path ?? '';
     // Remote capabilities
     _populateRemotes(d.remotes || {});
     document.getElementById('s-runpod-cost-limit').value = rp.global_cost_limit_usd ?? 0;
@@ -2055,6 +2066,8 @@ async function saveSettings(silent){
       default_gpu_type: document.getElementById('s-runpod-default-gpu').value.trim(),
       global_max_hourly_usd: parseFloat(document.getElementById('s-runpod-budget').value) || 0,
       registry_auth_id: document.getElementById('s-runpod-registry-auth').value.trim(),
+      network_volume_id: document.getElementById('s-runpod-volume').value.trim(),
+      volume_mount_path: document.getElementById('s-runpod-volume-mount').value.trim(),
       global_cost_limit_usd: parseFloat(document.getElementById('s-runpod-cost-limit').value) || 0,
       global_cost_period: document.getElementById('s-runpod-cost-period').value || 'unlimited',
       data_center: document.getElementById('s-runpod-dc').value.trim(),

codai/api/runpod_client.py

@@ -185,7 +185,8 @@ class RunpodClient:
                    bid_per_gpu: float = 0.0, data_center_id: str = "",
                    registry_auth_id: str = "",
                    entrypoint: Optional[list] = None,
-                   start_cmd: Optional[list] = None) -> str:
+                   start_cmd: Optional[list] = None,
+                   network_volume_id: str = "") -> str:
         """Provision a pod (on-demand or interruptible/spot). Returns the pod id.
 
         ``entrypoint``/``start_cmd`` override the image's ENTRYPOINT/CMD, which is
@@ -196,6 +197,7 @@ class RunpodClient:
         """
         if entrypoint or start_cmd:
             return self._create_pod_rest(
+                network_volume_id=network_volume_id,
                 name=name, image=image, gpu_type_id=gpu_type_id, port=port,
                 cloud_type=cloud_type, gpu_count=gpu_count,
                 container_disk_gb=container_disk_gb, volume_gb=volume_gb,
@@ -225,6 +227,11 @@ class RunpodClient:
         # images (vLLM, llama.cpp, a public coderai) need none.
         if registry_auth_id:
             common["containerRegistryAuthId"] = registry_auth_id
+        # A network volume persists across pods: weights downloaded once are
+        # there for every later pod, which is the difference between paying a
+        # cold model download per pod and paying it once.
+        if network_volume_id:
+            common["networkVolumeId"] = network_volume_id
         if is_spot:
             common["bidPerGpu"] = float(bid_per_gpu)
             mutation = "podRentInterruptable"
@@ -246,7 +253,8 @@ class RunpodClient:
     def _create_pod_rest(self, *, name, image, gpu_type_id, port, cloud_type,
                          gpu_count, container_disk_gb, volume_gb, volume_mount_path,
                          env, is_spot, bid_per_gpu, data_center_id,
-                         registry_auth_id, entrypoint, start_cmd) -> str:
+                         registry_auth_id, entrypoint, start_cmd,
+                         network_volume_id: str = "") -> str:
         """Create a pod through the REST API, which accepts entrypoint overrides."""
         import requests
         rest = (getattr(self._cfg, "rest_base", "") or "https://rest.runpod.io/v1").rstrip("/")
@@ -270,6 +278,8 @@ class RunpodClient:
             body["dataCenterIds"] = [data_center_id]
         if registry_auth_id:
             body["containerRegistryAuthId"] = registry_auth_id
+        if network_volume_id:
+            body["networkVolumeId"] = network_volume_id
         if is_spot:
             body["interruptible"] = True
             body["bidPerGpu"] = float(bid_per_gpu)
@@ -287,6 +297,27 @@ class RunpodClient:
             raise RunpodError(f"RunPod REST create returned no pod id: {str(data)[:300]}")
         return pod_id
 
+    def list_network_volumes(self) -> list:
+        """The account's network volumes: [{id, name, size, dataCenterId}, …].
+
+        Needed because a pod must run in the volume's OWN data center — a volume
+        in EU-RO-1 cannot be attached to a pod anywhere else, and the failure
+        arrives as an unhelpful capacity error.
+        """
+        import requests
+        rest = (getattr(self._cfg, "rest_base", "") or "https://rest.runpod.io/v1").rstrip("/")
+        try:
+            r = requests.get(f"{rest}/networkvolumes",
+                             headers={"Authorization": f"Bearer {self._api_key}"},
+                             timeout=30)
+            if r.status_code != 200:
+                return []
+            data = r.json()
+            return data if isinstance(data, list) else (data.get("data") or [])
+        except Exception as exc:
+            print(f"[runpod] could not list network volumes: {exc}", flush=True)
+            return []
+
     def get_pod(self, pod_id: str) -> dict:
         """Return {id, status, uptime_s, cost_per_hr, ports:[...], ready:bool, url?}."""
         q = """

codai/api/runpod_worker.py

@@ -60,6 +60,10 @@ class RunpodModelConfig:
     # RunPod container-registry credential id for a private image; blank falls
     # back to the account-wide one.
     registry_auth_id: str = ""
+    # Attach a RunPod network volume to this pool's pods. Blank falls back to the
+    # account-wide one. Weights, uploads and adapters on it outlive the pod.
+    network_volume_id: str = ""
+    volume_mount_path: str = ""              # blank = the account default
     # Share one set of pods with every other model naming the same pool, instead
     # of renting a card each. Only possible when the pod's server can serve more
     # than one model: a coderai pod is a whole coderai and picks the model from
@@ -449,6 +453,39 @@ def _add_staging(plan: dict, mcfg: "RunpodModelConfig", entry: dict, engine: str
     print(f"[runpod] pod will download before serving: {names}", flush=True)
 
 
+def volume_for(mcfg: "RunpodModelConfig", account) -> tuple:
+    """(volume_id, mount_path) for this pool, or ('', '')."""
+    vol = (getattr(mcfg, "network_volume_id", "") or "").strip() \
+        or (getattr(account, "network_volume_id", "") or "").strip()
+    if not vol:
+        return "", ""
+    mount = (getattr(mcfg, "volume_mount_path", "") or "").strip() \
+        or (getattr(account, "volume_mount_path", "") or "").strip() or "/workspace"
+    return vol, mount.rstrip("/") or "/workspace"
+
+
+def volume_env(mount: str) -> dict:
+    """Point everything that writes big files at the volume.
+
+    This is what makes a volume worth attaching. Without it a pod downloads its
+    weights to container disk and throws them away on teardown, so every cold pod
+    pays the download again; with it, the second pod finds them already there.
+    The same path is where uploaded models and LoRA adapters land, so a file sent
+    once is visible to every later pod — the "bounce host" role, without a third
+    party in the middle.
+    """
+    if not mount:
+        return {}
+    return {
+        "HF_HOME": f"{mount}/huggingface",
+        "HUGGINGFACE_HUB_CACHE": f"{mount}/huggingface/hub",
+        "TRANSFORMERS_CACHE": f"{mount}/huggingface/transformers",
+        "DIFFUSERS_CACHE": f"{mount}/diffusers",
+        "CODERAI_MODELS_DIR": f"{mount}/models",
+        "CODERAI_CACHE_DIR": f"{mount}/cache",
+    }
+
+
 def _plan(engine, image, args, mcfg, api_key, entry, served,
           seed_entries: list = None) -> dict:  # noqa: D401
     """Finish a plan for a coderai pod: auth plus the models it must serve."""
@@ -634,6 +671,12 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
     cfg.health_path = (b.get("health_path") or "").strip()
     cfg.docker_args = (b.get("docker_args") or "").strip()
     cfg.registry_auth_id = (b.get("registry_auth_id") or "").strip()
+    cfg.network_volume_id = (b.get("network_volume_id") or "").strip()
+    if cfg.network_volume_id:
+        # RunPod offers network volumes on Secure Cloud only; leaving COMMUNITY
+        # in the list just produces candidates the attach would reject.
+        cfg.cloud_types = ["SECURE"]
+    cfg.volume_mount_path = (b.get("volume_mount_path") or "").strip()
     cfg.pool = (b.get("pool") or "").strip().lower()
     cfg.quantization = (b.get("quantization") or "").strip()
     cfg.source = (b.get("source") or cfg.source).strip().lower() or "auto"
@@ -874,10 +917,21 @@ def _vllm_docker_args(mcfg: "RunpodModelConfig", served: str,
     return " ".join(args)
 
 
-#: Where a staged download lands inside the pod. Container disk, wiped with the
-#: pod — size `container_disk_gb` for what you are staging.
+#: Where a staged download lands inside the pod. Container disk by default —
+#: wiped with the pod, so size `container_disk_gb` for what you stage. With a
+#: network volume attached this moves onto it (see staging_dir), and a staged
+#: file then survives for every later pod.
 STAGE_DIR = "/runpod-volume/staged"
 
+
+def staging_dir(mcfg: "RunpodModelConfig" = None, account=None) -> str:
+    """Where staged downloads land: the volume when there is one, else container disk."""
+    try:
+        vol_id, mount = volume_for(mcfg, account) if mcfg is not None else ("", "")
+    except Exception:
+        vol_id, mount = "", ""
+    return f"{mount}/staged" if vol_id and mount else STAGE_DIR
+
 #: Fetch with whatever the image actually has. vLLM's image ships python3 but not
 #: always curl; llama.cpp's ships curl. Trying in order beats assuming.
 _FETCH = ('if command -v curl >/dev/null 2>&1; then curl -fsSL "$U" -o "$F"; '
@@ -1024,6 +1078,8 @@ class RunpodPodPool:
         self.entry = entry if isinstance(entry, dict) else None
         #: Models a CAPABILITY pod should register at boot (it serves many).
         self.seed_entries: list = []
+        #: Looked up once: the data center the network volume lives in.
+        self._volume_dc = None
         self.model_key = model_key
         self.account = account_cfg
         self.mcfg = mcfg
@@ -1082,6 +1138,27 @@ class RunpodPodPool:
         return ""
 
     # -- provisioning ----------------------------------------------------- #
+    def _data_center_for_volume(self, volume_id: str) -> str:
+        """The data center a network volume lives in, or ''.
+
+        A pod can only attach a volume in its own data center, so this pins
+        placement instead of letting GPU ranking pick a region where the attach
+        will simply fail.
+        """
+        try:
+            from codai.api.runpod_client import RunpodClient
+            for vol in RunpodClient(self.account).list_network_volumes():
+                if str(vol.get("id") or "") == str(volume_id):
+                    dc = str(vol.get("dataCenterId") or "")
+                    if dc:
+                        print(f"[runpod] volume {volume_id} is in {dc} — pinning pods "
+                              "there", flush=True)
+                    return dc
+        except Exception as exc:
+            print(f"[runpod] could not resolve the volume's data center: {exc}",
+                  flush=True)
+        return ""
+
     def _create_with_fallback(self, client, ranked, start=0):
         """Try ranked GPU candidates from ``start``; skip capacity misses. Returns
         (pod_id, sel, next_index). Raises if all fail."""
@@ -1094,6 +1171,16 @@ class RunpodPodPool:
         self.health_path = plan["health_path"]
         env = dict(self.mcfg.env or {})
         env.update(plan.get("env") or {})
+
+        # A network volume, when configured: caches and upload targets move onto
+        # it so weights survive the pod. RunPod requires Secure Cloud and the
+        # pod's data center to match the volume's, so both are forced here rather
+        # than surfacing later as an unexplained capacity error.
+        vol_id, vol_mount = volume_for(self.mcfg, self.account)
+        if vol_id:
+            env.update(volume_env(vol_mount))
+            if getattr(self, "_volume_dc", None) is None:
+                self._volume_dc = self._data_center_for_volume(vol_id)
         dc = getattr(self.account, "data_center", "") or ""
         last = None
         for i in range(start, len(ranked)):
@@ -1112,7 +1199,10 @@ class RunpodPodPool:
                     name=name, image=image, gpu_type_id=sel["gpu_type_id"], port=port,
                     cloud_type=sel["cloud_type"], container_disk_gb=self.mcfg.container_disk_gb,
                     volume_gb=self.mcfg.volume_gb, env=env, docker_args=args,
-                    is_spot=sel["is_spot"], bid_per_gpu=sel["bid"], data_center_id=dc,
+                    is_spot=sel["is_spot"], bid_per_gpu=sel["bid"],
+                    data_center_id=(getattr(self, "_volume_dc", "") or dc),
+                    network_volume_id=vol_id,
+                    volume_mount_path=(vol_mount or "/workspace"),
                     registry_auth_id=(self.mcfg.registry_auth_id
                                       or getattr(self.account, "registry_auth_id", "") or ""),
                     entrypoint=plan.get("entrypoint"), start_cmd=plan.get("start_cmd"))

codai/config.py

@@ -633,6 +633,14 @@ class RunpodConfig:
     # private registry (your own coderai image, typically). Per-model/capability
     # `registry_auth_id` overrides this. Blank = public images only.
     registry_auth_id: str = ""
+    # A RunPod network volume (created by hand in the RunPod console) that pods
+    # attach at launch. Weights downloaded onto it survive the pod, so the second
+    # pod does not re-download a 30 GB model — and it doubles as the place to put
+    # uploaded models and LoRA adapters, since every pod sees the same files.
+    # Constraints RunPod imposes: Secure Cloud only, and the pod MUST run in the
+    # volume's own data center.
+    network_volume_id: str = ""
+    volume_mount_path: str = "/workspace"    # where pods mount it
     # Stable tag baked into every pod name (coderai-<deployment_id>-<model>-<rand>) so the
     # reaper can identify OUR pods across restarts and never touch another deployment's.
     deployment_id: str = "default"
@@ -1218,6 +1226,8 @@ class ConfigManager:
                 "default_gpu_type": self.config.runpod.default_gpu_type,
                 "data_center": self.config.runpod.data_center,
                 "registry_auth_id": self.config.runpod.registry_auth_id,
+                "network_volume_id": self.config.runpod.network_volume_id,
+                "volume_mount_path": self.config.runpod.volume_mount_path,
                 "deployment_id": self.config.runpod.deployment_id,
                 "global_max_hourly_usd": self.config.runpod.global_max_hourly_usd,
                 "global_cost_limit_usd": self.config.runpod.global_cost_limit_usd,

docs/remote-execution.md

@@ -496,6 +496,51 @@ A model that resolves to none of those — a local path with no `hf_repo`,
 `model_url` or `source: upload` — is refused with that reason rather than
 registered as something the pod could never load.
 
+### Network volumes (optional)
+
+A pod's own disk dies with the pod, so every cold pod re-downloads its weights.
+A RunPod **network volume** — created by hand in the RunPod console, coderai
+never creates one — turns that into a one-off:
+
+```json
+{"runpod": {"network_volume_id": "abc123", "volume_mount_path": "/workspace"}}
+```
+
+Account-wide in Settings, or per pool in its pod block. What changes when one is
+attached:
+
+| | Without a volume | With one |
+|---|---|---|
+| Downloaded weights | re-fetched by every cold pod | fetched once, reused |
+| Uploaded models / LoRA adapters | re-uploaded to every new pod | sent once |
+| Staged downloads (`source: url`) | container disk | on the volume |
+
+coderai points `HF_HOME`, the diffusers/transformers caches, `CODERAI_MODELS_DIR`
+and the staging directory at the mount, which is what actually makes the volume
+do anything — attaching one without redirecting the caches changes nothing.
+
+**That also makes the volume your bounce host.** An upload lands on it and every
+later pod sees the file, so "upload a QLoRA or a local model once, use it from
+any pod" needs no third-party storage and no public IP.
+
+Two constraints RunPod imposes, both handled for you:
+
+* **Secure Cloud only** — `cloud_types` is forced to `SECURE`, because a
+  COMMUNITY candidate would just fail the attach.
+* **Same data center as the volume** — the volume's region is looked up once and
+  pods are pinned there, rather than letting GPU price ranking pick a region
+  where the attach fails as an unexplained capacity error.
+
+A volume is attached at deploy time and cannot be added to a running pod. Cost is
+about $0.07/GB/month. And RunPod warns that several workers **writing** the same
+volume at once can corrupt it — coderai's writes are write-once (content-addressed
+adapter blobs, named model files), but keep that in mind before pointing many
+pods at one volume for anything else.
+
+**Container images cannot come from a volume.** A pod pulls its image from a
+registry before any volume exists; the volume holds model weights and adapters,
+not the image.
+
 ### Keeping a pod warm
 
 `keep_warm` on a model's runpod block (or a capability's) keeps one pod running

tests/test_burst_placement.py

@@ -380,3 +380,55 @@ def test_a_borrowed_pod_is_released_not_terminated(monkeypatch):
     pool.pods = [borrowed]
     pool._terminate(borrowed, "idle")
     assert killed == [] and pool.pods == []
+
+
+# --------------------------------------------------------------------------- #
+# network volumes
+# --------------------------------------------------------------------------- #
+def test_a_volume_moves_the_caches_off_container_disk():
+    """The point of a volume: weights land somewhere that outlives the pod, so
+    the second pod does not re-download them."""
+    from codai.api.runpod_worker import volume_for, volume_env, staging_dir, \
+        parse_model_runpod
+
+    class _Acct:
+        network_volume_id = ""
+        volume_mount_path = "/workspace"
+
+    cfg = parse_model_runpod({"network_volume_id": "vol-abc"})
+    assert volume_for(cfg, _Acct()) == ("vol-abc", "/workspace")
+
+    env = volume_env("/workspace")
+    # Downloads, uploads and every cache go to the volume.
+    assert env["HF_HOME"] == "/workspace/huggingface"
+    assert env["CODERAI_MODELS_DIR"] == "/workspace/models"
+    assert staging_dir(cfg, _Acct()) == "/workspace/staged"
+
+    # Without one, nothing changes: container disk, wiped with the pod.
+    plain = parse_model_runpod({})
+    assert volume_for(plain, _Acct()) == ("", "")
+    assert staging_dir(plain, _Acct()).startswith("/runpod-volume")
+
+
+def test_an_account_wide_volume_applies_and_a_pod_can_override():
+    from codai.api.runpod_worker import volume_for, parse_model_runpod
+
+    class _Acct:
+        network_volume_id = "vol-account"
+        volume_mount_path = "/workspace"
+
+    assert volume_for(parse_model_runpod({}), _Acct())[0] == "vol-account"
+    assert volume_for(parse_model_runpod({"network_volume_id": "vol-own"}),
+                      _Acct())[0] == "vol-own"
+
+
+def test_a_volume_forces_secure_cloud():
+    """RunPod offers network volumes on Secure Cloud only: keeping COMMUNITY in
+    the list would just produce candidates the attach rejects."""
+    from codai.api.runpod_worker import parse_model_runpod
+
+    cfg = parse_model_runpod({"network_volume_id": "vol-abc",
+                              "cloud_types": ["SECURE", "COMMUNITY"]})
+    assert cfg.cloud_types == ["SECURE"]
+    # Without a volume the choice is left alone.
+    assert parse_model_runpod({"cloud_types": ["COMMUNITY"]}).cloud_types == ["COMMUNITY"]