runpod: optionally keep pod dependencies on the volume, not in the image

A capability image is ~7 GB and nearly all of it is torch, pulled again by
every cold pod. With a network volume those libraries can live there
instead: `venv_on_volume` boots a slim image — OS, Python and coderai
only — and runs coderai from a venv on the volume. The first pod builds
it; every pod after skips both the pull and the install.

Opt-in per pool and per model, never a default, because the trade is real
and unmeasured: the pull gets much smaller, but importing torch off
network storage is slower than off local disk, so some of the saving
comes back at import time. Whether that is a net win depends on how often
pods cold-start, which is the user's workload, not mine to assume.

Two hazards are handled rather than left to chance. The ready marker is
written AFTER the install, so a pod that dies mid-build leaves nothing a
later pod would import from. And a lock directory — mkdir is atomic —
makes a second pod wait for the first instead of two installs
interleaving into the same directory.

`venv_on_volume` without a volume is refused with the reason, rather than
booting a pod that installs 7 GB onto container disk and discards it.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/routes.py

@@ -2795,7 +2795,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                   "registry_auth_id", "api_key", "pool",
                   # where the POD gets the weights
                   "source", "hf_repo", "model_url", "quantization",
-                  "network_volume_id", "volume_mount_path"):
+                  "network_volume_id", "volume_mount_path",
+                  "venv_name", "slim_image"):
             v = src.get(k)
             if isinstance(v, str) and v.strip():
                 rpo[k] = v.strip()
@@ -2811,7 +2812,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
             rpo["cloud_types"] = ct
         # bools
         for k in ("allow_spot", "allow_open_pod", "sticky_sessions", "keep_warm",
-                  "model_url_is_tar"):
+                  "model_url_is_tar", "venv_on_volume"):
             if k in src and src.get(k) is not None and src.get(k) != "":
                 sv = src.get(k)
                 rpo[k] = (sv.lower() in ("1", "true", "on", "yes")

codai/admin/templates/models.html

@@ -800,6 +800,17 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               </label>
             </div>
           </div>
+          <div style="display:grid;grid-template-columns:1fr 1fr;gap:.5rem;margin-top:.5rem">
+            <div class="form-row" style="margin:0">
+              <label class="form-label">Network volume id <span class="muted">(this model)</span></label>
+              <input type="text" id="cfg-rp-volume" class="form-input" placeholder="blank = the account default, or none">
+              <span class="form-hint" style="font-size:11px">Weights, uploads and adapters land here and outlive the pod. Secure Cloud only; pods are pinned to the volume's region.</span>
+            </div>
+            <div class="form-row" style="margin:0;align-self:start">
+              <label style="display:flex;align-items:center;gap:.3rem;font-size:12px;margin-top:1.4rem"><input type="checkbox" id="cfg-rp-venv-volume" style="width:auto"> Keep dependencies on the volume</label>
+              <span class="form-hint" style="font-size:11px">Boots a small image and runs from a venv on the volume — the first pod builds it, later pods skip the 7 GB pull. Needs a volume; imports are slower off network storage.</span>
+            </div>
+          </div>
           <div class="form-row" style="margin:.5rem 0 0">
             <label class="form-label">Share pod pool <span class="muted">(optional name)</span></label>
             <input type="text" id="cfg-rp-pool" class="form-input" placeholder="blank = this model rents its own pods">
@@ -4207,6 +4218,8 @@ function _populateRunpod(s){
   set('cfg-rp-engine', rp.engine || 'auto');
   set('cfg-rp-hf-gguf', rp.hf_gguf || '');
   set('cfg-rp-pool', rp.pool || '');
+  set('cfg-rp-volume', rp.network_volume_id || '');
+  chk('cfg-rp-venv-volume', rp.venv_on_volume);
   set('cfg-rp-source', rp.source || 'auto');
   set('cfg-rp-hf-repo', rp.hf_repo || '');
   set('cfg-rp-model-url', rp.model_url || '');
@@ -4375,6 +4388,8 @@ function _collectRunpod(){
     engine:             val('cfg-rp-engine') || 'auto',
     hf_gguf:            (val('cfg-rp-hf-gguf') || '').trim(),
     pool:               (val('cfg-rp-pool') || '').trim(),
+    network_volume_id:  (val('cfg-rp-volume') || '').trim(),
+    venv_on_volume:     document.getElementById('cfg-rp-venv-volume').checked,
     source:             val('cfg-rp-source') || 'auto',
     hf_repo:            (val('cfg-rp-hf-repo') || '').trim(),
     model_url:          (val('cfg-rp-model-url') || '').trim(),

codai/api/runpod_worker.py

@@ -64,6 +64,14 @@ class RunpodModelConfig:
     # account-wide one. Weights, uploads and adapters on it outlive the pod.
     network_volume_id: str = ""
     volume_mount_path: str = ""              # blank = the account default
+    # Keep this pod's Python dependencies on the VOLUME instead of in the image,
+    # and boot a small image that uses them. The first pod builds the venv (a few
+    # minutes); every pod after skips both the 7 GB image pull and the install.
+    # Needs a network volume. Opt-in per pool — never a default, because it
+    # trades a fast pull for slower imports off network storage.
+    venv_on_volume: bool = False
+    venv_name: str = ""                      # blank = the profile/capability name
+    slim_image: str = ""                     # blank = the published slim image
     # Share one set of pods with every other model naming the same pool, instead
     # of renting a card each. Only possible when the pod's server can serve more
     # than one model: a coderai pod is a whole coderai and picks the model from
@@ -366,9 +374,9 @@ def pod_plan(mcfg: "RunpodModelConfig", served: str, model_key: str = "",
         # them meant a pod with an explicit image got no models and refused
         # everything with "not available".
         image = mcfg.image
+        capability_hint = model_capability(entry or {}, include_text=True)
         if not image:
-            cap = model_capability(entry or {}, include_text=True)
-            image = default_capability_image(cap) if cap else ""
+            image = default_capability_image(capability_hint) if capability_hint else ""
         if not image:
             raise RuntimeError(
                 f"RunPod {engine} pod: set `image` on the runpod block to the image "
@@ -376,7 +384,7 @@ def pod_plan(mcfg: "RunpodModelConfig", served: str, model_key: str = "",
                 f"{CAPABILITY_IMAGE_REPO}-<capability> image; this one has none, so "
                 "name it explicitly.")
         return _plan(engine, image, mcfg.docker_args, mcfg, api_key, entry, served,
-                     seed_entries)
+                     seed_entries, capability=capability_hint)
     else:
         image = mcfg.image or DEFAULT_POD_IMAGE
         args = mcfg.docker_args or _vllm_docker_args(mcfg, served, entry)
@@ -464,6 +472,55 @@ def volume_for(mcfg: "RunpodModelConfig", account) -> tuple:
     return vol, mount.rstrip("/") or "/workspace"
 
 
+#: The dependency-free image that runs coderai from a venv on a volume.
+SLIM_POD_IMAGE = os.environ.get("CODERAI_SLIM_POD_IMAGE",
+                                f"{CAPABILITY_IMAGE_REPO}-slim:{CAPABILITY_IMAGE_TAG}")
+
+
+def venv_bootstrap_script(mount: str, profile: str, venv_name: str = "",
+                          port: int = 8000) -> str:
+    """Start command for a slim pod: build the venv on the volume if needed, run.
+
+    The first pod to use a given venv pays for the install; every pod after finds
+    it and starts in seconds. A completion marker is written LAST and checked
+    first, so a pod that dies mid-install cannot leave a half-built venv that
+    later pods would import from and fail on in confusing ways.
+
+    Concurrent builders are the other hazard — two pods starting together would
+    install into the same directory. A lock directory (mkdir is atomic) makes the
+    second one wait for the first rather than interleave with it.
+    """
+    name = (venv_name or profile or "default").strip()
+    venv = f"{mount}/venvs/{name}"
+    reqs = "/opt/coderai/app/packaging/runpod/profiles"
+    return "\n".join([
+        "set -eu",
+        f'VENV={_sh_quote(venv)}',
+        f'LOCK={_sh_quote(venv + ".lock")}',
+        f'MARK={_sh_quote(venv + "/.ready")}',
+        # Wait for another pod that is already building this venv.
+        'for i in $(seq 1 180); do',
+        '  [ -f "$MARK" ] && break',
+        '  if mkdir "$LOCK" 2>/dev/null; then',
+        '    echo "[venv] building $VENV (first pod pays for this)"',
+        '    python -m venv "$VENV"',
+        f'    "$VENV/bin/python" -m pip install --upgrade pip',
+        f'    "$VENV/bin/python" -m pip install -r {reqs}/core.txt',
+        f'    "$VENV/bin/python" -m pip install -r {reqs}/{profile}.txt',
+        '    touch "$MARK"',        # last: a crash leaves no usable marker
+        '    rmdir "$LOCK" || true',
+        '    break',
+        '  fi',
+        '  echo "[venv] another pod is building $VENV — waiting"',
+        '  sleep 10',
+        'done',
+        '[ -f "$MARK" ] || { echo "[venv] $VENV never became ready"; exit 1; }',
+        'echo "[venv] using $VENV"',
+        f'exec "$VENV/bin/python" -m uvicorn codai.api.app:app --host 0.0.0.0 '
+        f'--port {int(port)}',
+    ])
+
+
 def volume_env(mount: str) -> dict:
     """Point everything that writes big files at the volume.
 
@@ -487,7 +544,7 @@ def volume_env(mount: str) -> dict:
 
 
 def _plan(engine, image, args, mcfg, api_key, entry, served,
-          seed_entries: list = None) -> dict:  # noqa: D401
+          seed_entries: list = None, capability: str = "") -> dict:  # noqa: D401
     """Finish a plan for a coderai pod: auth plus the models it must serve."""
     import json as _json
     env = {}
@@ -514,9 +571,38 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
             unique.append(s)
         env["CODERAI_SEED_MODELS"] = _json.dumps(unique)
         print(f"[runpod] pod will be told about {len(unique)} model(s)", flush=True)
-    return {"engine": engine, "image": image, "args": args, "env": env,
+    plan = {"engine": engine, "image": image, "args": args, "env": env,
             "health_path": mcfg.health_path or _HEALTH_PATHS.get(engine, "/healthz")}
 
+    # Dependencies on the volume instead of in the image: a small image, and a
+    # venv the first pod builds and every later pod reuses.
+    if getattr(mcfg, "venv_on_volume", False):
+        vol_id, mount = volume_for(mcfg, _account_hint())
+        if not vol_id:
+            raise RuntimeError(
+                "RunPod: `venv_on_volume` needs a network volume — the venv has "
+                "nowhere to live otherwise. Set network_volume_id (account-wide "
+                "or on this pod block), or turn venv_on_volume off.")
+        profile = (capability or model_capability(entry or {}, include_text=True)
+                   or "text")
+        plan["image"] = mcfg.slim_image or SLIM_POD_IMAGE
+        plan["entrypoint"] = ["/bin/sh", "-c"]
+        plan["start_cmd"] = [venv_bootstrap_script(
+            mount, profile, mcfg.venv_name, mcfg.port or 8000)]
+        print(f"[runpod] pod will run from a venv on the volume "
+              f"({mount}/venvs/{mcfg.venv_name or profile}) with image "
+              f"{plan['image']}", flush=True)
+    return plan
+
+
+def _account_hint():
+    """The active RunPod account config, for defaults the plan needs."""
+    try:
+        from codai.models.manager import get_active_runpod_config
+        return get_active_runpod_config()
+    except Exception:
+        return None
+
 
 def seed_model_env(entry: dict, served: str = "", source: str = "") -> str:
     """The model entry to register on a fresh pod, as JSON for CODERAI_SEED_MODELS.
@@ -677,6 +763,9 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
         # in the list just produces candidates the attach would reject.
         cfg.cloud_types = ["SECURE"]
     cfg.volume_mount_path = (b.get("volume_mount_path") or "").strip()
+    cfg.venv_on_volume = _as_bool(b.get("venv_on_volume"), cfg.venv_on_volume)
+    cfg.venv_name = (b.get("venv_name") or "").strip()
+    cfg.slim_image = (b.get("slim_image") or "").strip()
     cfg.pool = (b.get("pool") or "").strip().lower()
     cfg.quantization = (b.get("quantization") or "").strip()
     cfg.source = (b.get("source") or cfg.source).strip().lower() or "auto"

docs/remote-execution.md

@@ -537,6 +537,39 @@ volume at once can corrupt it — coderai's writes are write-once (content-addre
 adapter blobs, named model files), but keep that in mind before pointing many
 pods at one volume for anything else.
 
+#### Dependencies on the volume, small image in the registry
+
+A capability image is ~7 GB and nearly all of it is torch. With a volume those
+libraries can live there instead, and the pod boots a **slim** image — OS,
+Python and coderai only — that runs from a venv on the volume:
+
+```json
+{"runpod": {"network_volume_id": "abc123", "venv_on_volume": true}}
+```
+
+The first pod to use a given venv builds it (a few minutes of pip); every pod
+after finds it ready and skips both the big pull and the install. The venv is
+named after the capability by default, so `embeddings` and `video` pods get their
+own; `venv_name` overrides that to share or separate them deliberately.
+
+Build and push the slim image once:
+
+```bash
+PUSH=1 ./packaging/runpod/build_capability_image.sh slim ghcr.io/<you>/coderai-slim:latest
+```
+
+**The trade-off is real and unmeasured on your workload.** The pull gets much
+smaller, but importing torch from network-backed storage is slower than from
+local disk, so some of the saving comes back at import time. Whether it is a net
+win depends on how often your pods cold-start. It is opt-in per pool, never a
+default.
+
+Two failure modes are handled rather than left to chance: the ready marker is
+written **after** the install, so a pod that dies mid-build cannot leave a
+half-built venv that later pods import from; and a lock directory (mkdir is
+atomic) makes a second pod **wait** for the first rather than installing into the
+same directory at the same time.
+
 **Container images cannot come from a volume.** A pod pulls its image from a
 registry before any volume exists; the volume holds model weights and adapters,
 not the image.

packaging/runpod/Dockerfile.capability-slim

+# A coderai pod image with NO machine-learning dependencies.
+#
+# The full capability images are ~7 GB, nearly all of it torch and friends, and
+# every cold pod pulls that before it can answer anything. When a network volume
+# is attached, those libraries can live on the volume instead: this image carries
+# only the OS packages, Python and coderai itself, and the pod's start command
+# points it at a venv on the volume — built once by the first pod, reused by
+# every pod after.
+#
+# The trade is real but not free, and worth measuring for your own workload:
+#   * the image is small (hundreds of MB, not 7 GB), so the pull is quick;
+#   * but importing torch from network-backed storage is slower than from local
+#     disk, so some of what the pull saved comes back at import time.
+#
+# It is opt-in per pool (`venv_on_volume`), never a default.
+#
+#   docker build -f packaging/runpod/Dockerfile.capability-slim \
+#       -t ghcr.io/<you>/coderai-slim:latest .
+
+ARG BASE=python:3.12-slim
+FROM ${BASE}
+
+ENV PYTHONUNBUFFERED=1 \
+    CODERAI_HOME=/config
+
+# The same runtime libraries the full images carry: ffmpeg for audio/video,
+# libgl/libglib for opencv. These are OS packages, not Python ones, so they stay
+# in the image — a venv on a volume cannot provide them.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ffmpeg git curl ca-certificates libgl1 libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/coderai/app
+COPY . /opt/coderai/app
+
+# Nothing is imported at build time: the dependencies are not here yet. The pod's
+# start command builds (or reuses) the venv on the volume and runs coderai from
+# it — see venv_bootstrap_script() in codai/api/runpod_worker.py.
+EXPOSE 8000
+HEALTHCHECK --interval=30s --timeout=5s --start-period=600s \
+    CMD curl -fsS http://127.0.0.1:8000/healthz || exit 1

packaging/runpod/build_capability_image.sh

@@ -16,6 +16,20 @@ PROFILE="${1:-}"
 TAG="${2:-coderai-${PROFILE}:latest}"
 PROFILES_DIR="packaging/runpod/profiles"
 
+# The slim image carries no ML dependencies: it runs coderai from a venv on a
+# network volume (see venv_on_volume). Built from its own Dockerfile, with no
+# profile and no import smoke test — there is nothing to import yet.
+if [[ "$PROFILE" == "slim" ]]; then
+    TAG="${2:-coderai-slim:latest}"
+    echo "==> building $TAG (no ML dependencies; runs from a venv on a volume)"
+    DOCKER_BUILDKIT=1 docker build -f packaging/runpod/Dockerfile.capability-slim \
+        -t "$TAG" . || exit 1
+    size=$(docker image inspect "$TAG" --format '{{.Size}}')
+    echo "==> $TAG built ($(( size / 1000000 )) MB)"
+    if [[ "${PUSH:-0}" == "1" ]]; then docker push "$TAG"; fi
+    exit 0
+fi
+
 if [[ -z "$PROFILE" ]]; then
     echo "usage: $0 <profile> [tag]" >&2
     echo "profiles: $(ls "$PROFILES_DIR" | grep -v '^core.txt$' | sed 's/\.txt$//' | tr '\n' ' ')" >&2

tests/test_burst_placement.py

@@ -432,3 +432,56 @@ def test_a_volume_forces_secure_cloud():
     assert cfg.cloud_types == ["SECURE"]
     # Without a volume the choice is left alone.
     assert parse_model_runpod({"cloud_types": ["COMMUNITY"]}).cloud_types == ["COMMUNITY"]
+
+
+def test_venv_on_volume_uses_a_small_image_and_the_volumes_venv(monkeypatch):
+    """The 7 GB image is nearly all torch. With a volume, those libraries can
+    live there and the pod pulls a small image instead."""
+    import codai.api.runpod_worker as rw
+
+    class _Acct:
+        network_volume_id = "vol-abc"
+        volume_mount_path = "/workspace"
+        enabled = True
+
+    monkeypatch.setattr(rw, "_account_hint", lambda: _Acct())
+    cfg = rw.parse_model_runpod({"engine": "coderai", "venv_on_volume": True})
+    plan = rw.pod_plan(cfg, "embeddings", "capability:embeddings",
+                       api_key="tok", entry={"path": "org/e", 
+                                             "model_type": "embedding_models"})
+    assert plan["image"].endswith("coderai-slim:latest")
+    assert plan["entrypoint"] == ["/bin/sh", "-c"]
+    script = plan["start_cmd"][0]
+    assert "/workspace/venvs/embeddings" in script
+    assert "profiles/embeddings.txt" in script      # the right profile installed
+    assert script.rstrip().endswith("--port 8000")
+
+
+def test_venv_on_volume_without_a_volume_is_refused(monkeypatch):
+    """The venv would have nowhere to live; say so rather than boot a pod that
+    installs 7 GB onto container disk and throws it away."""
+    import codai.api.runpod_worker as rw
+
+    class _NoVol:
+        network_volume_id = ""
+        volume_mount_path = "/workspace"
+
+    monkeypatch.setattr(rw, "_account_hint", lambda: _NoVol())
+    cfg = rw.parse_model_runpod({"engine": "coderai", "image": "img",
+                                 "venv_on_volume": True})
+    with pytest.raises(RuntimeError, match="needs a network volume"):
+        rw.pod_plan(cfg, "x", "capability:embeddings", entry={})
+
+
+def test_the_venv_marker_is_written_last_and_builders_are_serialised():
+    """Two hazards: a pod that dies mid-install leaving a venv later pods import
+    from, and two pods installing into the same directory at once."""
+    from codai.api.runpod_worker import venv_bootstrap_script
+
+    s = venv_bootstrap_script("/workspace", "tts")
+    # The marker is created after the installs, not before.
+    assert s.index("pip install -r") < s.index('touch "$MARK"')
+    # And the ready check gates the exec.
+    assert s.index('touch "$MARK"') < s.index('[ -f "$MARK" ] ||')
+    # mkdir is the atomic lock; a loser waits instead of installing too.
+    assert 'mkdir "$LOCK"' in s and "another pod is building" in s