runpod: let several models share one pod until it saturates

Two models that both offload rented a card each, even when neither came
close to filling one. They can now name a shared `pool` — on the model's
runpod block or on a burst target — and land on the same pod; each
request carries its own model name, and the pool rents a second pod only
once the first is carrying scale_up_inflight_per_pod requests.

Sharing is refused, loudly, for pod servers pinned to one model at
launch: a vLLM pod is `--model X` and a llama.cpp pod `-hf one.gguf`.
Allowing it would send requests to a pod that answers with the wrong
model's weights, or 404s — a clear error at configuration time beats
debugging that. `engine: coderai` (a whole coderai, which picks the model
per request) is what makes a shared pool possible.

The first model to create a shared pool sets its shape; later joiners do
not overwrite it, so whichever model happens to load last cannot silently
redefine the budget everyone is sharing. Capability pools now use the
same rule instead of their own ad-hoc guard.

Settable in the UI on both the model's pod section and its burst target.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/routes.py

@@ -2787,7 +2787,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                   "endpoint_id", "cost_period",
                   # engine selection, private registries, and the pod's own token
                   "engine", "hf_gguf", "health_path", "docker_args",
-                  "registry_auth_id", "api_key"):
+                  "registry_auth_id", "api_key", "pool"):
             v = src.get(k)
             if isinstance(v, str) and v.strip():
                 rpo[k] = v.strip()

codai/admin/templates/models.html

@@ -773,6 +773,11 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               <span class="form-hint" style="font-size:11px">A local .gguf path means nothing on a rented machine — name the HF repo the pod should download.</span>
             </div>
           </div>
+          <div class="form-row" style="margin:.5rem 0 0">
+            <label class="form-label">Share pod pool <span class="muted">(optional name)</span></label>
+            <input type="text" id="cfg-rp-pool" class="form-input" placeholder="blank = this model rents its own pods">
+            <span class="form-hint" style="font-size:11px">Models naming the same pool share one set of pods and one budget until it saturates. Needs a <b>coderai</b> or custom multi-model image — a vLLM or llama.cpp pod is launched for ONE model and sharing is refused.</span>
+          </div>
           <div style="display:grid;grid-template-columns:2fr 1fr 1fr;gap:.5rem;margin-top:.5rem">
             <div class="form-row" style="margin:0"><label class="form-label">Pod image <span class="muted">(blank = engine default)</span></label><input type="text" id="cfg-rp-image" class="form-input" placeholder="vllm/vllm-openai:latest"></div>
             <div class="form-row" style="margin:0"><label class="form-label">Disk (GB)</label><input type="number" id="cfg-rp-disk" class="form-input" min="1" placeholder="40"></div>
@@ -911,7 +916,10 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               <div class="form-row" style="margin:0"><label class="form-label">Max pods</label><input type="number" id="cfg-spill-max-pods" class="form-input" min="1" placeholder="1"></div>
               <div class="form-row" style="margin:0"><label class="form-label">Idle timeout (s)</label><input type="number" id="cfg-spill-idle" class="form-input" min="0" placeholder="300"></div>
             </div>
-            <div class="form-row" style="margin:.5rem 0 0"><label class="form-label">Pod image <span class="muted">(coderai / custom engines)</span></label><input type="text" id="cfg-spill-image" class="form-input" placeholder="ghcr.io/you/coderai-images:0.2.0"></div>
+            <div style="display:grid;grid-template-columns:2fr 1fr;gap:.5rem;margin-top:.5rem">
+              <div class="form-row" style="margin:0"><label class="form-label">Pod image <span class="muted">(coderai / custom engines)</span></label><input type="text" id="cfg-spill-image" class="form-input" placeholder="ghcr.io/you/coderai-images:0.2.0"></div>
+              <div class="form-row" style="margin:0"><label class="form-label">Share pool <span class="muted">(optional)</span></label><input type="text" id="cfg-spill-pool" class="form-input" placeholder="blank = its own pods"></div>
+            </div>
           </div>
         </div>
       </div>
@@ -4148,6 +4156,7 @@ function _populateRunpod(s){
   set('cfg-rp-cost-period', rp.cost_period || 'unlimited');
   set('cfg-rp-engine', rp.engine || 'auto');
   set('cfg-rp-hf-gguf', rp.hf_gguf || '');
+  set('cfg-rp-pool', rp.pool || '');
   set('cfg-rp-api-key', rp.api_key || '');
   chk('cfg-rp-open', rp.allow_open_pod);
   set('cfg-rp-registry-auth', rp.registry_auth_id || '');
@@ -4184,6 +4193,7 @@ function _populatePlacement(s){
   set('cfg-spill-max-pods', tg.max_pods != null ? tg.max_pods : '');
   set('cfg-spill-idle', tg.idle_timeout_s != null ? tg.idle_timeout_s : '');
   set('cfg-spill-image', tg.image || '');
+  set('cfg-spill-pool', tg.pool || '');
   _updateSpillVisibility();
 }
 
@@ -4232,6 +4242,7 @@ function _collectSpillover(){
     target.max_pods      = intOrU('cfg-spill-max-pods');
     target.idle_timeout_s= intOrU('cfg-spill-idle');
     target.image         = (val('cfg-spill-image') || '').trim();
+    target.pool          = (val('cfg-spill-pool') || '').trim();
   }
   return {
     enabled: true,
@@ -4281,6 +4292,7 @@ function _collectRunpod(){
     cost_period:        val('cfg-rp-cost-period') || 'unlimited',
     engine:             val('cfg-rp-engine') || 'auto',
     hf_gguf:            (val('cfg-rp-hf-gguf') || '').trim(),
+    pool:               (val('cfg-rp-pool') || '').trim(),
     api_key:            (val('cfg-rp-api-key') || '').trim(),
     allow_open_pod:     document.getElementById('cfg-rp-open').checked,
     registry_auth_id:   (val('cfg-rp-registry-auth') || '').trim(),

codai/api/runpod_worker.py

@@ -60,6 +60,12 @@ class RunpodModelConfig:
     # RunPod container-registry credential id for a private image; blank falls
     # back to the account-wide one.
     registry_auth_id: str = ""
+    # Share one set of pods with every other model naming the same pool, instead
+    # of renting a card each. Only possible when the pod's server can serve more
+    # than one model: a coderai pod is a whole coderai and picks the model from
+    # the request, while a vLLM pod is launched `--model X` and a llama.cpp pod
+    # `-hf one.gguf` — those serve exactly one, and sharing is refused.
+    pool: str = ""
     # Bearer token the pod requires on every request. A RunPod proxy URL is
     # reachable by anyone who learns it, so a pod without one is an open GPU on
     # the public internet. Blank does NOT mean "no auth": the pool generates a
@@ -267,6 +273,7 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
     cfg.health_path = (b.get("health_path") or "").strip()
     cfg.docker_args = (b.get("docker_args") or "").strip()
     cfg.registry_auth_id = (b.get("registry_auth_id") or "").strip()
+    cfg.pool = (b.get("pool") or "").strip().lower()
     cfg.api_key = (b.get("api_key") or "").strip()
     cfg.allow_open_pod = _as_bool(b.get("allow_open_pod"), cfg.allow_open_pod)
     cfg.served_model = (b.get("served_model") or "").strip()
@@ -848,13 +855,48 @@ _pools: dict = {}          # model_key -> RunpodPodPool
 _scaler_started = False
 
 
-def get_pod_pool(model_key, account_cfg, mcfg: "RunpodModelConfig", served_name) -> RunpodPodPool:
+#: Pod servers that are pinned to a single model at launch, so several models
+#: can never share one. A coderai (or comparable) image picks the model per
+#: request and can.
+_SINGLE_MODEL_ENGINES = ("vllm", "llamacpp")
+
+
+def shared_pool_key(mcfg: "RunpodModelConfig", model_key: str,
+                    model_path: str = "") -> str:
+    """The pool key for this model: its own, or a shared one it opted into.
+
+    Raises when the configured engine cannot serve more than one model — better
+    a clear error now than a pod that answers every request with the wrong
+    model's weights (or a 404 from a server that was launched for another one).
+    """
+    shared = (getattr(mcfg, "pool", "") or "").strip().lower()
+    if not shared:
+        return str(model_key)
+    engine = resolve_pod_engine(mcfg, model_key, model_path)
+    if engine in _SINGLE_MODEL_ENGINES:
+        raise RuntimeError(
+            f"RunPod: model {model_key!r} asks to share pod pool {shared!r}, but a "
+            f"{engine} pod is launched for ONE model and cannot serve another. "
+            "Share a pool only between models on a coderai (or custom multi-model) "
+            "pod image, or drop the pool name so this model gets its own pods.")
+    return f"pool:{shared}"
+
+
+def get_pod_pool(model_key, account_cfg, mcfg: "RunpodModelConfig", served_name,
+                 shared: bool = False) -> RunpodPodPool:
+    """The pool for ``model_key``, created on first use.
+
+    ``shared`` marks a pool several models joined by name. The first model to
+    create it sets its shape (image, GPU class, budgets, scaling) and later
+    joiners do NOT overwrite it — otherwise whichever model happened to load
+    last would silently redefine the budget for everyone sharing the pod.
+    """
     with _pools_lock:
         pool = _pools.get(model_key)
         if pool is None:
             pool = RunpodPodPool(model_key, account_cfg, mcfg, served_name)
             _pools[model_key] = pool
-        else:
+        elif not shared:
             # refresh config each load so edits take effect
             pool.account, pool.mcfg, pool.served = account_cfg, mcfg, served_name
     _ensure_scaler()
@@ -890,15 +932,7 @@ def get_capability_pool(capability: str, block: dict):
     shared = str(b.get("pool") or "").strip().lower()
     mcfg = parse_model_runpod(b)
     key = f"capability:{shared or capability}"
-    # A shared pool is configured once (on whichever capability carries the
-    # settings) and referenced by the others, so don't let a bare `{"pool": …}`
-    # reference overwrite the real config with defaults.
-    if shared:
-        with _pools_lock:
-            existing = _pools.get(key)
-        if existing is not None and len(b) <= 2:
-            return existing
-    return get_pod_pool(key, acct, mcfg, shared or capability)
+    return get_pod_pool(key, acct, mcfg, shared or capability, shared=bool(shared))
 
 
 def _all_pools_hourly_rate() -> float:

codai/backends/runpod.py

@@ -101,8 +101,14 @@ class RunpodBackend(ModelBackend):
         else:
             # pods / auto: coderai-managed pool of remote GPU pods. (auto currently
             # behaves as pods; serverless-vs-pods auto-arbitration is a later refinement.)
+            # A model may join a shared pool rather than rent its own card; the
+            # request carries its own model name, so one multi-model pod serves
+            # them all until it saturates and the pool grows.
+            _pkey = runpod_worker.shared_pool_key(
+                self._mcfg, model_name, runpod_worker._model_path_for(model_name))
             self._pool = runpod_worker.get_pod_pool(
-                model_name, self._acct, self._mcfg, self._served_name)
+                _pkey, self._acct, self._mcfg, self._served_name,
+                shared=(_pkey != model_name))
             # A RunPod proxy URL is reachable by anyone who learns it, so the pod
             # is launched locked to a bearer token (generated when none is
             # configured) and every request we send must carry it.

codai/frontproxy/app.py

@@ -750,9 +750,16 @@ class FrontProxy:
         try:
             from codai.api import runpod_worker as _rw
             mcfg = _rw.parse_model_runpod(target)
-            key = "spill:" + (self._queue_key(model) or str(model))
             served = (target.get("served_model") or "").strip() or (model or "")
-            pool = _rw.get_pod_pool(key, rp, mcfg, served)
+            # Two models bursting to the same pool share its pods: the overflow
+            # from both lands on one card until it saturates, rather than each
+            # renting its own. Refused for engines pinned to a single model.
+            shared = (target.get("pool") or "").strip().lower()
+            key = f"spill:pool:{shared}" if shared else \
+                "spill:" + (self._queue_key(model) or str(model))
+            if shared:
+                _rw.shared_pool_key(mcfg, model or "", "")   # raises if not shareable
+            pool = _rw.get_pod_pool(key, rp, mcfg, served, shared=bool(shared))
             # Provisioning blocks for minutes on a cold pod; keep the loop free.
             handle, url = await _asyncio.to_thread(pool.acquire)
         except Exception as exc:

docs/remote-execution.md

@@ -372,6 +372,32 @@ Triggers, most eager first:
 gated, scaled, reaped, billed against the same caps) or `serverless` (your own
 RunPod endpoint id). `served_model` renames the model for the remote.
 
+### Several models on one pod
+
+Two models that would each rent a card can share one, until it saturates:
+
+```json
+{"runpod": {"pool": "shared", "engine": "coderai",
+            "image": "ghcr.io/<you>/coderai-images:0.2.0",
+            "max_pods": 2, "scale_up_inflight_per_pod": 4}}
+```
+
+Give the second model the same `pool` name. Both requests land on the same pod —
+each carries its own model name, and a coderai pod picks the model per request —
+and the pool rents a second pod only once the first is carrying
+`scale_up_inflight_per_pod` requests. The same `pool` field works on a burst
+target, so two models' overflow shares one card.
+
+**This needs a multi-model pod image.** A vLLM pod is launched `--model X` and a
+llama.cpp pod `-hf one.gguf`: they serve exactly one model, so sharing is refused
+with an explicit error rather than sending requests to a pod that would answer
+with the wrong weights. Use `engine: coderai` (or a custom multi-model image) for
+a shared pool.
+
+The first model to create a shared pool sets its shape — image, GPU class,
+budget, scaling — and later joiners do not overwrite it, so whichever model
+happens to load last can't silently redefine the budget everyone is sharing.
+
 The burst is a borrow: a pod is taken for one request and handed straight back,
 so an overlap that never recurs is reaped after `idle_timeout_s` rather than
 lingering. And a burst that cannot be opened — cold pod failed, budget cap hit,

tests/test_burst_placement.py

@@ -212,3 +212,67 @@ def test_a_pod_coderai_rejects_requests_without_the_token(monkeypatch):
     assert hmac.compare_digest("tok", "tok")
     from codai.api.ratelimit import _unauthorized
     assert _unauthorized().status_code == 401
+
+
+# --------------------------------------------------------------------------- #
+# sharing one pod between models
+# --------------------------------------------------------------------------- #
+def test_two_models_on_one_coderai_pod_share_a_pool(monkeypatch):
+    """Both models name the same pool, so the overflow from both lands on one
+    card until it saturates — rather than renting a GPU each."""
+    import codai.api.runpod_worker as rw
+
+    monkeypatch.setattr(rw, "_pools", {})
+    monkeypatch.setattr(rw, "_ensure_scaler", lambda: None)
+
+    class _Acct:
+        enabled = True
+
+    block = {"pool": "shared", "engine": "coderai", "image": "reg/coderai:base",
+             "max_pods": 2, "max_hourly_usd": 0.5}
+    a_cfg = rw.parse_model_runpod(block)
+    b_cfg = rw.parse_model_runpod({"pool": "shared", "engine": "coderai"})
+
+    ka = rw.shared_pool_key(a_cfg, "model-a")
+    kb = rw.shared_pool_key(b_cfg, "model-b")
+    assert ka == kb == "pool:shared"
+
+    pa = rw.get_pod_pool(ka, _Acct(), a_cfg, "model-a", shared=True)
+    pb = rw.get_pod_pool(kb, _Acct(), b_cfg, "model-b", shared=True)
+    assert pa is pb
+    # The first model's settings define the shared pod; a later joiner must not
+    # silently redefine the budget everyone is sharing.
+    assert pa.mcfg.max_hourly_usd == 0.5 and pa.mcfg.image == "reg/coderai:base"
+
+
+def test_a_model_without_a_pool_name_still_gets_its_own():
+    import codai.api.runpod_worker as rw
+    cfg = rw.parse_model_runpod({"engine": "coderai", "image": "i"})
+    assert rw.shared_pool_key(cfg, "model-a") == "model-a"
+
+
+def test_sharing_is_refused_for_single_model_pod_servers():
+    """A vLLM pod is launched `--model X` and llama.cpp `-hf one.gguf`: sharing
+    would answer with the wrong model's weights, or 404."""
+    import codai.api.runpod_worker as rw
+
+    vllm = rw.parse_model_runpod({"pool": "shared", "served_model": "org/model-a"})
+    with pytest.raises(RuntimeError, match="cannot serve another"):
+        rw.shared_pool_key(vllm, "model-a")
+
+    gguf = rw.parse_model_runpod({"pool": "shared", "hf_gguf": "u/r:Q4"})
+    with pytest.raises(RuntimeError, match="cannot serve another"):
+        rw.shared_pool_key(gguf, "model-b", "/AI/m.gguf")
+
+
+def test_a_shared_pod_grows_when_it_saturates():
+    """Sharing must not mean queueing forever: once the pod is carrying
+    scale_up_inflight_per_pod requests, the pool rents a second one."""
+    p = _pool(pool="shared", engine="coderai", image="i", max_pods=2,
+              scale_up_inflight_per_pod=3)
+    pod = _Pod("a", inflight=3)
+    p.pods = [pod]
+    assert p._should_grow([pod]) is True
+    quiet = _Pod("a", inflight=1)
+    p.pods = [quiet]
+    assert p._should_grow([quiet]) is False   # not saturated -> keep sharing