runpod: optionally give the dearest pod back first (v0.3.12)

A pool rents at whatever the market offered when each pod was started, so one
fleet can hold a $0.35 card and a $0.79 card doing identical work. Shedding the
cheap one saves less than half as much, and which to prefer is the operator's
call, not ours — hence a checkbox rather than a new default.

With scale_down_costliest_first the dearest surplus pod goes first, tie-broken
by the idlest and then the coldest cache. Cost decides WHICH pod goes, never
whether one goes, and a dear pod with requests on it is drained rather than
dropped — so choosing it costs time, not work. Worth weighing before switching
on: an on-demand pod costs more than a spot pod, so "dearest first" prefers to
keep the interruptible ones.

Also fixes the default tie-break, which was inverted when scale-down was
written: `-last_used` picked the pod used MOST recently, discarding the warmest
prefix cache of the set. It now retires the coldest.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.11"
+__version__ = "0.3.12"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/admin/routes.py

@@ -3155,6 +3155,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
         # ints
         for k in ("container_disk_gb", "volume_gb", "port", "ctx", "min_pods",
                   "max_pods", "scale_up_inflight_per_pod", "scale_down_after_s",
+                  "scale_down_costliest_first",
                   "max_inflight_per_pod",
                   "idle_timeout_s",
                   # how hard the POD's own server may be driven

codai/admin/templates/models.html

@@ -1115,6 +1115,7 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
             </div>
             <div class="form-row" style="margin:0;align-self:end">
               <label style="display:flex;align-items:center;gap:.3rem;font-size:12px"><input type="checkbox" id="cfg-rp-sticky" style="width:auto" checked> Sticky sessions</label>
+              <label style="display:flex;align-items:center;gap:.3rem;font-size:12px" title="When shedding a surplus pod, give back the dearest one first instead of the least loaded. Only a mixed fleet has anything to choose between — pods take whatever the market offered when they were rented. Note that an on-demand pod costs more than a spot one, so this prefers to keep the interruptible ones."><input type="checkbox" id="cfg-rp-costliest" style="width:auto"> Scale down the most expensive first</label>
               <span class="form-hint" style="font-size:11px">Send a conversation back to the pod whose cache still holds it.</span>
             </div>
           </div>
@@ -4934,6 +4935,7 @@ function _populateRunpod(s){
   set('cfg-rp-scale-down', rp.scale_down_after_s != null ? rp.scale_down_after_s : '');
   set('cfg-rp-max-inflight', rp.max_inflight_per_pod != null ? rp.max_inflight_per_pod : '');
   chk('cfg-rp-sticky', rp.sticky_sessions !== false);
+  chk('cfg-rp-costliest', !!rp.scale_down_costliest_first);
   _updateRunpodVisibility();
   _updateRunpodEngineHint();
   _populatePlacement(s);
@@ -5115,6 +5117,7 @@ function _collectRunpod(){
     scale_down_after_s:        intOrU('cfg-rp-scale-down'),
     max_inflight_per_pod:      intOrU('cfg-rp-max-inflight'),
     sticky_sessions:    document.getElementById('cfg-rp-sticky').checked,
+    scale_down_costliest_first: document.getElementById('cfg-rp-costliest').checked,
     schedule_enabled:   document.getElementById('cfg-rp-schedule-enabled').checked,
     schedule_start:     (val('cfg-rp-schedule-start') || '').trim(),
     schedule_end:       (val('cfg-rp-schedule-end') || '').trim(),

codai/api/runpod_worker.py

@@ -168,6 +168,17 @@ class RunpodModelConfig:
     # this long (a burst that dips for a moment must not cost a pod that is
     # needed again seconds later, and a cold pod takes minutes to replace).
     scale_down_after_s: int = 180
+    # Which surplus pod to give back first. Off, the pool retires the least
+    # loaded one (and among equals the one used least recently, whose prefix
+    # cache is coldest). On, PRICE leads: the dearest pod goes first.
+    #
+    # Only a mixed fleet has anything to choose between — pods rented at
+    # different times take whatever the market offered, so the same pool can hold
+    # a $0.35 card and a $0.79 one for the same work, and shedding the cheap one
+    # saves less than half as much. Note what "dearest" tends to mean: an
+    # on-demand pod costs more than a spot pod, so this prefers to keep the
+    # interruptible ones.
+    scale_down_costliest_first: bool = False
     # Hard ceiling on concurrent requests per pod (0 = no ceiling, let the remote
     # server's own batching absorb them). Above it, a request waits for a slot
     # instead of being piled on — use it for engines that degrade badly under
@@ -1652,6 +1663,8 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
     # 0 disables shedding entirely (pods then go only when idle_timeout_s says so).
     cfg.scale_down_after_s = max(0, _as_int(b.get("scale_down_after_s"),
                                             cfg.scale_down_after_s))
+    cfg.scale_down_costliest_first = _as_bool(b.get("scale_down_costliest_first"),
+                                              cfg.scale_down_costliest_first)
     cfg.max_inflight_per_pod = max(0, _as_int(b.get("max_inflight_per_pod"),
                                               cfg.max_inflight_per_pod))
     cfg.sticky_sessions = _as_bool(b.get("sticky_sessions"), cfg.sticky_sessions)
@@ -2832,6 +2845,27 @@ class RunpodPodPool:
         need = -(-total // threshold)          # ceil, in integers
         return max(self.effective_min_pods(), min(int(self.mcfg.max_pods), need))
 
+    def _scale_down_victim(self, serving: list):
+        """Which surplus pod to give back.
+
+        By default the least loaded, and among equals the one used least recently:
+        its prefix cache is the coldest, so retiring it throws away the least
+        work. (That tie-break was inverted when this was written — ``-last_used``
+        picked the pod used MOST recently, discarding the warmest cache in the
+        set.)
+
+        With ``scale_down_costliest_first`` price leads instead, which is the
+        whole point on a mixed fleet: pods are rented at whatever the market
+        offered, so one pool can hold a cheap card and an expensive one doing the
+        same work. Cost still only decides WHICH pod goes, never whether one
+        goes — and a dear pod with requests on it is drained, not dropped.
+        """
+        if getattr(self.mcfg, "scale_down_costliest_first", False):
+            # Dearest first; then the idlest, then the coldest cache.
+            return max(serving, key=lambda p: (float(p.hourly_usd or 0.0),
+                                               -p.inflight, -p.last_used))
+        return min(serving, key=lambda p: (p.inflight, p.last_used))
+
     def _scale_down(self, now: float) -> None:
         """Retire one surplus pod per pass, once the surplus has persisted.
 
@@ -2882,7 +2916,7 @@ class RunpodPodPool:
             # Hold the observation window again before the next one, so a pass
             # does not walk the fleet down in one sweep.
             self._surplus_since = now
-            victim = min(serving, key=lambda x: (x.inflight, -x.last_used))
+            victim = self._scale_down_victim(serving)
             if victim.inflight == 0:
                 victim.healthy = False
                 self.pods.remove(victim)

docs/remote-execution.md

@@ -772,6 +772,23 @@ Three rules make it safe to leave on:
 The warm floor (`min_pods`, or a client's `/v1/runpod/scale` lease) is a floor
 here too and is never scaled through.
 
+**Which surplus pod goes.** By default the least loaded, and among equals the one
+used least recently — its prefix cache is the coldest, so retiring it throws away
+the least work. With `"scale_down_costliest_first": true` price leads instead:
+
+```json
+{"runpod": {"max_pods": 64, "scale_down_after_s": 180,
+            "scale_down_costliest_first": true}}
+```
+
+Only a mixed fleet has anything to choose between: pods are rented at whatever
+the market offered when each one was started, so the same pool can hold a $0.35
+card and a $0.79 card doing identical work, and shedding the cheap one saves less
+than half as much. The rule decides **which** pod goes, never whether one goes,
+and a dear pod with requests on it is drained rather than dropped. One thing to
+weigh before switching it on: an on-demand pod costs more than a spot pod, so
+"dearest first" prefers to keep the interruptible ones.
+
 **This needs a multi-model pod image.** A vLLM pod is launched `--model X` and a
 llama.cpp pod `-hf one.gguf`: they serve exactly one model, so sharing is refused
 with an explicit error rather than sending requests to a pod that would answer

tests/test_runpod_scale_down.py

@@ -27,7 +27,8 @@ sys.path.insert(0, str(ROOT))
 from codai.api import runpod_worker as rw                         # noqa: E402
 
 
-def _pool(max_pods=8, min_pods=0, threshold=4, after=180, pods=()):
+def _pool(max_pods=8, min_pods=0, threshold=4, after=180, pods=(),
+          costliest_first=False):
     """A PodPool with its scaling state and nothing else."""
     pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
     import threading
@@ -42,6 +43,7 @@ def _pool(max_pods=8, min_pods=0, threshold=4, after=180, pods=()):
     cfg.min_pods = min_pods
     cfg.scale_up_inflight_per_pod = threshold
     cfg.scale_down_after_s = after
+    cfg.scale_down_costliest_first = costliest_first
     cfg.schedule_enabled = False
     pool.mcfg = cfg
     pool.effective_min_pods = lambda: min_pods
@@ -51,8 +53,8 @@ def _pool(max_pods=8, min_pods=0, threshold=4, after=180, pods=()):
     return pool
 
 
-def _pod(pid, inflight=0, last_used=0.0, draining=False):
-    return rw.PodHandle(pod_id=pid, url=f"http://{pid}", hourly_usd=0.6,
+def _pod(pid, inflight=0, last_used=0.0, draining=False, hourly_usd=0.6):
+    return rw.PodHandle(pod_id=pid, url=f"http://{pid}", hourly_usd=hourly_usd,
                         started_at=0.0, inflight=inflight, last_used=last_used,
                         draining=draining)
 
@@ -226,3 +228,70 @@ def test_one_request_is_enough_for_shedding_to_apply():
     pool = _pool(threshold=4, pods=[_pod("a", 1), _pod("b", 0)])
     _settle(pool)
     assert len(pool._killed) == 1
+
+
+# ------------------------------------------------- which of the surplus pods goes #
+# A pool rents at whatever the market offered, so one fleet can hold a $0.35 card
+# and a $0.79 one doing the same work. Shedding the cheap one saves less than half
+# as much, and the operator may want to say so.
+
+
+def test_by_default_the_least_loaded_pod_goes():
+    pool = _pool(threshold=10, pods=[_pod("busy", 3), _pod("quiet", 1)])
+    assert pool._scale_down_victim(pool.pods).pod_id == "quiet"
+
+
+def test_among_equals_the_coldest_cache_goes():
+    """Retiring the pod used least recently throws away the least work: its
+    prefix cache is the one with the least in it. This tie-break was inverted
+    when scale-down was written — it picked the pod used MOST recently."""
+    pool = _pool(pods=[_pod("warm", 0, last_used=900.0),
+                       _pod("cold", 0, last_used=100.0)])
+    assert pool._scale_down_victim(pool.pods).pod_id == "cold"
+
+
+def test_with_the_rule_on_the_dearest_pod_goes_first():
+    pool = _pool(costliest_first=True,
+                 pods=[_pod("cheap", 0, hourly_usd=0.35),
+                       _pod("dear", 0, hourly_usd=0.79)])
+    assert pool._scale_down_victim(pool.pods).pod_id == "dear"
+
+
+def test_price_leads_even_when_the_dear_pod_is_the_busier_one():
+    """It is drained, not dropped — so choosing it costs nothing but time, and
+    saves more than shedding the cheap one."""
+    pool = _pool(costliest_first=True, threshold=10,
+                 pods=[_pod("cheap", 0, hourly_usd=0.35),
+                       _pod("dear", 2, hourly_usd=0.79)])
+    assert pool._scale_down_victim(pool.pods).pod_id == "dear"
+
+
+def test_at_the_same_price_the_rule_falls_back_to_the_idlest():
+    pool = _pool(costliest_first=True, threshold=10,
+                 pods=[_pod("a", 2, hourly_usd=0.6), _pod("b", 0, hourly_usd=0.6)])
+    assert pool._scale_down_victim(pool.pods).pod_id == "b"
+
+
+def test_at_the_same_price_and_load_the_coldest_cache_goes():
+    pool = _pool(costliest_first=True,
+                 pods=[_pod("warm", 0, last_used=900.0, hourly_usd=0.6),
+                       _pod("cold", 0, last_used=100.0, hourly_usd=0.6)])
+    assert pool._scale_down_victim(pool.pods).pod_id == "cold"
+
+
+def test_the_rule_decides_which_pod_goes_never_whether_one_goes():
+    """A fleet the load still needs keeps every pod, dear ones included."""
+    pool = _pool(costliest_first=True, threshold=4,
+                 pods=[_pod("a", 4, hourly_usd=0.79), _pod("b", 4, hourly_usd=0.35)])
+    _settle(pool)
+    assert pool._killed == []
+    assert not any(p.draining for p in pool.pods)
+
+
+def test_a_dear_pod_with_work_is_still_drained_not_killed():
+    pool = _pool(costliest_first=True, threshold=10,
+                 pods=[_pod("cheap", 0, hourly_usd=0.35),
+                       _pod("dear", 2, hourly_usd=0.79)])
+    _settle(pool)
+    assert pool._killed == []
+    assert [p.pod_id for p in pool.pods if p.draining] == ["dear"]