max_pods must count pods that bill, not pods that are healthy (v0.4.27)

A pool configured max_pods=1, min_pods=1 held two A40s: a pod adopted at
17:41, a restart at 17:43, a second pod rented at 17:46. $1.18/hr for a
configuration that asked for $0.59, and the operator's explicit ceiling
exceeded by the very code path whose job is to hold exactly one pod warm.

The warm-floor top-up in maintain() computed

    in_hand = len([p for p in self.pods if p.healthy]) + self._booting

and tested BOTH the deficit and the max_pods cap against it. After a restart
the pool adopts the pod it had before. That pod is briefly not yet healthy, and
it was not booted by this process, so `_booting` is 0 and `in_hand` is 0 --
room under a cap of one, beside a pod that is already running and already
being charged for by the minute.

They are two questions and they need two counts. The deficit is in pods that
can SERVE, so it keeps counting healthy ones: an unhealthy pod does not satisfy
a warm floor whose entire purpose is that the next request finds a pod ready.
The cap is in pods that are CHARGED FOR, so it counts every pod in the pool
whatever its health -- which is what `_should_grow` has always done for its own
cap. This path disagreed with it, and the disagreement was the bug.

The existing test pinned the single `in_hand` by its source text; it now pins
both counts, including that the cap is the billing one, since that is the part
whose regression costs money.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KnfkxBpcZ1v6ypRFFyhrVM

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.4.26"
+__version__ = "0.4.27"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -4930,12 +4930,33 @@ class RunpodPodPool:
             # own `scale_up_max_parallel_boots`, which already means "how many
             # pods may be booting at once", and by max_pods.
             with self._cv:
-                in_hand = len([p for p in self.pods if p.healthy]) + self._booting
+                # Two different counts, because they answer two different
+                # questions and conflating them rents cards nobody asked for.
+                #
+                # The DEFICIT is in pods that can serve, so it counts healthy
+                # ones: an unhealthy pod does not satisfy a warm floor whose
+                # whole purpose is that the next request finds a pod ready.
+                #
+                # The CAP is in pods that BILL, so it counts every pod in the
+                # pool whatever its health, exactly as `_should_grow` does. A
+                # pod that exists occupies a max_pods slot; it is charged for by
+                # the minute while it boots and while it is sick.
+                #
+                # Counting only healthy pods against max_pods is how a pool with
+                # max_pods=1 came to hold two A40s: after a restart the pool
+                # ADOPTS the pod it had before, which is briefly not yet healthy
+                # and was not booted by this process, so `_booting` is 0 and
+                # `in_hand` computed 0 -- room under a cap of one, beside a pod
+                # already running. Observed 2026-10-18: a pod adopted at 17:41,
+                # a restart at 17:43, a second pod rented at 17:46, $1.18/hr for
+                # a configuration that asked for $0.59.
+                healthy_in_hand = len([p for p in self.pods if p.healthy]) + self._booting
+                billing_in_hand = len(self.pods) + self._booting
                 parallel = max(1, int(getattr(self.mcfg,
                                               "scale_up_max_parallel_boots", 4) or 4))
-                want = min(keep_min - in_hand,
+                want = min(keep_min - healthy_in_hand,
                            max(0, parallel - self._booting),
-                           max(0, int(self.mcfg.max_pods or 0) - in_hand))
+                           max(0, int(self.mcfg.max_pods or 0) - billing_in_hand))
                 if want <= 0:
                     return
                 self._booting += want

tests/test_runpod_client_scale.py

@@ -347,12 +347,24 @@ def test_the_warm_top_up_does_not_stack_on_a_booting_pod():
     """A floor of 1 must not become 2 by topping up against a pod already
     booting. 0.3.69 fills a floor in waves rather than one pod per pass, so
     the old `not self._booting` guard is gone and the intent is arithmetic:
-    `_booting` is counted in `in_hand`, so the deficit a wave may request is
-    already reduced by whatever is on the way."""
+    `_booting` is counted, so the deficit a wave may request is already reduced
+    by whatever is on the way.
+
+    The single `in_hand` this originally pinned became two counts in 0.4.27,
+    because the deficit and the max_pods cap are different questions -- the
+    deficit is in pods that can SERVE, the cap is in pods that BILL -- and
+    answering both with the healthy-only count let a pool with max_pods=1 rent
+    beside an adopted pod that was not yet healthy. Both counts still include
+    `_booting`, which is what this test is about.
+    """
     src = __import__("pathlib").Path(rw.__file__).read_text()
     blk = src.split("def maintain")[1].split("def close")[0]
-    assert "in_hand = len([p for p in self.pods if p.healthy]) + self._booting" in blk
-    assert "want = min(keep_min - in_hand," in blk
+    assert ("healthy_in_hand = len([p for p in self.pods if p.healthy]) "
+            "+ self._booting") in blk
+    assert "billing_in_hand = len(self.pods) + self._booting" in blk
+    assert "want = min(keep_min - healthy_in_hand," in blk
+    # the cap must be the billing count, or the regression returns
+    assert "- billing_in_hand)" in blk
     assert "sibling_is_provisioning" in blk
 
 

tests/test_runpod_warm_floor_respects_max_pods.py

+"""max_pods counts pods that BILL, not pods that are healthy.
+
+Observed 2026-10-18 on a pool configured max_pods=1, min_pods=1: a pod adopted
+at 17:41, a restart at 17:43, a second pod rented at 17:46 -- $1.18/hr for a
+configuration that asked for $0.59. The operator's explicit ceiling of one was
+exceeded by the code path whose job is to hold exactly one warm.
+
+Cause: the warm-floor top-up in maintain() computed
+
+    in_hand = len([p for p in self.pods if p.healthy]) + self._booting
+
+and tested BOTH the deficit and the max_pods cap against it. After a restart
+the pool adopts the pod it had before; that pod is briefly not yet healthy and
+was not booted by this process, so `_booting` is 0 and `in_hand` is 0 -- room
+under a cap of one, beside a pod that is already running and already billing.
+
+The two questions need two counts: the deficit is in pods that can SERVE, the
+cap is in pods that are CHARGED FOR. `_should_grow` already counted
+`len(self.pods) + self._booting` for its cap; this path disagreed.
+"""
+import pytest
+
+
+class _Pod:
+    def __init__(self, healthy, draining=False):
+        self.healthy = healthy
+        self.draining = draining
+
+
+def want_to_rent(pods, booting, keep_min, max_pods, parallel=4):
+    """Mirror of the corrected arithmetic in maintain()."""
+    healthy_in_hand = len([p for p in pods if p.healthy]) + booting
+    billing_in_hand = len(pods) + booting
+    return max(0, min(keep_min - healthy_in_hand,
+                      max(0, parallel - booting),
+                      max(0, max_pods - billing_in_hand)))
+
+
+# --- the exact regression ---
+
+def test_an_adopted_unhealthy_pod_still_holds_its_max_pods_slot():
+    """The 17:43 restart: one pod adopted, not yet healthy, nothing booting."""
+    assert want_to_rent([_Pod(healthy=False)], booting=0,
+                        keep_min=1, max_pods=1) == 0
+
+
+def test_the_old_arithmetic_would_have_rented():
+    """Guards the test itself: the bug really was reachable this way."""
+    pods, booting, keep_min, max_pods = [_Pod(healthy=False)], 0, 1, 1
+    old_in_hand = len([p for p in pods if p.healthy]) + booting
+    old_want = min(keep_min - old_in_hand, 4 - booting, max_pods - old_in_hand)
+    assert old_want == 1, "the regression must be reproducible, or this test proves nothing"
+    assert want_to_rent(pods, booting, keep_min, max_pods) == 0
+
+
+# --- the cap holds whatever state the existing pods are in ---
+
+@pytest.mark.parametrize("healthy", [True, False])
+def test_a_full_pool_never_rents(healthy):
+    assert want_to_rent([_Pod(healthy=healthy)], 0, 1, 1) == 0
+
+
+def test_a_sick_pod_does_not_free_a_slot_for_a_second_card():
+    """A pod that is sick is still charged for by the minute."""
+    assert want_to_rent([_Pod(healthy=True), _Pod(healthy=False)], 0,
+                        keep_min=2, max_pods=2) == 0
+
+
+def test_a_draining_pod_still_bills_and_holds_its_slot():
+    assert want_to_rent([_Pod(healthy=True, draining=True)], 0, 1, 1) == 0
+
+
+# --- and the floor is still filled when there IS room ---
+
+def test_an_empty_pool_rents_its_floor():
+    assert want_to_rent([], 0, keep_min=1, max_pods=1) == 1
+
+
+def test_a_deficit_under_the_cap_is_filled():
+    assert want_to_rent([_Pod(healthy=True)], 0, keep_min=4, max_pods=8) == 3
+
+
+def test_a_boot_already_on_the_way_is_not_duplicated():
+    assert want_to_rent([], booting=1, keep_min=1, max_pods=1) == 0
+
+
+def test_the_parallel_boot_cap_still_applies():
+    assert want_to_rent([], 0, keep_min=8, max_pods=8, parallel=2) == 2
+
+
+def test_the_cap_binds_before_the_deficit():
+    """A floor larger than max_pods cannot rent past max_pods."""
+    assert want_to_rent([], 0, keep_min=10, max_pods=3) == 3
+
+
+def test_an_unhealthy_pod_does_not_satisfy_the_floor_when_there_is_room():
+    """The deficit still counts health: a sick pod is not a warm pod. With a
+    cap of 2 there is room to rent the healthy one the floor asked for."""
+    assert want_to_rent([_Pod(healthy=False)], 0, keep_min=1, max_pods=2) == 1
+
+
+# --- the arithmetic above must be the arithmetic in the worker ---
+
+def test_worker_uses_two_distinct_counts():
+    src = open("codai/api/runpod_worker.py").read()
+    assert "billing_in_hand = len(self.pods) + self._booting" in src
+    assert "healthy_in_hand = len([p for p in self.pods if p.healthy]) + self._booting" in src
+    assert "max(0, int(self.mcfg.max_pods or 0) - billing_in_hand)" in src, \
+        "the max_pods cap must be tested against pods that bill"
+    assert "want = min(keep_min - healthy_in_hand," in src, \
+        "the deficit must still be tested against pods that can serve"
+
+
+def test_grow_and_warm_floor_agree_on_what_the_cap_counts():
+    """They disagreed, and that disagreement was the bug."""
+    src = open("codai/api/runpod_worker.py").read()
+    assert "in_hand = len(self.pods) + self._booting" in src, \
+        "_should_grow's cap count"
+    assert "len([p for p in self.pods if p.healthy]) + self._booting)" not in src.replace(
+        "healthy_in_hand = len([p for p in self.pods if p.healthy]) + self._booting", ""), \
+        "no remaining health-filtered count feeding a max_pods test"