runpod: two pods for a warm floor of one (v0.2.95)

Watched it happen on the production orchestrator: configuring qwen38-awq warm
during office hours provisioned TWO A40s a second apart.

A pod does not enter self.pods until it answers its health probe, which is
minutes away. ensure_ready() never set the _provisioning flag, and the warm
top-up in maintain() never read it — so the 15 s scaler saw "0 healthy, floor 1"
while a pod was already booting and rented another. It stopped at two only
because _provision_one blocks the scaler thread for the whole boot; a
non-blocking provision would have added one every 15 seconds.

The demand path (_should_grow) has always checked the flag. Now all three agree:
one on the way is one pod, not none, including one a sibling engine sharing the
pool is starting.

Also: /v1/runpod/scale answered with the internal pool key ("pool:digesta-qwen38")
rather than the model the client asked about. Several models share a pool, and
that string is not something the caller sent or can look up — so the answer
names the model and reports the pool beside it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.94"
+__version__ = "0.2.95"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -2309,12 +2309,23 @@ class RunpodPodPool:
         with self._cv:
             if any(p.healthy for p in self.pods):
                 return
+            # A pod takes MINUTES to answer its health probe, and it is not in
+            # self.pods until it does. Without this flag the 15 s scaler sees
+            # "0 healthy, floor 1" and rents a second one — two A40s where the
+            # configuration asked for one.
+            if self._provisioning:
+                return
+            self._provisioning = True
+        try:
             for _ in range(want):
                 try:
                     self._provision_one()
                 except Exception as exc:
                     print(f"[runpod] ensure_ready: {exc}", flush=True)
                     raise
+        finally:
+            with self._cv:
+                self._provisioning = False
 
     def _pick(self, healthy: list, affinity: str):
         """Which pod serves this request: the one that already holds the
@@ -2601,11 +2612,23 @@ class RunpodPodPool:
         # Keep min_pods warm.
         with self._cv:
             healthy_n = len([p for p in self.pods if p.healthy])
-        if healthy_n < keep_min and not self._closed:
+        if healthy_n < keep_min and not self._closed and not self._provisioning:
+            # One already on the way (from here, from ensure_ready, or from a
+            # sibling engine sharing this pool) is one pod, not none: topping up
+            # against a booting pod is how a warm floor of 1 became two.
+            if sibling_is_provisioning(str(self.model_key)):
+                return
+            with self._cv:
+                if self._provisioning:
+                    return
+                self._provisioning = True
             try:
                 self._provision_one()
             except Exception as exc:
                 print(f"[runpod] maintain: warm provision failed: {exc}", flush=True)
+            finally:
+                with self._cv:
+                    self._provisioning = False
 
     def close(self):
         self._closed = True
@@ -2949,7 +2972,13 @@ def set_client_pods(model: str, pods: int) -> dict:
     # Creating the pool is the point: asking for a warm floor on a model nobody
     # has called yet is exactly when it is worth pre-warming.
     pool = get_model_pod_pool(key, _model_entry(key), block)
-    return pool.set_client_floor(pods)
+    out = pool.set_client_floor(pods)
+    # Answer with the name the client asked about, not the internal pool key:
+    # several models may share one pool, and "pool:digesta-qwen38" is not
+    # something the caller sent or can look up.
+    out["pool"] = out["model"]
+    out["model"] = key
+    return out
 
 
 def pods_status() -> list:

tests/test_runpod_client_scale.py

@@ -320,3 +320,41 @@ def test_a_client_lease_can_warm_at_boot_even_out_of_hours():
               schedule_start="08:00", schedule_end="20:00")
     p.set_client_floor(2)
     assert p.effective_min_pods() >= 2
+
+
+# ------------------------------------- not renting two pods for a floor of one
+
+def test_ensure_ready_marks_the_pool_as_provisioning():
+    """A pod is not in self.pods until it answers its health probe, minutes
+    later. Without this flag the 15 s scaler reads "0 healthy, floor 1" and
+    rents a second one — which is exactly what happened on the first live
+    boot: two A40s where the configuration asked for one."""
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    blk = src.split("def ensure_ready")[1].split("def _pick")[0]
+    assert "self._provisioning = True" in blk
+    assert "finally:" in blk and "self._provisioning = False" in blk
+    assert blk.index("if self._provisioning:") < blk.index("self._provisioning = True")
+
+
+def test_the_warm_top_up_does_not_stack_on_a_booting_pod():
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    blk = src.split("def maintain")[1].split("def close")[0]
+    assert "not self._provisioning" in blk
+    assert "sibling_is_provisioning" in blk
+
+
+def test_the_demand_path_already_had_the_guard_and_keeps_it():
+    """_should_grow has checked it all along; this pins that it still does, so
+    the two paths cannot drift apart again."""
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    blk = src.split("def _should_grow")[1].split("def _grow_in_background")[0]
+    assert "or self._provisioning" in blk
+
+
+def test_the_answer_names_the_model_the_client_asked_about():
+    """Models sharing a pool answer with the pool key otherwise, which the
+    caller never sent and cannot look up."""
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    blk = src.split("def set_client_pods")[1]
+    assert 'out["model"] = key' in blk
+    assert 'out["pool"]' in blk