runpod: do not rent beside a pod that is still booting (v0.2.98)

The last hole in the warm path. sibling_is_provisioning already recognises a
pod registered at creation with no URL yet — and after a restart the recorded
pid no longer matches, so our own booting pod qualifies — but ensure_ready
never asked. Seen live: a pod created at 18:57 was six minutes into its image
pull when the 19:03 upgrade restarted the service, and the warm path rented a
second A40 beside it. The upgrade runs hourly and an image pull is ~6 minutes,
so that is not a rare alignment.

Order in ensure_ready is now: a healthy pod of our own, else adopt a ready one,
else wait for one that is booting, else rent.

Also prune the pod registry in reap_orphans: an id RunPod no longer has stayed
"known", which both shielded it from reaping and let the file grow for the life
of the install — including pods terminated out of band, of which this afternoon
produced five.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.97"
+__version__ = "0.2.98"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -2375,6 +2375,16 @@ class RunpodPodPool:
             # floor of one became two A40s at every boot.
             if self._adopt_shared_pod():
                 return
+            # Nothing ready to adopt — but a pod may be mid-boot, registered
+            # with no URL yet. That is one pod on the way, not none. It matters
+            # most right here: the warm floor is rebuilt at every boot, so an
+            # upgrade landing during the ~6 minutes a pod takes to pull its
+            # image would rent a second one, every time.
+            if sibling_is_provisioning(str(self.model_key)):
+                print(f"[runpod] a pod for {self.model_key!r} is already booting "
+                      f"— not renting another; the scaler will adopt it",
+                      flush=True)
+                return
             for _ in range(want):
                 try:
                     self._provision_one()
@@ -3259,6 +3269,12 @@ def reap_orphans() -> int:
         return 0
     known = _known_pod_ids()
     reaped = 0
+    # Drop registry entries for pods RunPod no longer has. They are shielded
+    # from reaping by being "known", and they accumulate for the life of the
+    # install — including pods terminated out of band.
+    live = {str(p.get("id")) for p in pods if p.get("id")}
+    for gone in (known - live):
+        unregister_pod(gone)
     for p in pods:
         pid, name = p.get("id"), (p.get("name") or "")
         status = str(p.get("status") or "").upper()

tests/test_runpod_client_scale.py

@@ -428,3 +428,46 @@ def test_the_registry_records_the_rate_so_adoption_can_read_it():
             rw._pod_health_ok = saved_health
     finally:
         rw._pod_registry_path = saved
+
+
+def test_the_warm_path_waits_for_a_pod_that_is_still_booting():
+    """A booting pod is registered with no URL, so there is nothing to adopt
+    yet — but it is one pod on the way, not none. An upgrade landing during the
+    ~6 minutes an image pull takes would otherwise rent a second one, and the
+    upgrade runs hourly."""
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    ready = src.split("def ensure_ready")[1].split("def _pick")[0]
+    assert "sibling_is_provisioning" in ready
+    assert ready.index("sibling_is_provisioning") < ready.index("self._provision_one()")
+
+
+def test_a_booting_pod_from_a_previous_process_counts():
+    """The whole point after a restart: the pid differs, so the entry is not
+    'ours', and it has no url yet."""
+    import json
+    import os
+    import tempfile
+    d = tempfile.mkdtemp()
+    saved = rw._pod_registry_path
+    rw._pod_registry_path = lambda: d + "/pods.json"
+    try:
+        json.dump({"p1": {"pool": "pool:x", "pid": os.getpid() + 1,
+                          "at": rw.time.time(), "url": "", "key": "k"}},
+                  open(d + "/pods.json", "w"))
+        assert rw.sibling_is_provisioning("pool:x") is True
+        # Stale beyond the grace must not block provisioning forever.
+        json.dump({"p1": {"pool": "pool:x", "pid": os.getpid() + 1,
+                          "at": rw.time.time() - rw.REAP_GRACE_SECONDS - 10,
+                          "url": "", "key": "k"}},
+                  open(d + "/pods.json", "w"))
+        assert rw.sibling_is_provisioning("pool:x") is False
+    finally:
+        rw._pod_registry_path = saved
+
+
+def test_the_reaper_prunes_entries_for_pods_that_are_gone():
+    """A terminated-but-registered id stays 'known', so it is shielded from
+    reaping and accumulates for the life of the install."""
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    blk = src.split("def reap_orphans")[1].split("def _scaler_loop")[0]
+    assert "known - live" in blk and "unregister_pod(gone)" in blk