runpod: a restart mid-boot froze the pool and hid its own pods (v0.3.68)

Observed live: six pods billing on the RunPod account, none in coderai's
interface, and "another engine is booting a pod for 'datalab-to/surya-ocr-2' --
waiting for it" every fifteen seconds for seven minutes. Nothing was booting. A
deploy had restarted coderai mid-boot, leaving that pod's URL-less registry
entry with no owner.

Three fixes, three different mistakes:

1. sibling_is_provisioning() judged the entry's liveness against
   REAP_GRACE_SECONDS (1200s). That constant answers how long the REAPER waits
   before killing an unowned pod -- a different question. The registry has a
   60s heartbeat, so a dead owner is detectable in seconds. Now
   PROVISION_STALE_SECONDS (180s = three missed beats).

2. The sibling check ran BEFORE _adopt_shared_pod(), so the recovery path for
   orphaned pods sat behind the marker that orphaning creates. Five running,
   registered, billing pods were never adopted. Adoption now runs first: a pod
   already serving this pool beats waiting for a boot and beats renting.

3. A pod registered at CREATION recorded no $/hr -- the rate was passed only at
   the ready-registration 80 lines later -- so a boot interrupted before ready
   could only be reaped with "the registry records no rate for it". The price is
   known at creation. Also, hourly_usd is now carried over on re-registration
   like image and gpu already were: an adoption that did not know the rate
   turned a billing pod free.

14 tests, including the exact live entry (url-less, 910s stale) as a case.
test_runpod_client_scale's adopt-before-rent assertion was pinned to the old
source spelling; updated to assert the intent, which is now stronger.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HXRMe9ZPUQCCCtBai9uqt3

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.67"
+__version__ = "0.3.68"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -3021,6 +3021,12 @@ class RunpodPodPool:
             # …and from every SIBLING engine's reaper, which cannot see the set
             # above: each engine has its own pools and its own reaper.
             register_pod(pod_id, str(self.model_key), api_key=self.api_key,
+                         # The rate is known HERE, not only once the pod is
+                         # ready. A boot interrupted by a restart or a crash
+                         # leaves an entry that bills at a real $/hr, and
+                         # without this the reaper could only say "the registry
+                         # records no rate for it" and drop the money.
+                         hourly_usd=float(sel.get("price") or 0.0),
                          image=str(self._image or ""),
                          gpu=str(sel["display_name"]))   # url once ready
             _warn = weight_transfer_warning(_model_entry(self.served or self.model_key),
@@ -3798,6 +3804,18 @@ class RunpodPodPool:
             if self._release_pending and healthy_n <= keep_min:
                 self._release_pending = False          # satisfied
         if healthy_n < keep_min and not self._closed and not self._booting:
+            # A pod that is ALREADY RUNNING for this pool beats waiting for one
+            # that is booting, and beats renting a second. Tried before the
+            # sibling check on purpose: after a restart the pods this deployment
+            # rented a minute ago are running, registered and billing, and
+            # deferring to a sibling's boot marker left them unowned -- the pool
+            # empty, the account full. Adoption is the recovery path, so it must
+            # not sit behind the thing it recovers from.
+            try:
+                if self._adopt_shared_pod():
+                    return
+            except Exception as exc:
+                print(f"[runpod] maintain: adoption failed: {exc}", flush=True)
             # One already on the way (from here, from ensure_ready, or from a
             # sibling engine sharing this pool) is one pod, not none: topping up
             # against a booting pod is how a warm floor of 1 became two.
@@ -3814,8 +3832,7 @@ class RunpodPodPool:
             print(f"[runpod] {self.model_key!r}: warm floor {keep_min}, "
                   f"{healthy_n} pod(s) healthy — renting one", flush=True)
             try:
-                if not self._adopt_shared_pod():
-                    self._provision_one()
+                self._provision_one()
             except Exception as exc:
                 print(f"[runpod] maintain: warm provision failed: {exc}", flush=True)
             finally:
@@ -4486,8 +4503,13 @@ def register_pod(pod_id: str, pool_key: str = "", url: str = "",
                           "image": str(image or (was or {}).get("image") or ""),
                           "gpu": str(gpu or (was or {}).get("gpu") or ""),
                           # So a pod adopted after a restart is still attributed
-                          # its real $/hr instead of looking free.
-                          "hourly_usd": float(hourly_usd or 0.0)}
+                          # its real $/hr instead of looking free. Carried over
+                          # like image/gpu: a re-registration that does not know
+                          # the rate must not overwrite one that did, or an
+                          # adoption turns a billing pod free.
+                          "hourly_usd": float(hourly_usd
+                                              or (was or {}).get("hourly_usd")
+                                              or 0.0)}
         return True
 
     _registry_update(_write)
@@ -4523,8 +4545,22 @@ def touch_registered_pods() -> int:
     return touched[0]
 
 
+#: How long a URL-less registry entry may go un-heartbeated before it stops
+#: meaning "a sibling is booting this". The heartbeat runs every 60s, so three
+#: missed beats is a dead owner -- not a slow one.
+#:
+#: This used to be REAP_GRACE_SECONDS (1200s), which conflated two unrelated
+#: questions: "is a sibling still booting?" (answered by the heartbeat) and "how
+#: long before the reaper may kill an unowned pod?" (deliberately generous). The
+#: cost was paid live on 2026-10-14: a deploy restarted coderai mid-boot, the
+#: previous incarnation's URL-less entry looked like a live sibling for twenty
+#: minutes, and because this check runs BEFORE adoption the pool could neither
+#: rent nor adopt -- six pods billing on the account and none in the interface.
+PROVISION_STALE_SECONDS = 180.0
+
+
 def sibling_is_provisioning(pool_key: str) -> bool:
-    """True when another process has a pod for this pool that is still booting.
+    """True when another LIVE process has a pod for this pool that is booting.
 
     Registered at creation with no URL yet. Without this the sibling rents its
     own during the ~2.5 minutes a pod takes to boot — which is exactly what
@@ -4539,9 +4575,10 @@ def sibling_is_provisioning(pool_key: str) -> bool:
             continue
         if _is_this_process(info) or info.get("url"):
             continue
-        # Only while it could plausibly still be booting; a stale entry from a
-        # dead process must not block provisioning forever.
-        if now - float(info.get("at") or 0) < REAP_GRACE_SECONDS:
+        # Only while its owner is still TALKING. A heartbeat that stopped means
+        # the process that was booting this pod is gone, and waiting on it is
+        # waiting forever -- while the pod it rented bills.
+        if now - float(info.get("at") or 0) < PROVISION_STALE_SECONDS:
             return True
     return False
 

docs/runpod.md

@@ -25,6 +25,37 @@ deployment — see [RunPod-only deployments](#runpod-only-deployments).
 
 ---
 
+## A restart mid-boot used to freeze the pool
+
+Observed live 2026-10-14: six pods billing on the account, none in coderai's
+interface, and this every fifteen seconds for seven minutes —
+
+```
+[runpod] another engine is booting a pod for 'datalab-to/surya-ocr-2' — waiting for it
+```
+
+Nothing was booting. A deploy had restarted coderai while a pod was mid-boot,
+leaving that pod's URL-less registry entry behind with no owner. Two separate
+decisions turned a dead marker into a frozen pool:
+
+* `sibling_is_provisioning()` judged the entry's liveness against
+  `REAP_GRACE_SECONDS` (1200s). That constant answers a different question — how
+  long the reaper waits before killing an unowned pod — and the registry has a
+  **60-second heartbeat**, so a dead owner is detectable in seconds. Now
+  `PROVISION_STALE_SECONDS` (180s, three missed beats).
+* the sibling check ran **before** `_adopt_shared_pod()`. So the recovery path
+  for orphaned pods sat behind the marker that orphaning creates, and five
+  running, registered, billing pods were never adopted. Adoption now runs first:
+  a pod already serving this pool beats both waiting for a boot and renting.
+
+Money, same incident: a pod registered at *creation* recorded no `$/hr` (the rate
+was passed only at the ready-registration), so a boot interrupted before ready
+could only ever be reaped with `the registry records no rate for it`. The price
+is known at creation and is recorded there. And `hourly_usd` is now carried over
+on re-registration like `image` and `gpu` were — an adoption that did not know
+the rate used to turn a billing pod free.
+
+
 ## 1. Account setup
 
 **Settings → RunPod** (the Settings page is split into category tabs; RunPod is

tests/test_runpod_client_scale.py

@@ -390,7 +390,13 @@ def test_the_warm_path_adopts_before_it_rents():
     assert "_adopt_shared_pod()" in ready
     assert ready.index("_adopt_shared_pod()") < ready.index("self._provision_one()")
     warm = src.split("def maintain")[1].split("def close")[0]
-    assert "if not self._adopt_shared_pod():" in warm
+    # Spelling changed in 0.3.68: adoption moved ABOVE the sibling-boot check
+    # and now returns early, because a restart's own orphans were sitting behind
+    # the stale boot marker that the restart created. The intent this test
+    # guards -- adopt before renting -- is stronger than before, so assert the
+    # intent rather than the old line. See tests/test_runpod_restart_orphans.py.
+    assert "_adopt_shared_pod()" in warm
+    assert warm.index("_adopt_shared_pod()") < warm.index("self._provision_one()")
 
 
 def test_adoption_is_one_implementation_for_both_callers():

tests/test_runpod_restart_orphans.py

+"""A deploy restarted coderai mid-boot and the pool went blind.
+
+Observed live 2026-10-14, 15:16-15:23: six pods billing on the RunPod account,
+none in coderai's interface, and this line every fifteen seconds —
+
+    [runpod] another engine is booting a pod for 'datalab-to/surya-ocr-2'
+             — waiting for it
+
+Nothing was booting. The restart had killed the process that was, leaving its
+URL-less registry entry behind. Two decisions turned that into a frozen pool:
+
+1. `sibling_is_provisioning` judged the entry's liveness against
+   REAP_GRACE_SECONDS (1200s), which answers a different question than the one
+   being asked. The registry has a 60s heartbeat, so a dead owner is detectable
+   in seconds; twenty minutes was the reaper's patience, borrowed by mistake.
+
+2. The sibling check ran BEFORE `_adopt_shared_pod()`. So the recovery path for
+   orphaned pods sat behind the very marker the orphaning created, and the five
+   running, registered, billing pods were never adopted.
+
+Plus the money: a pod registered at CREATION recorded no $/hr (the rate was only
+passed at the ready-registration), so a boot interrupted before ready could only
+ever be reaped with "the registry records no rate for it".
+"""
+
+import pathlib
+import sys
+import time
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from codai.api import runpod_worker as rw                       # noqa: E402
+
+
+def _entry(pool, *, url="", age=0.0, proc="someone-else", hourly=0.59):
+    return {"pool": pool, "proc": proc, "pid": 999999,
+            "at": time.time() - age, "created": time.time() - age,
+            "url": url, "key": "k", "image": "img", "gpu": "A40",
+            "hourly_usd": hourly}
+
+
+def _registry(monkeypatch, data):
+    monkeypatch.setattr(rw, "_registry_update", lambda fn: data)
+
+
+# ------------------------------------------------- the marker's lifetime #
+def test_a_fresh_sibling_boot_still_defers():
+    """The original purpose must survive the fix: two pods for one capability,
+    45 seconds apart, is what this check exists to prevent."""
+    import pytest                                              # noqa: F401
+    data = {"p1": _entry("m", url="", age=10.0)}
+    assert rw.PROVISION_STALE_SECONDS > 10.0
+
+
+def test_a_booting_sibling_within_the_heartbeat_window_defers(monkeypatch):
+    _registry(monkeypatch, {"p1": _entry("m", url="", age=30.0)})
+    assert rw.sibling_is_provisioning("m") is True
+
+
+def test_a_marker_whose_owner_stopped_heartbeating_does_not_block(monkeypatch):
+    """Three missed beats is a dead owner, not a slow one."""
+    _registry(monkeypatch, {"p1": _entry("m", url="", age=300.0)})
+    assert rw.sibling_is_provisioning("m") is False
+
+
+def test_the_window_is_the_heartbeat_not_the_reap_grace():
+    """1200s answered 'how long before the reaper may kill an unowned pod',
+    which is not the question. Twenty minutes of a frozen pool was the cost."""
+    assert rw.PROVISION_STALE_SECONDS < rw.REAP_GRACE_SECONDS
+    assert rw.PROVISION_STALE_SECONDS <= 300.0
+
+
+def test_the_exact_live_case_no_longer_blocks(monkeypatch):
+    """y3nndqmh9wqwc9: url-less, 910s stale, owner killed by a deploy."""
+    _registry(monkeypatch, {"y3nndqmh9wqwc9": _entry(
+        "datalab-to/surya-ocr-2", url="", age=910.0, hourly=0.0)})
+    assert rw.sibling_is_provisioning("datalab-to/surya-ocr-2") is False
+
+
+def test_a_pod_that_reached_ready_is_not_a_boot_in_progress(monkeypatch):
+    """A URL means it is serving, not booting; it is adoptable, not awaited."""
+    _registry(monkeypatch, {"p1": _entry("m", url="https://1.2.3.4:8000",
+                                         age=10.0)})
+    assert rw.sibling_is_provisioning("m") is False
+
+
+def test_our_own_entry_is_never_a_sibling(monkeypatch):
+    _registry(monkeypatch, {"p1": _entry("m", url="", age=10.0,
+                                         proc=rw._PROC_ID)})
+    assert rw.sibling_is_provisioning("m") is False
+
+
+def test_another_pools_boot_is_not_ours(monkeypatch):
+    _registry(monkeypatch, {"p1": _entry("other", url="", age=10.0)})
+    assert rw.sibling_is_provisioning("m") is False
+
+
+# --------------------------------------------------- adoption comes first #
+def test_maintain_adopts_before_deferring_to_a_sibling_boot():
+    """Source-anchored: the recovery path must not sit behind the marker that
+    the thing it recovers from leaves behind."""
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    body = src.split("# Keep min_pods warm.")[1].split("def close(")[0]
+    adopt = body.index("_adopt_shared_pod()")
+    sibling = body.index("sibling_is_provisioning(")
+    assert adopt < sibling, (
+        "sibling_is_provisioning is checked before adoption; a restart's own "
+        "orphans then stay unowned while they bill")
+
+
+def test_maintain_does_not_adopt_twice():
+    """Adoption moved above the sibling check; the old call below it must be
+    gone, or a warm top-up tries it again on every tick."""
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    body = src.split("# Keep min_pods warm.")[1].split("def close(")[0]
+    assert body.count("_adopt_shared_pod()") == 1
+
+
+# ------------------------------------------------------------ the money #
+def test_a_pod_records_its_rate_at_creation_not_only_when_ready():
+    """An interrupted boot bills at a real $/hr. Without the rate the reaper can
+    only say 'the registry records no rate for it' and drop the money."""
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    at_create = src.split("# url once ready")[0].rsplit("register_pod(", 1)[1]
+    assert "hourly_usd" in at_create, "creation-time registration omits the rate"
+
+
+def test_a_re_registration_cannot_erase_a_known_rate(monkeypatch, tmp_path):
+    """image and gpu were carried over; hourly_usd was not, so an adoption that
+    did not know the rate turned a billing pod free."""
+    store = {"p1": {"pool": "m", "hourly_usd": 0.59, "image": "img",
+                    "gpu": "A40", "created": 1000.0}}
+
+    def _upd(fn):
+        r = fn(store)
+        return store if r is False else store
+    monkeypatch.setattr(rw, "_registry_update", _upd)
+    rw.register_pod("p1", "m", "https://x", "k")        # no hourly_usd given
+    assert store["p1"]["hourly_usd"] == 0.59
+
+
+def test_a_supplied_rate_still_wins(monkeypatch):
+    store = {"p1": {"pool": "m", "hourly_usd": 0.59, "created": 1000.0}}
+    monkeypatch.setattr(rw, "_registry_update",
+                        lambda fn: (fn(store), store)[1])
+    rw.register_pod("p1", "m", "https://x", "k", hourly_usd=1.29)
+    assert store["p1"]["hourly_usd"] == 1.29
+
+
+def test_created_still_survives_a_re_registration(monkeypatch):
+    """Already true, asserted here because the ledger's hours come from it and
+    this change edits the same dict."""
+    store = {"p1": {"pool": "m", "hourly_usd": 0.59, "created": 1000.0}}
+    monkeypatch.setattr(rw, "_registry_update",
+                        lambda fn: (fn(store), store)[1])
+    rw.register_pod("p1", "m", "https://x", "k")
+    assert store["p1"]["created"] == 1000.0