runpod: a pid is not a process identity in a container (v0.2.99)

The root cause under the last three fixes. The pod registry recorded
os.getpid() and both readers skipped an entry whose pid matched ours as "our
own, the local pool handles it". Inside a container the engine comes up as the
SAME pid on every boot — 74, on the production host — so after a restart the
previous container's pod read as ours and was skipped by both
find_shared_pod (nothing to adopt) and sibling_is_provisioning (nothing
booting). The warm path then rented another A40 beside the one already running.
Three restarts this afternoon, three extra A40s, each billing until it was
terminated by hand.

Entries now carry a per-process nonce and identity is that. The pid is still
recorded, and an entry written before the nonce falls back to comparing it,
which is the old behaviour and no worse.

test_a_surviving_pods_token_is_remembered_across_a_restart simulated the new
process by monkeypatching os.getpid — the very thing that does not change. It
patches the nonce now, which is why it failed on this commit and passes on it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.98"
+__version__ = "0.2.99"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -3111,6 +3111,25 @@ def _registry_update(fn):
         return {}
 
 
+#: This process, distinctly. os.getpid() is NOT enough: inside a container the
+#: engine comes up as the same pid on every boot (74, here), so a pod registered
+#: by the PREVIOUS container read as "our own" and was skipped — the restart
+#: rented another A40 beside it, every time. Observed three times in a row.
+_PROC_ID = f"{os.getpid()}-{uuid.uuid4().hex[:12]}"
+
+
+def _is_this_process(info: dict) -> bool:
+    """Whether a registry entry was written by THIS process.
+
+    Falls back to the pid for an entry written before the nonce existed, which
+    is the old behaviour and no worse than it was.
+    """
+    proc = str(info.get("proc") or "")
+    if proc:
+        return proc == _PROC_ID
+    return info.get("pid") == os.getpid()
+
+
 def register_pod(pod_id: str, pool_key: str = "", url: str = "",
                  api_key: str = "", hourly_usd: float = 0.0) -> None:
     """Record a pod as owned by this deployment.
@@ -3128,7 +3147,8 @@ def register_pod(pod_id: str, pool_key: str = "", url: str = "",
     if not pod_id:
         return
     _registry_update(lambda d: d.__setitem__(
-        str(pod_id), {"pool": str(pool_key), "pid": os.getpid(), "at": time.time(),
+        str(pod_id), {"pool": str(pool_key), "pid": os.getpid(),
+                      "proc": _PROC_ID, "at": time.time(),
                       "url": str(url or ""), "key": str(api_key or ""),
                       # So a pod adopted after a restart is still attributed its
                       # real $/hr instead of looking free.
@@ -3146,12 +3166,11 @@ def sibling_is_provisioning(pool_key: str) -> bool:
     data = _registry_update(lambda d: False)
     if not isinstance(data, dict):
         return False
-    mine = os.getpid()
     now = time.time()
     for info in data.values():
         if not isinstance(info, dict) or info.get("pool") != str(pool_key):
             continue
-        if info.get("pid") == mine or info.get("url"):
+        if _is_this_process(info) or info.get("url"):
             continue
         # Only while it could plausibly still be booting; a stale entry from a
         # dead process must not block provisioning forever.
@@ -3182,11 +3201,10 @@ def find_shared_pod(pool_key: str, health_path: str = "/v1/models",
     data = _registry_update(lambda d: False)
     if not isinstance(data, dict):
         return None, "", ""
-    mine = os.getpid()
     for pod_id, info in data.items():
         if not isinstance(info, dict) or info.get("pool") != str(pool_key):
             continue
-        if info.get("pid") == mine:
+        if _is_this_process(info):
             continue                       # our own; the local pool handles it
         url = str(info.get("url") or "")
         if not url:

tests/test_burst_placement.py

@@ -536,7 +536,12 @@ def test_a_surviving_pods_token_is_remembered_across_a_restart(tmp_path, monkeyp
     # The pod the dead process rented, with the token it was launched with.
     rw.register_pod("pod-survivor", "capability:embeddings",
                     "http://pod-a:8000", "cra-original")
-    monkeypatch.setattr(rw.os, "getpid", lambda: 424242)   # a NEW process
+    # A NEW process. Patching only os.getpid() used to stand in for this, but
+    # a pid is not an identity: inside a container the engine comes up as the
+    # same pid every boot, which is how a restart came to rent a second pod
+    # beside the one it already had. The nonce is what distinguishes them.
+    monkeypatch.setattr(rw, "_PROC_ID", "424242-aaaaaaaaaaaa")
+    monkeypatch.setattr(rw.os, "getpid", lambda: 424242)
 
     pod_id, url, key, rate = rw.find_shared_pod("capability:embeddings",
                                           api_key="cra-freshly-generated")

tests/test_runpod_client_scale.py

@@ -471,3 +471,47 @@ def test_the_reaper_prunes_entries_for_pods_that_are_gone():
     src = __import__("pathlib").Path(rw.__file__).read_text()
     blk = src.split("def reap_orphans")[1].split("def _scaler_loop")[0]
     assert "known - live" in blk and "unregister_pod(gone)" in blk
+
+
+# ------------------------------- the pid in a container is not an identity
+
+def test_a_pod_from_a_previous_container_is_not_mistaken_for_ours():
+    """THE root cause of the repeated double-renting: inside a container the
+    engine comes up as the same pid every boot (74, on the production host), so
+    'pid == os.getpid()' read the PREVIOUS container's pod as our own and
+    skipped it — and the warm path rented another A40 beside it. Three restarts,
+    three extra A40s."""
+    import os
+    same_pid_other_process = {"pid": os.getpid(), "proc": "74-deadbeefcafe"}
+    assert rw._is_this_process(same_pid_other_process) is False
+    assert rw._is_this_process({"pid": os.getpid(), "proc": rw._PROC_ID}) is True
+
+
+def test_an_entry_from_before_the_nonce_falls_back_to_the_pid():
+    import os
+    assert rw._is_this_process({"pid": os.getpid()}) is True
+    assert rw._is_this_process({"pid": os.getpid() + 1}) is False
+
+
+def test_the_nonce_is_recorded_so_the_next_boot_can_tell():
+    import json
+    import tempfile
+    d = tempfile.mkdtemp()
+    saved = rw._pod_registry_path
+    rw._pod_registry_path = lambda: d + "/pods.json"
+    try:
+        rw.register_pod("p1", "pool:x", "", "tok")
+        rec = json.load(open(d + "/pods.json"))["p1"]
+        assert rec["proc"] == rw._PROC_ID
+        assert rec["pid"] == __import__("os").getpid()   # kept for compatibility
+    finally:
+        rw._pod_registry_path = saved
+
+
+def test_the_registry_helpers_no_longer_compare_pids_directly():
+    src = __import__("pathlib").Path(rw.__file__).read_text()
+    for fn, end in (("def sibling_is_provisioning", "def find_shared_pod"),
+                    ("def find_shared_pod", "def unregister_pod")):
+        blk = src.split(fn)[1].split(end)[0]
+        assert "_is_this_process(info)" in blk, fn
+        assert 'info.get("pid") == mine' not in blk, fn