runpod: a rented pod that was never recorded, and billed on (v0.2.96)

NameError in _provision_one: it read `dc`, a local computed inside
_create_with_fallback, from a function that has no such name. The sequence that
costs money is this — create_pod succeeds, the pod boots on RunPod, the
PodHandle raises, the pool never records the pod, the caller provisions again.
Four A40s existed for a warm floor of one before anyone looked.

Both paths now call pool._data_center(), which keeps the same precedence (the
volume's region, then the model's, then the account's) in one place.

The two tests that covered this asserted the broken expression as a SOURCE
STRING — 'data_center=(getattr(self, "_volume_dc", "") or dc or "")' — so they
passed on a NameError and would have gone on passing. They now call the
resolver, including with nothing configured anywhere, which is the shape that
raised.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.95"
+__version__ = "0.2.96"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -2099,10 +2099,6 @@ class RunpodPodPool:
                 print(f"[runpod] direct_tcp on a {plan.get('engine')} image: plain "
                       f"HTTP — that image cannot take a certificate; the bearer "
                       f"token travels unencrypted", flush=True)
-        # Region precedence: the volume's data centre (a pod elsewhere simply
-        # cannot attach it) > this model's setting > the account default.
-        dc = (self.mcfg.data_center
-              or getattr(self.account, "data_center", "") or "")
         last = None
         for i in range(start, len(ranked)):
             sel = ranked[i]
@@ -2126,7 +2122,7 @@ class RunpodPodPool:
                     cloud_type=sel["cloud_type"], container_disk_gb=_disk_gb,
                     volume_gb=self.mcfg.volume_gb, env=env, docker_args=args,
                     is_spot=sel["is_spot"], bid_per_gpu=sel["bid"],
-                    data_center_id=(getattr(self, "_volume_dc", "") or dc),
+                    data_center_id=self._data_center(),
                     network_volume_id=vol_id,
                     volume_mount_path=(vol_mount or "/workspace"),
                     registry_auth_id=(self.mcfg.registry_auth_id
@@ -2200,6 +2196,23 @@ class RunpodPodPool:
     # NEXT candidate rather than failing the request.
     MAX_PROVISION_ATTEMPTS = 3
 
+    def _data_center(self) -> str:
+        """Which RunPod region this pool's pods go to.
+
+        The volume's own data centre wins: a network volume lives in one region
+        and a pod elsewhere simply cannot attach it. Then the model's setting,
+        then the account default.
+
+        A method rather than a local because it was a local: ``dc`` was computed
+        inside _create_with_fallback and then read again in _provision_one,
+        which has no such name. The pod was created, the PodHandle raised
+        NameError, and the pool never recorded the pod it had just rented — so
+        provisioning retried and the untracked A40 billed on.
+        """
+        return (getattr(self, "_volume_dc", "")
+                or self.mcfg.data_center
+                or getattr(self.account, "data_center", "") or "")
+
     def _provision_one(self):
         """Create + boot one pod; append it healthy. Blocking (minutes).
 
@@ -2286,7 +2299,7 @@ class RunpodPodPool:
                           started_at=time.time(), gpu=sel["display_name"],
                           is_spot=sel["is_spot"], healthy=True, last_used=time.time(),
                           console_url=console,
-                          data_center=(getattr(self, "_volume_dc", "") or dc or ""),
+                          data_center=self._data_center(),
                           cloud_type=str(sel.get("cloud_type") or ""),
                           gpu_count=int(sel.get("gpu_count") or 1))
             with self._cv:

tests/test_runpod_client_scale.py

@@ -358,3 +358,17 @@ def test_the_answer_names_the_model_the_client_asked_about():
     blk = src.split("def set_client_pods")[1]
     assert 'out["model"] = key' in blk
     assert 'out["pool"]' in blk
+
+
+def test_a_pod_is_recorded_even_when_the_region_is_unset():
+    """The leak, reproduced at the level it happened: build the handle fields
+    the way _provision_one does, with nothing configured anywhere. If the
+    region expression raises, the pool never records the pod it just rented."""
+    class _Acct:
+        data_center = ""
+
+    pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
+    pool.account = _Acct()
+    pool.mcfg = rw.parse_model_runpod({})
+    pool._volume_dc = None
+    assert pool._data_center() == ""        # no exception, and a usable value

tests/test_runpod_fleet_status.py

@@ -64,7 +64,10 @@ def test_each_pod_reports_where_it_is_and_what_it_serves():
         assert field in blk, field
     # and the placement has to be carried on the pod to be reportable
     assert "data_center: str = \"\"" in src.split("class PodHandle")[1][:600]
-    assert 'data_center=(getattr(self, "_volume_dc", "") or dc or "")' in src
+    # The handle gets its region from the pool's own resolver. It used to read
+    # a local `dc` that did not exist in this function, so the pod was created
+    # and then the handle raised NameError — the A40 billed untracked.
+    assert "data_center=self._data_center()" in src
 
 
 def test_status_and_spend_are_the_same_document_behind_the_same_gate():

tests/test_runpod_schedule_region_spend.py

@@ -12,6 +12,8 @@ import re
 
 import pytest
 
+from codai.api import runpod_worker as rw
+
 ROOT = pathlib.Path(__file__).resolve().parents[1]
 
 
@@ -160,10 +162,36 @@ def test_region_is_normalised_and_per_model():
 
 
 def test_volume_region_still_wins_over_the_model_setting():
-    """A pod in another region cannot attach the volume, so the volume decides."""
+    """A pod in another region cannot attach the volume, so the volume decides.
+
+    Exercised rather than grepped: this used to assert the source text
+    `dc or ""`, which was a NameError in one of the two places that read it —
+    the pod was rented and then never recorded. A string match could not see
+    that; calling the method can.
+    """
+    class _Acct:
+        data_center = "EU-RO-1"
+
+    pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
+    pool.account = _Acct()
+    pool.mcfg = rw.parse_model_runpod({"data_center": "EU-SE-1"})
+
+    pool._volume_dc = None
+    assert pool._data_center() == "EU-SE-1"        # the model's setting
+    pool.mcfg.data_center = ""
+    assert pool._data_center() == "EU-RO-1"        # falls back to the account
+    pool._volume_dc = "EU-CZ-1"
+    assert pool._data_center() == "EU-CZ-1"        # the volume always wins
+    pool.mcfg.data_center = "EU-SE-1"
+    assert pool._data_center() == "EU-CZ-1"
+
+
+def test_both_provisioning_paths_ask_for_the_region_the_same_way():
+    """One computed it as a local, the other read that name from a scope it was
+    never in. Neither may go back to a local."""
     src = (ROOT / "codai/api/runpod_worker.py").read_text()
-    assert 'dc = (self.mcfg.data_center' in src
-    assert 'data_center_id=(getattr(self, "_volume_dc", "") or dc)' in src
+    assert src.count("self._data_center()") >= 2
+    assert "dc = (self.mcfg.data_center" not in src
 
 
 # --------------------------------------------------- orchestration: admin API