runpod: make the next pod failure diagnosable from the answer

The last three pod failures were each diagnosed the same way: find the
pod URL in the engine log, ask the pod by hand what it serves, work
backwards. That is a diagnosis loop measured in minutes of log
archaeology per attempt.

Boot is now reported in PHASES, because "it took 150s" is not a
diagnosis: port open after Ns (image pull + container start), serving
after N more. The two have different fixes — a bigger boot_timeout_s
versus a bigger load_timeout_s.

The moment a pod is up, its /v1/models is logged. An empty catalogue now
says so outright — "pod serves NO models — it will refuse every request"
— instead of surfacing minutes later as "Model 'x' is not available. Use
one of: " with nothing after the colon.

And a test run's answer carries the pod diagnostics: which pods are live,
their GPU, uptime, spend so far, URL, and the models each one lists. The
fact that mattered in every failure so far was the catalogue, and it is
now in the response rather than three commands away.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/api/model_test.py

@@ -147,6 +147,10 @@ async def test_model(req: ModelTestRequest, request: Request,
            "where": placement["where"], "target": placement["target"],
            "ran": path, "status": status, "ok": 200 <= status < 300,
            "seconds": seconds}
+    # What the remote actually is, and what it thinks it can serve. Without this
+    # a failure means reading engine logs to find the pod URL and asking it by
+    # hand — which is how the last three failures were diagnosed.
+    out["pods"] = _pod_diagnostics()
     if out["ok"]:
         out["sample"] = _sample(raw, capability)
         empty = _empty_result(raw, capability)
@@ -215,6 +219,52 @@ def _no_remote_reason(model: str, entry: dict, capability: str) -> str:
             f"its {capability or 'text'} capability")
 
 
+def _pod_diagnostics() -> list:
+    """Live pods and what each is serving, for the answer to stand on its own."""
+    try:
+        from codai.api.runpod_worker import pods_status, _pools, _pools_lock
+    except Exception:
+        return []
+    out = []
+    try:
+        for pod in pods_status():
+            row = {"pod": pod.get("pod_id"), "for": pod.get("model"),
+                   "gpu": pod.get("gpu"), "state": pod.get("state"),
+                   "uptime_s": pod.get("uptime_s"),
+                   "usd_so_far": pod.get("live_cost_usd")}
+            out.append(row)
+        # Add the URL and catalogue for the pods this process holds.
+        with _pools_lock:
+            pools = list(_pools.values())
+        urls = {}
+        for pool in pools:
+            with pool._cv:
+                for p in pool.pods:
+                    urls[p.pod_id] = (p.url, getattr(pool, "api_key", ""))
+        for row in out:
+            url, key = urls.get(row["pod"], ("", ""))
+            if not url:
+                continue
+            row["url"] = url
+            row["serves"] = _remote_catalogue(url, key)
+    except Exception as exc:
+        return [{"error": f"pod diagnostics unavailable: {exc}"}]
+    return out
+
+
+def _remote_catalogue(url: str, api_key: str) -> list:
+    """The model ids a remote lists, or a short reason it cannot be asked."""
+    import requests
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+    try:
+        r = requests.get(url.rstrip("/") + "/v1/models", headers=headers, timeout=15)
+        if r.status_code != 200:
+            return [f"(HTTP {r.status_code})"]
+        return [str(m.get("id")) for m in (r.json().get("data") or [])] or ["(none)"]
+    except Exception as exc:
+        return [f"({exc})"]
+
+
 def _describe(model: str):
     """(entry, capability) for a model — capability '' means a text/LLM model."""
     try:

codai/api/runpod_worker.py

@@ -1227,6 +1227,33 @@ class RunpodPodPool:
         return ""
 
     # -- provisioning ----------------------------------------------------- #
+    def _log_pod_catalogue(self, url: str) -> None:
+        """Log what the pod says it can serve, the moment it is up.
+
+        The one check that would have turned a confusing "Model 'x' is not
+        available. Use one of: " into an obvious "the pod registered nothing",
+        minutes earlier and without reading a single line of our own code.
+        """
+        import requests
+        headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+        try:
+            r = requests.get(url.rstrip("/") + "/v1/models", headers=headers, timeout=20)
+            if r.status_code != 200:
+                print(f"[runpod] pod catalogue unreadable: HTTP {r.status_code} "
+                      f"{(r.text or '')[:120]}", flush=True)
+                return
+            ids = [str(m.get("id")) for m in (r.json().get("data") or [])]
+        except Exception as exc:
+            print(f"[runpod] pod catalogue unreadable: {exc}", flush=True)
+            return
+        if ids:
+            print(f"[runpod] pod serves {len(ids)} model(s): "
+                  f"{', '.join(ids[:8])}{' …' if len(ids) > 8 else ''}", flush=True)
+        else:
+            print("[runpod] pod serves NO models — it will refuse every request. "
+                  "Seeding did not take effect (check CODERAI_SEED_MODELS in the "
+                  "pod's env and the pod's own startup log).", flush=True)
+
     def _data_center_for_volume(self, volume_id: str) -> str:
         """The data center a network volume lives in, or ''.
 
@@ -1361,13 +1388,25 @@ class RunpodPodPool:
                   flush=True)
             boot_to = float(self.mcfg.boot_timeout_s or self.PORT_TIMEOUT_S)
             load_to = float(self.mcfg.load_timeout_s or self.HEALTH_TIMEOUT_S)
+            # Phase timings. A cold pod takes minutes and "it was slow" is not a
+            # diagnosis: the image pull, the server start and the model load are
+            # different problems with different fixes.
+            t_created = time.time()
             try:
                 url = client.wait_ready(pod_id, port, ready_timeout=boot_to)
+                t_port = time.time()
+                print(f"[runpod] pod {pod_id}: port open after "
+                      f"{t_port - t_created:.0f}s (image pull + container start)",
+                      flush=True)
                 # Wait for the OpenAI server inside the pod (image pull + model load).
                 deadline = time.time() + load_to
                 while time.time() < deadline:
                     if _pod_health_ok(url, path=self.health_path,
                                       api_key=self.api_key):
+                        print(f"[runpod] pod {pod_id}: serving after "
+                              f"{time.time() - t_port:.0f}s more "
+                              f"({time.time() - t_created:.0f}s total)", flush=True)
+                        self._log_pod_catalogue(url)
                         break
                     time.sleep(5)
                 else:

tests/test_remote_execution.py

@@ -1411,3 +1411,32 @@ def test_registering_a_model_makes_it_usable_not_just_listed(monkeypatch):
     assert _MM.model_aliases["bge-m3"] == "BAAI/bge-m3"
     # …AND not filtered out by an engine assignment it was never part of.
     assert {"BAAI/bge-m3", "bge-m3"} <= registered["assigned"]
+
+
+def test_a_test_run_reports_what_the_pods_are_serving(monkeypatch, test_run):
+    """Three failures in a row were diagnosed by finding the pod URL in engine
+    logs and asking it by hand. The answer should carry that itself."""
+    import codai.api.runpod_worker as rw
+    from codai.api import model_test as mt
+
+    monkeypatch.setattr(rw, "pods_status", lambda: [
+        {"pod_id": "pod-1", "model": "capability:embeddings", "gpu": "RTX 2000 Ada",
+         "state": "ready", "uptime_s": 140, "live_cost_usd": 0.009}])
+
+    class _Pool:
+        api_key = "tok"
+        _cv = __import__("threading").Condition()
+        pods = [rw.PodHandle(pod_id="pod-1", url="http://pod-1:8000",
+                             hourly_usd=0.24, started_at=0.0)]
+
+    monkeypatch.setattr(rw, "_pools", {"capability:embeddings": _Pool()})
+    monkeypatch.setattr(mt, "_remote_catalogue",
+                        lambda url, key: ["BAAI/bge-m3", "bge-m3"])
+
+    out = _run_test("my-llm")
+    pods = out["pods"]
+    assert pods[0]["pod"] == "pod-1" and pods[0]["gpu"] == "RTX 2000 Ada"
+    assert pods[0]["url"] == "http://pod-1:8000"
+    # The catalogue is the fact that mattered: an empty list is the whole story.
+    assert pods[0]["serves"] == ["BAAI/bge-m3", "bge-m3"]
+    assert pods[0]["usd_so_far"] == 0.009