gateway: retry a fresh pod's first not-found/unavailable answers before believing them

Two verification runs failed with a 404 issued the same second the pod
was declared ready — tts on one card, ocr on another — and both passed
unchanged on a rerun: RunPod's proxy answers 404 until its route
settles, and a catalogue can lag its listing by a moment. A 404/502/503
from a base seen less than a minute ago is retried three times over
~20 s before it is returned. A base up for longer gets no such doubt.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKm7Z56iKjHgNkHc6VhWKz

codai/api/remote_gateway.py

@@ -40,6 +40,7 @@ branch and the body is never buffered.
 from codai.api import pod_http
 import json
 import os
+import time
 import re
 from typing import Optional, Tuple
 
@@ -301,6 +302,18 @@ def _model_from_body(body: bytes, content_type: str, field: str = "model") -> st
 _MODEL_FIELD = {"/v1/ocr": "engine", "/v1/ocr/batch": "engine"}
 
 
+#: base url -> when it first answered, for the first-minute retry above.
+_READY_AT: dict = {}
+
+
+def _pod_ready_at(base: str) -> float:
+    """When ``base`` was first handed out by a pool; now, if never seen."""
+    b = (base or "").rstrip("/")
+    if b not in _READY_AT:
+        _READY_AT[b] = time.time()
+    return _READY_AT[b]
+
+
 def _rewrite_request_field(body: bytes, headers: dict, field: str, value: str):
     """Replace one field's value in a JSON or multipart body. (body, headers).
 
@@ -615,6 +628,24 @@ class RemoteGatewayMiddleware:
                                     headers=fwd, stream=True, timeout=(30, 3600))
         try:
             resp = await asyncio.to_thread(_call)
+            # The first request into a pod that came up seconds ago can land
+            # before the far side is really serving — RunPod's proxy answers
+            # 404 until its route settles, a catalogue may lag its /v1/models
+            # listing by a moment. Two verification runs failed exactly this
+            # way, 0 s after "ready", and passed unchanged a minute later. A
+            # not-found/unavailable answer in the first minute is retried a
+            # few times before it is believed.
+            for _wait in (3, 6, 10):
+                if resp.status_code not in (404, 502, 503):
+                    break
+                if time.time() - _pod_ready_at(base) > 60:
+                    break
+                print(f"[remote-gateway] {base} answered {resp.status_code} "
+                      f"{time.time() - _pod_ready_at(base):.0f}s after coming up — "
+                      f"retrying in {_wait}s", flush=True)
+                resp.close()
+                await asyncio.sleep(_wait)
+                resp = await asyncio.to_thread(_call)
         except Exception as exc:
             return await _error(send, 502, f"remote endpoint {base} unreachable: {exc}")
 

tests/test_remote_execution.py

@@ -275,6 +275,17 @@ class _EchoRemote(BaseHTTPRequestHandler):
         type(self).hits.append({"path": self.path, "len": len(body),
                                 "ctype": self.headers.get("Content-Type", ""),
                                 "body": body})
+        # A remote that is "not ready yet": the first N requests 404, like
+        # RunPod's proxy before its route settles. See the readiness-race test.
+        if getattr(type(self), "not_ready", 0) > 0:
+            type(self).not_ready -= 1
+            out = b'{"detail":"Not Found"}'
+            self.send_response(404)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(out)))
+            self.end_headers()
+            self.wfile.write(out)
+            return
         out = json.dumps({"served_by": "remote"}).encode()
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
@@ -1939,3 +1950,32 @@ def test_a_host_model_holds_no_local_weights(monkeypatch):
     assert mgr._model_is_remote("m", {"backend": "runpod"}) is True
     monkeypatch.setattr(mgr, "resolve_engine_backend", lambda n: None)
     assert mgr._model_is_remote("m", {"backend": "vulkan"}) is False
+
+
+def test_a_fresh_pods_first_not_found_is_retried_not_believed(gateway_app, monkeypatch):
+    """Two verification runs failed with a 404 issued 0 s after the pod was
+    declared ready and passed unchanged a minute later: RunPod's proxy
+    answers 404 until its route settles. The first minute's not-found /
+    unavailable answers are retried before they are believed."""
+    import codai.api.remote_gateway as gw
+    client, gwm, url = gateway_app
+    gwm.capability_endpoints = lambda: {"images": url}
+    import asyncio as _aio
+    monkeypatch.setattr(_aio, "sleep", _fast_sleep)
+    gw._READY_AT.clear()
+    _EchoRemote.not_ready = 2
+    r = client.post("/v1/images/generations", json={"prompt": "x"})
+    assert r.status_code == 200 and r.json()["served_by"] == "remote"
+    assert len([h for h in _EchoRemote.hits if h["path"] == "/v1/images/generations"]) == 3
+
+    # A base that has been up for a while gets no such benefit of the doubt.
+    _EchoRemote.hits = []
+    _EchoRemote.not_ready = 1
+    gw._READY_AT[url.rstrip("/")] = 0.0
+    r = client.post("/v1/images/generations", json={"prompt": "x"})
+    assert r.status_code == 404
+    assert len(_EchoRemote.hits) == 1
+
+
+async def _fast_sleep(_s):
+    return None