runpod: direct_tcp — reach a pod at its public ip:port, bypassing the Cloudflare-fronted proxy

Every pod request went through <pod>-<port>.proxy.runpod.net, which sits
behind Cloudflare: a 100 s idle limit per request and a body-size cap. A
slow non-streaming render can be cut off with a 524 while the pod is
still working. direct_tcp (off by default) exposes the port as tcp and
talks to the public ip:port the runtime reports — plain HTTP, no limits.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/routes.py

@@ -2812,7 +2812,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
             rpo["cloud_types"] = ct
         # bools
         for k in ("allow_spot", "allow_open_pod", "sticky_sessions", "keep_warm",
-                  "model_url_is_tar", "venv_on_volume"):
+                  "model_url_is_tar", "venv_on_volume", "direct_tcp"):
             if k in src and src.get(k) is not None and src.get(k) != "":
                 sv = src.get(k)
                 rpo[k] = (sv.lower() in ("1", "true", "on", "yes")

codai/admin/templates/models.html

@@ -839,6 +839,10 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               <span class="form-hint" style="font-size:11px">Boots a small image and runs from a venv on the volume — the first pod builds it, later pods skip the 7 GB pull. Needs a volume; imports are slower off network storage.</span>
             </div>
           </div>
+          <div class="form-row" style="margin:.5rem 0 0">
+            <label style="display:flex;align-items:center;gap:.3rem;font-size:12px"><input type="checkbox" id="cfg-rp-direct-tcp" style="width:auto"> Direct TCP to the pod (bypass RunPod's proxy / Cloudflare)</label>
+            <span class="form-hint" style="font-size:11px">Default is RunPod's HTTPS proxy, which sits behind Cloudflare: TLS, but a 100 s idle limit per request and a body-size cap — a slow non-streaming render can be cut off with a 524. Direct TCP talks to the pod's public ip:port with neither limit, over plain HTTP (the bearer token is not encrypted in transit).</span>
+          </div>
           <div class="form-row" style="margin:.5rem 0 0">
             <label class="form-label">Share pod pool <span class="muted">(optional name)</span></label>
             <input type="text" id="cfg-rp-pool" class="form-input" placeholder="blank = this model rents its own pods">
@@ -4304,6 +4308,7 @@ function _populateRunpod(s){
   set('cfg-rp-volume', rp.network_volume_id || '');
   set('cfg-rp-volume-path', rp.volume_path || '');
   chk('cfg-rp-venv-volume', rp.venv_on_volume);
+  chk('cfg-rp-direct-tcp', rp.direct_tcp);
   set('cfg-rp-source', rp.source || 'auto');
   set('cfg-rp-hf-repo', rp.hf_repo || '');
   set('cfg-rp-model-url', rp.model_url || '');
@@ -4482,6 +4487,7 @@ function _collectRunpod(){
     network_volume_id:  (val('cfg-rp-volume') || '').trim(),
     volume_path:        (val('cfg-rp-volume-path') || '').trim(),
     venv_on_volume:     document.getElementById('cfg-rp-venv-volume').checked,
+    direct_tcp:         document.getElementById('cfg-rp-direct-tcp').checked,
     source:             val('cfg-rp-source') || 'auto',
     hf_repo:            (val('cfg-rp-hf-repo') || '').trim(),
     model_url:          (val('cfg-rp-model-url') || '').trim(),

codai/api/runpod_client.py

@@ -32,10 +32,36 @@ def _mask(key: str) -> str:
 
 
 # A pod's public HTTP proxy hostname is derived from its id + the container port.
+# That hostname is fronted by Cloudflare: TLS for free, but a 100 s idle limit
+# per request and a request-body cap. See direct_tcp_url for the other way in.
 def pod_proxy_url(pod_id: str, port: int) -> str:
     return f"https://{pod_id}-{int(port)}.proxy.runpod.net"
 
 
+def direct_tcp_url(ports: list, port: int) -> str:
+    """The pod's public ip:port for ``port`` exposed as `<port>/tcp`, or ''.
+
+    A TCP port is mapped straight to the machine — no RunPod proxy, no
+    Cloudflare in front, so no 100 s idle timeout on a slow non-streaming
+    request and no body-size cap. Plain HTTP, though: the bearer token
+    travels unencrypted, so use it where the network is trusted or the
+    request must not be cut off. The public port is random per pod, which is
+    why it is read from the runtime rather than derived.
+    """
+    for p in ports or []:
+        try:
+            if int(p.get("privatePort") or 0) != int(port):
+                continue
+        except (TypeError, ValueError):
+            continue
+        if str(p.get("type") or "").lower() != "tcp" or not p.get("isIpPublic"):
+            continue
+        ip, pub = p.get("ip"), p.get("publicPort")
+        if ip and pub:
+            return f"http://{ip}:{int(pub)}"
+    return ""
+
+
 # Deep link to the pod in the RunPod web console (container + vLLM logs live there).
 def pod_console_url(pod_id: str) -> str:
     return f"https://www.runpod.io/console/pods/{pod_id}"
@@ -186,9 +212,12 @@ class RunpodClient:
                    registry_auth_id: str = "",
                    entrypoint: Optional[list] = None,
                    start_cmd: Optional[list] = None,
-                   network_volume_id: str = "") -> str:
+                   network_volume_id: str = "", direct_tcp: bool = False) -> str:
         """Provision a pod (on-demand or interruptible/spot). Returns the pod id.
 
+        ``direct_tcp`` exposes the port as `<port>/tcp` (public ip + random
+        public port, no proxy, no Cloudflare) instead of `<port>/http`.
+
         ``entrypoint``/``start_cmd`` override the image's ENTRYPOINT/CMD, which is
         what lets a pod fetch something before its server starts — an image whose
         ENTRYPOINT is the server itself (vllm/vllm-openai) otherwise treats every
@@ -204,9 +233,9 @@ class RunpodClient:
                 volume_mount_path=volume_mount_path, env=env,
                 is_spot=is_spot, bid_per_gpu=bid_per_gpu,
                 data_center_id=data_center_id, registry_auth_id=registry_auth_id,
-                entrypoint=entrypoint, start_cmd=start_cmd)
+                entrypoint=entrypoint, start_cmd=start_cmd, direct_tcp=direct_tcp)
         env_list = [{"key": str(k), "value": str(v)} for k, v in (env or {}).items()]
-        ports = f"{int(port)}/http"
+        ports = f"{int(port)}/{'tcp' if direct_tcp else 'http'}"
         common = {
             "cloudType": (cloud_type or "SECURE").upper(),
             "gpuCount": int(gpu_count),
@@ -267,7 +296,7 @@ class RunpodClient:
             "containerDiskInGb": int(container_disk_gb),
             "volumeInGb": int(volume_gb),
             "volumeMountPath": volume_mount_path,
-            "ports": [f"{int(port)}/http"],
+            "ports": [f"{int(port)}/{'tcp' if direct_tcp else 'http'}"],
             "env": {str(k): str(v) for k, v in (env or {}).items()},
         }
         if entrypoint:
@@ -425,8 +454,12 @@ class RunpodClient:
         return out
 
     def wait_ready(self, pod_id: str, port: int, ready_timeout: float = 900.0,
-                   poll_every: float = 5.0) -> str:
-        """Poll until the pod exposes its proxy port; return the proxy base URL.
+                   poll_every: float = 5.0, direct_tcp: bool = False) -> str:
+        """Poll until the pod exposes its port; return the base URL to talk to.
+
+        The proxy URL is derivable the moment any port shows; a direct TCP
+        mapping is only usable once the runtime reports the public ip:port,
+        so the wait continues until it does.
 
         Only waits for the port to be exposed — the HTTP service inside (vLLM) is
         health-checked separately by the worker."""
@@ -453,7 +486,11 @@ class RunpodClient:
                     f"died immediately. RunPod says: "
                     f"{info.get('last_change') or '(no reason given)'}")
             if info.get("ready"):
-                return pod_proxy_url(pod_id, port)
+                if not direct_tcp:
+                    return pod_proxy_url(pod_id, port)
+                direct = direct_tcp_url(info.get("ports"), port)
+                if direct:
+                    return direct
             # A multi-GB image pull is minutes of silence otherwise, which reads
             # exactly like a hung pod. Say what is actually being waited on.
             if time.time() >= next_report:

codai/api/runpod_worker.py

@@ -71,6 +71,10 @@ class RunpodModelConfig:
     network_volume_id: str = ""
     volume_mount_path: str = ""              # blank = the account default
     volume_path: str = ""                    # weights already on the volume (file or dir)
+    # Reach the pod at its public ip:port instead of through RunPod's HTTP proxy.
+    # The proxy is fronted by Cloudflare: TLS, but a 100 s idle limit per request
+    # and a request-body cap. Direct TCP has neither — and no TLS either.
+    direct_tcp: bool = False
     # Keep this pod's Python dependencies on the VOLUME instead of in the image,
     # and boot a small image that uses them. The first pod builds the venv (a few
     # minutes); every pod after skips both the 7 GB image pull and the install.
@@ -1191,6 +1195,7 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
         cfg.cloud_types = ["SECURE"]
     cfg.volume_mount_path = (b.get("volume_mount_path") or "").strip()
     cfg.volume_path = (b.get("volume_path") or "").strip()
+    cfg.direct_tcp = _as_bool(b.get("direct_tcp"), False)
     cfg.venv_on_volume = _as_bool(b.get("venv_on_volume"), cfg.venv_on_volume)
     cfg.venv_name = (b.get("venv_name") or "").strip()
     cfg.slim_image = (b.get("slim_image") or "").strip()
@@ -1771,7 +1776,8 @@ class RunpodPodPool:
                     volume_mount_path=(vol_mount or "/workspace"),
                     registry_auth_id=(self.mcfg.registry_auth_id
                                       or getattr(self.account, "registry_auth_id", "") or ""),
-                    entrypoint=plan.get("entrypoint"), start_cmd=plan.get("start_cmd"))
+                    entrypoint=plan.get("entrypoint"), start_cmd=plan.get("start_cmd"),
+                    direct_tcp=bool(getattr(self.mcfg, "direct_tcp", False)))
                 return pod_id, sel, i + 1
             except RunpodError as exc:
                 msg = str(exc).lower()
@@ -1881,7 +1887,8 @@ class RunpodPodPool:
             t_created = time.time()
             url = ""
             try:
-                url = client.wait_ready(pod_id, port, ready_timeout=boot_to)
+                url = client.wait_ready(pod_id, port, ready_timeout=boot_to,
+                                        direct_tcp=bool(getattr(self.mcfg, "direct_tcp", False)))
                 t_port = time.time()
                 print(f"[runpod] pod {pod_id}: port open after "
                       f"{t_port - t_created:.0f}s (image pull + container start)",

docs/runpod.md

@@ -129,6 +129,7 @@ sync.) In `models.json` that is a `backend` pin plus a `runpod` block:
 | `served_model` | — | The HF repo id the pod actually serves. **Required for pods.** |
 | `image` | `vllm/vllm-openai:latest` | Container image. Must expose an OpenAI-compatible server. |
 | `port` | `8000` | Port the server listens on inside the container. |
+| `direct_tcp` | `false` | Talk to the pod at its public `ip:port` instead of through RunPod's HTTPS proxy. The proxy (`<pod>-<port>.proxy.runpod.net`) sits behind **Cloudflare**: TLS for free, but a 100 s idle limit per request — a slow *non-streaming* request (a long render, a cold 150 GB engine's first token) can be cut off with a 524 while the pod is still working — and a request-body cap. Direct TCP has neither limit, over plain HTTP: the bearer token is not encrypted in transit. |
 | `ctx` | — | Passed as vLLM's `--max-model-len`. |
 | `container_disk_gb` | `40` | Grown automatically to fit the weights unless a network volume holds them; an explicit larger value is respected. |
 | `weights_gb` | — | State the model's size and the estimate is skipped. The only honest answer for a URL download, an upload or a local file — the estimator cannot see those — and for an HF repo it beats a heuristic that errs high. Also accepted on the model entry itself. |
@@ -257,6 +258,7 @@ the RAM they need, not just the VRAM — `min_vram_gb` says nothing about RAM.
 
 ```
 client ──▶ front proxy ──▶ primary engine ──▶ RunpodBackend ──▶ https://<pod>-8000.proxy.runpod.net/v1
+                                                            (or http://<public-ip>:<port>/v1 with direct_tcp)
                                                             └─▶ https://api.runpod.ai/v2/<eid>/openai/v1
 ```
 

tests/test_burst_placement.py

@@ -1029,3 +1029,43 @@ def test_the_kt_worker_launches_from_its_own_venv(tmp_path, monkeypatch):
     # A venv that is configured but missing falls back rather than crashing.
     monkeypatch.setenv("CODERAI_KT_VENV", str(tmp_path / "missing"))
     assert kw._venv_python(_Cfg()) == sys.executable
+
+
+def test_direct_tcp_reaches_the_pod_without_the_proxy(monkeypatch):
+    """The proxy hostname is fronted by Cloudflare: a 100 s idle limit per
+    request cuts off a slow non-streaming render. direct_tcp exposes the
+    port as tcp and talks to the public ip:port instead — read from the
+    runtime, because the public port is random per pod."""
+    from codai.api import runpod_client as rc
+    from codai.api.runpod_worker import parse_model_runpod
+
+    assert parse_model_runpod({}).direct_tcp is False              # proxy by default
+    assert parse_model_runpod({"direct_tcp": True}).direct_tcp is True
+
+    ports = [{"ip": "203.0.113.7", "isIpPublic": True, "privatePort": 8000,
+              "publicPort": 41234, "type": "tcp"}]
+    assert rc.direct_tcp_url(ports, 8000) == "http://203.0.113.7:41234"
+    assert rc.direct_tcp_url(ports, 8001) == ""                    # wrong port
+    assert rc.direct_tcp_url([{**ports[0], "type": "http"}], 8000) == ""
+    assert rc.direct_tcp_url([{**ports[0], "isIpPublic": False}], 8000) == ""
+    assert rc.pod_proxy_url("abc", 8000) == "https://abc-8000.proxy.runpod.net"
+
+    # create_pod asks for the tcp mapping only when told to.
+    seen = {}
+    class _C(rc.RunpodClient):
+        def __init__(self): pass
+        def _gql(self, q, v, timeout=30.0):
+            seen["ports"] = v["input"]["ports"]; return {"podFindAndDeployOnDemand": {"id": "p1"}}
+    _C().create_pod(name="n", image="i", gpu_type_id="g", port=8000)
+    assert seen["ports"] == "8000/http"
+    _C().create_pod(name="n", image="i", gpu_type_id="g", port=8000, direct_tcp=True)
+    assert seen["ports"] == "8000/tcp"
+
+    # wait_ready keeps waiting until the runtime reports the public mapping.
+    infos = iter([{"status": "RUNNING", "ready": True, "ports": [], "uptime_s": 1},
+                  {"status": "RUNNING", "ready": True, "ports": ports, "uptime_s": 2}])
+    class _W(rc.RunpodClient):
+        def __init__(self): pass
+        def get_pod(self, pod_id): return next(infos)
+    monkeypatch.setattr(rc.time, "sleep", lambda s: None)
+    assert _W().wait_ready("p1", 8000, direct_tcp=True) == "http://203.0.113.7:41234"