runpod: direct TCP is the default for coderai images

Now that a coderai pod serves TLS on its public ip:port, the proxy's
100 s cutoff and body cap buy nothing for our images — and a voice
clone already died on it. direct_tcp becomes three-state: unset means
direct for coderai images and the proxy for vLLM / llama.cpp / custom
(the proxy is the only TLS those have); true/false force it. Model form,
save route, harness (--direct-tcp / --proxy) and docs follow.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKm7Z56iKjHgNkHc6VhWKz

codai/admin/routes.py

@@ -2812,7 +2812,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
             rpo["cloud_types"] = ct
         # bools
         for k in ("allow_spot", "allow_open_pod", "sticky_sessions", "keep_warm",
-                  "model_url_is_tar", "venv_on_volume", "direct_tcp"):
+                  "model_url_is_tar", "venv_on_volume"):
             if k in src and src.get(k) is not None and src.get(k) != "":
                 sv = src.get(k)
                 rpo[k] = (sv.lower() in ("1", "true", "on", "yes")
@@ -2837,6 +2837,13 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                     rpo[k] = float(v)
                 except (TypeError, ValueError):
                     pass
+        # direct_tcp is three-state: absent/"auto" = decide by image (direct
+        # for coderai images), "on"/"off" or a bool force it.
+        dt = src.get("direct_tcp")
+        if isinstance(dt, bool):
+            rpo["direct_tcp"] = dt
+        elif isinstance(dt, str) and dt.strip().lower() in ("on", "off", "true", "false", "1", "0"):
+            rpo["direct_tcp"] = dt.strip().lower() in ("on", "true", "1")
         # env dict (extra pod env)
         env = src.get("env")
         if isinstance(env, dict) and env:

codai/admin/templates/models.html

@@ -842,8 +842,13 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
             </div>
           </div>
           <div class="form-row" style="margin:.5rem 0 0">
-            <label style="display:flex;align-items:center;gap:.3rem;font-size:12px"><input type="checkbox" id="cfg-rp-direct-tcp" style="width:auto"> Direct TCP to the pod (bypass RunPod's proxy / Cloudflare)</label>
-            <span class="form-hint" style="font-size:11px">Default is RunPod's HTTPS proxy, which sits behind Cloudflare: TLS, but a 100 s idle limit per request and a body-size cap — a slow non-streaming render can be cut off with a 524. Direct TCP talks to the pod's public ip:port with neither limit. coderai pods serve HTTPS on it with a certificate from this install's own CA, so nothing travels in clear; vLLM / llama.cpp images cannot take a certificate and stay plain HTTP on this path.</span>
+            <label class="form-label">Reach the pod</label>
+            <select id="cfg-rp-direct-tcp" class="form-input">
+              <option value="auto">Auto — direct for coderai images, RunPod's proxy for vLLM / llama.cpp / custom</option>
+              <option value="on">Direct TCP — the pod's public ip:port (coderai images serve TLS on it)</option>
+              <option value="off">RunPod proxy — behind Cloudflare: 100 s idle cutoff per request, body-size cap</option>
+            </select>
+            <span class="form-hint" style="font-size:11px">Direct has no request cutoff and no body cap; coderai images bring a certificate from this install's own CA so nothing travels in clear. vLLM / llama.cpp images cannot take a certificate, so on the direct path they would be plain HTTP — the proxy is their TLS.</span>
           </div>
           <div class="form-row" style="margin:.5rem 0 0">
             <label class="form-label">Share pod pool <span class="muted">(optional name)</span></label>
@@ -4310,7 +4315,7 @@ function _populateRunpod(s){
   set('cfg-rp-volume', rp.network_volume_id || '');
   set('cfg-rp-volume-path', rp.volume_path || '');
   chk('cfg-rp-venv-volume', rp.venv_on_volume);
-  chk('cfg-rp-direct-tcp', rp.direct_tcp);
+  set('cfg-rp-direct-tcp', rp.direct_tcp === true ? 'on' : (rp.direct_tcp === false ? 'off' : 'auto'));
   set('cfg-rp-source', rp.source || 'auto');
   set('cfg-rp-hf-repo', rp.hf_repo || '');
   set('cfg-rp-model-url', rp.model_url || '');
@@ -4491,7 +4496,7 @@ function _collectRunpod(){
     network_volume_id:  (val('cfg-rp-volume') || '').trim(),
     volume_path:        (val('cfg-rp-volume-path') || '').trim(),
     venv_on_volume:     document.getElementById('cfg-rp-venv-volume').checked,
-    direct_tcp:         document.getElementById('cfg-rp-direct-tcp').checked,
+    direct_tcp:         val('cfg-rp-direct-tcp') || 'auto',
     source:             val('cfg-rp-source') || 'auto',
     hf_repo:            (val('cfg-rp-hf-repo') || '').trim(),
     model_url:          (val('cfg-rp-model-url') || '').trim(),

codai/api/runpod_worker.py

@@ -74,8 +74,11 @@ class RunpodModelConfig:
     volume_path: str = ""                    # weights already on the volume (file or dir)
     # Reach the pod at its public ip:port instead of through RunPod's HTTP proxy.
     # The proxy is fronted by Cloudflare: TLS, but a 100 s idle limit per request
-    # and a request-body cap. Direct TCP has neither — and no TLS either.
-    direct_tcp: bool = False
+    # and a request-body cap. Direct TCP has neither, and a coderai image brings
+    # its own TLS on it — so it is the DEFAULT for our images (None = decide by
+    # image: direct for coderai, proxy for vLLM/llama.cpp/custom, which cannot
+    # take a certificate). True/False force it either way.
+    direct_tcp: Optional[bool] = None
     # Keep this pod's Python dependencies on the VOLUME instead of in the image,
     # and boot a small image that uses them. The first pod builds the venv (a few
     # minutes); every pod after skips both the 7 GB image pull and the install.
@@ -423,6 +426,20 @@ def resolve_pod_engine(mcfg: "RunpodModelConfig", model_key: str = "",
     return "vllm"
 
 
+def direct_tcp_for(mcfg: "RunpodModelConfig", engine: str) -> bool:
+    """Whether this pod is reached at its public ip:port rather than the proxy.
+
+    An explicit setting wins. Otherwise a coderai image goes direct — it
+    serves TLS on that path, and the proxy's 100 s cutoff has already cost a
+    voice clone — while vLLM's, llama.cpp's and custom images stay behind the
+    proxy, which is the only TLS they have.
+    """
+    explicit = getattr(mcfg, "direct_tcp", None)
+    if explicit is not None:
+        return bool(explicit)
+    return (engine or "") == "coderai"
+
+
 def engine_pod_backend(entry: dict, mcfg: "RunpodModelConfig" = None) -> str:
     """The native engine a pod must run for this model, or ''.
 
@@ -1231,7 +1248,8 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
         cfg.cloud_types = ["SECURE"]
     cfg.volume_mount_path = (b.get("volume_mount_path") or "").strip()
     cfg.volume_path = (b.get("volume_path") or "").strip()
-    cfg.direct_tcp = _as_bool(b.get("direct_tcp"), False)
+    _dt = b.get("direct_tcp")
+    cfg.direct_tcp = None if _dt in (None, "", "auto") else _as_bool(_dt, False)
     cfg.venv_on_volume = _as_bool(b.get("venv_on_volume"), cfg.venv_on_volume)
     cfg.venv_name = (b.get("venv_name") or "").strip()
     cfg.slim_image = (b.get("slim_image") or "").strip()
@@ -1788,7 +1806,8 @@ class RunpodPodPool:
         # token. Only a coderai image knows what to do with it — vLLM's and
         # llama.cpp's images stay plain HTTP on this path, and say so.
         self._tls = False
-        if getattr(self.mcfg, "direct_tcp", False):
+        self._direct = direct_tcp_for(self.mcfg, plan.get("engine"))
+        if self._direct:
             # coderai pods only: boot.sh and the venv-on-volume bootstrap both
             # read the env; a custom image is anyone's guess and vLLM's and
             # llama.cpp's images ignore it, which would leave us speaking
@@ -1831,7 +1850,7 @@ class RunpodPodPool:
                     registry_auth_id=(self.mcfg.registry_auth_id
                                       or getattr(self.account, "registry_auth_id", "") or ""),
                     entrypoint=plan.get("entrypoint"), start_cmd=plan.get("start_cmd"),
-                    direct_tcp=bool(getattr(self.mcfg, "direct_tcp", False)))
+                    direct_tcp=bool(getattr(self, "_direct", False)))
                 return pod_id, sel, i + 1
             except RunpodError as exc:
                 msg = str(exc).lower()
@@ -1942,7 +1961,7 @@ class RunpodPodPool:
             url = ""
             try:
                 url = client.wait_ready(pod_id, port, ready_timeout=boot_to,
-                                        direct_tcp=bool(getattr(self.mcfg, "direct_tcp", False)),
+                                        direct_tcp=bool(getattr(self, "_direct", False)),
                                         tls=bool(getattr(self, "_tls", False)))
                 t_port = time.time()
                 print(f"[runpod] pod {pod_id}: port open after "

docs/runpod.md

@@ -129,7 +129,7 @@ sync.) In `models.json` that is a `backend` pin plus a `runpod` block:
 | `served_model` | — | The HF repo id the pod actually serves. **Required for pods.** |
 | `image` | `vllm/vllm-openai:latest` | Container image. Must expose an OpenAI-compatible server. |
 | `port` | `8000` | Port the server listens on inside the container. |
-| `direct_tcp` | `false` | Talk to the pod at its public `ip:port` instead of through RunPod's HTTPS proxy. The proxy (`<pod>-<port>.proxy.runpod.net`) sits behind **Cloudflare**: TLS for free, but a 100 s idle limit per request — a slow *non-streaming* request (a long render, a cold 150 GB engine's first token) can be cut off with a 524 while the pod is still working — and a request-body cap. Direct TCP has neither limit. **coderai pods bring their own TLS on this path**: a certificate from this install's CA (`<config>/runpod/tls/ca.pem`, created on first use with `openssl`) is issued per pod and sent as environment; the pod serves HTTPS on it and this side verifies against that CA — so the token and the payloads stay encrypted. vLLM's and llama.cpp's images cannot take the certificate and stay plain HTTP on this path (the log says so). |
+| `direct_tcp` | *auto* | **Auto = direct for coderai images, proxy for vLLM / llama.cpp / custom.** Talk to the pod at its public `ip:port` instead of through RunPod's HTTPS proxy. The proxy (`<pod>-<port>.proxy.runpod.net`) sits behind **Cloudflare**: TLS for free, but a 100 s idle limit per request — a slow *non-streaming* request (a long render, a cold 150 GB engine's first token) can be cut off with a 524 while the pod is still working — and a request-body cap. Direct TCP has neither limit. **coderai pods bring their own TLS on this path**: a certificate from this install's CA (`<config>/runpod/tls/ca.pem`, created on first use with `openssl`) is issued per pod and sent as environment; the pod serves HTTPS on it and this side verifies against that CA — so the token and the payloads stay encrypted. vLLM's and llama.cpp's images cannot take the certificate and stay plain HTTP on this path (the log says so). |
 | `ctx` | — | Passed as vLLM's `--max-model-len`. |
 | `container_disk_gb` | `40` | Grown automatically to fit the weights unless a network volume holds them; an explicit larger value is respected. |
 | `weights_gb` | — | State the model's size and the estimate is skipped. The only honest answer for a URL download, an upload or a local file — the estimator cannot see those — and for an HF repo it beats a heuristic that errs high. Also accepted on the model entry itself. |
@@ -273,8 +273,8 @@ the RAM they need, not just the VRAM — `min_vram_gb` says nothing about RAM.
 ## 3. Request flow
 
 ```
-client ──▶ front proxy ──▶ primary engine ──▶ RunpodBackend ──▶ https://<pod>-8000.proxy.runpod.net/v1
-                                                            (or http://<public-ip>:<port>/v1 with direct_tcp)
+client ──▶ front proxy ──▶ primary engine ──▶ RunpodBackend ──▶ https://<public-ip>:<port>/v1   (coderai images: direct, pod TLS)
+                                                            or https://<pod>-8000.proxy.runpod.net/v1 (vLLM / llama.cpp images)
                                                             └─▶ https://api.runpod.ai/v2/<eid>/openai/v1
 ```
 

tests/test_burst_placement.py

@@ -1039,8 +1039,18 @@ def test_direct_tcp_reaches_the_pod_without_the_proxy(monkeypatch):
     from codai.api import runpod_client as rc
     from codai.api.runpod_worker import parse_model_runpod
 
-    assert parse_model_runpod({}).direct_tcp is False              # proxy by default
-    assert parse_model_runpod({"direct_tcp": True}).direct_tcp is True
+    from codai.api.runpod_worker import direct_tcp_for
+    # Unset = decide by image: our images go direct (they serve TLS there),
+    # upstream images stay behind the proxy (it is the only TLS they have).
+    assert parse_model_runpod({}).direct_tcp is None
+    assert direct_tcp_for(parse_model_runpod({}), "coderai") is True
+    assert direct_tcp_for(parse_model_runpod({}), "vllm") is False
+    assert direct_tcp_for(parse_model_runpod({}), "llamacpp") is False
+    assert direct_tcp_for(parse_model_runpod({}), "custom") is False
+    # An explicit setting wins either way.
+    assert direct_tcp_for(parse_model_runpod({"direct_tcp": True}), "vllm") is True
+    assert direct_tcp_for(parse_model_runpod({"direct_tcp": False}), "coderai") is False
+    assert parse_model_runpod({"direct_tcp": "auto"}).direct_tcp is None
 
     ports = [{"ip": "203.0.113.7", "isIpPublic": True, "privatePort": 8000,
               "publicPort": 41234, "type": "tcp"}]
@@ -1119,6 +1129,7 @@ def test_a_direct_tcp_pod_speaks_tls_the_renting_coderai_can_verify(tmp_path, mo
     # Provisioning issues a cert only for coderai pods on the direct path.
     from codai.api.runpod_worker import parse_model_runpod
     assert parse_model_runpod({"direct_tcp": True}).direct_tcp is True
+    assert parse_model_runpod({"direct_tcp": "off"}).direct_tcp is False
     from pathlib import Path
     boot = Path("packaging/runpod/boot.sh").read_text()
     assert "CODERAI_TLS_CERT" in boot and "--ssl-certfile" in boot and "unset CODERAI_TLS_KEY" in boot

tools/runpod_model_test.py

@@ -111,7 +111,7 @@ _OCR_ENGINES = ("paddle", "doctr", "surya")
 
 def _pin_to_runpod(model: str, engine: str, vram: float, usd: float,
                    image: str = "", add_as: str = "",
-                   capability: str = "", direct_tcp: bool = False) -> bool:
+                   capability: str = "", direct_tcp=None) -> bool:
     """Give this ONE model a runpod block. True when the catalogue changed."""
     data = _load_models()
     section, index, entry = _find(data, model)
@@ -147,11 +147,9 @@ def _pin_to_runpod(model: str, engine: str, vram: float, usd: float,
         # that had already been published under that very tag. Production still
         # runs `:latest`; this override exists so a result means something.
         entry["runpod"]["image"] = image
-    if direct_tcp:
-        # The pod's public ip:port instead of the Cloudflare-fronted proxy —
-        # with TLS the pod brings itself on a coderai image. Exercised here
-        # because nothing else does until a request runs past 100 s.
-        entry["runpod"]["direct_tcp"] = True
+    if direct_tcp is not None:
+        # Force the path; None leaves the default (direct for coderai images).
+        entry["runpod"]["direct_tcp"] = bool(direct_tcp)
     data[section][index] = entry
     _save_models(data)
     os.utime(MODELS, None)           # the front watches this mtime to push a reload
@@ -222,8 +220,10 @@ def main() -> int:
                          "cached ':latest' and the result would not say which "
                          "build it exercised.")
     ap.add_argument("--direct-tcp", action="store_true",
-                    help="reach the pod at its public ip:port (TLS on coderai "
-                         "images) instead of RunPod's Cloudflare-fronted proxy")
+                    help="force the pod's public ip:port (the default for coderai "
+                         "images already); --proxy forces RunPod's proxy instead")
+    ap.add_argument("--proxy", action="store_true",
+                    help="force RunPod's Cloudflare-fronted proxy")
     args = ap.parse_args()
 
     backup = MODELS + ".testrun-backup"
@@ -247,7 +247,7 @@ def main() -> int:
             print(f"================ {model} ================", flush=True)
             if not _pin_to_runpod(model, args.engine, args.vram, args.usd,
                                   args.image, args.add_as, args.capability,
-                                  direct_tcp=args.direct_tcp):
+                                  direct_tcp=(True if args.direct_tcp else (False if args.proxy else None))):
                 results.append((model, {"error": "not in models.json"}))
                 continue
             problem = _wait_for_engine(model)