runpod: TLS on direct-TCP pods — both no-cutoff and encrypted

direct_tcp traded the proxy's 100 s cutoff for plain HTTP. Now a coderai
pod on that path brings its own TLS: a per-install CA (openssl, kept
beside the pod registry) signs a short-lived leaf per pod, sent as env
like the token; boot.sh and the venv-on-volume bootstrap start uvicorn
on it. This side verifies https-to-a-bare-IP against that CA only —
pod_http, a requests shim every pod-facing call now goes through. No
hostname check: the IP is unknown at issue time, and a foreign cert
fails the chain before a name is compared. Proven end to end in a test
with a real https server. vLLM/llama.cpp images stay plain on this path.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/admin/templates/models.html

@@ -841,7 +841,7 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
           </div>
           <div class="form-row" style="margin:.5rem 0 0">
             <label style="display:flex;align-items:center;gap:.3rem;font-size:12px"><input type="checkbox" id="cfg-rp-direct-tcp" style="width:auto"> Direct TCP to the pod (bypass RunPod's proxy / Cloudflare)</label>
-            <span class="form-hint" style="font-size:11px">Default is RunPod's HTTPS proxy, which sits behind Cloudflare: TLS, but a 100 s idle limit per request and a body-size cap — a slow non-streaming render can be cut off with a 524. Direct TCP talks to the pod's public ip:port with neither limit, over plain HTTP (the bearer token is not encrypted in transit).</span>
+            <span class="form-hint" style="font-size:11px">Default is RunPod's HTTPS proxy, which sits behind Cloudflare: TLS, but a 100 s idle limit per request and a body-size cap — a slow non-streaming render can be cut off with a 524. Direct TCP talks to the pod's public ip:port with neither limit. coderai pods serve HTTPS on it with a certificate from this install's own CA, so nothing travels in clear; vLLM / llama.cpp images cannot take a certificate and stay plain HTTP on this path.</span>
           </div>
           <div class="form-row" style="margin:.5rem 0 0">
             <label class="form-label">Share pod pool <span class="muted">(optional name)</span></label>

codai/api/host_worker.py

@@ -39,6 +39,7 @@ The pool wears the same acquire()/release() shape as RunpodPodPool so the
 gateway and the test run treat both identically.
 """
 
+from codai.api import pod_http
 import shlex
 import subprocess
 import threading
@@ -84,7 +85,7 @@ def _health_ok(url: str, path: str, api_key: str, timeout: float = 5.0) -> bool:
     import requests
     headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
     try:
-        r = requests.get(url.rstrip("/") + path, headers=headers, timeout=timeout)
+        r = pod_http.get(url.rstrip("/") + path, headers=headers, timeout=timeout)
         return r.status_code == 200
     except Exception:
         return False

codai/api/model_test.py

@@ -31,6 +31,7 @@ transcription and for voice cloning, PIL draws known words for OCR, and video is
 asked for the smallest clip it can make.
 """
 
+from codai.api import pod_http
 import io
 import json
 import os
@@ -548,7 +549,7 @@ def _remote_catalogue(url: str, api_key: str) -> list:
     import requests
     headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
     try:
-        r = requests.get(url.rstrip("/") + "/v1/models", headers=headers, timeout=15)
+        r = pod_http.get(url.rstrip("/") + "/v1/models", headers=headers, timeout=15)
         if r.status_code != 200:
             return [f"(HTTP {r.status_code})"]
         return [str(m.get("id")) for m in (r.json().get("data") or [])] or ["(none)"]

codai/api/pod_http.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+"""`requests`, but able to reach a pod over TLS it brought itself.
+
+A pod on the direct TCP path serves HTTPS with a certificate signed by this
+install's own CA (codai/api/pod_tls.py). No public CA knows it, and the pod
+is addressed by a public IP that was not known when the certificate was
+issued — so the default `requests` verification fails twice over. This
+module routes exactly one kind of URL differently: **https to a bare IP
+address** is verified against our CA with the hostname check off. Every
+other URL — the RunPod proxy, HuggingFace, a host you named — goes through
+`requests` untouched.
+
+Why that rule is safe: a certificate our CA did not sign fails the chain
+check before any name is compared, and only pods we provisioned hold one.
+Why no hostname check: the name would have to be the IP, which a pod does
+not have until it boots; checking a placeholder proves nothing extra.
+
+Drop-in: `pod_http.get/post/request(...)` take what `requests.*` take.
+"""
+
+import ipaddress
+import ssl
+import threading
+from urllib.parse import urlsplit
+
+import requests
+from requests.adapters import HTTPAdapter
+
+_lock = threading.Lock()
+_session = None
+
+
+class _PinnedAdapter(HTTPAdapter):
+    """An adapter whose pool trusts our CA and skips hostname matching."""
+
+    def __init__(self, ca_file: str, **kw):
+        self._ca_file = ca_file
+        super().__init__(**kw)
+
+    def init_poolmanager(self, *args, **kwargs):
+        ctx = ssl.create_default_context(cafile=self._ca_file)
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_REQUIRED
+        kwargs["ssl_context"] = ctx
+        return super().init_poolmanager(*args, **kwargs)
+
+    def cert_verify(self, conn, url, verify, cert):
+        # urllib3 would otherwise reset the context's CA to requests' bundle
+        # or, with verify=False, turn verification off; the context above is
+        # the whole point, so leave it alone.
+        conn.cert_reqs = "CERT_REQUIRED"
+        conn.ca_certs = self._ca_file
+        # urllib3 matches the hostname itself, independently of the context's
+        # check_hostname; the pod is addressed by an IP its cert cannot name.
+        conn.assert_hostname = False
+
+
+def is_pinned_url(url: str) -> bool:
+    """https:// to a bare IP address — a direct-TCP pod with our certificate."""
+    try:
+        parts = urlsplit(url or "")
+        if parts.scheme != "https" or not parts.hostname:
+            return False
+        ipaddress.ip_address(parts.hostname)
+        return True
+    except ValueError:
+        return False
+
+
+def _pinned_session() -> requests.Session:
+    global _session
+    with _lock:
+        if _session is None:
+            from codai.api.pod_tls import ensure_ca
+            s = requests.Session()
+            s.mount("https://", _PinnedAdapter(ensure_ca()))
+            _session = s
+        return _session
+
+
+def request(method: str, url: str, **kw):
+    if is_pinned_url(url):
+        kw.pop("verify", None)
+        return _pinned_session().request(method, url, **kw)
+    return requests.request(method, url, **kw)
+
+
+def get(url: str, **kw):
+    return request("GET", url, **kw)
+
+
+def post(url: str, **kw):
+    return request("POST", url, **kw)
+
+
+def head(url: str, **kw):
+    return request("HEAD", url, **kw)

codai/api/pod_tls.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+"""TLS for pods reached directly, so `direct_tcp` is not a choice between
+"no 100 s cutoff" and "the token travels in clear".
+
+RunPod's HTTP proxy terminates TLS itself and expects plain HTTP on the
+container port, so TLS on the pod only makes sense on the direct TCP path.
+There, nothing sits between this machine and the pod's public ip:port — so
+the pod has to bring its own certificate, and this side has to know which
+certificate to trust before the pod exists (the same problem as the bearer
+token, solved the same way: decided here, sent as pod environment).
+
+One CA per install, made once with the local openssl and kept beside the pod
+registry. Each pod gets a leaf certificate signed by it at provision time,
+sent as CODERAI_TLS_CERT / CODERAI_TLS_KEY; boot.sh writes them out and starts
+uvicorn on them. This side verifies pod connections against the CA only —
+no hostname check, because the pod's public IP is not known until after it
+boots, and it adds nothing anyway: a certificate this CA did not sign fails
+before a name is ever compared. See codai/api/pod_http.py for the client.
+"""
+
+import os
+import subprocess
+import tempfile
+import threading
+
+_lock = threading.Lock()
+
+#: Environment variables the pod reads (boot.sh). PEM text, not paths.
+ENV_CERT = "CODERAI_TLS_CERT"
+ENV_KEY = "CODERAI_TLS_KEY"
+
+
+def tls_dir() -> str:
+    """Where the CA lives: the config dir, beside runpod-pods.json."""
+    try:
+        from codai.api.runpod_worker import _pod_registry_path
+        base = os.path.dirname(_pod_registry_path())
+    except Exception:
+        base = os.path.expanduser("~/.coderai")
+    return os.path.join(base, "runpod", "tls")
+
+
+def ca_path() -> str:
+    return os.path.join(tls_dir(), "ca.pem")
+
+
+def _run(args, cwd=None) -> None:
+    r = subprocess.run(["openssl", *args], cwd=cwd, capture_output=True, text=True,
+                       timeout=60)
+    if r.returncode != 0:
+        raise RuntimeError(f"openssl {args[0]} failed: {(r.stderr or r.stdout).strip()[:300]}")
+
+
+def ensure_ca() -> str:
+    """Create the per-install CA if missing; return the path of ca.pem."""
+    d = tls_dir()
+    ca, key = os.path.join(d, "ca.pem"), os.path.join(d, "ca.key")
+    with _lock:
+        if os.path.isfile(ca) and os.path.isfile(key):
+            return ca
+        os.makedirs(d, exist_ok=True)
+        os.chmod(d, 0o700)
+        _run(["req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1",
+              "-nodes", "-days", "3650", "-subj", "/CN=coderai pod CA",
+              "-addext", "basicConstraints=critical,CA:TRUE",
+              "-addext", "keyUsage=critical,keyCertSign,cRLSign",
+              "-keyout", key, "-out", ca])
+        os.chmod(key, 0o600)
+        print(f"[runpod] created the pod TLS CA at {ca}", flush=True)
+        return ca
+
+
+def issue_pod_cert(pod_name: str, days: int = 30) -> dict:
+    """A leaf certificate for one pod, signed by the CA: {cert, key} as PEM.
+
+    Short-lived by design — a pod lives hours, and a leaked key from a pod
+    that has since been reaped should not stay useful. The SAN is a fixed
+    name, never checked (see the module docstring); it exists because some
+    TLS stacks refuse a certificate with no SAN at all.
+    """
+    ca = ensure_ca()
+    ca_key = os.path.join(tls_dir(), "ca.key")
+    name = "".join(ch if ch.isalnum() or ch in "-._" else "-" for ch in pod_name)[:60] or "pod"
+    with tempfile.TemporaryDirectory(prefix="coderai-pod-tls-") as tmp:
+        key, csr, crt = (os.path.join(tmp, n) for n in ("pod.key", "pod.csr", "pod.pem"))
+        ext = os.path.join(tmp, "ext.cnf")
+        with open(ext, "w") as f:
+            f.write("subjectAltName=DNS:coderai-pod\n"
+                    "extendedKeyUsage=serverAuth\n"
+                    "basicConstraints=CA:FALSE\n")
+        _run(["req", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1",
+              "-nodes", "-subj", f"/CN={name}", "-keyout", key, "-out", csr])
+        _run(["x509", "-req", "-in", csr, "-CA", ca, "-CAkey", ca_key, "-CAcreateserial",
+              "-days", str(int(days)), "-extfile", ext, "-out", crt])
+        with open(crt) as f:
+            cert_pem = f.read()
+        with open(key) as f:
+            key_pem = f.read()
+    return {"cert": cert_pem, "key": key_pem}
+
+
+def pod_env(pod_name: str) -> dict:
+    """The environment that turns TLS on in a coderai pod."""
+    leaf = issue_pod_cert(pod_name)
+    return {ENV_CERT: leaf["cert"], ENV_KEY: leaf["key"]}

codai/api/remote_gateway.py

@@ -37,6 +37,7 @@ Cost: when nothing is configured as remote the middleware returns on its first
 branch and the body is never buffered.
 """
 
+from codai.api import pod_http
 import json
 import os
 import re
@@ -610,7 +611,7 @@ class RemoteGatewayMiddleware:
 
         def _call():
             import requests
-            return requests.request(scope.get("method", "POST"), dest, data=body or None,
+            return pod_http.request(scope.get("method", "POST"), dest, data=body or None,
                                     headers=fwd, stream=True, timeout=(30, 3600))
         try:
             resp = await asyncio.to_thread(_call)
@@ -735,10 +736,10 @@ def sync_loras(base: str, api_key: str, body: bytes) -> bytes:
             with open(local, "rb") as fh:
                 data = fh.read()
             digest = hashlib.sha256(data).hexdigest()
-            have = requests.get(f"{base}/v1/loras/blob/{digest}",
+            have = pod_http.get(f"{base}/v1/loras/blob/{digest}",
                                 headers=headers, timeout=30)
             if have.status_code != 200:
-                up = requests.post(f"{base}/v1/loras/upload", data=data,
+                up = pod_http.post(f"{base}/v1/loras/upload", data=data,
                                    headers={**headers,
                                             "Content-Type": "application/octet-stream"},
                                    timeout=1800)
@@ -780,9 +781,9 @@ def push_lora_file(base: str, api_key: str, path: str) -> str:
         with open(path, "rb") as fh:
             data = fh.read()
         digest = hashlib.sha256(data).hexdigest()
-        have = requests.get(f"{base}/v1/loras/blob/{digest}", headers=headers, timeout=30)
+        have = pod_http.get(f"{base}/v1/loras/blob/{digest}", headers=headers, timeout=30)
         if have.status_code != 200:
-            up = requests.post(f"{base}/v1/loras/upload", data=data,
+            up = pod_http.post(f"{base}/v1/loras/upload", data=data,
                                headers={**headers,
                                         "Content-Type": "application/octet-stream"},
                                timeout=1800)
@@ -899,7 +900,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool:
     if api_key:
         headers["Authorization"] = f"Bearer {api_key}"
     try:
-        r = requests.post(f"{base}/v1/models/register", data=payload,
+        r = pod_http.post(f"{base}/v1/models/register", data=payload,
                           headers=headers, timeout=60)
         if not r.ok:
             print(f"[remote-gateway] {base} refused {model!r}: "
@@ -935,7 +936,7 @@ def ensure_model_uploaded(base: str, api_key: str, entry: dict) -> None:
             sum(os.path.getsize(os.path.join(r, f))
                 for r, _, fs in os.walk(path) for f in fs))
     try:
-        have = requests.get(f"{base}/v1/models/uploaded",
+        have = pod_http.get(f"{base}/v1/models/uploaded",
                             params={"name": name, "bytes": 0 if is_dir else size},
                             headers=headers, timeout=30)
         if have.status_code == 200:
@@ -957,13 +958,13 @@ def ensure_model_uploaded(base: str, api_key: str, entry: dict) -> None:
             proc = subprocess.Popen(
                 ["tar", "-cf", "-", "-C", os.path.dirname(path), name],
                 stdout=subprocess.PIPE)
-            resp = requests.post(f"{base}/v1/models/upload", params=params,
+            resp = pod_http.post(f"{base}/v1/models/upload", params=params,
                                  data=proc.stdout, headers=headers, timeout=None)
             proc.stdout.close()
             proc.wait()
         else:
             with open(path, "rb") as fh:
-                resp = requests.post(f"{base}/v1/models/upload", params=params,
+                resp = pod_http.post(f"{base}/v1/models/upload", params=params,
                                      data=fh, headers=headers, timeout=None)
         resp.raise_for_status()
         _UPLOADED.add((base, name))

codai/api/runpod_client.py

@@ -38,7 +38,7 @@ def pod_proxy_url(pod_id: str, port: int) -> str:
     return f"https://{pod_id}-{int(port)}.proxy.runpod.net"
 
 
-def direct_tcp_url(ports: list, port: int) -> str:
+def direct_tcp_url(ports: list, port: int, tls: bool = False) -> str:
     """The pod's public ip:port for ``port`` exposed as `<port>/tcp`, or ''.
 
     A TCP port is mapped straight to the machine — no RunPod proxy, no
@@ -58,7 +58,7 @@ def direct_tcp_url(ports: list, port: int) -> str:
             continue
         ip, pub = p.get("ip"), p.get("publicPort")
         if ip and pub:
-            return f"http://{ip}:{int(pub)}"
+            return f"{'https' if tls else 'http'}://{ip}:{int(pub)}"
     return ""
 
 
@@ -454,7 +454,8 @@ class RunpodClient:
         return out
 
     def wait_ready(self, pod_id: str, port: int, ready_timeout: float = 900.0,
-                   poll_every: float = 5.0, direct_tcp: bool = False) -> str:
+                   poll_every: float = 5.0, direct_tcp: bool = False,
+                   tls: bool = False) -> str:
         """Poll until the pod exposes its port; return the base URL to talk to.
 
         The proxy URL is derivable the moment any port shows; a direct TCP
@@ -488,7 +489,7 @@ class RunpodClient:
             if info.get("ready"):
                 if not direct_tcp:
                     return pod_proxy_url(pod_id, port)
-                direct = direct_tcp_url(info.get("ports"), port)
+                direct = direct_tcp_url(info.get("ports"), port, tls=tls)
                 if direct:
                     return direct
             # A multi-GB image pull is minutes of silence otherwise, which reads

codai/api/runpod_worker.py

@@ -24,6 +24,7 @@ The account settings (API key, endpoints, global caps) live in
 :class:`codai.config.RunpodConfig`; this module reads the per-model block.
 """
 
+from codai.api import pod_http
 import os
 from collections import OrderedDict as _OrderedDict
 from dataclasses import dataclass, field
@@ -727,8 +728,15 @@ def venv_bootstrap_script(mount: str, profile: str, venv_name: str = "",
         'done',
         '[ -f "$MARK" ] || { echo "[venv] $VENV never became ready"; exit 1; }',
         'echo "[venv] using $VENV"',
+        # TLS from env, as boot.sh does it (this script replaces boot.sh).
+        'TLS=""; if [ -n "$CODERAI_TLS_CERT" ] && [ -n "$CODERAI_TLS_KEY" ]; then '
+        'mkdir -p /run/coderai-tls && chmod 700 /run/coderai-tls; '
+        'printf "%s\\n" "$CODERAI_TLS_CERT" > /run/coderai-tls/cert.pem; '
+        'printf "%s\\n" "$CODERAI_TLS_KEY" > /run/coderai-tls/key.pem; '
+        'chmod 600 /run/coderai-tls/key.pem; unset CODERAI_TLS_KEY; '
+        'TLS="--ssl-certfile /run/coderai-tls/cert.pem --ssl-keyfile /run/coderai-tls/key.pem"; fi',
         f'exec "$VENV/bin/python" -m uvicorn codai.api.app:app --host 0.0.0.0 '
-        f'--port {int(port)}',
+        f'--port {int(port)} $TLS',
     ])
 
 
@@ -1308,7 +1316,7 @@ def _pod_health_ok(url: str, timeout: float = 4.0, path: str = "/v1/models",
     import requests
     headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
     try:
-        r = requests.get(url.rstrip("/") + "/" + path.lstrip("/"), timeout=timeout,
+        r = pod_http.get(url.rstrip("/") + "/" + path.lstrip("/"), timeout=timeout,
                          headers=headers)
         return r.status_code == 200
     except Exception:
@@ -1683,7 +1691,7 @@ class RunpodPodPool:
         import requests
         headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
         try:
-            r = requests.get(url.rstrip("/") + "/v1/models", headers=headers, timeout=20)
+            r = pod_http.get(url.rstrip("/") + "/v1/models", headers=headers, timeout=20)
             if r.status_code != 200:
                 print(f"[runpod] pod catalogue unreadable: HTTP {r.status_code} "
                       f"{(r.text or '')[:120]}", flush=True)
@@ -1747,6 +1755,24 @@ class RunpodPodPool:
             env.update(volume_env(vol_mount))
             if getattr(self, "_volume_dc", None) is None:
                 self._volume_dc = self._data_center_for_volume(vol_id)
+        # Direct TCP puts nothing between here and the pod, so the pod brings
+        # its own TLS: a certificate from this install's CA, sent like the
+        # token. Only a coderai image knows what to do with it — vLLM's and
+        # llama.cpp's images stay plain HTTP on this path, and say so.
+        self._tls = False
+        if getattr(self.mcfg, "direct_tcp", False):
+            # coderai pods only: boot.sh and the venv-on-volume bootstrap both
+            # read the env; a custom image is anyone's guess and vLLM's and
+            # llama.cpp's images ignore it, which would leave us speaking
+            # https to a plain-HTTP server.
+            if plan.get("engine") == "coderai":
+                from codai.api.pod_tls import pod_env as _tls_env
+                env.update(_tls_env(f"{_deploy_tag(self.account)}-{self.model_key}"))
+                self._tls = True
+            else:
+                print(f"[runpod] direct_tcp on a {plan.get('engine')} image: plain "
+                      f"HTTP — that image cannot take a certificate; the bearer "
+                      f"token travels unencrypted", flush=True)
         dc = getattr(self.account, "data_center", "") or ""
         last = None
         for i in range(start, len(ranked)):
@@ -1817,7 +1843,7 @@ class RunpodPodPool:
                 import requests
                 headers = ({"Authorization": f"Bearer {self.api_key}"}
                            if self.api_key else {})
-                r = requests.get(url.rstrip("/") + "/boot", headers=headers, timeout=15)
+                r = pod_http.get(url.rstrip("/") + "/boot", headers=headers, timeout=15)
                 if r.status_code == 200:
                     data = r.json()
                     lines = list(data.get("container") or []) + list(data.get("phases") or [])
@@ -1888,7 +1914,8 @@ class RunpodPodPool:
             url = ""
             try:
                 url = client.wait_ready(pod_id, port, ready_timeout=boot_to,
-                                        direct_tcp=bool(getattr(self.mcfg, "direct_tcp", False)))
+                                        direct_tcp=bool(getattr(self.mcfg, "direct_tcp", False)),
+                                        tls=bool(getattr(self, "_tls", False)))
                 t_port = time.time()
                 print(f"[runpod] pod {pod_id}: port open after "
                       f"{t_port - t_created:.0f}s (image pull + container start)",

codai/backends/remote_openai.py

@@ -27,6 +27,7 @@ Model entry fields:
   api_key         bearer token for the remote, when it wants one.
 """
 
+from codai.api import pod_http
 import os
 from typing import Dict, Optional
 
@@ -110,7 +111,7 @@ class RemoteOpenAIBackend(RunpodBackend):
     def health(self) -> Optional[dict]:
         import requests
         try:
-            r = requests.get(self._base() + "/models", headers=self._headers, timeout=10)
+            r = pod_http.get(self._base() + "/models", headers=self._headers, timeout=10)
             return r.json() if r.ok else None
         except Exception:
             return None

codai/backends/runpod.py

@@ -20,6 +20,7 @@ Phase 2: the serverless path is live. Pods mode raises a clear error until the
 pod pool lands in the next phase.
 """
 
+from codai.api import pod_http
 import asyncio
 import threading
 from typing import AsyncGenerator, Dict, List, Optional
@@ -199,7 +200,7 @@ class RunpodBackend(ModelBackend):
                 payload["response_format"] = {"type": "json_object"}
             if tools:
                 payload["tools"] = tools
-            r = requests.post(base + "/chat/completions", json=payload,
+            r = pod_http.post(base + "/chat/completions", json=payload,
                               headers=self._headers, timeout=3600)
             r.raise_for_status()
             data = r.json()
@@ -275,7 +276,7 @@ class RunpodBackend(ModelBackend):
         def _worker():
             import requests
             try:
-                with requests.post(url, json=payload, headers=headers,
+                with pod_http.post(url, json=payload, headers=headers,
                                    stream=True, timeout=3600) as r:
                     r.raise_for_status()
                     buf = b""

docs/runpod.md

@@ -129,7 +129,7 @@ sync.) In `models.json` that is a `backend` pin plus a `runpod` block:
 | `served_model` | — | The HF repo id the pod actually serves. **Required for pods.** |
 | `image` | `vllm/vllm-openai:latest` | Container image. Must expose an OpenAI-compatible server. |
 | `port` | `8000` | Port the server listens on inside the container. |
-| `direct_tcp` | `false` | Talk to the pod at its public `ip:port` instead of through RunPod's HTTPS proxy. The proxy (`<pod>-<port>.proxy.runpod.net`) sits behind **Cloudflare**: TLS for free, but a 100 s idle limit per request — a slow *non-streaming* request (a long render, a cold 150 GB engine's first token) can be cut off with a 524 while the pod is still working — and a request-body cap. Direct TCP has neither limit, over plain HTTP: the bearer token is not encrypted in transit. |
+| `direct_tcp` | `false` | Talk to the pod at its public `ip:port` instead of through RunPod's HTTPS proxy. The proxy (`<pod>-<port>.proxy.runpod.net`) sits behind **Cloudflare**: TLS for free, but a 100 s idle limit per request — a slow *non-streaming* request (a long render, a cold 150 GB engine's first token) can be cut off with a 524 while the pod is still working — and a request-body cap. Direct TCP has neither limit. **coderai pods bring their own TLS on this path**: a certificate from this install's CA (`<config>/runpod/tls/ca.pem`, created on first use with `openssl`) is issued per pod and sent as environment; the pod serves HTTPS on it and this side verifies against that CA — so the token and the payloads stay encrypted. vLLM's and llama.cpp's images cannot take the certificate and stay plain HTTP on this path (the log says so). |
 | `ctx` | — | Passed as vLLM's `--max-model-len`. |
 | `container_disk_gb` | `40` | Grown automatically to fit the weights unless a network volume holds them; an explicit larger value is respected. |
 | `weights_gb` | — | State the model's size and the estimate is skipped. The only honest answer for a URL download, an upload or a local file — the estimator cannot see those — and for an HF repo it beats a heuristic that errs high. Also accepted on the model entry itself. |

packaging/runpod/Dockerfile.capability

@@ -107,7 +107,7 @@ EXPOSE 8000
 # /healthz is what a `coderai` pod is probed on, and it answers before any model
 # is loaded — which is correct: the pod is ready to be asked.
 HEALTHCHECK --interval=30s --timeout=5s --start-period=120s \
-    CMD curl -fsS http://127.0.0.1:8000/healthz || exit 1
+    CMD curl -fsS http://127.0.0.1:8000/healthz || curl -fsSk https://127.0.0.1:8000/healthz || exit 1
 
 COPY packaging/runpod/boot.sh /opt/coderai/boot.sh
 RUN chmod +x /opt/coderai/boot.sh

packaging/runpod/Dockerfile.capability-engines

@@ -115,7 +115,7 @@ RUN python -c "import codai.api.app; print('coderai imports OK for profile ${PRO
 
 EXPOSE 8000
 HEALTHCHECK --interval=30s --timeout=5s --start-period=120s \
-    CMD curl -fsS http://127.0.0.1:8000/healthz || exit 1
+    CMD curl -fsS http://127.0.0.1:8000/healthz || curl -fsSk https://127.0.0.1:8000/healthz || exit 1
 
 COPY packaging/runpod/boot.sh /opt/coderai/boot.sh
 RUN chmod +x /opt/coderai/boot.sh

packaging/runpod/Dockerfile.capability-slim

@@ -38,4 +38,4 @@ COPY . /opt/coderai/app
 # it — see venv_bootstrap_script() in codai/api/runpod_worker.py.
 EXPOSE 8000
 HEALTHCHECK --interval=30s --timeout=5s --start-period=600s \
-    CMD curl -fsS http://127.0.0.1:8000/healthz || exit 1
+    CMD curl -fsS http://127.0.0.1:8000/healthz || curl -fsSk https://127.0.0.1:8000/healthz || exit 1

packaging/runpod/boot.sh

@@ -53,6 +53,22 @@ else
     phase "gpu: nvidia-smi not present"
 fi
 
-phase "starting uvicorn on 0.0.0.0:8000"
+# TLS, when the renting coderai sent a certificate: it reaches this pod at a
+# public ip:port with nothing in between (direct_tcp), so the pod terminates
+# TLS itself. The cert is signed by that install's own CA and only it
+# verifies it. PEM in env → files, because uvicorn wants paths. Absent env
+# means the RunPod proxy is in front, which speaks plain HTTP to us.
+TLS_ARGS=""
+if [ -n "$CODERAI_TLS_CERT" ] && [ -n "$CODERAI_TLS_KEY" ]; then
+    mkdir -p /run/coderai-tls && chmod 700 /run/coderai-tls
+    printf '%s\n' "$CODERAI_TLS_CERT" > /run/coderai-tls/cert.pem
+    printf '%s\n' "$CODERAI_TLS_KEY" > /run/coderai-tls/key.pem
+    chmod 600 /run/coderai-tls/key.pem
+    unset CODERAI_TLS_KEY
+    TLS_ARGS="--ssl-certfile /run/coderai-tls/cert.pem --ssl-keyfile /run/coderai-tls/key.pem"
+    phase "tls: serving https with the certificate the renting coderai sent"
+fi
+
+phase "starting uvicorn on 0.0.0.0:8000${TLS_ARGS:+ (https)}"
 exec python -m uvicorn codai.api.app:app --host 0.0.0.0 --port 8000 \
-     --log-level "${CODERAI_LOG_LEVEL:-info}"
+     --log-level "${CODERAI_LOG_LEVEL:-info}" $TLS_ARGS

tests/test_burst_placement.py

@@ -1069,3 +1069,56 @@ def test_direct_tcp_reaches_the_pod_without_the_proxy(monkeypatch):
         def get_pod(self, pod_id): return next(infos)
     monkeypatch.setattr(rc.time, "sleep", lambda s: None)
     assert _W().wait_ready("p1", 8000, direct_tcp=True) == "http://203.0.113.7:41234"
+
+
+def test_a_direct_tcp_pod_speaks_tls_the_renting_coderai_can_verify(tmp_path, monkeypatch):
+    """direct_tcp removed the 100 s proxy cutoff at the price of plain HTTP.
+    Now the pod brings TLS: a leaf from this install's CA, sent as env, and
+    this side verifies against that CA alone — no hostname check, since the
+    pod's IP is unknown when the cert is issued and a foreign cert fails the
+    chain before any name is compared."""
+    import http.server, json, ssl, threading
+    from codai.api import pod_tls, pod_http, runpod_client as rc
+
+    monkeypatch.setattr(pod_tls, "tls_dir", lambda: str(tmp_path / "tls"))
+    env = pod_tls.pod_env("deploy-mymodel")
+    assert env["CODERAI_TLS_CERT"].startswith("-----BEGIN CERTIFICATE-----")
+    assert "PRIVATE KEY" in env["CODERAI_TLS_KEY"]
+    assert (tmp_path / "tls" / "ca.pem").exists()
+
+    # A "pod": an https server on 127.0.0.1 using exactly what boot.sh writes.
+    cert, key = tmp_path / "cert.pem", tmp_path / "key.pem"
+    cert.write_text(env["CODERAI_TLS_CERT"]); key.write_text(env["CODERAI_TLS_KEY"])
+    class _H(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            body = json.dumps({"ok": True, "auth": self.headers.get("Authorization")}).encode()
+            self.send_response(200); self.send_header("Content-Length", str(len(body)))
+            self.end_headers(); self.wfile.write(body)
+        def log_message(self, *a): pass
+    srv = http.server.HTTPServer(("127.0.0.1", 0), _H)
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER); ctx.load_cert_chain(str(cert), str(key))
+    srv.socket = ctx.wrap_socket(srv.socket, server_side=True)
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    url = f"https://127.0.0.1:{srv.server_address[1]}"
+
+    # The URL a direct-TCP TLS pod gets: https to its bare public IP.
+    ports = [{"ip": "127.0.0.1", "isIpPublic": True, "privatePort": 8000,
+              "publicPort": srv.server_address[1], "type": "tcp"}]
+    assert rc.direct_tcp_url(ports, 8000, tls=True) == url
+    assert pod_http.is_pinned_url(url) and not pod_http.is_pinned_url("https://x-8000.proxy.runpod.net")
+
+    monkeypatch.setattr(pod_http, "_session", None)
+    r = pod_http.get(url + "/healthz", headers={"Authorization": "Bearer t"}, timeout=5)
+    assert r.status_code == 200 and r.json()["auth"] == "Bearer t"
+    # Plain requests would refuse it — that is the whole reason pod_http exists.
+    import requests, pytest
+    with pytest.raises(requests.exceptions.SSLError):
+        requests.get(url + "/healthz", timeout=5)
+    srv.shutdown()
+
+    # Provisioning issues a cert only for coderai pods on the direct path.
+    from codai.api.runpod_worker import parse_model_runpod
+    assert parse_model_runpod({"direct_tcp": True}).direct_tcp is True
+    from pathlib import Path
+    boot = Path("packaging/runpod/boot.sh").read_text()
+    assert "CODERAI_TLS_CERT" in boot and "--ssl-certfile" in boot and "unset CODERAI_TLS_KEY" in boot