ocr: the gateway has to trust a pod's certificate like everything else (v0.3.16)

First real document, first real pod: it came up at https://194.68.245.79:22198
and every page came back

  502 — OCR upstream pod failed: [SSL: CERTIFICATE_VERIFY_FAILED]
        unable to get local issuer certificate

A direct-TCP pod serves the certificate coderai's OWN CA signed, on a bare IP
that certificate can never name. codai.api.pod_http has always known this: it
pins to pod_tls.ensure_ca() and switches hostname checking off. The fan-out
gateway is async and could not reuse that requests-based module, and I gave it
a bare httpx client — so the one hop that now carries every OCR page was the
one hop with no pod trust.

It keeps TWO clients per event loop, chosen by pod_http.is_pinned_url: the
pinned context trusts only our CA, so using it for everything would reject a
cluster node whose certificate a public CA signed, and sharing one client would
let whichever upstream connected first decide the policy for both.

Worth recording: this was never fan-out-specific. With serve_fanout off, surya
attaches straight to that same https:// IP with its own httpx and fails
identically — `model` mode has never worked against a TLS pod. The gateway is
what makes it fixable in one place.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.15"
+__version__ = "0.3.16"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/ocr/gateway.py

@@ -70,7 +70,12 @@ _KEEPALIVE_EXPIRY_S = 60.0
 
 _lock = threading.Lock()
 _leases: dict = {}              # lease token -> {"model_id", "served_name"}
-_clients: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()  # loop -> client
+#: loop -> {pinned?: client}. TWO clients per loop, because the upstreams need
+#: different trust: a direct-TCP pod serves OUR certificate on a bare IP, which
+#: only coderai's CA can verify and whose hostname can never match, while a
+#: cluster node has an ordinary certificate a public CA signed. One client cannot
+#: hold both policies, and using the pinned one everywhere would reject the node.
+_clients: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()
 
 
 def issue(model_id: str, served_name: str) -> str:
@@ -125,33 +130,62 @@ def engine_url(token: str) -> str:
 
 
 # --------------------------------------------------------------------- routes #
-def _upstream_client():
-    """The shared client for this event loop, created on first use."""
+def _pinned_ssl_context():
+    """The TLS policy a direct-TCP pod needs: our CA, no hostname match.
+
+    The same policy :mod:`codai.api.pod_http` applies for every other request to
+    a pod — it is only separate here because that module is built on requests and
+    this hop is async. Getting it wrong is not subtle: the pod answered
+    "CERTIFICATE_VERIFY_FAILED: unable to get local issuer certificate" to every
+    page the first time a real document went through.
+    """
+    import ssl
+
+    from codai.api.pod_tls import ensure_ca
+    ctx = ssl.create_default_context(cafile=ensure_ca())
+    # The pod is addressed by an IP its certificate cannot name.
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_REQUIRED
+    return ctx
+
+
+def _upstream_client(url: str = ""):
+    """The shared client for this event loop and this upstream's trust policy."""
     import asyncio
 
     import httpx
 
+    try:
+        from codai.api.pod_http import is_pinned_url
+        pinned = bool(is_pinned_url(url))
+    except Exception:
+        pinned = False
     try:
         key = asyncio.get_running_loop()
     except RuntimeError:
         key = _upstream_client            # a stable stand-in when there is no loop
     with _lock:
-        got = _clients.get(key)
+        bucket = _clients.get(key)
+        if bucket is None:
+            bucket = {}
+            _clients[key] = bucket
+        got = bucket.get(pinned)
         if got is not None and not got.is_closed:
             return got
+        verify = _pinned_ssl_context() if pinned else True
         got = httpx.AsyncClient(
-            timeout=UPSTREAM_TIMEOUT_S,
+            timeout=UPSTREAM_TIMEOUT_S, verify=verify,
             limits=httpx.Limits(max_connections=_MAX_CONNECTIONS,
                                 max_keepalive_connections=_MAX_KEEPALIVE,
                                 keepalive_expiry=_KEEPALIVE_EXPIRY_S))
-        _clients[key] = got
+        bucket[pinned] = got
         return got
 
 
 async def close_upstream_clients() -> None:
     """Close the pooled clients (process shutdown, or a test tidying up)."""
     with _lock:
-        clients = list(_clients.values())
+        clients = [c for bucket in _clients.values() for c in bucket.values()]
         _clients.clear()
     for c in clients:
         try:
@@ -239,7 +273,7 @@ async def chat_completions(token: str, request: Request):
         if chosen.api_key:
             headers["Authorization"] = f"Bearer {chosen.api_key}"
         url = chosen.base.rstrip("/") + "/chat/completions"
-        reply = await _upstream_client().post(url, content=body, headers=headers)
+        reply = await _upstream_client(url).post(url, content=body, headers=headers)
         # The reply goes back as it came: a re-serialised response is how
         # logprobs got lost in the first place.
         media = reply.headers.get("content-type", "application/json")

tests/test_ocr_fanout.py

@@ -1121,3 +1121,102 @@ def test_local_is_no_longer_what_an_unconfigured_install_gets():
     from codai.config import OcrConfig
     assert OcrConfig().surya_serve == "model"
     assert _surya(serve="")._serve_mode() == "model"
+
+
+# ------------------------------------------- a pod serves OUR certificate, on an IP #
+# The first real document found this: the pod came up at https://194.68.245.79:22198
+# and every page came back "CERTIFICATE_VERIFY_FAILED: unable to get local issuer
+# certificate". A direct-TCP pod serves the certificate coderai's own CA signed,
+# on a bare IP that certificate can never name — which is why pod_http pins to
+# that CA and switches hostname checking off. This hop is async, so it could not
+# reuse that module and had to be told the same thing.
+
+
+def test_a_pod_url_gets_the_pinned_trust_policy(monkeypatch):
+    _reset()
+    import asyncio
+    import httpx
+    built = []
+
+    class _Counting(_CapturingClient):
+        def __init__(self, *a, **kw):
+            built.append(kw)
+            super().__init__(*a, **kw)
+
+        @property
+        def is_closed(self):
+            return False
+
+    monkeypatch.setattr(httpx, "AsyncClient", _Counting)
+    monkeypatch.setattr(gateway, "_pinned_ssl_context", lambda: "PINNED-CTX")
+    _no_node_load(monkeypatch)
+    _fixed_candidates(monkeypatch, [_cand("pod:m", kind="pod",
+                                          base="https://194.68.245.79:22198/v1")])
+    token = gateway.issue("m", "m")
+    asyncio.run(gateway.chat_completions(token, _Req(SURYA_BODY)))
+    assert built and built[0]["verify"] == "PINNED-CTX"
+
+
+def test_an_ordinary_https_upstream_keeps_normal_verification(monkeypatch):
+    """A cluster node has a certificate a public CA signed; the pinned context
+    trusts ONLY our CA, so using it everywhere would reject the node."""
+    _reset()
+    import asyncio
+    import httpx
+    built = []
+
+    class _Counting(_CapturingClient):
+        def __init__(self, *a, **kw):
+            built.append(kw)
+            super().__init__(*a, **kw)
+
+        @property
+        def is_closed(self):
+            return False
+
+    monkeypatch.setattr(httpx, "AsyncClient", _Counting)
+    _no_node_load(monkeypatch)
+    _fixed_candidates(monkeypatch, [_cand("node:a", base="https://node.lan:8443/v1")])
+    token = gateway.issue("m", "m")
+    asyncio.run(gateway.chat_completions(token, _Req(SURYA_BODY)))
+    assert built and built[0]["verify"] is True
+
+
+def test_the_two_policies_do_not_share_a_client(monkeypatch):
+    """Otherwise whichever upstream came first would decide the trust for both."""
+    _reset()
+    import asyncio
+    import httpx
+    built = []
+
+    class _Counting(_CapturingClient):
+        def __init__(self, *a, **kw):
+            built.append(kw)
+            super().__init__(*a, **kw)
+
+        @property
+        def is_closed(self):
+            return False
+
+    monkeypatch.setattr(httpx, "AsyncClient", _Counting)
+    monkeypatch.setattr(gateway, "_pinned_ssl_context", lambda: "PINNED-CTX")
+
+    async def _both():
+        _fixed_candidates(monkeypatch, [_cand("node:a", base="https://node.lan/v1")])
+        await gateway.chat_completions(tok, _Req(SURYA_BODY))
+        _fixed_candidates(monkeypatch, [_cand("pod:m", kind="pod",
+                                              base="https://10.0.0.5:8000/v1")])
+        await gateway.chat_completions(tok, _Req(SURYA_BODY))
+
+    _no_node_load(monkeypatch)
+    tok = gateway.issue("m", "m")
+    asyncio.run(_both())
+    assert [b["verify"] for b in built] == [True, "PINNED-CTX"]
+
+
+def test_a_plain_http_upstream_is_not_pinned(monkeypatch):
+    """A local vLLM on loopback has no certificate at all."""
+    from codai.api.pod_http import is_pinned_url
+    assert is_pinned_url("http://127.0.0.1:8000/v1") is False
+    assert is_pinned_url("https://194.68.245.79:22198/v1") is True
+    assert is_pinned_url("https://node.lan:8443/v1") is False