ten connections is a browser's number, not a fan-out's (v0.3.101)

One shared session serves every pod, and `requests`' stock adapter keeps ten
connections per host. `ocr.max_concurrency` defaults to 64 and is 128 on the
install this came from, so from the ELEVENTH concurrent page onward urllib3
opened a connection, used it, found the pool full on release and threw it
away:

    urllib3.connectionpool: Connection pool is full, discarding connection:
    194.68.245.54. Connection pool size: 10

74 of those in five minutes — about fourteen a minute against one pod — while
that run's health checks were timing out. Every discarded connection means the
next page pays a full TLS handshake to a machine in another country: several
round trips of latency each, and handshake CPU on the orchestrator at precisely
the moment it is busiest. It does not CAUSE a slow run, but it multiplies one,
and it is the part that is ours.

The pool is now sized from the concurrency the install is configured for:
`ocr.max_concurrency`, floored at 64 and capped at 256. Read at session
creation rather than at import, because the config is not loaded when this
module is imported — and because an operator raising max_concurrency should not
have to know a connection pool exists. An idle connection costs a socket and
nothing else, which is the cheap side of this trade.

The adapter also carries the floor as its own default, so a `_PinnedAdapter`
built anywhere cannot silently fall back to ten.

Routing is deliberately unchanged: only https-to-a-bare-IP uses the pinned
session, everything else still goes through `requests` untouched, so callers
and tests that patch `requests.get` keep working.

Not a fix for the slowness that exposed it. That host was running MariaDB at
534% CPU, four Postgres parallel workers, and a backup that had been streaming
over SFTP for three hours, against coderai's own 8.8% of one core out of
sixteen — the orchestrator was waiting for a CPU, not using one.

9 tests. Suite: 3151 passed, the same 13 known-stale failures.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KnfkxBpcZ1v6ypRFFyhrVM

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.100"
+__version__ = "0.3.101"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/pod_http.py

@@ -37,11 +37,48 @@ _lock = threading.Lock()
 _session = None
 
 
+#: Connections kept alive PER POD. requests' own default is 10, and one shared
+#: session serves every pod, so the pool is per host either way — but 10 is a
+#: number for a browser talking to a website, not for an OCR run fanning
+#: `ocr.max_concurrency` pages at ONE pod.
+#:
+#: Past the tenth concurrent request urllib3 opens a connection anyway, uses it,
+#: finds the pool full on release and discards it. Every page after the tenth
+#: therefore pays a fresh TLS handshake to a machine in another country: several
+#: round trips of latency each, and handshake CPU on the orchestrator at exactly
+#: the moment it is busiest. Measured on a live run: 74 "Connection pool is
+#: full, discarding connection" warnings in five minutes, ~14 a minute, against
+#: a single pod IP.
+#:
+#: Idle connections cost a socket and nothing else, so sizing this to the
+#: concurrency the install is configured for is the cheap side of the trade.
+_DEFAULT_POOL = 64
+_MAX_POOL = 256
+
+
+def _pool_size() -> int:
+    """How many connections to keep per pod — from the configured concurrency.
+
+    Read at session creation, not import: the config is not loaded yet when
+    this module is imported, and an install that raises `ocr.max_concurrency`
+    should not have to know that a connection pool exists.
+    """
+    try:
+        from codai.config import config_manager
+        cfg = config_manager.config.ocr if config_manager and config_manager.config else None
+        want = int(getattr(cfg, "max_concurrency", 0) or 0)
+    except Exception:
+        want = 0
+    return max(_DEFAULT_POOL, min(_MAX_POOL, want))
+
+
 class _PinnedAdapter(HTTPAdapter):
     """An adapter whose pool trusts our CA and skips hostname matching."""
 
     def __init__(self, ca_file: str, **kw):
         self._ca_file = ca_file
+        kw.setdefault("pool_connections", _DEFAULT_POOL)
+        kw.setdefault("pool_maxsize", _DEFAULT_POOL)
         super().__init__(**kw)
 
     def init_poolmanager(self, *args, **kwargs):
@@ -80,7 +117,9 @@ def _pinned_session() -> requests.Session:
         if _session is None:
             from codai.api.pod_tls import ensure_ca
             s = requests.Session()
-            s.mount("https://", _PinnedAdapter(ensure_ca()))
+            size = _pool_size()
+            s.mount("https://", _PinnedAdapter(ensure_ca(), pool_connections=size,
+                                               pool_maxsize=size))
             _session = s
         return _session
 

tests/test_pod_http_pool.py

+"""Ten connections is a browser's number, not a fan-out's.
+
+One shared session serves every pod, and `requests`' stock adapter keeps 10
+connections per host. `ocr.max_concurrency` defaults to 64 and was 128 on the
+install where this showed up, so from the eleventh concurrent page onward
+urllib3 opened a connection, used it, found the pool full on release and threw
+it away — a full TLS handshake per page to a pod in another country, and
+handshake CPU on the orchestrator exactly when it had none to spare.
+
+Measured live 2026-10-15: 74 "Connection pool is full, discarding connection:
+<pod ip>. Connection pool size: 10" in five minutes, ~14 a minute, while an OCR
+run's health checks were timing out.
+"""
+
+import types
+
+import pytest
+
+from codai.api import pod_http
+
+
+def _ca() -> str:
+    """A real PEM — the adapter builds its SSL context at construction, so a
+    placeholder path raises before the pool is ever sized."""
+    import certifi
+    return certifi.where()
+
+
+def test_the_adapter_no_longer_takes_requests_default_of_ten():
+    a = pod_http._PinnedAdapter(_ca())
+    assert a._pool_connections >= 64
+    assert a._pool_maxsize >= 64
+
+
+def test_an_explicit_size_wins_over_the_default():
+    a = pod_http._PinnedAdapter(_ca(), pool_connections=7, pool_maxsize=7)
+    assert a._pool_maxsize == 7
+
+
+# ------------------------------------------------------- sizing from config #
+def _cfg(monkeypatch, max_concurrency):
+    mgr = types.SimpleNamespace(
+        config=types.SimpleNamespace(
+            ocr=types.SimpleNamespace(max_concurrency=max_concurrency)))
+    import codai.config
+    monkeypatch.setattr(codai.config, "config_manager", mgr, raising=False)
+
+
+def test_the_pool_follows_the_configured_concurrency(monkeypatch):
+    """The install that hit this runs 128 pages at once."""
+    _cfg(monkeypatch, 128)
+    assert pod_http._pool_size() == 128
+
+
+def test_a_small_concurrency_still_gets_a_usable_floor(monkeypatch):
+    """Pod traffic is not only OCR — health probes, blob pushes, forwarded
+    chat all share this session."""
+    _cfg(monkeypatch, 4)
+    assert pod_http._pool_size() == pod_http._DEFAULT_POOL
+
+
+def test_an_absurd_concurrency_is_capped(monkeypatch):
+    _cfg(monkeypatch, 100000)
+    assert pod_http._pool_size() == pod_http._MAX_POOL
+
+
+def test_no_config_yet_is_not_an_error(monkeypatch):
+    """Read at session creation precisely because import happens first, but a
+    caller before the config loads must still get a session."""
+    import codai.config
+    monkeypatch.setattr(codai.config, "config_manager", None, raising=False)
+    assert pod_http._pool_size() == pod_http._DEFAULT_POOL
+
+
+def test_a_broken_config_object_falls_back(monkeypatch):
+    class _Boom:
+        @property
+        def config(self):
+            raise RuntimeError("not loaded")
+    import codai.config
+    monkeypatch.setattr(codai.config, "config_manager", _Boom(), raising=False)
+    assert pod_http._pool_size() == pod_http._DEFAULT_POOL
+
+
+# ------------------------------------------------------------ the session #
+def test_the_session_mounts_the_sized_adapter(monkeypatch, tmp_path):
+    monkeypatch.setattr(pod_http, "_session", None, raising=False)
+    import codai.api.pod_tls as pod_tls
+    monkeypatch.setattr(pod_tls, "ensure_ca", _ca)
+    _cfg(monkeypatch, 96)
+    try:
+        s = pod_http._pinned_session()
+    finally:
+        pod_http._session = None
+    a = s.get_adapter("https://10.0.0.1/")
+    assert a._pool_maxsize == 96
+
+
+def test_routing_is_unchanged():
+    """Only pods get the pinned session; everything else stays on requests, so
+    callers and tests that patch requests.get keep working."""
+    assert pod_http.is_pinned_url("https://194.68.245.54:22031/v1/ocr")
+    assert not pod_http.is_pinned_url("https://huggingface.co/x")
+    assert not pod_http.is_pinned_url("http://194.68.245.54:8000/v1/ocr")