ocr: surya_instances is a process count, and a busy probe is not a death (v0.3.42)

A batch of documents gave one flawless 19-second burst — 22 documents, correct
page counts, clean accents — and then no completions at all for nine minutes,
while the rented A40 sat at 2% KV-cache usage with `Waiting: 0`. The GPU was
never the bottleneck. Two local mistakes compounded:

`surya_instances: 64` reads like GPU concurrency and is not: surya, paddle and
docTR spawn ONE OS PROCESS per instance, in their own venv. 64 of them meant 37
GB of RSS, swap exhausted and load 25 on a 16-core host that also runs some
thirty other containers. Nine hours after the batch ended, 66 of those workers
were still resident with 19 seconds of CPU time each.

Then it fed itself. The workers' /health probe (5s, against coderai's OWN
gateway) began timing out under the load the workers were creating, and a
timed-out probe counted as "the endpoint is gone" — so each worker was killed
and respawned, which added load, which timed out more probes.

  - the process-per-instance engines are capped at one worker per core and
    never more than half of free RAM, with a log line naming the figure and the
    reason. CODERAI_OCR_INSTANCE_CEILING overrides it for a measured host. The
    in-process engines (chandra, olmOCR) are untouched: an instance there is a
    concurrency slot, not a process;
  - a health probe now means "gone" only on a definite 404 "unknown OCR engine
    lease", which is the case re-resolving was written for: a front that
    restarted. A timeout means busy and the worker stays up, because respawning
    cannot fix a front that is down — the replacement asks the same front for a
    lease. A 503 means the pool has nothing rented this second, which the lease
    does not care about.

GPU-side concurrency remains runpod.scale_up_inflight_per_pod and max_pods; the
local worker pool is a hard ceiling on how many pods surya can keep fed, which
is the standing argument for running it in-process.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.41"
+__version__ = "0.3.42"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/ocr/manager.py

@@ -28,6 +28,7 @@ bounds the whole subsystem.
 """
 
 import asyncio
+import os
 import threading
 import time
 from typing import Dict, List, Optional
@@ -205,12 +206,74 @@ def _engine_enabled(cfg, name: str) -> bool:
     return False
 
 
+#: Engines that spawn ONE OS PROCESS per instance, in their own venv. For these
+#: ``*_instances`` is not a concurrency slot count — it is a process count, and
+#: each process costs ~600 MB of RSS and a core's worth of page rendering. The
+#: in-process engines (chandra, olmOCR) cost a slot and nothing more.
+_PROCESS_PER_INSTANCE = ("surya", "paddle", "doctr")
+
+#: Measured RSS of one idle surya worker on Aruba, 2026-10-12: 586-677 MB across
+#: 64 of them. Rounded up for headroom, because the figure that matters is the
+#: one *after* a page has been rendered in it.
+_WORKER_RSS_GB = 0.8
+
+
+def _host_instance_ceiling(name: str) -> int:
+    """How many worker PROCESSES this host can actually run for one engine.
+
+    `surya_instances: 64` was set on Aruba as though it were the GPU-side
+    concurrency (it reads like it), and on 2026-10-12 it did exactly what it
+    says: 64 worker processes, 37 GB of RSS, swap exhausted, load 25 on 16
+    cores — on a box that also runs ~30 other containers. A batch then behaved
+    precisely as the operator reported: one perfect 19-second burst, then no
+    completions at all for nine minutes while the rented A40 sat at 2% KV-cache
+    usage with `Waiting: 0`. The GPU was never the bottleneck; the host was.
+
+    GPU-side concurrency is `runpod.scale_up_inflight_per_pod` and
+    `runpod.max_pods`. This number is a local process pool and has to be bounded
+    by local resources, so it is: one worker per core, and never more than half
+    of free RAM. `CODERAI_OCR_INSTANCE_CEILING` overrides it for an operator who
+    has measured their own host.
+    """
+    forced = (os.environ.get("CODERAI_OCR_INSTANCE_CEILING") or "").strip()
+    if forced:
+        try:
+            return max(1, int(forced))
+        except ValueError:
+            pass
+    if name not in _PROCESS_PER_INSTANCE:
+        return 0                      # in-process: a slot, not a process
+    try:
+        by_cpu = max(1, os.cpu_count() or 4)
+    except Exception:
+        by_cpu = 4
+    by_ram = by_cpu
+    try:
+        # Free RAM, not total: the rest of the host is someone else's working set.
+        with open("/proc/meminfo") as fh:
+            avail_kb = next(int(l.split()[1]) for l in fh
+                            if l.startswith("MemAvailable:"))
+        by_ram = max(1, int((avail_kb / 1048576.0) * 0.5 / _WORKER_RSS_GB))
+    except Exception:
+        pass
+    return max(1, min(by_cpu, by_ram))
+
+
 def _engine_instances(cfg, name: str) -> int:
     n = getattr(cfg, f"{name}_instances", 1)
     try:
-        return max(1, int(n))
+        want = max(1, int(n))
     except Exception:
-        return 1
+        want = 1
+    ceiling = _host_instance_ceiling(name)
+    if ceiling and want > ceiling:
+        print(f"[ocr] engine {name!r}: {want} instances asked for, running "
+              f"{ceiling} — each one is a worker PROCESS (~{_WORKER_RSS_GB} GB) "
+              f"and this host has {os.cpu_count()} cores. GPU-side concurrency "
+              f"is runpod.scale_up_inflight_per_pod / max_pods; raise "
+              f"CODERAI_OCR_INSTANCE_CEILING to override.", flush=True)
+        return ceiling
+    return want
 
 
 def _evict_for_ocr(needed_gb: float) -> None:

codai/ocr/surya.py

@@ -223,14 +223,44 @@ class SuryaEngine(SubprocessOcrEngine):
     # ------------------------------------------------------------------ leases
 
     def _endpoint_alive(self) -> bool:
-        """Whether the server this worker was pinned to still answers /health."""
+        """Whether the address this worker was pinned to is still OURS to use.
+
+        A `False` here is expensive: the caller kills this worker process and
+        spawns a replacement, which costs ~600 MB and several seconds of torch
+        import. So only a definite answer may return False.
+
+        In `model` mode the address is always coderai's own OCR gateway, never a
+        pod directly — the gateway is what holds the pool and drops a reaped pod.
+        It answers 404 "unknown OCR engine lease" when the lease really is gone
+        (the front restarted), and that is the one case re-resolving fixes. A 503
+        means no upstream *right now*; the lease is fine.
+
+        A probe that times out is not an answer at all, and it used to return
+        False. On 2026-10-12, with 64 of these workers on a 16-core host, the
+        5-second probe began timing out under the load the workers themselves
+        were creating — so coderai killed and respawned them, which added load,
+        which timed out more probes. The batch stalled for nine minutes with the
+        GPU idle. Respawning cannot fix a front that is down (the replacement
+        asks the same front for a lease), so a timeout now means "busy, keep
+        working" and the page's real error is reported as it stands.
+        """
         import requests
         from codai.ocr.served_model import health_url
-        try:
-            r = requests.get(health_url(self._server_url) + "/health", timeout=5)
-            return r.status_code == 200
-        except Exception:
-            return False
+        url = health_url(self._server_url) + "/health"
+        last = None
+        for timeout in (10, 20):
+            try:
+                r = requests.get(url, timeout=timeout)
+            except Exception as exc:
+                last = exc
+                continue
+            if r.status_code == 404:
+                return False              # the lease is genuinely gone
+            return True                   # 200, or 503 "no upstream yet"
+        print(f"[ocr] surya: {url} did not answer the health probe "
+              f"({type(last).__name__ if last else 'timeout'}) — treating it as "
+              f"busy, not gone; the worker stays up", flush=True)
+        return True
 
     def recognize_image(self, image):
         """One page, re-resolving the endpoint if the lease expired underneath us.

docs/ocr.md

@@ -968,6 +968,46 @@ The address is a lease: a secret minted per attachment, carried in the URL (the
 engine's `/health` and `/v1/models` probes send no headers, so it cannot be a
 bearer alone) and valid for nothing but that one gateway path.
 
+#### `*_instances` is a local worker count, not GPU concurrency
+
+The two numbers look interchangeable on the OCR card and are not:
+
+| setting | what it bounds | where it costs |
+|---|---|---|
+| `runpod.scale_up_inflight_per_pod`, `runpod.max_pods` | concurrent pages **per rented GPU**, and how many GPUs | the pod |
+| `ocr.surya_instances` (also `paddle`, `doctr`) | **worker processes on this machine** | ~600 MB RSS and a core's rendering, each |
+| `ocr.chandra_instances`, `ocr.olmocr_instances` | concurrency slots, in-process | a slot |
+
+Only surya, paddle and docTR spawn a process per instance — they run in their own
+venv, driven over a pipe. `surya_instances: 64` was set on Aruba as though it were
+the GPU-side figure, and on 2026-10-12 it did exactly what it says: **64 worker
+processes, 37 GB of RSS, swap exhausted, load 25 on 16 cores**, on a host also
+running some thirty other containers. A batch then behaved as the operator
+reported — one flawless 19-second burst of 22 documents, then **no completions at
+all for nine minutes** — while the rented A40 sat at 2% KV-cache usage with
+`Waiting: 0`. The GPU was never the bottleneck. The host was.
+
+It got worse on its own, because the workers' `/health` probe (5s) started timing
+out under the load the workers themselves were creating, and a timed-out probe
+counted as *"the endpoint is gone"*: coderai killed each worker and spawned a
+replacement, which added load, which timed out more probes.
+
+Both halves are now bounded:
+
+* the process-per-instance engines are capped at **one worker per core, and never
+  more than half of free RAM**, with a log line naming the figure it is running
+  and why (`CODERAI_OCR_INSTANCE_CEILING` overrides it for an operator who has
+  measured their own host);
+* a health probe only means *gone* on a definite **404 "unknown OCR engine
+  lease"** — which is the real case it was written for, a front that restarted. A
+  timeout means busy, and the worker stays up: respawning cannot fix a front that
+  is down, since the replacement asks that same front for its lease. A 503 means
+  the pool has nothing rented this second, which the lease does not care about.
+
+To raise OCR throughput, raise the pod-side numbers, not these. The local worker
+pool is a hard ceiling on how many pods surya can keep fed at all — which is the
+standing argument for running it in-process, as chandra and olmOCR already do.
+
 ```bash
 curl -F file=@sentenza.pdf -F structured=true -F detect=both -F schema=italian_sentenza \
      http://localhost:8776/v1/ocr

tests/test_ocr_served_model.py

@@ -511,3 +511,128 @@ def test_runpod_wins_over_a_node_that_also_lists_it(monkeypatch):
     assert (lease.base, lease.api_key) == ("https://pod/v1", "podtok")
     assert pool_taken == [1]
     lease.release()
+
+
+# ------------------- a busy gateway is not a vanished one, and 64 is not 64 #
+# On 2026-10-12 a batch of documents produced one perfect 19-second burst and
+# then no completions for nine minutes, while the rented A40 sat at 2% KV-cache
+# usage with `Waiting: 0`. Two mistakes compounded, both local to the host:
+#
+#   * `surya_instances: 64` reads like GPU concurrency but is a count of worker
+#     PROCESSES: 64 of them, 37 GB of RSS, swap exhausted, load 25 on 16 cores;
+#   * the 5s `/health` probe then began timing out under the load those workers
+#     were creating, and a timed-out probe counted as "the endpoint is gone", so
+#     coderai killed and respawned each worker — adding load, timing out more
+#     probes. The GPU was never the bottleneck.
+def _probe_engine(monkeypatch, replies):
+    """A surya engine whose health probe gets ``replies`` (status or exception)."""
+    import requests
+    from codai.ocr.surya import SuryaEngine
+
+    eng = SuryaEngine.__new__(SuryaEngine)
+    eng._server_url = "http://127.0.0.1:8780/v1/ocr/attach/tok/v1"
+    seen = []
+
+    def _get(url, timeout=None):
+        seen.append(timeout)
+        got = replies[min(len(seen) - 1, len(replies) - 1)]
+        if isinstance(got, Exception):
+            raise got
+
+        class _R:
+            status_code = got
+        return _R()
+
+    monkeypatch.setattr(requests, "get", _get)
+    return eng, seen
+
+
+def test_a_timed_out_health_probe_keeps_the_worker(monkeypatch):
+    """Respawning cannot fix a front that is down — the replacement asks the same
+    front for a lease. So a probe that never answered must not cost a process."""
+    import requests
+    eng, seen = _probe_engine(monkeypatch, [requests.Timeout("too busy")])
+    assert eng._endpoint_alive() is True          # "busy, keep working"
+    assert len(seen) == 2                         # retried, with a longer budget
+    assert seen[1] > seen[0]
+
+
+def test_only_an_unknown_lease_means_the_endpoint_is_gone(monkeypatch):
+    """404 is the one answer re-resolving fixes: the front restarted and does not
+    know this lease any more."""
+    eng, _ = _probe_engine(monkeypatch, [404])
+    assert eng._endpoint_alive() is False
+
+
+def test_no_upstream_yet_is_not_a_dead_lease(monkeypatch):
+    """503 means the pool has nothing rented this second. The lease is fine, and
+    killing the worker would not rent anything."""
+    eng, _ = _probe_engine(monkeypatch, [503])
+    assert eng._endpoint_alive() is True
+
+
+def test_a_healthy_gateway_answers_once(monkeypatch):
+    eng, seen = _probe_engine(monkeypatch, [200])
+    assert eng._endpoint_alive() is True
+    assert len(seen) == 1                         # no retry needed
+
+
+def test_a_slow_probe_that_recovers_is_not_a_death(monkeypatch):
+    import requests
+    eng, seen = _probe_engine(monkeypatch, [requests.Timeout("busy"), 200])
+    assert eng._endpoint_alive() is True
+    assert len(seen) == 2
+
+
+def test_the_worker_pool_is_bounded_by_the_host_not_the_config(monkeypatch):
+    """`surya_instances` is a process count. 64 of them is 37 GB and 64 cores'
+    worth of rendering, whatever the operator meant by the number."""
+    from codai.ocr import manager
+
+    monkeypatch.delenv("CODERAI_OCR_INSTANCE_CEILING", raising=False)
+    monkeypatch.setattr(manager.os, "cpu_count", lambda: 16)
+
+    class _Cfg:
+        surya_instances = 64
+        chandra_instances = 128
+
+    got = manager._engine_instances(_Cfg(), "surya")
+    assert 1 <= got <= 16
+    assert got < 64
+
+
+def test_an_in_process_engine_is_not_clamped(monkeypatch):
+    """chandra and olmOCR cost a concurrency slot, not a process: 128 is fine."""
+    from codai.ocr import manager
+
+    monkeypatch.delenv("CODERAI_OCR_INSTANCE_CEILING", raising=False)
+
+    class _Cfg:
+        chandra_instances = 128
+
+    assert manager._engine_instances(_Cfg(), "chandra") == 128
+
+
+def test_an_operator_who_has_measured_can_override(monkeypatch):
+    from codai.ocr import manager
+
+    monkeypatch.setenv("CODERAI_OCR_INSTANCE_CEILING", "40")
+
+    class _Cfg:
+        surya_instances = 64
+
+    assert manager._engine_instances(_Cfg(), "surya") == 40
+
+
+def test_the_ceiling_never_goes_below_one(monkeypatch):
+    """A busy host must still be able to read a page."""
+    from codai.ocr import manager
+
+    monkeypatch.delenv("CODERAI_OCR_INSTANCE_CEILING", raising=False)
+    monkeypatch.setattr(manager.os, "cpu_count", lambda: 0)
+    assert manager._host_instance_ceiling("surya") >= 1
+
+    class _Cfg:
+        surya_instances = 8
+
+    assert manager._engine_instances(_Cfg(), "surya") >= 1