ocr: say how many pods the pages imply, not how many max_pods allows

Comparing 64 pages against max_pods x inflight (64 x 16 = 1024 on the Digesta
orchestrator) would never have fired, and would have been reassuring if it
had: the fleet's ceiling is not what serves a burst. A pod takes ~12 minutes
to become ready, so the burst is served by the pods already warm. What is
worth saying at setup time is the arithmetic itself — these pages need that
many pods — plus the case where it exceeds max_pods and the surplus can only
queue.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/ocr/manager.py

@@ -276,13 +276,11 @@ def _engine_instances(cfg, name: str) -> int:
     return want
 
 
-def _fleet_page_slots(name: str):
-    """How many pages the remote fleet can actually have in flight, or None.
+def _fleet_shape(name: str):
+    """``(inflight_per_pod, max_pods)`` for the model this engine reads from.
 
-    ``max_pods x scale_up_inflight_per_pod``. Not a limit we impose — the pool
-    queues the excess perfectly correctly — but a number worth comparing with
-    ours, because pages beyond it do not go faster, they only wait, and a pod
-    takes ~12 minutes to appear so the fleet cannot grow into a burst.
+    Taken from the model's own ``runpod`` block, which is where per-pod
+    concurrency lives — one knob per model, not an OCR setting.
     """
     try:
         from codai.models.manager import _model_entry_for
@@ -298,42 +296,53 @@ def _fleet_page_slots(name: str):
             mcfg = parse_model_runpod(block)
             if (mcfg.mode or "").lower() != "pods":
                 continue
-            pods = max(1, int(getattr(mcfg, "max_pods", 1) or 1))
             per = int(getattr(mcfg, "scale_up_inflight_per_pod", 0) or 0)
             if per <= 0:
                 per = int(getattr(mcfg, "max_inflight_per_pod", 0) or 0)
             if per > 0:
-                return pods * per
+                return per, max(1, int(getattr(mcfg, "max_pods", 1) or 1))
         except Exception:
             continue
     return None
 
 
-#: The served model names an engine may be reading from, for the slot estimate.
+#: The served model names an engine may be reading from, for the fleet estimate.
 _SERVED_NAMES = {"surya": ("datalab-to/surya-ocr-2", "surya-ocr-2"),
                  "chandra": ("datalab-to/chandra-ocr-2", "chandra-ocr-2"),
                  "olmocr": ("allenai/olmOCR-2-7B-1025-FP8", "olmocr")}
 
+#: Measured on the Digesta orchestrator: ~7 min image pull + ~5 min warm-up.
+_POD_READY_MINUTES = 12
+
 
 def _warn_if_oversubscribed(name: str, pages: int) -> None:
-    """Say so when we fan out more pages than the fleet can serve at once.
-
-    Oversubscription does not lose work and it does not go faster: the surplus
-    waits in the gateway, every page's latency rises by the queue, and because a
-    client usually counts finished DOCUMENTS, spreading N pages over many
-    documents at once means they all advance and none completes until near the
-    end. That reads from outside as "one opening burst, then silence" — which is
-    exactly how the 2026-10-13 batch was reported, while the pod sat at 18
-    concurrent requests, `Waiting: 0` and 5-7% KV cache the entire time.
+    """Say how many pods this many pages implies, at the moment they are set up.
+
+    Not a limit — the pool queues the surplus perfectly correctly. But pages
+    beyond what the READY pods can hold do not go faster, they only wait, and
+    because a pod takes ~12 minutes to appear the fleet cannot be scaled into a
+    burst: the first burst is served by whatever is already warm, however high
+    ``max_pods`` is. That matters most to a client counting finished DOCUMENTS,
+    which completes none of them until one document's LAST page lands — so 64
+    pages spread over sixteen documents reads from outside as a stall while
+    every pod is busy. Exactly how the 2026-10-13 batch was reported, with the
+    pod at 18 concurrent requests, ``Waiting: 0`` and 5-7% KV cache throughout.
     """
-    slots = _fleet_page_slots(name)
-    if not slots or pages <= slots:
+    shape = _fleet_shape(name)
+    if not shape:
         return
-    print(f"[ocr] engine {name!r}: {pages} pages at once, but the fleet serves "
-          f"{slots} (max_pods x scale_up_inflight_per_pod). The surplus waits in "
-          f"the gateway — it does not go faster, and a pod takes ~12 min to "
-          f"appear, so a burst cannot be scaled into. Lower instances/lanes, or "
-          f"raise the pod's inflight ceiling.", flush=True)
+    per, max_pods = shape
+    need = (pages + per - 1) // per
+    tail = (f"a pod takes ~{_POD_READY_MINUTES} min to become ready, so the "
+            f"first burst is served by whatever is already warm")
+    if need > max_pods:
+        print(f"[ocr] engine {name!r}: {pages} pages at once needs {need} pods at "
+              f"{per} in flight each, but max_pods is {max_pods} — "
+              f"{max_pods * per} will be served and the rest will queue. "
+              f"Lower instances/lanes, or raise max_pods.", flush=True)
+    elif need > 1:
+        print(f"[ocr] engine {name!r}: {pages} pages at once spreads over {need} "
+              f"pods ({per} in flight each); {tail}.", flush=True)
 
 
 def _engine_lanes(eng) -> int:

docs/ocr.md

@@ -1033,16 +1033,26 @@ pool queues pages beyond `max_pods x scale_up_inflight_per_pod` and serves them
 perfectly correctly, but every page's latency rises by the queue — and a client
 that counts finished DOCUMENTS sees nothing at all until the first document's
 LAST page lands. Spread 64 pages over sixteen documents and all sixteen advance
-while none completes. That reads exactly like a stall. The pool logs the
-mismatch at build time:
+while none completes. That reads exactly like a stall. The pool therefore says, when the
+pages are set up, how many pods they imply:
 
 ```
-[ocr] engine 'surya': 64 pages at once, but the fleet serves 32
-      (max_pods x scale_up_inflight_per_pod). The surplus waits in the gateway —
-      it does not go faster, and a pod takes ~12 min to appear, so a burst
-      cannot be scaled into.
+[ocr] engine 'surya': 64 pages at once spreads over 4 pods (16 in flight each);
+      a pod takes ~12 min to become ready, so the first burst is served by
+      whatever is already warm.
 ```
 
+and, when the arithmetic exceeds `max_pods`, that the surplus simply queues:
+
+```
+[ocr] engine 'surya': 64 pages at once needs 4 pods at 16 in flight each, but
+      max_pods is 2 — 32 will be served and the rest will queue.
+```
+
+A high `max_pods` does not help a burst: it is the pods already **warm** that
+serve it, because the twelve minutes are spent before the first of the new ones
+answers anything.
+
 **The defect: 2,644 generated tokens for a page whose text is ~400.** Surya
 re-runs a page whose generation repeated itself — `_should_retry` in
 `surya.inference.backends.openai_client` — up to `max_retries` (3) more times,

tests/test_surya_retry_budget.py

@@ -251,44 +251,57 @@ def test_local_mode_gets_no_client_parallel():
 
 
 # ------------------------------------------------- the oversubscription note #
-def test_more_pages_than_the_fleet_can_serve_is_reported(capsys, monkeypatch):
-    """64 pages into 2 pods x 16 slots does not go faster — it queues, every
-    page's latency rises, and a client counting DOCUMENTS sees one opening
-    burst then silence."""
+def test_how_many_pods_this_many_pages_implies(capsys, monkeypatch):
+    """The useful number. 64 pages at 16 per pod is 4 pods — and a pod takes
+    ~12 minutes, so the first burst gets whatever is already warm, however
+    high max_pods is."""
     from codai.ocr import manager as m
-    monkeypatch.setattr(m, "_fleet_page_slots", lambda name: 32)
+    monkeypatch.setattr(m, "_fleet_shape", lambda name: (16, 64))
     m._warn_if_oversubscribed("surya", 64)
     out = capsys.readouterr().out
-    assert "64 pages at once" in out and "fleet serves 32" in out
+    assert "64 pages at once spreads over 4 pods" in out
+    assert "12 min" in out and "already warm" in out
 
 
-def test_a_fleet_that_can_take_it_says_nothing(capsys, monkeypatch):
+def test_more_pods_than_max_pods_is_called_out(capsys, monkeypatch):
+    """Here the surplus genuinely cannot be served: it queues until a pod frees
+    up, and no amount of waiting grows the fleet past max_pods."""
     from codai.ocr import manager as m
-    monkeypatch.setattr(m, "_fleet_page_slots", lambda name: 64)
+    monkeypatch.setattr(m, "_fleet_shape", lambda name: (16, 2))
     m._warn_if_oversubscribed("surya", 64)
+    out = capsys.readouterr().out
+    assert "needs 4 pods" in out and "max_pods is 2" in out and "32 will be served" in out
+
+
+def test_one_pod_is_worth_no_remark(capsys, monkeypatch):
+    """8 pages at 16 per pod fits one pod. Saying so would be noise."""
+    from codai.ocr import manager as m
+    monkeypatch.setattr(m, "_fleet_shape", lambda name: (16, 64))
+    m._warn_if_oversubscribed("surya", 8)
     assert capsys.readouterr().out == ""
 
 
 def test_an_unknown_fleet_says_nothing(capsys, monkeypatch):
     """No RunPod config, a local engine, a lookup that failed — none of those
-    are oversubscription, and guessing would cry wolf on every build."""
+    are a fleet, and guessing would cry wolf on every build."""
     from codai.ocr import manager as m
-    monkeypatch.setattr(m, "_fleet_page_slots", lambda name: None)
+    monkeypatch.setattr(m, "_fleet_shape", lambda name: None)
     m._warn_if_oversubscribed("surya", 9999)
     assert capsys.readouterr().out == ""
 
 
-def test_the_slot_estimate_reads_the_models_own_runpod_block(monkeypatch):
+def test_the_estimate_reads_the_models_own_runpod_block(monkeypatch):
+    """Per-pod concurrency is one knob per model, not an OCR setting."""
     from codai.ocr import manager as m
     import codai.models.manager as mm
     monkeypatch.setattr(mm, "_model_entry_for",
                         lambda name: {"runpod": {"mode": "pods", "max_pods": 4,
                                                  "scale_up_inflight_per_pod": 16}},
                         raising=False)
-    assert m._fleet_page_slots("surya") == 64
+    assert m._fleet_shape("surya") == (16, 4)
 
 
-def test_a_serverless_model_has_no_pod_slots(monkeypatch):
+def test_a_serverless_model_has_no_pod_shape(monkeypatch):
     """The arithmetic is about pods; serverless scales on its own terms."""
     from codai.ocr import manager as m
     import codai.models.manager as mm
@@ -297,4 +310,4 @@ def test_a_serverless_model_has_no_pod_slots(monkeypatch):
                                                  "max_pods": 4,
                                                  "scale_up_inflight_per_pod": 16}},
                         raising=False)
-    assert m._fleet_page_slots("surya") is None
+    assert m._fleet_shape("surya") is None