ocr: a ruined page is not an error, and nothing was catching it (v0.3.79)

From last night's run: ~97% of a 1,000-document batch came out truncated,
with no error anywhere. The engine answered 200 every time. Its decode had
fallen into a loop, run to the token cap emitting the same fragment, and
the resumer correctly refused to continue a reply that was already
repeating itself -- so the page was flagged, returned, and indexed.

`fallback_engines` never fired, because it only fires on an exception.
The same pages read cleanly on another engine (38,055 characters against a
truncated stub), so the recovery was there the whole time and nothing
reached for it.

`ocr.retry_degraded` (default "auto") re-reads ONLY the degraded pages on
another engine and keeps the replacement only if it is not degraded in
turn. Bounded by construction: a clean document costs nothing, a ruined
one costs at most a second pass over its ruined pages, and
`retry_degraded_max_pages` caps even that. The page keeps its own index
and carries `degraded_retry` saying who re-read it and why; the response
reports `recovered_pages`.

Degraded means looping or truncated -- the two the engine itself attests.
NOT a short page, a blank page or an unreadable one: those have their own
meaning and re-reading them spends a card to be told the same thing.

Which leaves the other half of the report: olmOCR returned 4,046
characters where chandra read 30,983 on the same page, ending cleanly well
short of the cap, flagging nothing. The only evidence an early stop leaves
is the company it keeps, so `retry_short_pages_below` compares a page with
the DOCUMENT'S OWN median rather than a character count -- a threshold in
characters would need re-tuning for every language, DPI and layout.
Default 0, OFF, and deliberately: this signal is inferred where the others
are attested, and a genuine title or signature page looks identical. A
short page is also only replaced by a LONGER one.

Off on pods: only the orchestrator can reach another engine's pods, so a
pod would re-read on the engine that just produced the flag.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HXRMe9ZPUQCCCtBai9uqt3

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.78"
+__version__ = "0.3.79"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/admin/routes.py

@@ -4180,6 +4180,10 @@ def build_settings_dict(c, gpu_cards):
             "vlm_max_num_seqs": c.ocr.vlm_max_num_seqs,
             "vlm_max_model_len": c.ocr.vlm_max_model_len,
             "fallback_engines": c.ocr.fallback_engines,
+            "retry_degraded": c.ocr.retry_degraded,
+            "retry_degraded_max_pages": c.ocr.retry_degraded_max_pages,
+            "retry_short_pages_below": c.ocr.retry_short_pages_below,
+            "retry_short_pages_min_doc": c.ocr.retry_short_pages_min_doc,
             "olmocr_enabled": c.ocr.olmocr_enabled,
             "olmocr_instances": c.ocr.olmocr_instances,
             "olmocr_serve": c.ocr.olmocr_serve,
@@ -5043,6 +5047,10 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
         if "build_retry_cooldown_s" in d: o.build_retry_cooldown_s = _f("build_retry_cooldown_s", o.build_retry_cooldown_s, 0.0)
         if "vlm_gpu_memory_utilization" in d: o.vlm_gpu_memory_utilization = _f("vlm_gpu_memory_utilization", o.vlm_gpu_memory_utilization, 0.0)
         if "fallback_engines" in d: o.fallback_engines = (d.get("fallback_engines") or "auto").strip()
+        if "retry_degraded" in d: o.retry_degraded = (d.get("retry_degraded") or "auto").strip()
+        if "retry_degraded_max_pages" in d: o.retry_degraded_max_pages = max(0, int(d.get("retry_degraded_max_pages") or 0))
+        if "retry_short_pages_below" in d: o.retry_short_pages_below = max(0.0, min(1.0, float(d.get("retry_short_pages_below") or 0.0)))
+        if "retry_short_pages_min_doc" in d: o.retry_short_pages_min_doc = max(3, int(d.get("retry_short_pages_min_doc") or 4))
         if "vlm_max_num_batched_tokens" in d: o.vlm_max_num_batched_tokens = _i("vlm_max_num_batched_tokens", o.vlm_max_num_batched_tokens, 0)
         if "vlm_max_num_seqs" in d: o.vlm_max_num_seqs = _i("vlm_max_num_seqs", o.vlm_max_num_seqs, 0)
         if "vlm_max_model_len" in d: o.vlm_max_model_len = _i("vlm_max_model_len", o.vlm_max_model_len, 0)

codai/admin/templates/settings.html

@@ -1200,6 +1200,28 @@
         <div class="form-hint">Tried when the chosen engine can't serve, so a document
           is never refused: <code>auto</code>, <code>off</code>, or an explicit order.</div>
       </div>
+      <div class="form-group" style="flex:1;min-width:170px">
+        <label class="form-label">Re-read degraded pages on</label>
+        <input type="text" id="s-ocr-retry-degraded" class="form-input" placeholder="auto">
+        <div class="form-hint">A page can come back looping or cut off at the token cap
+          with the engine answering <code>200</code>, so <em>Fallback engines</em> never
+          fires. These re-read only those pages elsewhere and keep whichever answer
+          isn't degraded: <code>auto</code>, <code>off</code>, or an explicit order.</div>
+      </div>
+      <div class="form-group" style="flex:1;min-width:150px">
+        <label class="form-label">Re-read cap (pages/doc)</label>
+        <input type="number" id="s-ocr-retry-degraded-max" class="form-input" min="0" placeholder="0">
+        <div class="form-hint">0 = no limit. A scan the whole fleet reads badly would
+          otherwise pay twice for every page to arrive in the same place.</div>
+      </div>
+      <div class="form-group" style="flex:1;min-width:150px">
+        <label class="form-label">Re-read pages under × median</label>
+        <input type="number" id="s-ocr-retry-short" class="form-input" min="0" max="1" step="0.05" placeholder="0">
+        <div class="form-hint">0 = off. Catches an engine that stops early without
+          flagging anything, by comparing a page with the document's own median.
+          <strong>Inferred, not attested</strong> — a real title or signature page looks
+          the same, so leave it off unless your documents have none.</div>
+      </div>
     </div>
 
     <div class="card-title" style="font-size:13px;margin-top:.4rem">PaddleOCR</div>
@@ -2232,6 +2254,9 @@ async function loadSettings(){
     document.getElementById('s-ocr-retry-cooldown').value = ocr.build_retry_cooldown_s ?? 60;
     document.getElementById('s-ocr-vlm-gmu').value = ocr.vlm_gpu_memory_utilization ?? 0;
     document.getElementById('s-ocr-fallback').value = ocr.fallback_engines ?? 'auto';
+    document.getElementById('s-ocr-retry-degraded').value = ocr.retry_degraded ?? 'auto';
+    document.getElementById('s-ocr-retry-degraded-max').value = ocr.retry_degraded_max_pages ?? 0;
+    document.getElementById('s-ocr-retry-short').value = ocr.retry_short_pages_below ?? 0;
     document.getElementById('s-ocr-vlm-mnbt').value = ocr.vlm_max_num_batched_tokens ?? 4096;
     document.getElementById('s-ocr-vlm-mns').value = ocr.vlm_max_num_seqs ?? 16;
     document.getElementById('s-ocr-vlm-mml').value = ocr.vlm_max_model_len ?? 0;
@@ -2533,6 +2558,9 @@ async function saveSettings(silent){
       build_retry_cooldown_s: parseFloat(document.getElementById('s-ocr-retry-cooldown').value) || 0,
       vlm_gpu_memory_utilization: parseFloat(document.getElementById('s-ocr-vlm-gmu').value) || 0,
       fallback_engines: document.getElementById('s-ocr-fallback').value.trim() || 'auto',
+      retry_degraded: document.getElementById('s-ocr-retry-degraded').value.trim() || 'auto',
+      retry_degraded_max_pages: parseInt(document.getElementById('s-ocr-retry-degraded-max').value) || 0,
+      retry_short_pages_below: parseFloat(document.getElementById('s-ocr-retry-short').value) || 0,
       vlm_max_num_batched_tokens: parseInt(document.getElementById('s-ocr-vlm-mnbt').value) || 0,
       vlm_max_num_seqs: parseInt(document.getElementById('s-ocr-vlm-mns').value) || 0,
       vlm_max_model_len: parseInt(document.getElementById('s-ocr-vlm-mml').value) || 0,

codai/api/runpod_worker.py

@@ -1336,6 +1336,11 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
         # This image carries NO docTR -- `auto` would route a failed page to an
         # engine that is not there. This engine or nothing, said out loud.
         env["CODERAI_OCR_FALLBACK_ENGINES"] = "off"
+        # Same reasoning for the degraded-page re-read, and one more: only the
+        # ORCHESTRATOR can reach another engine's pods. A pod re-reading its own
+        # degraded page would re-read it on the one engine that just produced
+        # it, which is the retry that already failed.
+        env["CODERAI_OCR_RETRY_DEGRADED"] = "off"
         # And the TUNING. Every one of these was set in response to a measured
         # failure, and a pod reading pages at surya's defaults instead would be
         # indistinguishable from the "pre-fix pod image" that got blamed three

codai/config.py

@@ -1084,6 +1084,30 @@ class OcrConfig:
     # "off"/"none" = no fallback (fail with the original error); or an explicit
     # comma/space separated order.
     fallback_engines: str = "auto"
+    # A page can come back DEGRADED without the engine ever failing: the reply
+    # looped until it hit the token cap, or it stopped at the cap and could not
+    # be resumed. `fallback_engines` never fires for those -- the call returned
+    # 200 -- so a run can come out almost entirely truncated with no error
+    # anywhere. These re-read ONLY the degraded pages on another engine and keep
+    # whichever answer is not degraded. "auto" = the fallback order; "off" = keep
+    # what the first engine produced; or an explicit comma/space separated order.
+    retry_degraded: str = "auto"
+    # Stop re-reading after this many pages of one document, 0 = no limit. A
+    # scan the whole fleet reads badly would otherwise pay twice for every page
+    # to arrive at the same place.
+    retry_degraded_max_pages: int = 0
+    # Re-read a page whose text is below this FRACTION of the document's median
+    # page, even though the engine flagged nothing. 0 = off, which is the
+    # default, because unlike a looping or truncated page this one is INFERRED:
+    # the engine says the page is fine and we are disagreeing with it from the
+    # shape of the document around it. A real early stop looks exactly like a
+    # title page or a signature page, and only the operator knows which their
+    # corpus has. 0.25 catches an engine that returned an eighth of what its
+    # neighbours did without touching a document of genuinely uneven pages.
+    retry_short_pages_below: float = 0.0
+    # ...and only in a document with at least this many pages, since a median
+    # taken over two pages is not a median.
+    retry_short_pages_min_doc: int = 4
 
     # --- PaddleOCR / PP-Structure ---
     paddle_enabled: bool = True
@@ -2011,6 +2035,10 @@ class ConfigManager:
                 "vlm_max_num_seqs": self.config.ocr.vlm_max_num_seqs,
                 "vlm_max_model_len": self.config.ocr.vlm_max_model_len,
                 "fallback_engines": self.config.ocr.fallback_engines,
+                "retry_degraded": self.config.ocr.retry_degraded,
+                "retry_degraded_max_pages": self.config.ocr.retry_degraded_max_pages,
+                "retry_short_pages_below": self.config.ocr.retry_short_pages_below,
+                "retry_short_pages_min_doc": self.config.ocr.retry_short_pages_min_doc,
                 "olmocr_enabled": self.config.ocr.olmocr_enabled,
                 "olmocr_instances": self.config.ocr.olmocr_instances,
                 "olmocr_serve": self.config.ocr.olmocr_serve,
@@ -2162,7 +2190,8 @@ class ConfigManager:
                                 ("CODERAI_OCR_CHANDRA_MODEL", "chandra_model"),
                                 ("CODERAI_OCR_CHANDRA_MODEL_ID", "chandra_model_id"),
                                 ("CODERAI_OCR_CHANDRA_SERVER_URL", "chandra_server_url"),
-                                ("CODERAI_OCR_FALLBACK_ENGINES", "fallback_engines")):
+                                ("CODERAI_OCR_FALLBACK_ENGINES", "fallback_engines"),
+                                ("CODERAI_OCR_RETRY_DEGRADED", "retry_degraded")):
                 _val = (_os.environ.get(_env) or "").strip()
                 if _val:
                     setattr(self.config.ocr, _attr, _val)