ocr: an empty byte trace must not be read as a verdict (v0.3.26)

v0.3.25 shipped the trace as a bare list of findings, and defined the empty list
as "the bytes were sound, so the loss is in the serving path". That is wrong
whenever the upstream returned no logprobs: there was nothing to analyse, and
the emptiness would have read as a confident conclusion drawn from missing data
-- exactly the class of mistake the trace exists to stop.

So the trace is now a self-describing envelope:

  {"calls": 7, "with_logprobs": 7, "conclusive": true,
   "kinds": {"invalid-utf8": 29}, "why": "...", "findings": [...]}

and `unreadable.blame` is one of "the model", "the serving path" or "unknown".
`conclusive` is false when no calls were recorded (the hook never fired) or when
none carried logprobs, and then blame is "unknown" rather than defaulting to
either culprit.

surya asks for logprobs by default (SURYA_INFERENCE_LOGPROBS = True) and the OCR
gateway forwards the body verbatim precisely so they survive, so the normal case
is conclusive. But an upstream that declines them is a configuration, not a bug,
and it has to be visible instead of silently inverted into an answer.

Also: a trace analysis that throws now reports {"conclusive": false, "why":
"trace analysis failed: ..."} rather than no key at all, so "the analysis broke"
and "the bytes were fine" cannot look the same either.

tests/test_ocr_token_trace.py: 37 (was 30), including the inconclusive paths.
825 passed across the OCR/config/admin/runpod selection, same 10 pre-existing
failures.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.25"
+__version__ = "0.3.26"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/ocr/base.py

@@ -147,7 +147,7 @@ class OcrPage:
             # The byte-level trace is reported ONCE, document-wide, by
             # unreadable_report(). Repeating it on every page would put the same
             # findings in the response twice.
-            meta = {k: v for k, v in self.meta.items() if k != "unreadable_tokens"}
+            meta = {k: v for k, v in self.meta.items() if k != "unreadable_trace"}
             if meta:
                 out["meta"] = meta
         n = count_unreadable(self.text)
@@ -376,12 +376,21 @@ def unreadable_report(pages: "List[OcrPage]") -> dict:
     # one, because surya-ocr-2's vocabulary is byte-level and either byte of an
     # accented letter decodes alone to exactly one marker. The tokens tell them
     # apart; the text never can.
-    tokens = []
+    traces = []
     for page in pages or []:
-        for finding in (getattr(page, "meta", {}) or {}).get("unreadable_tokens", []):
-            entry = dict(finding)
+        trace = (getattr(page, "meta", {}) or {}).get("unreadable_trace")
+        if trace:
+            entry = dict(trace)
             entry["page"] = page.index
-            tokens.append(entry)
-    if tokens:
-        report["tokens"] = tokens
+            traces.append(entry)
+    if traces:
+        report["trace"] = traces
+        # One line a human can act on without reading the findings: the whole
+        # point is to stop guessing whose fault this is.
+        if all(t.get("conclusive") for t in traces):
+            blamed = ("the model" if any(t.get("findings") for t in traces)
+                      else "the serving path")
+            report["blame"] = blamed
+        else:
+            report["blame"] = "unknown"
     return report

codai/ocr/subprocess_engine.py

@@ -207,7 +207,7 @@ class SubprocessOcrEngine(OcrEngine):
         # The worker's byte-level diagnosis of an unreadable page, when it made one.
         # Carried on meta rather than a field of its own so engines that cannot
         # produce it change nothing about the response shape.
-        trace = d.get("unreadable_tokens")
+        trace = d.get("unreadable_trace")
         if trace:
-            page.meta["unreadable_tokens"] = trace
+            page.meta["unreadable_trace"] = trace
         return page

codai/ocr/workers/ocr_worker.py

@@ -307,6 +307,57 @@ def _tok_entry(entry) -> dict:
     return out
 
 
+def _trace_report(calls) -> dict:
+    """``_trace_findings`` plus the one fact that makes it safe to read.
+
+    An empty finding list has two completely different meanings:
+
+      * logprobs ARRIVED and the byte stream was sound -> the loss happened after
+        the model, so the serving stack is at fault. A real verdict.
+      * logprobs never arrived -> there was nothing to analyse, and the emptiness
+        says nothing at all.
+
+    Reporting a bare list would make those indistinguishable, and the second one
+    would read as the first: a confident conclusion drawn from missing data. So
+    the envelope always says how many calls were seen and how many carried
+    logprobs, and ``conclusive`` is false whenever they did not.
+
+    (surya asks for them by default -- ``SURYA_INFERENCE_LOGPROBS = True`` -- and
+    coderai's OCR gateway forwards the body verbatim precisely so they survive.
+    But an upstream that declines them is a configuration, not a bug, and has to
+    be visible rather than silently inverted into a verdict.)
+    """
+    calls = list(calls or [])
+    with_lp = sum(1 for c in calls if c)
+    report = {
+        "calls": len(calls),
+        "with_logprobs": with_lp,
+        "findings": _trace_findings(calls) if with_lp else [],
+    }
+    if not calls:
+        report["conclusive"] = False
+        report["why"] = ("no model calls were recorded -- the trace hook did not "
+                         "fire, so this says nothing about the cause")
+    elif not with_lp:
+        report["conclusive"] = False
+        report["why"] = ("the model server returned no logprobs, so the bytes "
+                         "could not be read -- this says nothing about the cause")
+    else:
+        report["conclusive"] = True
+        if not report["findings"]:
+            report["why"] = ("the byte stream decoded cleanly, so the characters "
+                             "were lost AFTER the model: the fault is in the "
+                             "serving path, not the model")
+        else:
+            kinds = {}
+            for f in report["findings"]:
+                kinds[f.get("kind")] = kinds.get(f.get("kind"), 0) + 1
+            report["kinds"] = kinds
+            report["why"] = ("the model's own byte stream is broken at the places "
+                             "listed: the model is at fault, not the serving path")
+    return report
+
+
 def _trace_findings(calls) -> list:
     """Where in the model's own token stream the text stopped being readable.
 
@@ -555,11 +606,15 @@ class SuryaWorker:
         # to what it has always returned.
         if self._trace and UNREADABLE in page["text"]:
             try:
-                findings = _trace_findings(self._calls)
+                page["unreadable_trace"] = _trace_report(self._calls)
             except Exception as e:
-                findings = [{"error": f"trace analysis failed: {e}"}]
-            if findings:
-                page["unreadable_tokens"] = findings
+                # Always a report, never an absence: "the analysis broke" and "the
+                # bytes were fine" must not look the same.
+                page["unreadable_trace"] = {
+                    "conclusive": False,
+                    "why": f"trace analysis failed: {e}",
+                    "findings": [],
+                }
         return page
 
 

docs/ocr.md

@@ -519,29 +519,44 @@ wrapper only reads, and swallows every error — a diagnostic that could break a
 be far worse than no diagnostic.
 
 With it on, a document that lost characters carries its own diagnosis under
-`unreadable.tokens`:
+`unreadable`, with a one-word `blame` for the impatient:
 
 ```json
 { "unreadable": {
     "count": 29,
-    "tokens": [
-      { "page": 0, "call": 3, "index": 41, "kind": "invalid-utf8",
-        "token":   {"token": "Ã", "bytes": [195], "logprob": -0.0021},
-        "why":     "byte 0xC3 does not form valid UTF-8 here",
-        "context": [ {"token": "p", "bytes": [112]}, {"token": "i", "bytes": [105]},
-                     {"token": "Ã", "bytes": [195]}, {"token": " ", "bytes": [32]} ] }
-    ] } }
+    "blame": "the model",
+    "trace": [
+      { "page": 0, "calls": 7, "with_logprobs": 7, "conclusive": true,
+        "kinds": {"invalid-utf8": 29},
+        "why": "the model's own byte stream is broken at the places listed: ...",
+        "findings": [
+          { "call": 3, "index": 41, "kind": "invalid-utf8",
+            "token":   {"token": "Ã", "bytes": [195], "logprob": -0.0021},
+            "why":     "byte 0xC3 does not form valid UTF-8 here",
+            "context": [ {"token": "p", "bytes": [112]}, {"token": "i", "bytes": [105]},
+                         {"token": "Ã", "bytes": [195]}, {"token": " ", "bytes": [32]} ] }
+        ] } ] } }
 ```
 
 How to read it:
 
-| what you see | verdict |
-|---|---|
-| `kind: "invalid-utf8"`, a lone `[195]` with nothing to complete it | the **model** emitted a broken sequence — report it to Datalab |
-| `kind: "model-emitted"`, the token *is* the replacement character | the model chose the marker, or something upstream substituted it before we saw a byte |
-| **no `tokens` at all** on a page full of markers | the bytes were sound and the loss happened after them — **ours**, in the serving stack |
-
-That last row is why an empty trace is a result and not a failure. The scan reassembles
+| `conclusive` | findings | verdict (`blame`) |
+|---|---|---|
+| `true` | `kind: "invalid-utf8"` — a lone `[195]` with nothing to complete it | **the model** emitted a broken sequence — report it to Datalab |
+| `true` | `kind: "model-emitted"` — the token *is* the replacement character | **the model** chose the marker, or something upstream substituted it before we saw a byte |
+| `true` | **none**, on a page full of markers | **the serving path** — the bytes were sound and the loss happened after them, so it is ours |
+| `false` | — | **unknown**. `with_logprobs: 0` means the upstream returned none, so there was nothing to read |
+
+**That last row is the point of the envelope.** An empty finding list has two
+incompatible meanings — "the bytes were sound" and "there were no bytes to look at" —
+and reporting a bare list would make the second read as the first: a confident verdict
+drawn from missing data. So the trace always says how many calls it saw, how many
+carried logprobs, and whether it is `conclusive`. surya asks for logprobs by default
+(`SURYA_INFERENCE_LOGPROBS = True`) and coderai's gateway forwards the body verbatim
+precisely so they survive, but an upstream that declines them is a configuration rather
+than a bug, and has to be visible instead of silently inverted into a conclusion.
+
+The scan reassembles
 the whole byte stream and walks it as UTF-8 rather than judging tokens one at a time: per
 token is meaningless here, since one character legitimately arrives as two or three
 tokens and each is "invalid" alone. The first version of this did judge them pairwise and

tests/test_ocr_token_trace.py

@@ -204,7 +204,7 @@ def test_a_clean_page_reports_nothing(monkeypatch):
     from PIL import Image
     page = w.ocr(Image.new("RGB", (10, 10)))
     # ...but the text has no marker, so there is nothing to diagnose.
-    assert "unreadable_tokens" not in page
+    assert "unreadable_trace" not in page
 
 
 def test_an_unreadable_page_carries_the_diagnosis(monkeypatch):
@@ -220,7 +220,9 @@ def test_an_unreadable_page_carries_the_diagnosis(monkeypatch):
                              "confidence": 0.4, "reading_order": 0}]}]
     monkeypatch.setattr(w, "_predict", _predict)
     page = w.ocr(Image.new("RGB", (10, 10)))
-    assert page["unreadable_tokens"][0]["token"]["bytes"] == [195]
+    tr = page["unreadable_trace"]
+    assert tr["conclusive"] is True
+    assert tr["findings"][0]["token"]["bytes"] == [195]
 
 
 def test_the_trace_is_cleared_between_pages(monkeypatch):
@@ -244,7 +246,7 @@ def test_analysis_failure_is_reported_not_raised(monkeypatch):
 
     def _boom(calls):
         raise RuntimeError("bad trace")
-    monkeypatch.setattr(W, "_trace_findings", _boom)
+    monkeypatch.setattr(W, "_trace_report", _boom)
 
     def _predict(img):
         return [{"blocks": [{"html": "<p>pi�</p>",
@@ -252,26 +254,89 @@ def test_analysis_failure_is_reported_not_raised(monkeypatch):
                              "confidence": 0.4, "reading_order": 0}]}]
     monkeypatch.setattr(w, "_predict", _predict)
     page = w.ocr(Image.new("RGB", (10, 10)))
-    assert "bad trace" in page["unreadable_tokens"][0]["error"]
+    tr = page["unreadable_trace"]
+    assert tr["conclusive"] is False
+    assert "bad trace" in tr["why"]
+
+
+# --------------------------------------------- an empty trace must not lie #
+def test_missing_logprobs_are_not_reported_as_a_clean_byte_stream():
+    """The flaw this envelope exists to close. With no logprobs there is nothing
+    to analyse, and a bare empty finding list would read as "the bytes were sound,
+    so the serving path is at fault" — a confident verdict from missing data."""
+    rep = W._trace_report([None, None])
+    assert rep["findings"] == []
+    assert rep["conclusive"] is False
+    assert "no logprobs" in rep["why"]
+    assert rep["with_logprobs"] == 0
+
+
+def test_a_hook_that_never_fired_is_also_inconclusive():
+    rep = W._trace_report([])
+    assert rep["conclusive"] is False
+    assert "did not" in rep["why"]
+
+
+def test_a_sound_stream_with_logprobs_present_blames_the_serving_path():
+    """The real verdict, and the one that would mean we can fix this ourselves."""
+    rep = W._trace_report([[_tok("p", [112]), _tok("\u00c3", [195]),
+                            _tok("\u0142", [160])]])
+    assert rep["conclusive"] is True
+    assert rep["findings"] == []
+    assert "serving path" in rep["why"]
+
+
+def test_a_broken_stream_blames_the_model():
+    rep = W._trace_report([[_tok("p", [112]), _tok("\u00c3", [195])]])
+    assert rep["conclusive"] is True
+    assert rep["kinds"] == {"invalid-utf8": 1}
+    assert "the model is at fault" in rep["why"]
+
+
+def test_the_envelope_always_counts_the_calls():
+    rep = W._trace_report([None, [_tok("a", [97])], []])
+    assert rep["calls"] == 3
+    assert rep["with_logprobs"] == 1
 
 
 # ------------------------------------------------------ and out through the API #
-def test_the_report_carries_the_tokens():
+def test_the_report_carries_the_trace():
     from codai.ocr.base import OcrPage, OcrLine, unreadable_report
-    page = OcrPage(index=0, text="pi�")
-    page.lines = [OcrLine(text="pi�", bbox=[0, 0, 1, 1], conf=0.4)]
-    page.meta["unreadable_tokens"] = [{"index": 2, "token": {"bytes": [195]},
-                                       "why": "x", "context": []}]
+    page = OcrPage(index=0, text="pi\ufffd")
+    page.lines = [OcrLine(text="pi\ufffd", bbox=[0, 0, 1, 1], conf=0.4)]
+    page.meta["unreadable_trace"] = {
+        "calls": 1, "with_logprobs": 1, "conclusive": True,
+        "findings": [{"index": 2, "kind": "invalid-utf8",
+                      "token": {"bytes": [195]}, "why": "x", "context": []}],
+    }
     rep = unreadable_report([page])
     assert rep["count"] == 1
-    assert rep["tokens"][0]["token"]["bytes"] == [195]
-    assert rep["tokens"][0]["page"] == 0
+    assert rep["trace"][0]["findings"][0]["token"]["bytes"] == [195]
+    assert rep["trace"][0]["page"] == 0
+    assert rep["blame"] == "the model"
+
+
+def test_the_blame_line_says_the_serving_path_when_the_bytes_were_sound():
+    from codai.ocr.base import OcrPage, unreadable_report
+    page = OcrPage(index=0, text="pi\ufffd")
+    page.meta["unreadable_trace"] = {"calls": 1, "with_logprobs": 1,
+                                     "conclusive": True, "findings": []}
+    assert unreadable_report([page])["blame"] == "the serving path"
+
+
+def test_the_blame_line_says_unknown_when_the_trace_could_not_be_read():
+    """Rather than defaulting to either culprit."""
+    from codai.ocr.base import OcrPage, unreadable_report
+    page = OcrPage(index=0, text="pi\ufffd")
+    page.meta["unreadable_trace"] = {"calls": 2, "with_logprobs": 0,
+                                     "conclusive": False, "findings": []}
+    assert unreadable_report([page])["blame"] == "unknown"
 
 
 def test_a_clean_document_has_no_report_at_all():
     from codai.ocr.base import OcrPage, unreadable_report
     page = OcrPage(index=0, text="tutto bene")
-    page.meta["unreadable_tokens"] = [{"anything": 1}]
+    page.meta["unreadable_trace"] = {"conclusive": True, "findings": []}
     assert unreadable_report([page]) == {}
 
 
@@ -279,18 +344,18 @@ def test_the_trace_is_not_repeated_on_every_page():
     """It is reported once, document-wide. Repeating it per page would put the
     same findings in the response twice."""
     from codai.ocr.base import OcrPage
-    page = OcrPage(index=0, text="pi�")
-    page.meta["unreadable_tokens"] = [{"index": 2}]
+    page = OcrPage(index=0, text="pi\ufffd")
+    page.meta["unreadable_trace"] = {"findings": [{"index": 2}]}
     page.meta["primary_language"] = "it"
     d = page.to_dict()
-    assert "unreadable_tokens" not in d.get("meta", {})
+    assert "unreadable_trace" not in d.get("meta", {})
     assert d["meta"]["primary_language"] == "it"      # other metadata survives
 
 
 def test_a_page_whose_only_meta_was_the_trace_reports_no_meta():
     from codai.ocr.base import OcrPage
     page = OcrPage(index=0, text="x")
-    page.meta["unreadable_tokens"] = [{"index": 1}]
+    page.meta["unreadable_trace"] = {"findings": [{"index": 1}]}
     assert "meta" not in page.to_dict()