surya reports a page confidence, and whether anything measured it (v0.4.15)

chandra and olmOCR each set `meta.confidence` and `meta.confidence_measured`.
surya set neither: its predictor reports confidence per LINE, so a caller
switching engines was handed a different kind of answer, and nothing could tell
"no line reported a confidence" from "the model was certain it was unsure".

Now it reports both. Weighted by CHARACTERS rather than per line, because the
other two average over tokens -- that is, over text -- and a plain line-mean
lets a two-word line count as much as a full one: on
[(100 chars, 0.95), (2 chars, 0.1)] it answers 0.9333 instead of 0.525.

`confidence_measured` is the half that earns its place, and the reason the
aggregate reads the RAW value rather than the per-line field: that field has to
coerce a missing confidence to 0.0 for its own sake, which would make

    nothing reported  -> 0.0
    a real zero       -> 0.0

indistinguishable. One such 0.0 has already produced a confidently wrong
verdict in this subsystem. So:

    nothing reported  -> (0.0, False)
    a real zero       -> (0.0, True)
    mixed             -> unreported lines skipped, never counted as zero

Both keys reach the caller with no new plumbing: `_page_from_dict` forwards
unknown keys by EXCLUSION (a choice made after that reassembly silently dropped
`accents_repaired`, and then `model_attempts`), and both are already in the
pod->orchestrator whitelist, so a page read on a rented pod reports what a
local one does. Tested, including that `False` is not mistaken for the empty
value that exclusion filter skips.

The aggregate is extracted as `_page_confidence()` so it is testable without
surya installed.

Worth recording what this is NOT: on the page where the accent bug was found,
surya reported ~96 % per line while every accent was a replacement character --
confident, and right to be, having emitted the correct tokens. Confidence is
not a corruption detector; the byte trace is.

13 new tests. 3447 passed, 13 pre-existing failures (broker, studio, manual
client) untouched.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KnfkxBpcZ1v6ypRFFyhrVM

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.4.14"
+__version__ = "0.4.15"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/vllm_worker.py

@@ -298,10 +298,17 @@ def _launch_cmd(py, cfg, host: str, port: int, model_path: str,
     return out
 
 
-#: Served names (and path fragments) whose vocabulary is BYTE-level, so a lost
-#: character can only be recovered from token ids. Matched loosely because a
-#: model arrives here as a repo id, a served name or a local path.
-_BYTE_LEVEL_OCR = ("surya", "chandra", "olmocr")
+#: Served names (and path fragments) that NEED token ids: a byte-level
+#: vocabulary whose lost characters can only be recovered from them, AND a
+#: repair that actually consumes them.
+#:
+#: surya only, deliberately. chandra and olmOCR are served by coderai's own
+#: client, read their logprobs for confidence (`mean_token_prob`, a VALUE) and
+#: have no id-based repair, so for them the flag would change the `token`
+#: string that nothing reads -- and chandra demonstrably does not have the
+#: fault: on the page where surya returned 25 U+FFFD and zero 0xC3 bytes,
+#: chandra returned 40 0xC3 bytes with every accent intact.
+_BYTE_LEVEL_OCR = ("surya",)
 
 
 def _wants_token_ids(cfg, served_name: str = "", model_path: str = "") -> bool:

codai/ocr/workers/ocr_worker.py

@@ -458,6 +458,37 @@ class _LastMime(threading.local):
 _LAST_MIME = _LastMime()
 
 
+def _page_confidence(confs):
+    """``(confidence, measured)`` for a page, from its per-line confidences.
+
+    chandra and olmOCR report one page number, the mean of ``exp(logprob)`` over
+    TOKENS. surya's predictor reports per LINE instead, so the comparable
+    aggregate weights each line by its CHARACTERS: a two-word line must not
+    count as much as a full one, or a page of mostly long confident lines is
+    dragged down by a single short uncertain one.
+
+    ``measured`` is the half that matters. A line whose confidence was never
+    reported arrives here as ``None``, and coercing that to 0.0 -- which the
+    per-line code does, by necessity, for its own field -- would be
+    indistinguishable from a model that really was certain it was unsure. One
+    such 0.0 has already produced a confidently wrong verdict in this
+    subsystem. So a page where NOTHING reported a confidence says so, rather
+    than claiming zero.
+    """
+    measured = [(int(n or 0), float(c)) for n, c in (confs or [])
+                if isinstance(c, (int, float))]
+    if not measured:
+        return 0.0, False
+    chars = sum(n for n, _ in measured)
+    if chars > 0:
+        score = sum(n * c for n, c in measured) / chars
+    else:
+        # Every line empty: fall back to a plain mean rather than divide by
+        # zero, because the confidences themselves are still real.
+        score = sum(c for _, c in measured) / len(measured)
+    return round(float(score), 4), True
+
+
 def _oc_repair_on() -> bool:
     """Whether the accent repair is active in this process, for reporting."""
     try:
@@ -1675,6 +1706,13 @@ class SuryaWorker:
         res = preds[0] if preds else None
         lines = []
         tl = getattr(res, "text_lines", None)
+        #: ``(characters, confidence-as-reported)`` per line. The RAW value, not
+        #: the coerced one: ``float(conf or 0.0)`` turns "not reported" into a
+        #: confident-looking zero, and a 0.0 that means "nothing measured this"
+        #: is indistinguishable from one that means "the model was certain it
+        #: was unsure" -- a confusion that has already produced one confidently
+        #: wrong verdict in this subsystem.
+        _confs = []
         if tl is None and isinstance(res, dict):
             tl = res.get("text_lines")
         if tl:
@@ -1691,6 +1729,7 @@ class SuryaWorker:
                 if len(bb) < 4:
                     bb = [0.0, 0.0, 0.0, 0.0]
                 lines.append({"text": str(text), "bbox": bb, "conf": float(conf or 0.0)})
+                _confs.append((len(str(text)), conf))
         else:
             # Surya2 (>=0.20): result.blocks[].{html,polygon,confidence,label,reading_order}
             blocks = getattr(res, "blocks", None)
@@ -1709,9 +1748,23 @@ class SuryaWorker:
                     conf = b.get("confidence", conf)
                 lines.append({"text": text, "bbox": _poly_to_xyxy(poly or [0, 0, 0, 0]),
                               "conf": float(conf or 0.0)})
+                _confs.append((len(text), conf))
         page = {"index": 0, "width": img.width, "height": img.height,
                 "lines": lines, "regions": [], "tables": [],
                 "text": "\n".join(l["text"] for l in lines)}
+        # A PAGE-LEVEL confidence, so surya reports the same shape chandra and
+        # olmOCR do and a caller switching engines is not handed a different
+        # kind of answer. Theirs is the mean of exp(logprob) over TOKENS;
+        # surya's predictor reports per LINE instead, so the comparable
+        # aggregate is weighted by characters -- a two-word line must not count
+        # as much as a full one.
+        #
+        # `confidence_measured` is the half that matters: it is false when NO
+        # line reported a confidence, which is the only honest way to tell that
+        # case apart from a page the model really was unsure about. The
+        # per-line numbers stay exactly as they were; this only adds the page
+        # summary.
+        page["confidence"], page["confidence_measured"] = _page_confidence(_confs)
         # Only when something actually went unread: a clean page stays byte-identical
         # to what it has always returned.
         if repairs:

tests/test_ocr_surya_confidence.py

+"""A page-level confidence for surya, in the shape the other engines use.
+
+chandra and olmOCR each set `meta.confidence` and `meta.confidence_measured`
+from the mean of `exp(logprob)` over tokens. surya set NEITHER: its predictor
+reports confidence per LINE, so a caller switching engines got a different kind
+of answer, and nothing could tell "no line reported a confidence" from "the
+model was certain it was unsure".
+
+That distinction is not academic. On the night the accent bug was found, surya
+reported ~96 % per line while every accent in the text was a replacement
+character -- the model was confident and RIGHT to be, because it emitted the
+correct tokens and the loss happened after it. A 0.0 that means "nothing
+measured this" being read as a verdict has already produced one confidently
+wrong conclusion in this subsystem.
+"""
+
+import pytest
+
+from codai.ocr.workers.ocr_worker import _page_confidence
+
+
+# ------------------------------------------------------------- the aggregate #
+
+def test_nothing_reported_is_not_a_confidence_of_zero():
+    """The case the flag exists for."""
+    assert _page_confidence([(10, None), (20, None)]) == (0.0, False)
+    assert _page_confidence([]) == (0.0, False)
+    assert _page_confidence(None) == (0.0, False)
+
+
+def test_a_real_zero_is_reported_as_measured():
+    """Indistinguishable from the above without the second value."""
+    assert _page_confidence([(10, 0.0)]) == (0.0, True)
+
+
+def test_it_is_weighted_by_characters_not_by_lines():
+    """chandra's and olmOCR's number is a mean over TOKENS, i.e. over text. A
+    plain mean over lines would let a two-word line count as much as a full
+    one."""
+    weighted = _page_confidence([(100, 0.95), (2, 0.1)])[0]
+    plain_mean = (0.95 + 0.1) / 2
+    assert weighted > plain_mean
+    assert weighted == pytest.approx((100 * 0.95 + 2 * 0.1) / 102, abs=1e-4)
+
+
+def test_lines_without_a_confidence_are_skipped_not_counted_as_zero():
+    assert _page_confidence([(10, 0.8), (10, None)]) == (0.8, True)
+
+
+def test_empty_lines_fall_back_to_a_plain_mean():
+    """Zero characters everywhere must not divide by zero, and the
+    confidences themselves are still real."""
+    assert _page_confidence([(0, 0.5), (0, 0.9)]) == (0.7, True)
+
+
+def test_a_uniform_page_reports_that_value():
+    assert _page_confidence([(10, 0.9), (30, 0.9)]) == (0.9, True)
+
+
+def test_strings_and_rubbish_are_not_confidences():
+    assert _page_confidence([(10, "0.9"), (5, None), (5, {})]) == (0.0, False)
+
+
+# ---------------------------------------------- it reaches the page and meta #
+
+def test_the_worker_sets_both_keys_on_the_page():
+    import inspect
+
+    from codai.ocr.workers.ocr_worker import SuryaWorker
+    src = inspect.getsource(SuryaWorker.ocr)
+    assert 'page["confidence"], page["confidence_measured"] = _page_confidence(' in src
+
+
+def test_the_raw_value_is_collected_not_the_coerced_one():
+    """`float(conf or 0.0)` is what the per-line field needs and what would
+    destroy the distinction if the aggregate read it."""
+    import inspect
+
+    from codai.ocr.workers.ocr_worker import SuryaWorker
+    src = inspect.getsource(SuryaWorker.ocr)
+    assert "_confs.append((len(str(text)), conf))" in src
+    assert "_confs.append((len(text), conf))" in src
+
+
+def test_both_keys_survive_the_reassembly_into_meta():
+    from codai.ocr.subprocess_engine import SubprocessOcrEngine
+    page = SubprocessOcrEngine._page_from_dict({
+        "index": 0, "text": "ciao", "width": 10, "height": 10,
+        "lines": [{"text": "ciao", "bbox": [0, 0, 1, 1], "conf": 0.9}],
+        "confidence": 0.9, "confidence_measured": True,
+    })
+    assert page.meta["confidence"] == 0.9
+    assert page.meta["confidence_measured"] is True
+
+
+def test_a_false_measured_flag_is_not_swallowed_as_empty():
+    """The reassembly skips keys whose value is None/""/[]/{} -- False must not
+    be mistaken for one of those, or the honest case disappears."""
+    from codai.ocr.subprocess_engine import SubprocessOcrEngine
+    page = SubprocessOcrEngine._page_from_dict({
+        "index": 0, "text": "x", "width": 1, "height": 1, "lines": [],
+        "confidence": 0.0, "confidence_measured": False,
+    })
+    assert page.meta.get("confidence_measured") is False
+    assert page.meta.get("confidence") == 0.0
+
+
+def test_the_keys_cross_the_pod_boundary():
+    """A page read on a rented pod must report the same shape as a local one."""
+    import inspect
+
+    from codai.ocr import manager
+    src = inspect.getsource(manager._page_from_pod)
+    assert "confidence" in src and "confidence_measured" in src
+
+
+def test_all_three_engines_now_report_the_same_shape():
+    import inspect
+
+    from codai.ocr import chandra, olmocr
+    for mod in (chandra, olmocr):
+        src = inspect.getsource(mod)
+        assert "confidence_measured" in src
+    from codai.ocr.workers import ocr_worker
+    assert "confidence_measured" in inspect.getsource(ocr_worker.SuryaWorker.ocr)