runpod: let a pod serve the VLM OCR engines it was configured for (v0.2.29)

A pod asked for surya or olmOCR was downgraded to docTR UNCONDITIONALLY. The reasoning
was sound — both are VLMs needing a model server, locally they ride coderai's own vLLM
engine, and a pod image will not get one (that venv alone is ~10.5 GB against a ~12 GB
ceiling, and it would be a second torch beside the one docTR and Surya already need) —
but the conclusion was too broad. In their SERVED modes these engines only speak HTTP,
and PIL and requests are already in the image. So a pod can run them whenever the
operator has pointed them at a reachable endpoint, including at a vLLM pod, which is the
composition to prefer over a fat image.

- _ocr_external_server(engine) decides it: olmocr_serve="server" + olmocr_server_url, or
  surya_serve="llamacpp" + surya_server_url. The local-only modes ("vllm", "model") still
  downgrade, and the log now names the setting to fix instead of only the symptom.
- The mode, URL and model ids are seeded to the pod. config.py gained the env overrides
  for surya_serve/surya_server_url and fallback_engines; only olmOCR's existed.
- fallback_engines is pinned to the engines the image actually carries: "auto" would hand
  a failed page to an engine with no server on that pod, which the fallback added in
  0.2.28 made newly reachable.
- Engine selection is alias-aware (`alias or served or path`, the convention used
  throughout the manager). An entry aliased "olmocr" over an HF repo path was read as an
  unknown engine and silently downgraded. Aliases are how one model carries several
  configs and how several models share an endpoint, so this path has to honour them.
- The licence decision now survives a downgrade: `wanted` was reassigned to "doctr"
  before the `wanted == "surya"` check, making that branch's own documented intent
  ("asking for surya by name IS the decision") unreachable.

olmOCR needs no pip dependency anywhere — it imports only PIL and requests — which is
why nothing flagged its absence from the profiles; ocr.txt now records that and why a
pod has no vLLM.

19 new tests; 278 pass across the OCR, front-gate, vLLM-share, cluster, remote-execution,
placement and routing suites.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.28"
+__version__ = "0.2.29"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -886,6 +886,39 @@ def _surya_accepted() -> bool:
         return False
 
 
+def _ocr_cfg():
+    """This install's OcrConfig, or None. The pod has no config of its own — every
+    OCR setting it runs with is seeded from here."""
+    try:
+        from codai.admin.routes import config_manager
+        return getattr(getattr(config_manager, "config", None), "ocr", None)
+    except Exception:
+        return None
+
+
+def _ocr_external_server(engine: str):
+    """The off-box OpenAI-compatible endpoint configured for a VLM OCR engine, or "".
+
+    This is what decides whether a pod can actually serve surya/olmOCR. Both are VLMs
+    and need a model server: locally they ride coderai's own vLLM, which a pod has
+    none of, and baking one in would mean a SECOND torch in the image (the vLLM venv
+    alone is ~10.5 GB against a ~12 GB ceiling). But in the served modes the engine
+    only speaks HTTP — PIL and requests, both already in the image — so a pod CAN run
+    them when the operator has pointed them at a server, including at a vLLM pod."""
+    o = _ocr_cfg()
+    if o is None:
+        return ""
+    if engine == "olmocr":
+        mode = str(getattr(o, "olmocr_serve", "") or "").strip().lower()
+        url = str(getattr(o, "olmocr_server_url", "") or "").strip()
+        return url if (mode == "server" and url) else ""
+    if engine == "surya":
+        mode = str(getattr(o, "surya_serve", "") or "").strip().lower()
+        url = str(getattr(o, "surya_server_url", "") or "").strip()
+        return url if (mode == "llamacpp" and url) else ""
+    return ""
+
+
 def _plan(engine, image, args, mcfg, api_key, entry, served,
           seed_entries: list = None, capability: str = "") -> dict:  # noqa: D401
     """Finish a plan for a coderai pod: auth plus the models it must serve."""
@@ -961,19 +994,54 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
 
     if cap == "ocr":
         env["CODERAI_OCR_ENABLED"] = "1"
-        wanted = str((entry or {}).get("path") or "").strip().lower()
+        # Which engine this pod is for. An entry is routable by its ALIAS before its
+        # path (the `alias or path or id` convention used throughout the manager), and
+        # aliases are how one model carries several configs and how several models sit
+        # behind one endpoint — so an aliased OCR entry must not be misread as "no
+        # engine named that" and silently downgraded. Take the first candidate that
+        # names a real engine; fall back to the path, as before.
+        _cands = [str((entry or {}).get("alias") or "").strip().lower(),
+                  str(served or "").strip().lower(),
+                  str((entry or {}).get("path") or "").strip().lower()]
+        wanted = next((c for c in _cands if c in ("paddle", "doctr", "surya", "olmocr")),
+                      _cands[-1])
+        # Remember what was ASKED for: the licence decision below is made by asking
+        # for surya by name, and that has to survive a downgrade to docTR.
+        _asked = wanted
         if wanted in ("surya", "olmocr"):
-            # Surya 0.22 is a VLM ("surya-ocr-2") and olmOCR-2 is one outright,
-            # not a self-contained OCR library: both need a server — vLLM, which
-            # surya spawns IN DOCKER (a pod cannot), or llama-server on a GGUF,
-            # which the image does not ship. Locally they ride coderai's own vLLM
-            # engine; a pod has none. So a pod asked for either serves the request
-            # with docTR, which is in the image and has passed — rather than
-            # loading it, reaching inference, and dying on "docker binary not
-            # found". (An olmOCR pod would want olmocr_serve="server" pointed at a
-            # model server it can actually reach; nothing provisions one here.)
-            print(f"[runpod] {wanted} on a pod needs a VLM server the image does not "
-                  "carry — this OCR request will be served by docTR", flush=True)
+            # Surya 0.22 is a VLM ("surya-ocr-2") and olmOCR-2 is one outright, not a
+            # self-contained OCR library: both need a model server. Locally they ride
+            # coderai's own vLLM engine, which a pod has none of, and baking one in
+            # would put a SECOND torch in the image (the vLLM venv alone is ~10.5 GB
+            # against a ~12 GB ceiling).
+            #
+            # But in their SERVED modes these engines only speak HTTP, and PIL and
+            # requests are already in the image. So when the operator has pointed the
+            # engine at a reachable endpoint — including at a vLLM pod, which is the
+            # composition this is for — the pod runs the engine it was asked for, and
+            # the settings are seeded below. Only with nothing configured does it fall
+            # back, because loading a VLM with no server reaches inference and dies.
+            _srv = _ocr_external_server(wanted)
+            if _srv:
+                _o = _ocr_cfg()
+                if wanted == "olmocr":
+                    env["CODERAI_OCR_OLMOCR_SERVE"] = "server"
+                    env["CODERAI_OCR_OLMOCR_SERVER_URL"] = _srv
+                    for _v, _a in (("CODERAI_OCR_OLMOCR_MODEL", "olmocr_model"),
+                                   ("CODERAI_OCR_OLMOCR_MODEL_ID", "olmocr_model_id")):
+                        _val = str(getattr(_o, _a, "") or "").strip()
+                        if _val:
+                            env[_v] = _val
+                else:
+                    env["CODERAI_OCR_SURYA_SERVE"] = "llamacpp"
+                    env["CODERAI_OCR_SURYA_SERVER_URL"] = _srv
+                print(f"[runpod] {wanted} on a pod will use the configured server at "
+                      f"{_srv} (the image carries no VLM server of its own)", flush=True)
+            else:
+                print(f"[runpod] {wanted} on a pod needs a VLM server the image does "
+                      f"not carry and none is configured (set "
+                      f"ocr.{wanted}_server_url and the served mode) — this OCR "
+                      f"request will be served by docTR", flush=True)
                 wanted = "doctr"
         if wanted not in ("paddle", "doctr", "surya", "olmocr"):
             # The pod images ship docTR, the only engine that runs in-process.
@@ -987,7 +1055,14 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
             # CUDA runtime. Point the engine at that interpreter rather than
             # letting it try to build a venv on a rented machine.
             env["CODERAI_OCR_PADDLE_VENV"] = "/opt/coderai/venvs/paddleocr"
-        if wanted == "surya" or _surya_accepted():
+        # The fallback chain (ocr.fallback_engines) must not name an engine this image
+        # cannot serve: "auto" would send a failed page to surya/olmocr, which on a pod
+        # have no server unless one was seeded just above. Pin it to what is present.
+        _fb = ["doctr"] if wanted != "paddle" else ["paddle", "doctr"]
+        if wanted not in _fb:
+            _fb.insert(0, wanted)
+        env["CODERAI_OCR_FALLBACK_ENGINES"] = ",".join(_fb)
+        if _asked == "surya" or _surya_accepted():
             # Surya is GPL and gated on an explicit acceptance. The pod inherits
             # the decision made HERE — it cannot make it for itself, and asking
             # for surya by name is that decision.

codai/config.py

@@ -1552,10 +1552,16 @@ class ConfigManager:
                     setattr(self.config.ocr, f"{name}_venv", venv)
             # olmOCR needs to be told WHERE its model is; a pod that carries no vLLM
             # is given a server URL (or the id of a vision model it does hold).
+            # Surya's served mode is seeded the same way, and so is the fallback
+            # chain: on a pod "auto" would hand a failed page to an engine the image
+            # cannot serve, so the renting install pins it to what is present.
             for _env, _attr in (("CODERAI_OCR_OLMOCR_SERVE", "olmocr_serve"),
                                 ("CODERAI_OCR_OLMOCR_MODEL", "olmocr_model"),
                                 ("CODERAI_OCR_OLMOCR_MODEL_ID", "olmocr_model_id"),
-                                ("CODERAI_OCR_OLMOCR_SERVER_URL", "olmocr_server_url")):
+                                ("CODERAI_OCR_OLMOCR_SERVER_URL", "olmocr_server_url"),
+                                ("CODERAI_OCR_SURYA_SERVE", "surya_serve"),
+                                ("CODERAI_OCR_SURYA_SERVER_URL", "surya_server_url"),
+                                ("CODERAI_OCR_FALLBACK_ENGINES", "fallback_engines")):
                 _val = (_os.environ.get(_env) or "").strip()
                 if _val:
                     setattr(self.config.ocr, _attr, _val)

docs/ocr.md

@@ -166,6 +166,26 @@ of 0.9 (or 0.6) can be more than the card has left. Otherwise: give the card les
 resident work, or serve the engine with `model`/`server` mode, which needs no instance of
 its own at all.
 
+### On a rented pod
+
+A pod image carries docTR and Surya in its venv and PaddleOCR in a baked one
+(`packaging/runpod/profiles/ocr.txt`, `ocr-paddle.*`). What it does **not** carry is a
+model server: Surya-2 and olmOCR-2 are VLMs, and locally they ride coderai's own vLLM
+engine. Baking that in would mean a second torch in the image — the vLLM venv alone is
+~10.5 GB against a ~12 GB ceiling — so it is deliberately absent.
+
+Those engines are still remotizable, through their **served** modes, which need nothing
+but HTTP: set `ocr.olmocr_serve = "server"` with `olmocr_server_url`, or
+`ocr.surya_serve = "llamacpp"` with `surya_server_url`, pointing at an endpoint the pod
+can reach — a vLLM pod, for instance. The renting install seeds the mode, the URL and the
+model ids to the pod, and pins `fallback_engines` to the engines the image actually has,
+so a failed page is never handed to one that cannot run there. Ask for a VLM engine with
+no server configured and the pod says so in its log and serves the request with docTR
+instead of dying at inference.
+
+Engine selection on a pod follows the usual `alias or path or id`, so an entry aliased
+`olmocr` over an HF repo path is recognised.
+
 ### A document is never refused because one engine is down
 
 `ocr.fallback_engines` (default `auto`) makes a failing engine hand the document to the

packaging/runpod/profiles/ocr.txt

@@ -27,3 +27,13 @@ jsonschema>=4.0.0
 # SuryaEngine still runs as a subprocess (it is a SubprocessOcrEngine), so the
 # pod points ocr.surya_venv at this interpreter rather than building one.
 surya-ocr==0.22.1
+
+# olmOCR-2 needs NOTHING here. It is a document VLM, not a library: it imports only
+# PIL and requests, and reaches its model one of three ways — coderai's own model
+# manager ("model"), coderai's vLLM engine ("vllm"), or any OpenAI-compatible
+# endpoint ("server"). A pod has no vLLM of its own and will not get one: that venv
+# is ~10.5 GB against a ~12 GB image ceiling, and it would be a second torch beside
+# the one docTR and Surya already need here. So on a pod olmOCR runs in "server"
+# mode against an endpoint the renting install configures — a vLLM pod, for
+# instance. runpod_worker seeds olmocr_serve/olmocr_server_url when they are set and
+# falls back to docTR only when they are not.

tests/test_runpod_ocr_remotization.py

+"""A rented pod must be able to serve the OCR engines this install is configured with.
+
+Surya-2 and olmOCR-2 are VLMs: locally they ride coderai's own vLLM engine, which a pod
+has none of, and a pod image will not get one (that venv is ~10.5 GB against a ~12 GB
+ceiling, and it would be a second torch beside the one docTR and Surya already need).
+A pod asked for either therefore used to be downgraded to docTR UNCONDITIONALLY — even
+when the operator had pointed the engine at a reachable endpoint, which is all the served
+modes need (PIL + requests, both already in the image).
+
+So: keep the configured engine when a server is configured, seed it, and pin the fallback
+chain to engines the image can actually serve.
+"""
+
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from codai.api import runpod_worker as rw
+from codai.config import OcrConfig
+
+
+@pytest.fixture
+def ocr_cfg(monkeypatch):
+    """Point runpod_worker's config accessor at an OcrConfig we control."""
+    holder = {}
+
+    def _set(**kw):
+        holder["cfg"] = OcrConfig(**kw)
+        return holder["cfg"]
+
+    monkeypatch.setattr(rw, "_ocr_cfg", lambda: holder.get("cfg"))
+    monkeypatch.setattr(rw, "_surya_accepted",
+                        lambda: bool(getattr(holder.get("cfg"), "surya_accept_license", False)))
+    return _set
+
+
+# ------------------------------------------------------ the server-URL decision
+def test_olmocr_in_server_mode_is_servable_on_a_pod(ocr_cfg):
+    ocr_cfg(olmocr_serve="server", olmocr_server_url="https://api.example/v1")
+    assert rw._ocr_external_server("olmocr") == "https://api.example/v1"
+
+
+def test_olmocr_in_vllm_mode_is_not(ocr_cfg):
+    """vllm mode means coderai's own engine — the thing a pod does not have."""
+    ocr_cfg(olmocr_serve="vllm", olmocr_server_url="https://api.example/v1")
+    assert rw._ocr_external_server("olmocr") == ""
+
+
+def test_olmocr_in_model_mode_is_not(ocr_cfg):
+    ocr_cfg(olmocr_serve="model")
+    assert rw._ocr_external_server("olmocr") == ""
+
+
+def test_server_mode_without_a_url_is_not(ocr_cfg):
+    ocr_cfg(olmocr_serve="server", olmocr_server_url="")
+    assert rw._ocr_external_server("olmocr") == ""
+
+
+def test_surya_against_an_external_server_is_servable(ocr_cfg):
+    ocr_cfg(surya_serve="llamacpp", surya_server_url="http://box:8080/v1")
+    assert rw._ocr_external_server("surya") == "http://box:8080/v1"
+
+
+def test_surya_on_the_local_vllm_is_not(ocr_cfg):
+    ocr_cfg(surya_serve="vllm")
+    assert rw._ocr_external_server("surya") == ""
+
+
+def test_an_engine_that_needs_no_server_has_none(ocr_cfg):
+    ocr_cfg()
+    assert rw._ocr_external_server("paddle") == ""
+    assert rw._ocr_external_server("doctr") == ""
+
+
+def test_no_config_at_all_is_tolerated(monkeypatch):
+    monkeypatch.setattr(rw, "_ocr_cfg", lambda: None)
+    assert rw._ocr_external_server("olmocr") == ""
+
+
+# ------------------------------------------------------ what the pod is told
+class _Mcfg:
+    """A RunpodModelConfig stand-in: _plan reads a handful of fields off it and we
+    care about none of them here, so anything unset reads as None."""
+
+    def __getattr__(self, _name):
+        return None
+
+
+def _seed(entry_path: str, alias: str = "", served: str = None) -> dict:
+    """Run the OCR branch of the pod seeder and return the env it produced."""
+    entry = {"path": entry_path}
+    if alias:
+        entry["alias"] = alias
+    return rw._plan(engine="", image="img", args={}, mcfg=_Mcfg(),
+                    api_key="k", entry=entry,
+                    served=(entry_path if served is None else served),
+                    capability="ocr").get("env", {})
+
+
+def test_a_configured_olmocr_pod_keeps_olmocr_and_is_told_where_to_look(ocr_cfg):
+    ocr_cfg(olmocr_serve="server", olmocr_server_url="https://api.example/v1",
+            olmocr_model_id="olmocr-2")
+    env = _seed("olmocr")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "olmocr"
+    assert env["CODERAI_OCR_OLMOCR_ENABLED"] == "1"
+    assert env["CODERAI_OCR_OLMOCR_SERVE"] == "server"
+    assert env["CODERAI_OCR_OLMOCR_SERVER_URL"] == "https://api.example/v1"
+    assert env["CODERAI_OCR_OLMOCR_MODEL_ID"] == "olmocr-2"
+
+
+def test_an_unconfigured_olmocr_pod_still_falls_back_to_doctr(ocr_cfg):
+    ocr_cfg(olmocr_serve="vllm")          # the local-only mode
+    env = _seed("olmocr")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "doctr"
+    assert "CODERAI_OCR_OLMOCR_SERVER_URL" not in env
+
+
+def test_a_configured_surya_pod_keeps_surya(ocr_cfg):
+    ocr_cfg(surya_serve="llamacpp", surya_server_url="http://box:8080/v1",
+            surya_accept_license=True)
+    env = _seed("surya")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "surya"
+    assert env["CODERAI_OCR_SURYA_SERVE"] == "llamacpp"
+    assert env["CODERAI_OCR_SURYA_SERVER_URL"] == "http://box:8080/v1"
+    assert env["CODERAI_OCR_SURYA_ACCEPT_LICENSE"] == "1"
+
+
+def test_the_licence_decision_is_still_made_by_the_renting_install(ocr_cfg):
+    ocr_cfg(surya_serve="vllm")            # no server -> downgraded
+    env = _seed("surya")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "doctr"
+    # Asking for surya by name IS the licence decision, downgrade or not.
+    assert env["CODERAI_OCR_SURYA_ACCEPT_LICENSE"] == "1"
+
+
+# ------------------------------------------------------ the fallback chain
+def test_the_fallback_chain_only_names_engines_the_image_has(ocr_cfg):
+    """"auto" would hand a failed page to an engine with no server on a pod."""
+    ocr_cfg(olmocr_serve="server", olmocr_server_url="https://api.example/v1")
+    chain = _seed("olmocr")["CODERAI_OCR_FALLBACK_ENGINES"].split(",")
+    assert chain[0] == "olmocr"            # the engine asked for leads
+    assert "doctr" in chain                # and docTR is in the image
+    assert "surya" not in chain and "paddle" not in chain
+
+
+def test_a_paddle_pod_falls_back_to_paddle_then_doctr(ocr_cfg):
+    ocr_cfg()
+    env = _seed("paddle")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "paddle"
+    assert env["CODERAI_OCR_PADDLE_VENV"] == "/opt/coderai/venvs/paddleocr"
+    assert env["CODERAI_OCR_FALLBACK_ENGINES"].split(",") == ["paddle", "doctr"]
+
+
+def test_the_seeded_names_are_the_ones_config_reads(ocr_cfg, monkeypatch):
+    """The seeder and the reader must agree, or the pod silently ignores all of it."""
+    ocr_cfg(olmocr_serve="server", olmocr_server_url="https://api.example/v1",
+            surya_serve="llamacpp", surya_server_url="http://box:8080/v1")
+    seeded = set(_seed("olmocr")) | set(_seed("surya"))
+    src = Path("codai/config.py").read_text()
+    for var in seeded:
+        if var.startswith("CODERAI_OCR_") and not var.endswith("_ENABLED"):
+            assert var in src, f"{var} is seeded to pods but config.py never reads it"
+
+
+# ------------------------------------------------------ aliases
+def test_an_aliased_entry_is_read_by_its_alias(ocr_cfg):
+    """An entry is routable by `alias or path or id` throughout the manager, and
+    aliases are how one model carries several configs and how several models sit
+    behind one endpoint. An aliased OCR entry must not be misread and downgraded."""
+    ocr_cfg(olmocr_serve="server", olmocr_server_url="https://api.example/v1")
+    env = _seed("allenai/olmOCR-2-7B-1025-FP8", alias="olmocr", served="olmocr")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "olmocr"
+    assert env["CODERAI_OCR_OLMOCR_SERVER_URL"] == "https://api.example/v1"
+
+
+def test_the_served_name_also_identifies_the_engine(ocr_cfg):
+    ocr_cfg()
+    env = _seed("some/doctr-checkpoint", served="doctr")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "doctr"
+
+
+def test_an_alias_that_names_no_engine_falls_back_to_the_path(ocr_cfg):
+    ocr_cfg()
+    env = _seed("paddle", alias="italian-documents")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "paddle"
+    assert env["CODERAI_OCR_FALLBACK_ENGINES"].split(",") == ["paddle", "doctr"]
+
+
+def test_an_aliased_surya_still_carries_the_licence(ocr_cfg):
+    ocr_cfg(surya_serve="llamacpp", surya_server_url="http://box:8080/v1")
+    env = _seed("datalab-to/surya-ocr-2", alias="surya", served="surya")
+    assert env["CODERAI_OCR_DEFAULT_ENGINE"] == "surya"
+    assert env["CODERAI_OCR_SURYA_ACCEPT_LICENSE"] == "1"