resolve (and load) the chat model off the event loop (v0.4.52)

The v0.4.51 stack dump named the OCR pod freeze. chat_completions called
multi_model_manager.get_model_for_request() on the loop. For a model not
loaded yet that goes _load_default_model -> load_model ->
vllm_worker.ensure_service(), which waits for vLLM in a time.sleep(2) loop.
On a pod it froze the whole process for 324 s at boot, /healthz included,
and for minutes whenever it decided to (re)start vLLM, so the orchestrator
took busy pods for dead ones.

chat_completions and completions now resolve the model in a thread. The
manager's load lock already prevents duplicate loads. The no-blocking-calls
AST guard now also forbids get_model_for_request, _load_default_model and
ensure_service in a coroutine; it fails on the old code and passes on the new.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KnfkxBpcZ1v6ypRFFyhrVM

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.4.51"
+__version__ = "0.4.52"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/text.py

@@ -1283,7 +1283,15 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
                 _instance_idx = None
                 _candidate = None
         if _candidate is None:
-            _candidate = multi_model_manager.get_model_for_request(requested_model)
+            # OFF THE LOOP. Resolving a model that is not loaded yet LOADS it,
+            # and a vLLM load waits for readiness in a `time.sleep(2)` loop --
+            # on an OCR pod that froze the whole process for 324 s at boot and
+            # for minutes whenever it decided to (re)start vLLM: no request of
+            # any kind served, /healthz included, so the orchestrator took a
+            # busy pod for a dead one. Duplicate loads are already prevented by
+            # the manager's load lock.
+            _candidate = await asyncio.to_thread(
+                multi_model_manager.get_model_for_request, requested_model)
         if _candidate is None and model_manager.backend is not None:
             _candidate = model_manager
         # Validate the candidate has a working backend before accepting it
@@ -3390,7 +3398,9 @@ async def completions(request: CompletionRequest):
     if _acq:
         _instance_idx, mm = _acq
     else:
-        mm = multi_model_manager.get_model_for_request(requested_model)
+        # Off the loop: this can load the model (see chat_completions).
+        mm = await asyncio.to_thread(
+            multi_model_manager.get_model_for_request, requested_model)
 
     def _release_instance():
         if _instance_idx is not None and _model_key:

tests/test_no_blocking_calls_on_the_loop.py

@@ -50,6 +50,12 @@ BLOCKING = {
     "placement.choose": "ends in a pool acquire: up to 1200 s, may rent inline",
     "pod_http.post": "sync HTTP — the thread twin of apost",
     "pod_http.get": "sync HTTP — the thread twin of aget",
+    # Resolving a model that is not loaded LOADS it, and a vLLM load waits for
+    # readiness in a sleep loop: 324 s with the whole process frozen on an OCR
+    # pod, /healthz included, so the orchestrator took a busy pod for a dead one.
+    "get_model_for_request": "may load the model: minutes, e.g. a vLLM start",
+    "_load_default_model": "loads the model: minutes, e.g. a vLLM start",
+    "ensure_service": "starts vLLM and sleeps until it is ready",
 }
 
 #: Method names that block when the receiver is not an asyncio primitive.
@@ -230,3 +236,12 @@ def test_it_does_not_flag_a_coroutine_handed_to_the_loop():
 
 def test_it_catches_a_thread_join_but_not_a_path_join():
     assert _hits("async def f(thread):\n    thread.join()\n")
+
+
+def test_it_catches_a_model_load_on_the_loop():
+    """The OCR pod freeze: chat_completions resolved its model on the loop."""
+    assert _hits("async def f():\n"
+                 "    m = multi_model_manager.get_model_for_request('x')\n")
+    assert not _hits("async def f():\n"
+                     "    m = await asyncio.to_thread(\n"
+                     "        multi_model_manager.get_model_for_request, 'x')\n")