models: two defects the first real test run exposed

Running the new test against the live instance found both immediately,
which is the point of it.

A FORCED REMOTE TEST COULD PASS LOCALLY. /v1/chat/completions is
deliberately outside the gateway (RemoteOpenAIBackend owns it), so the
x-coderai-placement header is ignored on that path: a text model with no
remote configuration was served HERE while the result reported
where=remote, ok=true. A test that passes in the wrong place is worse
than no test. Forcing remote is now verified before anything is
dispatched, and refuses with what is actually missing.

A 200 WITH NO OUTPUT WAS A PASS. The text probe allowed 8 tokens, which
a reasoning model spends thinking — Qwen3.5-9B answered with
completion_tokens=0, finish_reason=stop, and the test called it ok. The
budget is now 64, and an empty result is a failure carrying the numbers
that explain it, for text, embeddings, images and rerank alike.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/api/model_test.py

@@ -52,8 +52,11 @@ class ModelTestRequest(BaseModel):
 #: exercises a real generation.
 def _probe_for(capability: str, model: str):
     if capability in ("", "text"):
+        # 64, not 8: a reasoning model spends its budget thinking and answers
+        # with nothing at all if the cap is tight — which looks like a pass
+        # (HTTP 200) while proving the model produced no output.
         return "/v1/chat/completions", {
-            "model": model, "max_tokens": 8, "temperature": 0.0,
+            "model": model, "max_tokens": 64, "temperature": 0.0,
             "messages": [{"role": "user", "content": "Reply with exactly: OK"}]}
     if capability == "images":
         return "/v1/images/generations", {
@@ -96,6 +99,18 @@ async def test_model(req: ModelTestRequest, request: Request,
     entry, capability = _describe(req.model)
     placement = _placement_summary(req.model, entry, force)
 
+    # Forcing remote has to be verified HERE, not left to the gateway. Chat and
+    # completions are deliberately outside the gateway (RemoteOpenAIBackend owns
+    # them), so a forced header is ignored on that path and the request would be
+    # served locally while this reported "remote" — a test passing in the wrong
+    # place, which is worse than no test at all.
+    if force == "remote":
+        why = _no_remote_reason(req.model, entry, capability)
+        if why:
+            return {"model": req.model, "capability": capability or "text",
+                    "where": "remote", "target": "(not configured)",
+                    "ran": "", "ok": False, "seconds": 0.0, "error": why}
+
     path, body = _probe_for(capability, req.model)
     if not path:
         reason = _REACHABILITY_ONLY.get(capability, "no cheap probe for this kind")
@@ -134,11 +149,72 @@ async def test_model(req: ModelTestRequest, request: Request,
            "seconds": seconds}
     if out["ok"]:
         out["sample"] = _sample(raw, capability)
+        empty = _empty_result(raw, capability)
+        if empty:
+            # A 200 that carried no output is not a working model. Reporting it
+            # as a pass is worse than reporting nothing.
+            out["ok"] = False
+            out["error"] = empty
     else:
         out["error"] = _error_text(raw)
     return out
 
 
+def _empty_result(raw: bytes, capability: str) -> str:
+    """'' when the response carries real output, else why it does not."""
+    try:
+        obj = json.loads(raw)
+    except Exception:
+        return "" if raw else "the response was empty"
+    if capability in ("", "text"):
+        choice = (obj.get("choices") or [{}])[0]
+        content = ((choice.get("message") or {}).get("content") or "").strip()
+        if content:
+            return ""
+        produced = (obj.get("usage") or {}).get("completion_tokens", 0)
+        return (f"the model returned no text (completion_tokens={produced}, "
+                f"finish_reason={choice.get('finish_reason')!r}) — it answered the "
+                "request with nothing")
+    if capability == "embeddings":
+        try:
+            return "" if obj["data"][0]["embedding"] else "the embedding was empty"
+        except Exception:
+            return "no embedding in the response"
+    if capability == "images":
+        try:
+            d = (obj.get("data") or [{}])[0]
+            return "" if (d.get("b64_json") or d.get("url")) else "no image in the response"
+        except Exception:
+            return "no image in the response"
+    if capability == "rerank":
+        return "" if (obj.get("results") or obj.get("data")) else "no ranking returned"
+    return ""
+
+
+def _no_remote_reason(model: str, entry: dict, capability: str) -> str:
+    """'' when this model really is configured to run remotely, else why not."""
+    entry = entry or {}
+    if str(entry.get("service_url") or "").strip():
+        return ""
+    if str(entry.get("backend") or "").strip().lower() == "runpod":
+        return ""
+    if isinstance(entry.get("runpod"), dict) and entry["runpod"]:
+        return ""
+    try:
+        from codai.api.remote_gateway import capability_endpoints, capability_pods
+        if capability and (capability in capability_endpoints()
+                           or capability in capability_pods()):
+            return ""
+    except Exception:
+        pass
+    if not entry:
+        return (f"{model!r} is not in the model list, so nothing can say where it "
+                "should run")
+    return (f"nothing configures {model!r} to run remotely — set a service_url, or "
+            "backend 'runpod' with a runpod block on the model, or a remote for "
+            f"its {capability or 'text'} capability")
+
+
 def _describe(model: str):
     """(entry, capability) for a model — capability '' means a text/LLM model."""
     try:

tests/test_remote_execution.py

@@ -1173,7 +1173,9 @@ def test_a_test_run_can_force_where_it_runs(test_run):
 
     _run_test("my-llm", where="local")
     assert test_run["headers"][PLACEMENT_HEADER] == "local"
-    _run_test("my-llm", where="runpod")
+    # Forcing remote only dispatches for a model that really is configured
+    # remote; my-embed has a service_url.
+    _run_test("my-embed", where="runpod")
     assert test_run["headers"][PLACEMENT_HEADER] == "remote"
 
 
@@ -1212,3 +1214,41 @@ def test_the_placement_override_is_honoured_by_the_router():
     with pytest.raises(RuntimeError, match="nothing configures"):
         resolve_target("/v1/images/generations", "POST", "", body,
                        "application/json", force="remote")
+
+
+def test_forcing_runpod_fails_when_nothing_is_configured_remote(test_run):
+    """The bug this exists to prevent: /v1/chat/completions is outside the
+    gateway, so a forced header is ignored there — a text model with no remote
+    config would have been served LOCALLY while the result said "remote"."""
+    out = _run_test("my-llm", where="runpod")
+    assert out["ok"] is False
+    assert "nothing configures" in out["error"]
+    assert out["ran"] == ""          # nothing was dispatched at all
+
+    # A model that IS configured remote still runs.
+    out2 = _run_test("my-video", where="runpod")
+    assert out2["where"] == "runpod" and "nothing configures" not in str(out2)
+
+
+def test_a_200_with_no_output_is_not_a_pass(test_run, monkeypatch):
+    """A reasoning model can answer with nothing when the token budget is tight.
+    HTTP 200 plus zero content is a failure, not a pass."""
+    import codai.broker.asgi_bridge as bridge
+
+    async def empty_reply(request, *, method, path, headers=None, body=b""):
+        return {"status_code": 200, "body": json.dumps(
+            {"choices": [{"message": {"content": ""}, "finish_reason": "stop"}],
+             "usage": {"completion_tokens": 0}}).encode()}
+
+    monkeypatch.setattr(bridge, "execute_api_request", empty_reply)
+    out = _run_test("my-llm")
+    assert out["status"] == 200
+    assert out["ok"] is False
+    assert "no text" in out["error"] and "completion_tokens=0" in out["error"]
+
+
+def test_the_text_probe_leaves_room_for_a_reasoning_model():
+    from codai.api.model_test import _probe_for
+
+    _, body = _probe_for("text", "m")
+    assert body["max_tokens"] >= 64