test-run: stop three probes from failing models for the harness's own mistakes

The first RunPod pass over every capability produced four failures, three of
which were ours:

The TTS probe sent voice "default", which no shipped model has — Kokoro loaded
fine on a rented pod and then answered "Voice default not found in available
voices". TTSRequest already defaults to a real voice, so the probe stops
inventing one.

OCR engines are chosen by name (paddle|doctr|surya) and are not models.json
entries, so the test refused with "'surya' is not in the model list". _describe
now recognises them.

For a text model the refusal offered "a remote for its text capability" as a
fix. It is not one: /v1/chat/completions is outside the gateway, so that config
routes nothing and the test would then report a remote pass for a request served
locally — the exact failure the forced-remote check exists to prevent.
Co-Authored-By: 's avatarClaude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY
parent 83d637f3
...@@ -70,7 +70,11 @@ def _probe_for(capability: str, model: str): ...@@ -70,7 +70,11 @@ def _probe_for(capability: str, model: str):
return "/v1/rerank", {"model": model, "query": "cat", return "/v1/rerank", {"model": model, "query": "cat",
"documents": ["a cat", "a car"]} "documents": ["a cat", "a car"]}
if capability == "tts": if capability == "tts":
return "/v1/audio/speech", {"model": model, "input": "test", "voice": "default"} # No `voice`: TTSRequest's own default is a voice the shipped models
# actually have. Passing "default" failed on Kokoro with "Voice default
# not found in available voices" — the probe inventing a voice name, not
# the model being broken.
return "/v1/audio/speech", {"model": model, "input": "test"}
return "", None return "", None
...@@ -350,9 +354,18 @@ def _no_remote_reason(model: str, entry: dict, capability: str) -> str: ...@@ -350,9 +354,18 @@ def _no_remote_reason(model: str, entry: dict, capability: str) -> str:
if not entry: if not entry:
return (f"{model!r} is not in the model list, so nothing can say where it " return (f"{model!r} is not in the model list, so nothing can say where it "
"should run") "should run")
if not capability:
# A text/LLM model. Do NOT offer a 'text' capability remote as the fix:
# /v1/chat/completions is deliberately outside the gateway (the backend
# owns it), so configuring one would route nothing and the test would
# then report a remote pass for a request served locally.
return (f"nothing configures {model!r} to run remotely — set a service_url "
"on the model, or backend 'runpod' with a runpod block on it. A "
"capability remote does not apply to chat: that path is served by "
"the backend, not the gateway")
return (f"nothing configures {model!r} to run remotely — set a service_url, or " return (f"nothing configures {model!r} to run remotely — set a service_url, or "
"backend 'runpod' with a runpod block on the model, or a remote for " "backend 'runpod' with a runpod block on the model, or a remote for "
f"its {capability or 'text'} capability") f"its {capability} capability")
def _pod_diagnostics() -> list: def _pod_diagnostics() -> list:
...@@ -401,6 +414,12 @@ def _remote_catalogue(url: str, api_key: str) -> list: ...@@ -401,6 +414,12 @@ def _remote_catalogue(url: str, api_key: str) -> list:
return [f"({exc})"] return [f"({exc})"]
#: OCR engines are selected by name, not registered in models.json like a model.
#: Without this an OCR test answered "'surya' is not in the model list, so
#: nothing can say where it should run" — true of the catalogue, and useless.
_OCR_ENGINES = ("paddle", "doctr", "surya")
def _describe(model: str): def _describe(model: str):
"""(entry, capability) for a model — capability '' means a text/LLM model.""" """(entry, capability) for a model — capability '' means a text/LLM model."""
try: try:
...@@ -408,6 +427,8 @@ def _describe(model: str): ...@@ -408,6 +427,8 @@ def _describe(model: str):
entry = _model_entry_for(model) or {} entry = _model_entry_for(model) or {}
except Exception: except Exception:
entry = {} entry = {}
if not entry and (model or "").strip().lower() in _OCR_ENGINES:
return {"path": model, "model_type": "ocr"}, "ocr"
try: try:
from codai.api.runpod_worker import model_capability from codai.api.runpod_worker import model_capability
return entry, model_capability(entry) return entry, model_capability(entry)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment