ask the backend whether it can time words, and do not hand a CTC pipeline a None (v0.5.3)

Follow-up to v0.5.2, from measuring the fix instead of trusting it. Asking the
running install for word timings on 4.3 seconds of Italian it had just
synthesised showed two more faults in the same chain, both of which answer 200
with the timings missing.

A CTC pipeline cannot take `generate_kwargs=None`: it iterates it and raises
`TypeError: 'NoneType' object is not iterable`. The backend passed exactly
that whenever a model had no generate kwargs, which is every wav2vec2/MMS
checkpoint -- and the retry below it then turned the failure into a text-only
success, so the only trace was one line in the log. `wav2vec2-it` returned 0
words before this and the same 14 the pipeline always produced after. The
kwarg is now omitted when empty rather than passed as None.

And the pick has to consider the language, not just the backend: a model that
declares `languages` REFUSES the rest with a 400, so an install with
`wav2vec2-it` and `wav2vec2-en` has two models that each report word timings
and each reject the other's language. Picking one for the wrong language would
have replaced one silent fallback with another.

Measured, on sixteen configured audio models: ten cannot time words at all.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S8LAP3ueNzQJtwzhgVbq66

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.5.2"
+__version__ = "0.5.3"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/compose.py

@@ -704,19 +704,50 @@ def _stt_words(loop, audio_path: str, model: str, language: Optional[str]) -> Li
     return list((res or {}).get("words") or [])
 
 
-def _stt_backend(model_id: str) -> str:
-    """The configured ``backend`` of an audio model, without loading it."""
+def _stt_entry(model_id: str) -> dict:
+    """An audio model's own models.json entry, without loading the model."""
     try:
         from codai.models.manager import multi_model_manager
         cfg = multi_model_manager._config_for_model(model_id) or {}
         raw = cfg.get("_raw_cfg") if isinstance(cfg, dict) else None
-        entry = raw if isinstance(raw, dict) else cfg
-        return str((entry or {}).get("backend") or "").strip().lower()
+        return raw if isinstance(raw, dict) else (cfg if isinstance(cfg, dict) else {})
     except Exception:
-        return ""
+        return {}
+
+
+def _stt_backend(model_id: str) -> str:
+    """The configured ``backend`` of an audio model, without loading it."""
+    return str(_stt_entry(model_id).get("backend") or "").strip().lower()
+
+
+def _stt_speaks(model_id: str, languages) -> bool:
+    """Whether this model accepts every language the render narrates in.
+
+    A model that declares no ``languages`` accepts anything. One that declares
+    them REFUSES the rest with a 400 (see transcriptions._validate_language),
+    and a per-language model is easy to pick by accident: an install with
+    ``wav2vec2-it`` and ``wav2vec2-en`` has two models that each report word
+    timings and each reject the other's language.
+    """
+    declared = {str(x).strip().lower()
+                for x in (_stt_entry(model_id).get("languages") or [])}
+    if not declared:
+        return True
+    return all(str(l).strip().lower() in declared for l in languages if l)
+
+
+def _request_languages(req) -> list:
+    """The languages the narration is in, in order of first appearance."""
+    out = []
+    for sc in getattr(req, "scenes", None) or []:
+        lang = (getattr(sc, "voice", None) or req.voice).language
+        lang = str(lang or "").strip().lower()
+        if lang and lang not in out:
+            out.append(lang)
+    return out
 
 
-def _pick_stt_model(caps: CaptionSpec) -> Optional[str]:
+def _pick_stt_model(caps: CaptionSpec, languages=()) -> Optional[str]:
     """An audio model to time the words with: one that CAN time words, first.
 
     The previous order preferred anything named "whisper", then canary. Both are
@@ -734,14 +765,15 @@ def _pick_stt_model(caps: CaptionSpec) -> Optional[str]:
         return None
     import codai.api.stt_backends as stt_backends
     able = [i for i in ids if stt_backends.reports_word_timings(_stt_backend(i))]
+    speaks = [i for i in able if _stt_speaks(i, languages)] or able
     # Among those, a whisper-family model reads running speech best; wav2vec2 is
     # per-language and vosk is a small CPU model, so they come after.
     for want in ("whisper", "wav2vec", "vosk"):
-        for i in able:
+        for i in speaks:
             if want in i.lower():
                 return i
-    if able:
-        return able[0]
+    if speaks:
+        return speaks[0]
     # Nothing configured can report words. Still return something: the timings
     # will be estimated, the renderer says so, and the transcript is unused.
     return ids[0]
@@ -829,7 +861,8 @@ def _run_job(job_id: str, req: ComposeRequest, loop, base_url: str) -> None:
         if caps and caps.enabled:
             stage("captions", 30.0, "timing the words")
             timing = (caps.timing or "auto").lower()
-            model = _pick_stt_model(caps) if timing in ("auto", "stt") else None
+            model = (_pick_stt_model(caps, _request_languages(req))
+                     if timing in ("auto", "stt") else None)
             if timing == "stt" and not model:
                 raise rnd.RenderError("captions.timing=stt but no speech model is configured")
             if timing == "auto" and not model:

codai/api/stt_backends.py

@@ -199,9 +199,18 @@ class _Wav2Vec2Backend:
             src = audio_path
 
         def _run(ts):
-            if ts is None:
-                return self._pipe(src, generate_kwargs=gen_kwargs or None)
-            return self._pipe(src, return_timestamps=ts, generate_kwargs=gen_kwargs or None)
+            # `generate_kwargs` is OMITTED when empty rather than passed as
+            # None: a CTC pipeline (wav2vec2, MMS) iterates it unconditionally
+            # and raises `TypeError: 'NoneType' object is not iterable`, which
+            # the retry below then swallowed into a text-only answer. So every
+            # wav2vec2 request for word timings returned none of them, and the
+            # only trace was one line in the log.
+            kw = {}
+            if gen_kwargs:
+                kw["generate_kwargs"] = gen_kwargs
+            if ts is not None:
+                kw["return_timestamps"] = ts
+            return self._pipe(src, **kw)
 
         try:
             out = _run(mode)

docs/video-compose.md

@@ -103,10 +103,13 @@ so a mis-heard word cannot reach the screen. `timing`:
 * `estimate` — never load a speech model.
 
 Not every speech backend reports *per-word* timings: the HF pipelines
-(`wav2vec2`, `whisper_hf`, CrisperWhisper), `vosk` and plain faster-whisper do;
-a `whisper-server` (whisper.cpp) model and the NeMo worker behind canary return
-text only. `auto` therefore picks a model that can, whatever it is called, and
-`captions.stt_model` overrides the pick. A model that returns no timings is not
+(`wav2vec2`, `whisper_hf`, CrisperWhisper), `vosk` and plain faster-whisper do
+— measured, on one install: CrisperWhisper 14 words for a 4.3 s Italian line,
+wav2vec2 the same 14 — while a `whisper-server` (whisper.cpp) model and the
+NeMo worker behind canary return text only. `auto` therefore picks a model that can, whatever it is called, and
+`captions.stt_model` overrides the pick. A model that declares
+`languages` refuses the rest with a `400`, so the pick also skips a
+per-language model when the narration is in a language it does not declare. A model that returns no timings is not
 an error — the render continues with estimated ones — but it is reported in the
 job's `warnings`, because the captions will drift against the voice.
 

tests/test_caption_timings_and_status_polls.py

@@ -216,15 +216,26 @@ class _Model:
 
 
 def _audio_catalogue(monkeypatch, entries):
-    """``entries``: {model id: backend}. Patched at the manager boundary."""
+    """``entries``: {model id: backend} or {model id: (backend, [languages])}.
+
+    Patched at the manager boundary, in the shape the manager really returns:
+    runtime kwargs with the models.json entry nested under ``_raw_cfg``.
+    """
     import codai.api.compose as comp
     from codai.models import manager as mgr
 
+    def _raw(name):
+        spec = entries.get(name, "")
+        backend, langs = spec if isinstance(spec, tuple) else (spec, None)
+        entry = {"backend": backend}
+        if langs is not None:
+            entry["languages"] = langs
+        return {"_raw_cfg": entry}
+
     monkeypatch.setattr(mgr.multi_model_manager, "list_models",
                         lambda *a, **k: [_Model(i) for i in entries], raising=False)
     monkeypatch.setattr(mgr.multi_model_manager, "_config_for_model",
-                        lambda name: {"_raw_cfg": {"backend": entries.get(name, "")}},
-                        raising=False)
+                        _raw, raising=False)
     return comp
 
 
@@ -286,3 +297,115 @@ def test_the_renderer_warns_when_a_model_returns_no_timings():
     src = inspect.getsource(comp._run_job)
     assert "returned no word timings" in src
     assert "_no_words_warned" in src
+
+
+def test_a_per_language_model_is_not_picked_for_a_language_it_refuses(monkeypatch):
+    """`wav2vec2-it` reports word timings AND answers 400 for English.
+
+    A model that declares `languages` refuses the rest outright, so picking one
+    for the wrong language turns the whole feature into the warning path again
+    -- with a 400 this time instead of an AttributeError.
+    """
+    comp = _audio_catalogue(monkeypatch, {
+        "wav2vec2-it": ("wav2vec2", ["it"]),
+        "wav2vec2-en": ("wav2vec2", ["en"]),
+    })
+    from codai.api.compose import CaptionSpec
+    spec = CaptionSpec(enabled=True)
+    assert comp._pick_stt_model(spec, ["en"]) == "wav2vec2-en"
+    assert comp._pick_stt_model(spec, ["it"]) == "wav2vec2-it"
+
+
+def test_a_multilingual_model_wins_when_the_render_mixes_languages(monkeypatch):
+    comp = _audio_catalogue(monkeypatch, {
+        "wav2vec2-it": ("wav2vec2", ["it"]),
+        "whisper3-it": ("crisperwhisper", ["it"]),
+        "crisperwhisper": ("crisperwhisper", None),   # declares none = any
+    })
+    from codai.api.compose import CaptionSpec
+    spec = CaptionSpec(enabled=True)
+    assert comp._pick_stt_model(spec, ["it", "en"]) == "crisperwhisper"
+    # ...and with one language, the one tuned for it is still fine to pick
+    assert comp._pick_stt_model(spec, ["it"]) in ("whisper3-it", "crisperwhisper")
+
+
+def test_no_language_covers_it_still_returns_a_model(monkeypatch):
+    """Estimated timings beat refusing the render; the warning says which."""
+    comp = _audio_catalogue(monkeypatch, {"wav2vec2-it": ("wav2vec2", ["it"])})
+    from codai.api.compose import CaptionSpec
+    assert comp._pick_stt_model(CaptionSpec(enabled=True), ["de"]) == "wav2vec2-it"
+
+
+def test_the_languages_come_from_the_request(monkeypatch):
+    """Each scene's voice language, in order, de-duplicated."""
+    import codai.api.compose as comp
+    from codai.api.compose import ComposeRequest
+    req = ComposeRequest(
+        canvas={"width": 1080, "height": 1920},
+        voice={"language": "it"},
+        scenes=[{"text": "uno", "visual": {"type": "color", "color": "#000"}},
+                {"text": "two", "voice": {"language": "en"},
+                 "visual": {"type": "color", "color": "#000"}},
+                {"text": "tre", "visual": {"type": "color", "color": "#000"}}])
+    assert comp._request_languages(req) == ["it", "en"]
+
+
+# ===========================================================================
+#  1c. and the backend's own call, which threw the timings away
+# ===========================================================================
+#  Measured on a running install: `wav2vec2-it` answered 200 with 14 words'
+#  worth of text and no timings at all. `generate_kwargs=None` is what a CTC
+#  pipeline cannot take -- `TypeError: 'NoneType' object is not iterable` --
+#  and the retry turned that into a text-only success.
+
+class _FakePipe:
+    """Stands in for a transformers CTC pipeline, with its real fussiness."""
+
+    def __init__(self):
+        self.calls = []
+
+    def __call__(self, src, **kw):
+        self.calls.append(kw)
+        if "generate_kwargs" in kw and kw["generate_kwargs"] is None:
+            raise TypeError("'NoneType' object is not iterable")
+        if kw.get("return_timestamps") in ("word", "char"):
+            return {"text": "questa frase",
+                    "chunks": [{"text": "questa", "timestamp": (0.04, 0.28)},
+                               {"text": "frase", "timestamp": (0.34, 0.6)}]}
+        return {"text": "questa frase"}
+
+
+def _ctc_backend(monkeypatch, tmp_path):
+    import codai.api.stt_backends as stt
+    be = stt._Wav2Vec2Backend("jonatasgrosman/wav2vec2-large-xlsr-53-italian",
+                                None, {})
+    pipe = _FakePipe()
+    be._pipe = pipe
+    monkeypatch.setattr(be, "_ensure", lambda: None)
+    # The real backend decodes to 16 kHz mono first; keep the fixture a path.
+    monkeypatch.setattr(stt, "_to_wav_16k_mono", lambda p: str(p))
+    audio = tmp_path / "narration.wav"
+    audio.write_bytes(b"RIFF....WAVEfmt ")
+    return be, pipe, str(audio)
+
+
+def test_a_ctc_pipeline_is_not_handed_a_none_generate_kwargs(monkeypatch, tmp_path):
+    be, pipe, audio = _ctc_backend(monkeypatch, tmp_path)
+    out = be.transcribe(audio, language="it", word_timestamps=True)
+    assert [w["word"] for w in out["words"]] == ["questa", "frase"]
+    assert out["words"][0]["start"] == 0.04
+    # One call, first time: no TypeError, so no retry that drops the timings.
+    assert len(pipe.calls) == 1
+    assert "generate_kwargs" not in pipe.calls[0]
+    assert pipe.calls[0]["return_timestamps"] == "word"
+
+
+def test_a_whisper_checkpoint_still_gets_its_generate_kwargs(monkeypatch, tmp_path):
+    """The seq2seq path needs language/task; only the EMPTY dict is dropped."""
+    import codai.api.stt_backends as stt
+    be, pipe, audio = _ctc_backend(monkeypatch, tmp_path)
+    be._is_whisper = True
+    out = be.transcribe(audio, language="it", word_timestamps=True)
+    assert out["words"]
+    assert pipe.calls[0]["generate_kwargs"] == {"task": "transcribe",
+                                                "language": "it"}