speaker: pyannote needs matplotlib — and a venv check has to reach the real load

The HF token fix worked: pyannote got past the gate and failed one layer deeper,
instantiating the pipeline. Reproduced by running the real service with a GPU:
ModuleNotFoundError: No module named 'matplotlib'. pyannote's segmentation tasks
import matplotlib.pyplot at module load; it is not declared, and 'import
pyannote.audio' never reaches it, so the build's import check passed and the
venv shipped broken.

This is the fourth venv whose bare import check passed while the real load
failed. A <profile>.venv-<name>.check script now runs at build time in the
venv's own python and imports what the REAL load reaches — pipeline modules,
model classes — without a GPU or a token, which a build has neither of. The
pyannote check fails on the shipped image and passes with the fix; the XTTS
check passes as-is, so that venv is genuinely complete.

Also: every isolated-venv worker reported failures as the LAST FIVE LINES of the
subprocess tail. A Python traceback's last five lines are frame headers — the
exception is the final line, and cutting to five dropped it whenever the trace
was deep. Ten workers said where a process died and never why. Fixed in all of
them.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/api/canary_worker.py

@@ -176,7 +176,7 @@ def ensure_service(model_name: str, config: dict = None,
         _services[model_name] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         return f". Last output: {joined}" if joined else ""
 
     deadline = time.time() + ready_timeout

codai/api/colibri_worker.py

@@ -198,7 +198,7 @@ def _run_logged(cmd, cwd, label, tail, **kw):
             print(f"[colibri] {line}", flush=True)
     proc.wait()
     if proc.returncode != 0:
-        joined = " | ".join(list(tail)[-5:])
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip())
         raise RuntimeError(f"{label} failed (exit {proc.returncode}). {joined}")
 
 
@@ -252,7 +252,7 @@ def ensure_built(cfg, family: str = "glm") -> Path:
     if not binary.exists():
         raise RuntimeError(
             f"colibri {family} build completed but {binary} is missing. Last output: "
-            + " | ".join(list(tail)[-5:]))
+            + " | ".join(l.strip()[:300] for l in list(tail) if l.strip()))
     _built = True
     print(f"[colibri] built {binary}", flush=True)
     return binary
@@ -398,7 +398,7 @@ class MuxEngine:
             pass
 
     def log_tail(self) -> str:
-        return " | ".join(list(self._log_tail)[-5:]).strip()
+        return " | ".join(l.strip()[:300] for l in list(self._log_tail) if l.strip()).strip()
 
     def is_alive(self) -> bool:
         return (not self.closed and self.dispatcher_error is None

codai/api/crisperwhisper_worker.py

@@ -156,7 +156,7 @@ def ensure_service(model_name: str, config: dict = None,
         _services[model_name] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         return f". Last output: {joined}" if joined else ""
 
     deadline = time.time() + ready_timeout

codai/api/ds4_worker.py

@@ -87,7 +87,7 @@ def _run_logged(cmd, cwd, label, tail, **kw):
             print(f"[ds4] {line}", flush=True)
     proc.wait()
     if proc.returncode != 0:
-        joined = " | ".join(list(tail)[-5:])
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip())
         raise RuntimeError(f"{label} failed (exit {proc.returncode}). {joined}")
 
 
@@ -120,7 +120,7 @@ def ensure_built(cfg) -> Path:
     if not binary.exists():
         raise RuntimeError(
             f"ds4 build completed but {binary} is missing. Last output: "
-            + " | ".join(list(tail)[-5:]))
+            + " | ".join(l.strip()[:300] for l in list(tail) if l.strip()))
     _built = True
     print(f"[ds4] built {binary}", flush=True)
     return binary
@@ -279,7 +279,7 @@ def ensure_model(cfg, model_name: Optional[str] = None) -> Optional[str]:
     if not default_model.exists():
         raise RuntimeError(
             f"ds4 download_model.sh finished but {default_model} is missing. "
-            "Last output: " + " | ".join(list(tail)[-5:]))
+            "Last output: " + " | ".join(l.strip()[:300] for l in list(tail) if l.strip()))
     final = _relocate_into_cache(str(default_model))
     _register_downloaded_model(cfg, final, model_name)
     return final
@@ -473,7 +473,7 @@ def ensure_service(cfg, model_file: Optional[str] = None,
         _services[svc_key] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         return f". Last output: {joined}" if joined else ""
 
     deadline = time.time() + ready_timeout

codai/api/h3_worker.py

@@ -326,7 +326,7 @@ def ensure_service(model_path: str, config: dict = None,
         _services[model_path] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         return f". Last output: {joined}" if joined else ""
 
     deadline = time.time() + ready_timeout

codai/api/k3_worker.py

@@ -100,7 +100,7 @@ def ensure_built(cfg) -> Path:
     binary = _engine_bin(install_dir)
     if not binary.exists():
         raise RuntimeError("k3 build completed but bin/k3 is missing. Last output: "
-                           + " | ".join(list(tail)[-5:]))
+                           + " | ".join(l.strip()[:300] for l in list(tail) if l.strip()))
     print(f"[k3] built {binary}", flush=True)
     return binary
 

codai/api/kt_worker.py

@@ -94,7 +94,7 @@ def ensure_built(cfg) -> None:
         raise RuntimeError(
             "ktransformers: SGLang still not importable after pip install. Install "
             "SGLang + kt-kernel manually per the ktransformers docs. Last output: "
-            + " | ".join(list(tail)[-5:]))
+            + " | ".join(l.strip()[:300] for l in list(tail) if l.strip()))
 
 
 def resolve_service_key(cfg, model_path: Optional[str] = None):
@@ -179,7 +179,7 @@ def ensure_service(cfg, model_path: Optional[str] = None,
         _services[svc_key] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         return f". Last output: {joined}" if joined else ""
 
     deadline = time.time() + ready_timeout

codai/api/melotts_worker.py

@@ -181,7 +181,7 @@ def ensure_service(model_name: str, ready_timeout: float = 1800.0,
         _services[model_name] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         if "offline" in joined.lower() or "not" in joined.lower() and "found" in joined.lower():
             return (f". The model isn't in coderai's cache — download "
                     f"'{model_name}' from the model interface first. ({joined})")

codai/api/parler_worker.py

@@ -173,7 +173,7 @@ def ensure_service(model_name: str, ready_timeout: float = 1800.0) -> str:
         _services[model_name] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        joined = " | ".join(l.strip()[:300] for l in list(tail) if l.strip()).strip()
         if "offline" in joined.lower() or "not" in joined.lower() and "found" in joined.lower():
             return (f". The model isn't in coderai's cache — download "
                     f"'{model_name}' from the model interface first. ({joined})")

codai/api/pyannote_worker.py

@@ -203,7 +203,13 @@ def ensure_service(model_name: str = None, config: dict = None,
         _services[model_name] = {"proc": proc, "port": port, "url": url}
 
     def _tail_msg():
-        joined = " | ".join(list(tail)[-5:]).strip()
+        # The whole tail, not the last five lines. A Python traceback's last
+        # five lines are frame headers — the exception itself is the FINAL
+        # line, and cutting to five dropped it whenever the trace was deep: a
+        # pod reported 'File ".../mixins.py", line 3' and nothing else, which
+        # said where it died and not why. Long lines are cut, the count is not.
+        lines = [l.strip()[:300] for l in list(tail) if l.strip()]
+        joined = " | ".join(lines)
         return f". Last output: {joined}" if joined else ""
 
     deadline = time.time() + ready_timeout

packaging/runpod/Dockerfile.capability

@@ -80,6 +80,10 @@ RUN set -eux; \
             "$vdir/bin/pip" install --no-cache-dir --no-deps -r "$nodeps"; \
         fi; \
         "$vdir/bin/python" -c "import ${name}; print('${name}', getattr(${name}, '__version__', 'ok'), 'in its own venv')"; \
+        check="/tmp/venv-profiles/${PROFILE}.venv-${name}.check"; \
+        if [ -f "$check" ]; then \
+            "$vdir/bin/python" "$check" || { echo "ERROR: ${name} venv failed its deep check" >&2; exit 1; }; \
+        fi; \
     done; \
     rm -rf /tmp/venv-profiles
 

packaging/runpod/profiles/speaker.venv-pyannote.check

+# Deep check for the pyannote venv, run at build time by the venv's own python.
+#
+# 'import pyannote.audio' passes without matplotlib; instantiating a pipeline
+# does not. A bare import check is what let this venv ship broken, so this
+# imports the modules the REAL load reaches — without a GPU or a token, which a
+# build has neither of. If pyannote adds another undeclared import, this is
+# where it fails: in the build, not on a rented card.
+from pyannote.audio import Pipeline                                   # noqa
+from pyannote.audio.tasks.segmentation.multilabel import MultiLabelSegmentation  # noqa
+import pyannote.audio.pipelines.speaker_diarization                   # noqa
+import speechbrain                                                    # noqa
+print("pyannote deep check: pipeline modules import")

packaging/runpod/profiles/speaker.venv-pyannote.txt

@@ -9,3 +9,8 @@ huggingface_hub<1.0
 soundfile>=0.12.0
 # speechbrain (speaker ID) lives here too: this venv is where the torch is.
 speechbrain>=1.0.0
+# pyannote's segmentation tasks import matplotlib.pyplot at module load — not
+# declared, and not reached by 'import pyannote.audio', so the build's import
+# check passed and the pod died instantiating the pipeline with
+# ModuleNotFoundError. Reproduced by running the real service with a GPU.
+matplotlib>=3.7

packaging/runpod/profiles/tts-xtts.venv-TTS.check

+# Deep check for the coqui-tts venv: reach the XTTS model class, not just TTS.api.
+from TTS.api import TTS                                   # noqa
+from TTS.tts.models.xtts import Xtts                      # noqa
+import click                                              # noqa  (undeclared, found the hard way)
+print("coqui-tts deep check: XTTS model class imports")