audio: CrisperWhisper support (isolated venv, verbatim + word timestamps)

CrisperWhisper does verbatim ASR with precise word timestamps, but its custom
generation config crashes on the server's transformers 5.x. Run it in its own
venv worker (pinned torch 2.5.1+cu124 + transformers 4.46.3 — last pre-5.x with
working Whisper word timestamps and a py3.13 tokenizers wheel):
- codai/api/crisperwhisper_worker.py + tools/crisperwhisper_service.py
  (return_timestamps="word" + CrisperWhisper pause redistribution).
- requirements-crisperwhisper.txt.
- stt_backends.py: `crisperwhisper` is its own family -> _RemoteCrisperWhisperBackend
  (HTTP to the worker); generic `whisper-hf` still runs in the shared venv
  (text-only default + graceful timestamp fallback); HF-ASR path now converts any
  container (mp4/m4a/webm) to 16k mono wav first and uses model-correct
  return_timestamps (Whisper True/"word", CTC "chunk"/"word").
- transcriptions.py: crisperwhisper dispatched as a GPU worker.
- manager skip-download + admin backend option for whisper-hf/crisperwhisper.

Verified: mp4 -> verbatim text + per-word timestamps, HTTP 200, on the 3090.
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R6iugoNeEgt9PkajKrFyq4
parent d51f46d2
......@@ -557,6 +557,7 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
<option value="kt">ktransformers (SGLang, CPU+GPU)</option>
<option value="vllm">vLLM (high-concurrency, CUDA)</option>
<option value="wav2vec2">wav2vec2 / HF-ASR (speech-to-text)</option>
<option value="whisper-hf">whisper-hf / CrisperWhisper (HF transformers STT)</option>
<option value="vosk">Vosk (offline STT, CPU, per-language dir)</option>
<option value="nemo">NVIDIA NeMo — Canary / Parakeet (isolated venv)</option>
</select>
......@@ -3850,7 +3851,7 @@ function _collectColibri(){
function _updateSttVisibility(){
const be = (document.getElementById('cfg-backend').value || '');
const row = document.getElementById('cfg-stt-row');
if (row) row.style.display = (be === 'wav2vec2' || be === 'vosk' || be === 'nemo') ? '' : 'none';
if (row) row.style.display = (be === 'wav2vec2' || be === 'vosk' || be === 'nemo' || be === 'whisper-hf') ? '' : 'none';
}
async function saveModelConfig(){
......
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
"""Fully-managed CrisperWhisper worker (isolated venv).
CrisperWhisper needs its verbatim word-timestamp generation, which breaks on the
server's transformers 5.x. So it runs in its own venv (pinned transformers 4.x)
behind a small HTTP service (tools/crisperwhisper_service.py); coderai talks to it
as a remote STT backend. Mirrors codai.api.canary_worker / pyannote_worker.
"""
import os
import socket
import subprocess
import sys
import threading
import time
from pathlib import Path
_REPO_ROOT = Path(__file__).resolve().parents[2]
_SERVICE_SCRIPT = _REPO_ROOT / "tools" / "crisperwhisper_service.py"
_REQUIREMENTS = _REPO_ROOT / "requirements-crisperwhisper.txt"
_BAKED_VENV = Path("/opt/coderai/crisperwhisper_venv")
_lock = threading.RLock()
_services: dict = {} # model_name -> {"proc","port","url"}
_bootstrapped = False
def resolve_venv_dir() -> Path:
explicit = os.environ.get("CODERAI_CRISPERWHISPER_VENV")
if explicit:
return Path(explicit)
if (_BAKED_VENV / ("Scripts" if os.name == "nt" else "bin")).exists():
return _BAKED_VENV
return Path(os.path.expanduser("~/.coderai/crisperwhisper_venv"))
def _venv_python(venv: Path) -> Path:
return venv / ("Scripts" if os.name == "nt" else "bin") / (
"python.exe" if os.name == "nt" else "python")
def _pip_ok(py: Path) -> bool:
try:
return subprocess.run(
[str(py), "-c", "import transformers, torch, soundfile"],
capture_output=True).returncode == 0
except Exception:
return False
def _bootstrap_venv() -> Path:
global _bootstrapped
venv = resolve_venv_dir()
py = _venv_python(venv)
if _bootstrapped and py.exists():
return py
if py.exists() and _pip_ok(py):
_bootstrapped = True
return py
if not py.exists():
print(f"[crisperwhisper] creating isolated venv at {venv} …", flush=True)
venv.parent.mkdir(parents=True, exist_ok=True)
subprocess.run([sys.executable, "-m", "venv", str(venv)], check=True)
py = _venv_python(venv)
if not _pip_ok(py):
if not _REQUIREMENTS.exists():
raise RuntimeError(f"CrisperWhisper requirements missing: {_REQUIREMENTS}")
print("[crisperwhisper] installing pinned transformers+torch into the isolated "
"venv (first run downloads several GB) …", flush=True)
subprocess.run([str(py), "-m", "pip", "install", "-U", "pip"], check=True)
subprocess.run([str(py), "-m", "pip", "install", "-r", str(_REQUIREMENTS)],
check=True)
if not _pip_ok(py):
raise RuntimeError("CrisperWhisper venv install did not yield importable deps")
_bootstrapped = True
return py
def _free_port() -> int:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("127.0.0.1", 0))
port = s.getsockname()[1]
s.close()
return port
def _pump_logs(proc: subprocess.Popen, tail):
for line in proc.stdout:
line = line.rstrip()
if line:
tail.append(line)
print(f"[crisperwhisper] {line}", flush=True)
def _health_ok(url: str) -> bool:
import requests
try:
r = requests.get(url + "/health", timeout=3)
return r.ok and bool(r.json().get("ok"))
except Exception:
return False
def ensure_service(model_name: str, config: dict = None,
ready_timeout: float = 1800.0) -> str:
config = config or {}
with _lock:
svc = _services.get(model_name)
if svc and svc["proc"].poll() is None and _health_ok(svc["url"]):
return svc["url"]
if svc and svc["proc"].poll() is not None:
_services.pop(model_name, None)
py = _bootstrap_venv()
port = _free_port()
url = f"http://127.0.0.1:{port}"
env = dict(os.environ)
try:
from codai.models.cache import get_hf_hub_cache_dir
hub = get_hf_hub_cache_dir()
env["HF_HUB_CACHE"] = hub
env["HUGGINGFACE_HUB_CACHE"] = hub
except Exception:
pass
gpu = config.get("gpu_device")
if gpu is not None:
env["CUDA_VISIBLE_DEVICES"] = str(gpu)
proc = subprocess.Popen(
[str(py), str(_SERVICE_SCRIPT), "--model", model_name,
"--host", "127.0.0.1", "--port", str(port)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
bufsize=1, env=env, cwd=str(_REPO_ROOT))
import collections
tail = collections.deque(maxlen=15)
threading.Thread(target=_pump_logs, args=(proc, tail), daemon=True).start()
_services[model_name] = {"proc": proc, "port": port, "url": url}
def _tail_msg():
joined = " | ".join(list(tail)[-5:]).strip()
return f". Last output: {joined}" if joined else ""
deadline = time.time() + ready_timeout
while time.time() < deadline:
if proc.poll() is not None:
raise RuntimeError(
f"CrisperWhisper worker exited (code {proc.returncode}) before ready"
+ _tail_msg())
if _health_ok(url):
print(f"[crisperwhisper] service ready for {model_name} at {url}", flush=True)
return url
time.sleep(2)
stop_service(model_name)
raise RuntimeError(f"CrisperWhisper worker for {model_name} did not become ready"
+ _tail_msg())
def stop_service(model_name: str) -> None:
with _lock:
svc = _services.pop(model_name, None)
if not svc:
return
proc = svc["proc"]
if proc.poll() is None:
try:
proc.terminate()
proc.wait(timeout=10)
except Exception:
pass
if proc.poll() is None:
try:
proc.kill()
except Exception:
pass
print(f"[crisperwhisper] service for {model_name} stopped", flush=True)
def stop_all() -> None:
for name in list(_services.keys()):
stop_service(name)
import atexit as _atexit
_atexit.register(stop_all)
......@@ -58,6 +58,10 @@ def _family(model_name: str, config: Optional[dict]) -> str:
b = (cfg.get("backend") or "").strip().lower()
if b in ("wav2vec2", "wav2vec", "transformers-asr", "hf-asr"):
return "wav2vec2"
if b == "crisperwhisper":
return "crisperwhisper"
if b in ("whisper-hf", "whisper-transformers"):
return "whisper_hf"
if b == "vosk":
return "vosk"
if b in ("nemo", "canary", "parakeet"):
......@@ -67,6 +71,13 @@ def _family(model_name: str, config: Optional[dict]) -> str:
return "vosk"
if any(x in n for x in ("canary", "parakeet", "nemo", ".nemo")):
return "nemo"
# CrisperWhisper runs in its own isolated venv worker (verbatim + word
# timestamps need a transformers version the shared 5.x venv can't provide).
if "crisperwhisper" in n:
return "crisperwhisper"
# Other HF Whisper checkpoints = seq2seq Whisper via the shared-venv pipeline.
if "whisper" in n:
return "whisper_hf"
if any(x in n for x in ("wav2vec", "hubert", "mms-", "seamless", "wavlm")):
return "wav2vec2"
# Default for an unknown non-whisper audio model: treat as an HF ASR model.
......@@ -118,6 +129,10 @@ class _Wav2Vec2Backend:
self.model_name = model_name
self.config = config or {}
self._pipe = None
# Whisper HF checkpoints (e.g. CrisperWhisper) are seq2seq and take
# generate kwargs (language/task); CTC models (wav2vec2) don't.
self._is_whisper = bool(self.config.get("_is_whisper")) or \
"whisper" in (model_name or "").lower()
def _ensure(self):
if self._pipe is not None:
......@@ -155,16 +170,58 @@ class _Wav2Vec2Backend:
word_timestamps: bool = False) -> dict:
self._ensure()
gen_kwargs = {}
if self._is_whisper:
# Whisper seq2seq: choose transcribe vs translate, pass source language.
gen_kwargs["task"] = "translate" if target_language else "transcribe"
if language:
gen_kwargs["language"] = language
elif language and self.config.get("language_aware"):
# MMS / Seamless accept a target language; plain wav2vec2 ignores it.
if language and self.config.get("language_aware"):
gen_kwargs["language"] = language
# CTC models emit per-word timestamps with return_timestamps="word".
mode = "word" if word_timestamps else "chunk"
# return_timestamps: CTC uses "chunk" / "word". Whisper uses "word" or
# True (segment) — but some checkpoints (e.g. CrisperWhisper) ship a custom
# generation config whose timestamp postprocessing breaks on transformers
# 5.x. So default Whisper to TEXT-ONLY (None) and only attempt timestamps
# when explicitly requested, falling back to text on any failure.
if word_timestamps:
mode = "word"
elif self._is_whisper:
mode = None
else:
mode = "chunk"
# Decode to 16 kHz mono WAV first so any container (mp4/m4a/webm/…) is
# handled uniformly — feeding a raw mp4 path can yield empty audio.
wav = None
try:
out = self._pipe(audio_path, return_timestamps=mode,
generate_kwargs=gen_kwargs or None)
except (TypeError, ValueError):
out = self._pipe(audio_path)
wav = _to_wav_16k_mono(audio_path)
src = wav
except Exception:
src = audio_path
def _run(ts):
if ts is None:
return self._pipe(src, generate_kwargs=gen_kwargs or None)
return self._pipe(src, return_timestamps=ts, generate_kwargs=gen_kwargs or None)
try:
out = _run(mode)
except Exception as e:
if mode is None:
raise
# Timestamp path failed (checkpoint/transformers incompatibility) —
# retry text-only so transcription still succeeds.
print(f"[stt] {self.model_name}: return_timestamps={mode!r} failed "
f"({type(e).__name__}); retrying text-only")
try:
out = _run(None)
except Exception:
out = self._pipe(src)
finally:
if wav:
try:
os.unlink(wav)
except OSError:
pass
text = (out.get("text") if isinstance(out, dict) else str(out)) or ""
segments, words = [], []
for ch in (out.get("chunks") or []) if isinstance(out, dict) else []:
......@@ -313,6 +370,51 @@ class _RemoteNemoBackend:
pass
# --------------------------------------------------------------------------- #
# CrisperWhisper — isolated-venv worker (verbatim + word timestamps)
# --------------------------------------------------------------------------- #
class _RemoteCrisperWhisperBackend:
"""Talks to the managed CrisperWhisper worker (isolated venv, pinned
transformers) — verbatim transcription with precise word timestamps."""
family = "crisperwhisper"
def __init__(self, model_name: str, config: dict, service_url: Optional[str] = None):
self.model_name = model_name
self.config = config or {}
self.service_url = (service_url or self.config.get("service_url") or "").rstrip("/")
def transcribe(self, audio_path: str, language: Optional[str] = None,
prompt: Optional[str] = None, temperature: float = 0.0,
target_language: Optional[str] = None,
word_timestamps: bool = False) -> dict:
import requests
params = {"task": "translate" if target_language else "transcribe"}
if language:
params["language"] = language
with open(audio_path, "rb") as f:
resp = requests.post(f"{self.service_url}/transcribe", params=params,
data=f.read(), timeout=1800,
headers={"Content-Type": "application/octet-stream"})
if resp.status_code != 200:
raise RuntimeError(f"CrisperWhisper worker error {resp.status_code}: {resp.text[:300]}")
j = resp.json()
if j.get("error"):
raise RuntimeError(j["error"])
return {"text": (j.get("text") or "").strip(),
"segments": j.get("segments") or [],
"words": j.get("words") or [],
"language": j.get("language") or language}
def cleanup(self):
try:
from codai.api import crisperwhisper_worker
crisperwhisper_worker.stop_service(self.model_name)
except Exception:
pass
# --------------------------------------------------------------------------- #
# factory
# --------------------------------------------------------------------------- #
......@@ -337,11 +439,20 @@ def load_stt_backend(model_name: str, model_path: Optional[str], config: Optiona
from codai.api import canary_worker
url = canary_worker.ensure_service(real, config)
return _RemoteNemoBackend(real, {**config, "service_url": url}, service_url=url)
if fam == "crisperwhisper":
if config.get("service_url"):
return _RemoteCrisperWhisperBackend(real, config)
from codai.api import crisperwhisper_worker
url = crisperwhisper_worker.ensure_service(real, config)
return _RemoteCrisperWhisperBackend(real, {**config, "service_url": url},
service_url=url)
if fam == "whisper_hf":
return _Wav2Vec2Backend(real, real, {**config, "_is_whisper": True})
return _Wav2Vec2Backend(real, real, config)
# Family names that this module (not whisper-server / faster-whisper) handles.
STT_FAMILIES = {"wav2vec2", "vosk", "nemo"}
STT_FAMILIES = {"wav2vec2", "vosk", "nemo", "whisper_hf", "crisperwhisper"}
def resolve_family(model_name: str, config: Optional[dict]) -> str:
......
......@@ -594,13 +594,20 @@ async def _run_transcription_inner(
if family == 'nemo':
uses_gpu = True
needed_gb = float(stt_cfg.get('used_vram_gb') or 6.0)
elif family == 'wav2vec2':
elif family == 'crisperwhisper':
# Isolated-venv worker on the 3090 (whisper-large FP16).
uses_gpu = True
needed_gb = float(stt_cfg.get('used_vram_gb') or 3.0)
elif family in ('wav2vec2', 'whisper_hf'):
try:
import torch
uses_gpu = bool(torch.cuda.is_available())
except Exception:
uses_gpu = False
needed_gb = float(stt_cfg.get('used_vram_gb') or 2.0) if uses_gpu else 0.0
# whisper HF (e.g. CrisperWhisper large) is a bit bigger than
# wav2vec2; reserve a touch more by default.
_dflt = 3.0 if family == 'whisper_hf' else 2.0
needed_gb = float(stt_cfg.get('used_vram_gb') or _dflt) if uses_gpu else 0.0
def _run_backend():
def _loader():
......
......@@ -1799,7 +1799,8 @@ class MultiModelManager:
# transformers pipeline pulls lazily on first request. None should be
# eagerly downloaded here (that would block engine startup), unlike a
# faster-whisper size which set_audio_model resolves/caches below.
if _be in ("vosk", "nemo", "canary", "parakeet", "wav2vec2", "wav2vec"):
if _be in ("vosk", "nemo", "canary", "parakeet", "wav2vec2", "wav2vec",
"whisper-hf", "whisper-transformers", "crisperwhisper"):
print(f"Registered {_be} audio model: {model_name}")
return
......
# Isolated venv for CrisperWhisper (nyrahealth/CrisperWhisper).
#
# CrisperWhisper's whole value is VERBATIM transcription with precise WORD
# timestamps. Its custom generation config (forced_decoder_ids) is incompatible
# with the coderai server's transformers 5.x — the word/segment timestamp
# postprocessing crashes there. So it runs in its own venv pinned to a
# transformers version whose Whisper word-timestamp path works, and coderai talks
# to it over HTTP (codai/api/crisperwhisper_worker.py + tools/crisperwhisper_service.py).
# Do NOT merge these into requirements.txt.
#
# CUDA build for the 3090; cu124 wheels run on this box's newer driver.
--extra-index-url https://download.pytorch.org/whl/cu124
torch==2.5.1+cu124
# 4.46.x is the last pre-5.x transformers (Whisper word-timestamp path still
# works, unlike 5.x) AND allows tokenizers 0.20 which ships py3.13 wheels — 4.44
# pinned an older tokenizers with no 3.13 wheel, forcing a Rust build that fails.
transformers==4.46.3
accelerate>=0.33.0
soundfile>=0.12.0
#!/usr/bin/env python3
"""Standalone CrisperWhisper ASR HTTP microservice — run in its OWN venv.
CrisperWhisper does verbatim transcription with precise word timestamps, but its
generation config breaks on transformers 5.x. So it runs here in a venv pinned to
a compatible transformers, and coderai talks to it as a remote STT backend
(codai.api.crisperwhisper_worker).
Endpoints:
GET /health -> {"ok": true, "model": ...}
POST /transcribe -> {"text", "words":[{word,start,end}], "segments", "language"}
body: raw audio bytes; query: ?language=&task=transcribe|translate
"""
import argparse
import json
import os
import subprocess
import tempfile
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlparse, parse_qs
def _to_wav_16k_mono(raw: bytes) -> str:
fd, src = tempfile.mkstemp(suffix=".input"); os.write(fd, raw); os.close(fd)
fd, out = tempfile.mkstemp(suffix=".wav"); os.close(fd)
try:
p = subprocess.run(["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1",
"-c:a", "pcm_s16le", "-f", "wav", out],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.returncode != 0:
raise RuntimeError("ffmpeg conversion failed: " +
p.stderr.decode("utf-8", "replace")[-300:])
return out
finally:
try:
os.unlink(src)
except OSError:
pass
def _adjust_pauses(chunks, split_threshold=0.12):
"""CrisperWhisper's pause redistribution for accurate verbatim word bounds."""
out = [dict(c) for c in chunks]
for i in range(len(out) - 1):
cs, ce = out[i].get("timestamp") or (None, None)
ns, ne = out[i + 1].get("timestamp") or (None, None)
if None in (ce, ns):
continue
pause = ns - ce
if pause > 0:
d = (split_threshold / 2) if pause > split_threshold else (pause / 2)
out[i]["timestamp"] = (cs, ce + d)
out[i + 1]["timestamp"] = (ns - d, ne)
return out
class _Engine:
def __init__(self, model_name: str):
from transformers import pipeline
import torch
self.model_name = model_name
dev = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.float16 if dev == "cuda" else torch.float32
self._pipe = pipeline(
"automatic-speech-recognition", model=model_name,
torch_dtype=dtype, device=dev,
chunk_length_s=30, stride_length_s=5)
def transcribe(self, wav_path: str, language: str = None, task: str = None) -> dict:
gen = {"task": task or "transcribe"}
if language:
gen["language"] = language
out = self._pipe(wav_path, return_timestamps="word", generate_kwargs=gen)
text = (out.get("text") if isinstance(out, dict) else str(out)) or ""
chunks = _adjust_pauses(out.get("chunks") or []) if isinstance(out, dict) else []
words = []
for ch in chunks:
ts = ch.get("timestamp") or (None, None)
words.append({"word": (ch.get("text") or "").strip(),
"start": ts[0] or 0.0, "end": ts[1] or 0.0})
# One segment spanning all words (CrisperWhisper is word-first).
segments = []
if words:
segments = [{"start": words[0]["start"], "end": words[-1]["end"],
"text": text.strip()}]
return {"text": text.strip(), "words": words, "segments": segments,
"language": language}
ENGINE = None
class Handler(BaseHTTPRequestHandler):
def _send(self, code, body=b"", ctype="application/json"):
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if body:
self.wfile.write(body)
def log_message(self, fmt, *args):
pass
def do_GET(self):
if urlparse(self.path).path == "/health":
self._send(200, json.dumps({"ok": True, "model": ENGINE.model_name}).encode())
else:
self._send(404, b'{"error":"not found"}')
def do_POST(self):
parsed = urlparse(self.path)
if parsed.path != "/transcribe":
self._send(404, b'{"error":"not found"}')
return
wav = None
try:
n = int(self.headers.get("Content-Length", 0))
raw = self.rfile.read(n) if n else b""
if not raw:
self._send(400, b'{"error":"empty audio"}')
return
q = parse_qs(parsed.query)
def _q(k):
v = q.get(k)
return v[0] if v else None
wav = _to_wav_16k_mono(raw)
result = ENGINE.transcribe(wav, language=_q("language"), task=_q("task"))
self._send(200, json.dumps(result).encode())
except Exception as e:
import traceback
traceback.print_exc()
self._send(500, json.dumps({"error": str(e)}).encode())
finally:
if wav:
try:
os.unlink(wav)
except OSError:
pass
def main(argv=None):
ap = argparse.ArgumentParser(description="Standalone CrisperWhisper HTTP service")
ap.add_argument("--model", default="nyrahealth/CrisperWhisper")
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--port", type=int, default=8126)
args = ap.parse_args(argv)
global ENGINE
print(f"Loading CrisperWhisper {args.model} …")
ENGINE = _Engine(args.model)
print(f"Ready: {args.model} — serving on http://{args.host}:{args.port}")
ThreadingHTTPServer((args.host, args.port), Handler).serve_forever()
if __name__ == "__main__":
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment