feat: add full-quality audio ml pipeline scaffolding

README.md

@@ -123,6 +123,23 @@ pip install f5-tts    # Voice cloning (F5-TTS)
 pip install seed-vc   # Voice conversion / singing SVC
 ```
 
+### Full-Quality Audio ML Stack
+
+```bash
+pip install demucs deepfilternet rnnoise voicefixer
+```
+
+Use this stack when you want:
+- real ML stem separation for `/v1/audio/stems`
+- learned restoration for `/v1/audio/cleanup`
+- the strongest available backend path for `/v1/pipelines/audio-music-dub`
+
+Notes:
+- `demucs` is the primary separator for vocals/instrumental and multi-stem workflows.
+- `deepfilternet` is the primary learned cleanup backend.
+- `rnnoise` and `voicefixer` are optional alternates / complements.
+- Full music-dub quality depends on separation plus singing-capable conversion; even with this stack, output quality still depends heavily on source material and model/runtime availability.
+
 ### Face Swap
 
 ```bash

codai/api/__init__.py

@@ -15,6 +15,12 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 # codai.api - FastAPI application module
-from .app import app
 
-__all__ = ['app']
+__all__ = ['app']
\ No newline at end of file
+
+
+def __getattr__(name):
+    if name == 'app':
+        from .app import app
+        return app
+    raise AttributeError(name)

codai/api/app.py

@@ -121,6 +121,8 @@ app.include_router(tts_router)
 app.include_router(text_router)
 app.include_router(video_router)
 app.include_router(audio_gen_router)
+app.include_router(audio_stems_router)
+app.include_router(audio_clean_router)
 app.include_router(embeddings_router)
 app.include_router(pipelines_router)
 app.include_router(custom_pipelines_router)

codai/api/audio_backends.py

+import importlib.util
+from functools import lru_cache
+
+
+def _has_module(name: str) -> bool:
+    return importlib.util.find_spec(name) is not None
+
+
+@lru_cache(maxsize=1)
+def detect_audio_backends() -> dict:
+    demucs_ok = _has_module("demucs")
+    deepfilter_ok = _has_module("df") or _has_module("deepfilternet")
+    rnnoise_ok = _has_module("rnnoise")
+    voicefixer_ok = _has_module("voicefixer")
+
+    restoration_engine = None
+    if deepfilter_ok:
+        restoration_engine = "deepfilternet"
+    elif rnnoise_ok:
+        restoration_engine = "rnnoise"
+    elif voicefixer_ok:
+        restoration_engine = "voicefixer"
+
+    return {
+        "separation": {
+            "available": demucs_ok,
+            "engine": "demucs" if demucs_ok else None,
+            "candidates": ["demucs"],
+        },
+        "restoration": {
+            "available": bool(restoration_engine),
+            "engine": restoration_engine,
+            "candidates": ["deepfilternet", "rnnoise", "voicefixer"],
+        },
+    }
+
+
+def reset_audio_backend_cache() -> None:
+    detect_audio_backends.cache_clear()

codai/api/audio_clean.py

+import base64
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+import uuid
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, ConfigDict
+
+from codai.api.audio_backends import detect_audio_backends
+
+router = APIRouter()
+
+global_args = None
+global_file_path = None
+
+
+def set_global_args(args):
+    global global_args
+    global_args = args
+
+
+def set_global_file_path(path):
+    global global_file_path
+    global_file_path = path
+
+
+def _decode_audio(data: str) -> bytes:
+    if data.startswith("data:"):
+        _, enc = data.split(",", 1)
+        return base64.b64decode(enc)
+    return base64.b64decode(data)
+
+
+def _ffmpeg_binary() -> str:
+    ffmpeg = shutil.which("ffmpeg")
+    if not ffmpeg:
+        raise HTTPException(status_code=501, detail="ffmpeg is required for native audio cleanup")
+    return ffmpeg
+
+
+def _base_url(http_request: Request) -> str:
+    url_setting = getattr(global_args, "url", "auto") if global_args else "auto"
+    if url_setting != "auto":
+        return url_setting.rstrip("/")
+    host = http_request.headers.get("host", "127.0.0.1") if http_request else "127.0.0.1"
+    if ":" in host:
+        parts = host.split(":")
+        if len(parts) == 2 and parts[1].isdigit():
+            host = parts[0]
+    proto = "https" if getattr(global_args, "https", False) else "http"
+    port = getattr(global_args, "port", 8000) if global_args else 8000
+    return f"{proto}://{host}:{port}"
+
+
+def _persist_file(path: str, suffix: str, http_request: Request) -> dict:
+    data = Path(path).read_bytes()
+    if global_file_path:
+        os.makedirs(global_file_path, exist_ok=True)
+        filename = f"{uuid.uuid4().hex}{suffix}"
+        out_path = os.path.join(global_file_path, filename)
+        with open(out_path, "wb") as handle:
+            handle.write(data)
+        return {"url": f"{_base_url(http_request)}/v1/files/{filename}"}
+    return {f"b64_{suffix.lstrip('.')}": base64.b64encode(data).decode("ascii")}
+
+
+def _run_ffmpeg(command):
+    proc = subprocess.run(command, capture_output=True, text=True)
+    if proc.returncode != 0:
+        detail = proc.stderr.strip() or proc.stdout.strip() or "ffmpeg command failed"
+        raise HTTPException(status_code=500, detail=detail)
+
+
+def restore_with_provider(audio_bytes: bytes, options: dict, workdir: str) -> dict:
+    raise HTTPException(status_code=501, detail="ML audio restoration backend not installed")
+
+
+def _cleanup_audio(audio_bytes: bytes, options: dict, workdir: str) -> dict:
+    ffmpeg = _ffmpeg_binary()
+    src = os.path.join(workdir, "input.wav")
+    dst = os.path.join(workdir, "cleaned.wav")
+    with open(src, "wb") as handle:
+        handle.write(audio_bytes)
+
+    filters = []
+    applied = []
+    if options.get("noise_reduction"):
+        filters.append("afftdn=nf=-25")
+        applied.append("noise_reduction")
+    if options.get("remove_hum"):
+        filters.append("highpass=f=60,lowpass=f=15000")
+        applied.append("remove_hum")
+    if options.get("repair_clicks"):
+        filters.append("adeclick=t=40")
+        applied.append("repair_clicks")
+    if options.get("normalize"):
+        filters.append("loudnorm=I=-16:TP=-1.5:LRA=11")
+        applied.append("normalize")
+
+    if not filters:
+        raise HTTPException(status_code=400, detail="Select at least one cleanup operation")
+
+    _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", ",".join(filters), dst])
+    return {
+        "path": dst,
+        "engine": "ffmpeg-filter-chain",
+        "applied": applied,
+        "limitations": [
+            "best-effort cleanup only",
+            "not equivalent to spectral or ML restoration",
+            "heavy damage may remain audible",
+        ],
+    }
+
+
+class AudioCleanupRequest(BaseModel):
+    audio: str
+    noise_reduction: Optional[bool] = True
+    normalize: Optional[bool] = False
+    remove_hum: Optional[bool] = False
+    repair_clicks: Optional[bool] = False
+    response_format: Optional[str] = "url"
+    fallback_mode: Optional[bool] = False
+    model_config = ConfigDict(extra="allow")
+
+
+@router.post("/v1/audio/cleanup")
+async def cleanup_audio(request: AudioCleanupRequest, http_request: Request = None):
+    try:
+        audio_bytes = _decode_audio(request.audio)
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=f"Invalid audio payload: {exc}")
+
+    options = {
+        "noise_reduction": bool(request.noise_reduction),
+        "normalize": bool(request.normalize),
+        "remove_hum": bool(request.remove_hum),
+        "repair_clicks": bool(request.repair_clicks),
+    }
+    with tempfile.TemporaryDirectory(prefix="codai-clean-") as workdir:
+        backend_info = detect_audio_backends()["restoration"]
+        if request.fallback_mode:
+            result = _cleanup_audio(audio_bytes, options, workdir)
+            quality = "best-effort"
+            dependency = "ffmpeg"
+            model_name = None
+        else:
+            result = restore_with_provider(audio_bytes, options, workdir)
+            quality = "ml"
+            dependency = "python"
+            model_name = result.get("model")
+        payload = _persist_file(result["path"], ".wav", http_request)
+        return {
+            "created": int(time.time()),
+            "backend": {
+                "engine": result["engine"],
+                "model": model_name,
+                "quality": quality,
+                "dependency": dependency,
+                "ml_backend_available": backend_info["available"],
+                "preferred_engine": backend_info["engine"],
+            },
+            "applied": result["applied"],
+            "limitations": result["limitations"],
+            "data": [payload],
+        }

codai/api/audio_stems.py

+import base64
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+import uuid
+from pathlib import Path
+from typing import List, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, ConfigDict
+
+from codai.api.audio_backends import detect_audio_backends
+
+router = APIRouter()
+
+global_args = None
+global_file_path = None
+
+
+def set_global_args(args):
+    global global_args
+    global_args = args
+
+
+def set_global_file_path(path):
+    global global_file_path
+    global_file_path = path
+
+
+def _decode_audio(data: str) -> bytes:
+    if data.startswith("data:"):
+        _, enc = data.split(",", 1)
+        return base64.b64decode(enc)
+    return base64.b64decode(data)
+
+
+def _ffmpeg_binary() -> str:
+    ffmpeg = shutil.which("ffmpeg")
+    if not ffmpeg:
+        raise HTTPException(status_code=501, detail="ffmpeg is required for native stem separation")
+    return ffmpeg
+
+
+def _base_url(http_request: Request) -> str:
+    url_setting = getattr(global_args, "url", "auto") if global_args else "auto"
+    if url_setting != "auto":
+        return url_setting.rstrip("/")
+    host = http_request.headers.get("host", "127.0.0.1") if http_request else "127.0.0.1"
+    if ":" in host:
+        parts = host.split(":")
+        if len(parts) == 2 and parts[1].isdigit():
+            host = parts[0]
+    proto = "https" if getattr(global_args, "https", False) else "http"
+    port = getattr(global_args, "port", 8000) if global_args else 8000
+    return f"{proto}://{host}:{port}"
+
+
+def _persist_file(path: str, suffix: str, http_request: Request) -> dict:
+    data = Path(path).read_bytes()
+    if global_file_path:
+        os.makedirs(global_file_path, exist_ok=True)
+        filename = f"{uuid.uuid4().hex}{suffix}"
+        out_path = os.path.join(global_file_path, filename)
+        with open(out_path, "wb") as handle:
+            handle.write(data)
+        return {"url": f"{_base_url(http_request)}/v1/files/{filename}"}
+    return {f"b64_{suffix.lstrip('.')}": base64.b64encode(data).decode("ascii")}
+
+
+def _run_ffmpeg(command: List[str]):
+    proc = subprocess.run(command, capture_output=True, text=True)
+    if proc.returncode != 0:
+        detail = proc.stderr.strip() or proc.stdout.strip() or "ffmpeg command failed"
+        raise HTTPException(status_code=500, detail=detail)
+
+
+def separate_with_provider(audio_bytes: bytes, stem_mode: str, workdir: str) -> dict:
+    raise HTTPException(status_code=501, detail="ML stem separation backend not installed")
+
+
+def _split_audio(audio_bytes: bytes, mode: str, workdir: str) -> dict:
+    ffmpeg = _ffmpeg_binary()
+    src = os.path.join(workdir, "input.wav")
+    with open(src, "wb") as handle:
+        handle.write(audio_bytes)
+
+    if mode == "vocals-instrumental":
+        vocal_path = os.path.join(workdir, "vocals.wav")
+        instrumental_path = os.path.join(workdir, "instrumental.wav")
+        _run_ffmpeg([
+            ffmpeg,
+            "-y",
+            "-i",
+            src,
+            "-af",
+            "pan=mono|c=0.5*FL+0.5*FR,highpass=f=120",
+            vocal_path,
+        ])
+        _run_ffmpeg([
+            ffmpeg,
+            "-y",
+            "-i",
+            src,
+            "-af",
+            "pan=stereo|c0=FL-0.5*FC|c1=FR-0.5*FC,lowpass=f=14000",
+            instrumental_path,
+        ])
+        return {
+            "stem_mode": mode,
+            "artifacts": [
+                {"name": "vocals", "path": vocal_path, "role": "lead-vocal-estimate"},
+                {"name": "instrumental", "path": instrumental_path, "role": "backing-mix-estimate"},
+            ],
+            "engine": "ffmpeg-mid-side-estimate",
+            "limitations": [
+                "best-effort heuristic only",
+                "works best on center-panned vocals",
+                "not equivalent to ML demixing",
+            ],
+        }
+
+    if mode == "drums-bass-other":
+        drums_path = os.path.join(workdir, "drums.wav")
+        bass_path = os.path.join(workdir, "bass.wav")
+        other_path = os.path.join(workdir, "other.wav")
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "highpass=f=80,lowpass=f=220", bass_path])
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "highpass=f=1800", drums_path])
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "highpass=f=220,lowpass=f=1800", other_path])
+        return {
+            "stem_mode": mode,
+            "artifacts": [
+                {"name": "drums", "path": drums_path, "role": "high-frequency-transient-band"},
+                {"name": "bass", "path": bass_path, "role": "low-frequency-band"},
+                {"name": "other", "path": other_path, "role": "mid-band-residual"},
+            ],
+            "engine": "ffmpeg-band-split",
+            "limitations": [
+                "frequency-band approximation only",
+                "not isolated stems",
+                "bleed between sources is expected",
+            ],
+        }
+
+    if mode == "4-stem":
+        drums_path = os.path.join(workdir, "drums.wav")
+        bass_path = os.path.join(workdir, "bass.wav")
+        vocals_path = os.path.join(workdir, "vocals.wav")
+        other_path = os.path.join(workdir, "other.wav")
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "highpass=f=80,lowpass=f=220", bass_path])
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "highpass=f=1800", drums_path])
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "pan=mono|c=0.5*FL+0.5*FR,highpass=f=120", vocals_path])
+        _run_ffmpeg([ffmpeg, "-y", "-i", src, "-af", "highpass=f=220,lowpass=f=1800", other_path])
+        return {
+            "stem_mode": mode,
+            "artifacts": [
+                {"name": "vocals", "path": vocals_path, "role": "lead-vocal-estimate"},
+                {"name": "drums", "path": drums_path, "role": "high-frequency-transient-band"},
+                {"name": "bass", "path": bass_path, "role": "low-frequency-band"},
+                {"name": "other", "path": other_path, "role": "mid-band-residual"},
+            ],
+            "engine": "ffmpeg-hybrid-estimate",
+            "limitations": [
+                "hybrid heuristic split only",
+                "not phase-accurate demixing",
+                "use dedicated ML separators for production quality",
+            ],
+        }
+
+    raise HTTPException(status_code=400, detail=f"Unsupported stem_mode: {mode}")
+
+
+class AudioStemRequest(BaseModel):
+    audio: str
+    stem_mode: Optional[str] = "vocals-instrumental"
+    response_format: Optional[str] = "url"
+    fallback_mode: Optional[bool] = False
+    model_config = ConfigDict(extra="allow")
+
+
+@router.post("/v1/audio/stems")
+async def separate_stems(request: AudioStemRequest, http_request: Request = None):
+    try:
+        audio_bytes = _decode_audio(request.audio)
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=f"Invalid audio payload: {exc}")
+
+    with tempfile.TemporaryDirectory(prefix="codai-stems-") as workdir:
+        backend_info = detect_audio_backends()["separation"]
+        if request.fallback_mode:
+            result = _split_audio(audio_bytes, request.stem_mode or "vocals-instrumental", workdir)
+            quality = "best-effort"
+            dependency = "ffmpeg"
+            model_name = None
+        else:
+            result = separate_with_provider(audio_bytes, request.stem_mode or "vocals-instrumental", workdir)
+            quality = "ml"
+            dependency = "python"
+            model_name = result.get("model")
+        data = []
+        for artifact in result["artifacts"]:
+            payload = _persist_file(artifact["path"], ".wav", http_request)
+            payload.update({"name": artifact["name"], "role": artifact["role"]})
+            data.append(payload)
+        return {
+            "created": int(time.time()),
+            "stem_mode": result["stem_mode"],
+            "backend": {
+                "engine": result["engine"],
+                "model": model_name,
+                "quality": quality,
+                "dependency": dependency,
+                "ml_backend_available": backend_info["available"],
+                "preferred_engine": backend_info["engine"],
+            },
+            "limitations": result["limitations"],
+            "data": data,
+        }

codai/main.py

@@ -580,6 +580,16 @@ def main():
     if global_file_path:
         set_audiogen_file_path(global_file_path)
 
+    from codai.api.audio_stems import set_global_args as set_astems_global_args, set_global_file_path as set_astems_file_path
+    set_astems_global_args(global_args)
+    if global_file_path:
+        set_astems_file_path(global_file_path)
+
+    from codai.api.audio_clean import set_global_args as set_aclean_global_args, set_global_file_path as set_aclean_file_path
+    set_aclean_global_args(global_args)
+    if global_file_path:
+        set_aclean_file_path(global_file_path)
+
     # Set voice clone module global args
     from codai.api.voice_clone import set_global_args as set_vc_global_args, set_global_file_path as set_vc_file_path
     set_vc_global_args(global_args)

requirements-nvidia.txt

@@ -38,6 +38,15 @@ f5-tts>=1.1.0
 # Voice conversion / singing voice conversion (Seed-VC — preserves pitch/melody)
 seed-vc>=0.4.0
 
+# Audio ML separation / restoration
+# Demucs is the primary high-quality separator backend.
+demucs>=4.0.1
+# DeepFilterNet is the primary learned cleanup/restoration backend.
+deepfilternet>=0.5.6
+# Optional alternates.
+rnnoise>=0.2.1
+voicefixer>=0.1.3
+
 # Face swap (insightface INSwapper — downloads inswapper_128.onnx on first use)
 insightface>=0.7.3
 onnxruntime-gpu>=1.20.0        # GPU-accelerated ONNX runtime for insightface

requirements-vulkan.txt

@@ -22,6 +22,13 @@ whispercpp>=0.0.17  # For GGUF-based Whisper transcription without PyTorch
 # Voice cloning (F5-TTS zero-shot voice cloning)
 f5-tts>=1.1.0
 
+# Audio ML separation / restoration
+# These run outside the Vulkan text backend path but are required for full-quality audio workflows.
+demucs>=4.0.1
+deepfilternet>=0.5.6
+rnnoise>=0.2.1
+voicefixer>=0.1.3
+
 # Face swap (insightface INSwapper — downloads inswapper_128.onnx on first use)
 insightface>=0.7.3
 onnxruntime>=1.20.0            # CPU ONNX runtime (use onnxruntime-gpu for GPU acceleration)

requirements.txt

@@ -81,6 +81,15 @@ f5-tts>=1.1.0
 # Voice conversion / singing voice conversion (Seed-VC — preserves pitch/melody)
 seed-vc>=0.4.0
 
+# Audio ML separation / restoration
+# Demucs provides real source separation for vocals/instrumental and multi-stem workflows.
+demucs>=4.0.1
+# DeepFilterNet provides learned denoise/restoration for higher-quality cleanup than ffmpeg-only filters.
+deepfilternet>=0.5.6
+# Optional alternate restoration backends.
+rnnoise>=0.2.1
+voicefixer>=0.1.3
+
 # Face swap (insightface INSwapper — downloads inswapper_128.onnx on first use)
 insightface>=0.7.3
 onnxruntime-gpu>=1.20.0        # GPU-accelerated ONNX runtime for insightface

tests/test_audio_ml_backends.py

+from pathlib import Path
+import sys
+
+sys.path.insert(0, '/storage/coderai')
+
+from codai.api.audio_backends import detect_audio_backends, reset_audio_backend_cache
+
+
+def test_detect_audio_backends_reports_missing_providers(monkeypatch):
+    reset_audio_backend_cache()
+    monkeypatch.setattr("importlib.util.find_spec", lambda name: None)
+
+    backends = detect_audio_backends()
+
+    assert backends["separation"]["available"] is False
+    assert backends["separation"]["engine"] is None
+    assert backends["restoration"]["available"] is False
+    assert backends["restoration"]["engine"] is None
+
+
+def test_detect_audio_backends_prefers_deepfilter_for_restoration(monkeypatch):
+    reset_audio_backend_cache()
+
+    def fake_find_spec(name: str):
+        if name in {"demucs", "df"}:
+            return object()
+        return None
+
+    monkeypatch.setattr("importlib.util.find_spec", fake_find_spec)
+
+    backends = detect_audio_backends()
+
+    assert backends["separation"] == {
+        "available": True,
+        "engine": "demucs",
+        "candidates": ["demucs"],
+    }
+    assert backends["restoration"]["available"] is True
+    assert backends["restoration"]["engine"] == "deepfilternet"
+    assert "voicefixer" in backends["restoration"]["candidates"]
+
+
+def test_requirements_include_audio_ml_dependencies():
+    req = Path('/storage/coderai/requirements.txt').read_text().lower()
+    assert 'demucs' in req
+    assert 'deepfilternet' in req or 'df[' in req

tests/test_audio_ml_endpoints.py

+import base64
+import os
+import sys
+import wave
+from io import BytesIO
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+
+@pytest.fixture
+def sample_wav_b64():
+    buf = BytesIO()
+    with wave.open(buf, "wb") as wav_file:
+        wav_file.setnchannels(1)
+        wav_file.setsampwidth(2)
+        wav_file.setframerate(8000)
+        wav_file.writeframes(b"\x00\x00" * 800)
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+def test_audio_stems_uses_provider_output(monkeypatch, tmp_path, sample_wav_b64):
+    import importlib.util
+
+    module_path = Path(__file__).resolve().parents[1] / "codai" / "api" / "audio_stems.py"
+    spec = importlib.util.spec_from_file_location("test_audio_stems_module", module_path)
+    audio_stems = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(audio_stems)
+
+    app = FastAPI()
+    app.include_router(audio_stems.router)
+    audio_stems.set_global_file_path(str(tmp_path))
+
+    vocals = tmp_path / "vocals.wav"
+    inst = tmp_path / "inst.wav"
+    vocals.write_bytes(b"wav")
+    inst.write_bytes(b"wav")
+
+    monkeypatch.setattr(audio_stems, "separate_with_provider", lambda *args, **kwargs: {
+        "engine": "demucs",
+        "model": "htdemucs",
+        "stem_mode": "vocals-instrumental",
+        "artifacts": [
+            {"name": "vocals", "path": str(vocals), "role": "vocals"},
+            {"name": "instrumental", "path": str(inst), "role": "instrumental"},
+        ],
+        "limitations": [],
+    })
+    monkeypatch.setattr(audio_stems, "detect_audio_backends", lambda: {
+        "separation": {"available": True, "engine": "demucs", "candidates": ["demucs"]}
+    })
+
+    client = TestClient(app)
+    response = client.post(
+        "/v1/audio/stems",
+        json={"audio": sample_wav_b64, "stem_mode": "vocals-instrumental", "response_format": "url"},
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["backend"]["engine"] == "demucs"
+    assert body["backend"]["model"] == "htdemucs"
+    assert body["backend"]["quality"] == "ml"
+    assert len(body["data"]) == 2
+    assert body["data"][0]["url"].endswith(".wav")
+
+
+def test_audio_cleanup_uses_restore_provider(monkeypatch, tmp_path, sample_wav_b64):
+    import importlib.util
+
+    module_path = Path(__file__).resolve().parents[1] / "codai" / "api" / "audio_clean.py"
+    spec = importlib.util.spec_from_file_location("test_audio_clean_module", module_path)
+    audio_clean = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(audio_clean)
+
+    app = FastAPI()
+    app.include_router(audio_clean.router)
+    audio_clean.set_global_file_path(str(tmp_path))
+
+    cleaned = tmp_path / "cleaned.wav"
+    cleaned.write_bytes(b"wav")
+
+    monkeypatch.setattr(audio_clean, "restore_with_provider", lambda *args, **kwargs: {
+        "engine": "deepfilternet",
+        "model": "DeepFilterNet3",
+        "path": str(cleaned),
+        "applied": ["denoise", "normalize"],
+        "limitations": [],
+    })
+    monkeypatch.setattr(audio_clean, "detect_audio_backends", lambda: {
+        "restoration": {"available": True, "engine": "deepfilternet", "candidates": ["deepfilternet", "rnnoise", "voicefixer"]}
+    })
+
+    client = TestClient(app)
+    response = client.post(
+        "/v1/audio/cleanup",
+        json={
+            "audio": sample_wav_b64,
+            "noise_reduction": True,
+            "normalize": True,
+            "remove_hum": False,
+            "repair_clicks": False,
+            "response_format": "url",
+        },
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["backend"]["engine"] == "deepfilternet"
+    assert body["backend"]["model"] == "DeepFilterNet3"
+    assert body["backend"]["quality"] == "ml"
+    assert body["applied"] == ["denoise", "normalize"]
+    assert body["data"][0]["url"].endswith(".wav")

tests/test_studio_composed_surfaces.py

+import base64
+import os
+import sys
+import wave
+from io import BytesIO
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+
+@pytest.fixture
+def studio_client(tmp_path):
+    from codai.api import audio_clean, audio_stems, custom_pipelines, transcriptions, tts, text, embeddings
+    from codai.admin import routes as admin_routes
+
+    audio_stems.set_global_file_path(str(tmp_path))
+    audio_clean.set_global_file_path(str(tmp_path))
+
+    app = FastAPI()
+    app.include_router(audio_stems.router)
+    app.include_router(audio_clean.router)
+    app.include_router(custom_pipelines.router)
+    app.include_router(transcriptions.router)
+    app.include_router(tts.router)
+    app.include_router(text.router)
+    app.include_router(embeddings.router)
+    app.include_router(admin_routes.router)
+    app.dependency_overrides[admin_routes.require_auth] = lambda: "tester"
+    return TestClient(app)
+
+
+@pytest.fixture
+def sample_wav_b64():
+    buf = BytesIO()
+    with wave.open(buf, "wb") as wav_file:
+        wav_file.setnchannels(1)
+        wav_file.setsampwidth(2)
+        wav_file.setframerate(8000)
+        wav_file.writeframes(b"\x00\x00" * 800)
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+def test_audio_understanding_composes_transcript_and_summary(monkeypatch, studio_client):
+    from codai.api import custom_pipelines
+
+    async def fake_run_step(step, context, http_request):
+        if step["type"] == "stt":
+            return {"output": "meeting transcript text", "text": "meeting transcript text"}
+        if step["type"] == "text_gen":
+            assert context["step0"]["output"] == "meeting transcript text"
+            prompt = step["params"]["prompt"]
+            assert "Summarize action items" in prompt
+            assert "{{step0.output}}" in prompt
+            return {"output": "summary from transcript"}
+        raise AssertionError(f"unexpected step type {step['type']}")
+
+    monkeypatch.setattr(custom_pipelines, "_run_step", fake_run_step)
+
+    response = studio_client.post(
+        "/v1/pipelines/audio-understand",
+        json={
+            "input": "Summarize action items",
+            "audio_model": "whisper-small",
+            "text_model": "qwen-text",
+            "audio": "ZmFrZQ==",
+            "language": "en",
+        },
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["pipeline"] == "audio-understand"
+    assert body["transcript"] == "meeting transcript text"
+    assert body["summary"] == "summary from transcript"
+    assert [step["type"] for step in body["steps"]] == ["stt", "text_gen"]
+
+
+def test_audio_understanding_returns_transcript_only_without_text_model(monkeypatch, studio_client):
+    from codai.api import custom_pipelines
+
+    async def fake_run_step(step, context, http_request):
+        assert step["type"] == "stt"
+        return {"output": "raw transcript", "text": "raw transcript"}
+
+    monkeypatch.setattr(custom_pipelines, "_run_step", fake_run_step)
+
+    response = studio_client.post(
+        "/v1/pipelines/audio-understand",
+        json={
+            "input": "Describe the call",
+            "audio_model": "whisper-small",
+            "audio": "ZmFrZQ==",
+        },
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["transcript"] == "raw transcript"
+    assert body["summary"] is None
+    assert len(body["steps"]) == 1
+
+
+def test_audio_understanding_requires_audio_source(studio_client):
+    response = studio_client.post(
+        "/v1/pipelines/audio-understand",
+        json={"audio_model": "whisper-small", "input": "Summarize"},
+    )
+
+    assert response.status_code == 422
+    assert "audio" in response.text.lower()
+
+
+def test_music_dub_pipeline_returns_full_stage_outputs(monkeypatch, studio_client):
+    from codai.api import custom_pipelines
+
+    async def fake_run_full_music_dub(request, http_request):
+        return {
+            "vocals": {"path": "vocals.wav"},
+            "instrumental": {"path": "inst.wav"},
+            "transcript": "lyrics",
+            "translated_lyrics": "translated lyrics",
+            "converted_vocals": {"path": "dub.wav"},
+            "final_mix": {"path": "mix.wav"},
+            "steps": [
+                {"step": 0, "type": "stems"},
+                {"step": 1, "type": "stt"},
+                {"step": 2, "type": "translate"},
+                {"step": 3, "type": "voice_convert"},
+                {"step": 4, "type": "remix"},
+            ],
+        }
+
+    monkeypatch.setattr(custom_pipelines, "run_full_music_dub", fake_run_full_music_dub)
+
+    response = studio_client.post(
+        "/v1/pipelines/audio-music-dub",
+        json={
+            "audio_model": "whisper-small",
+            "audio": "ZmFrZQ==",
+            "target_lang": "es",
+            "notes": "Prefer singability",
+        },
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["pipeline"] == "audio-music-dub"
+    assert body["status"] == "available"
+    assert body["transcript"] == "lyrics"
+    assert body["translated_lyrics"] == "translated lyrics"
+    assert body["final_mix"]["path"] == "mix.wav"
+    assert [step["type"] for step in body["steps"]] == ["stems", "stt", "translate", "voice_convert", "remix"]
+
+
+def test_stem_separation_returns_artifacts_and_limitations(monkeypatch, studio_client, sample_wav_b64, tmp_path):
+    from codai.api import audio_stems
+
+    stem_paths = []
+    for name in ("vocals.wav", "instrumental.wav"):
+        path = tmp_path / name
+        path.write_bytes(b"wav")
+        stem_paths.append(str(path))
+
+    def fake_split(audio_bytes, mode, workdir):
+        assert audio_bytes
+        assert mode == "vocals-instrumental"
+        return {
+            "stem_mode": mode,
+            "artifacts": [
+                {"name": "vocals", "path": stem_paths[0], "role": "lead-vocal"},
+                {"name": "instrumental", "path": stem_paths[1], "role": "backing-mix"},
+            ],
+            "engine": "ffmpeg-phase-invert",
+            "limitations": ["center-panned-only"],
+        }
+
+    monkeypatch.setattr(audio_stems, "_split_audio", fake_split)
+
+    response = studio_client.post(
+        "/v1/audio/stems",
+        json={"audio": sample_wav_b64, "stem_mode": "vocals-instrumental", "response_format": "url", "fallback_mode": True},
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["stem_mode"] == "vocals-instrumental"
+    assert body["backend"]["engine"] == "ffmpeg-phase-invert"
+    assert body["backend"]["quality"] == "best-effort"
+    assert len(body["data"]) == 2
+    assert "/v1/files/" in body["data"][0]["url"]
+    assert "/v1/files/" in body["data"][1]["url"]
+    assert "center-panned-only" in body["limitations"]
+
+
+def test_audio_cleanup_returns_artifact_and_applied_operations(monkeypatch, studio_client, sample_wav_b64, tmp_path):
+    from codai.api import audio_clean
+
+    cleaned_path = tmp_path / "cleaned.wav"
+    cleaned_path.write_bytes(b"wav")
+
+    def fake_cleanup(audio_bytes, options, workdir):
+        assert audio_bytes
+        assert options["noise_reduction"] is True
+        assert options["normalize"] is True
+        return {
+            "path": str(cleaned_path),
+            "engine": "ffmpeg-afftdn",
+            "applied": ["noise_reduction", "normalize"],
+            "limitations": ["not-ml-restoration"],
+        }
+
+    monkeypatch.setattr(audio_clean, "_cleanup_audio", fake_cleanup)
+
+    response = studio_client.post(
+        "/v1/audio/cleanup",
+        json={
+            "audio": sample_wav_b64,
+            "noise_reduction": True,
+            "normalize": True,
+            "remove_hum": False,
+            "repair_clicks": False,
+            "response_format": "url",
+            "fallback_mode": True,
+        },
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["backend"]["engine"] == "ffmpeg-afftdn"
+    assert body["backend"]["quality"] == "best-effort"
+    assert body["applied"] == ["noise_reduction", "normalize"]
+    assert "/v1/files/" in body["data"][0]["url"]
+    assert "not-ml-restoration" in body["limitations"]
+
+
+def test_chat_template_wires_preview_shells_for_new_runnable_panels():
+    template_path = "/storage/coderai/codai/admin/templates/chat.html"
+    text = open(template_path, "r", encoding="utf-8").read()
+
+    assert "id=\"at-preview\"" in text
+    assert "id=\"as-preview\"" in text
+    assert "id=\"ig-preview\"" in text
+    assert "id=\"em-preview\"" in text
+    assert "id=\"ast-preview\"" in text
+    assert "id=\"ac-preview\"" in text
+    assert "'aud-tts':" in text
+    assert "'aud-stt':" in text
+    assert "'aud-stems':" in text
+    assert "'aud-clean':" in text
+    assert "buildAudioUnderstandPreviewData" in text
+    assert "buildMusicDubPreviewData" in text
+    assert "buildStemPreviewData" in text
+    assert "buildCleanupPreviewData" in text
+
+
+def test_chat_template_marks_full_quality_audio_panels_with_runtime_backend_metadata():
+    template_path = "/storage/coderai/codai/admin/templates/chat.html"
+    text = open(template_path, "r", encoding="utf-8").read()
+
+    assert "audioBackendHealth" in text
+    assert "renderAudioBackendHealth" in text
+    assert "aud-music-dub" in text
+    assert "aud-stems" in text
+    assert "aud-clean" in text
+
+
+def test_chat_template_exposes_ml_preview_and_artifact_markers():
+    template_path = "/storage/coderai/codai/admin/templates/chat.html"
+    text = open(template_path, "r", encoding="utf-8").read()
+
+    assert "buildStemPreviewData" in text
+    assert "buildCleanupPreviewData" in text
+    assert "buildMusicDubPreviewData" in text
+    assert "pushArtifactHistory({" in text
+    assert "backend?.model" in text
+    assert "translated_lyrics" in text