longcat: weight variants and the audio-driven avatar tasks (v0.2.35)

Step 6. Read from upstream's avatar demo rather than assumed, because almost nothing
carries over from the base model:

- avatar is a DIFFERENT pipeline class, LongCatVideoAvatarPipeline, with its own methods
  generate_at2v / generate_ai2v / generate_avc;
- it takes TWO guidance scales — text and audio — where the base pipeline takes one;
- its distilled pass is dmd_lora at 8 steps with lora_network_dim=128/alpha=64 and
  dit.enable_loras(["dmd"]), NOT cfg_step_lora at 16;
- use_int8 is a separate load_quantized_dit(subfolder="base_model_int8"), not a dtype;
- both INT8 and the DMD LoRA exist only for avatar-1.5, which also REQUIRES distilled
  sampling;
- the audio is librosa.load(sr=16000) then pipe.get_audio_embedding(...), and the encoder
  differs per family (chinese-wav2vec2-base for Avatar, whisper-large-v3 for Avatar-1.5).

Unsupported combinations are refused BEFORE loading, because the alternative is quietly
serving something that is not what was asked for: INT8 or DMD on anything but
avatar-1.5, avatar-1.5 without distilled sampling, a base-model mode on avatar weights or
the reverse, and GGUF, which needs ComfyUI's gguf loader and says so.

FP8 loads the DiT as float8_e4m3fn. Community layouts differ between publishers (Kijai's
scaled format is not a plain e4m3 dump), so a repo that needs its own loader fails with
that reason rather than producing noise.

Also: librosa is a real new dependency and went into requirements-longcat.txt, not only
into the code that imports it. And the avatar audio comes from the request's `audio_file`
field — `add_audio` is a BOOLEAN for the post-processing path, and using it would have
passed True into the base64 encoder.

35 new tests, 188 across the six LongCat suites.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.34"
+__version__ = "0.2.35"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/video.py

@@ -4128,12 +4128,15 @@ async def _generate_longcat(request: VideoGenerationRequest, model_name: str,
     mode = (request.mode or "t2v").lower()
     _task = {"t2v": "t2v", "": "t2v",
              "i2v": "i2v", "ti2v": "i2v",
-             "extend": "vc", "vc": "vc", "v2v": "vc"}.get(mode)
+             "extend": "vc", "vc": "vc", "v2v": "vc",
+             # Audio-driven avatar: a different pipeline class upstream, with its own
+             # methods and two guidance scales. 'at2v' is audio+text, 'ai2v' adds a
+             # reference image.
+             "at2v": "at2v", "ai2v": "ai2v", "avatar": "at2v"}.get(mode)
     if _task is None:
         raise HTTPException(status_code=400, detail=(
-            f"LongCat-Video does not support mode '{mode}'. Use 't2v', 'i2v' or "
-            f"'extend' (continuation); the audio-driven avatar tasks are a separate "
-            f"increment."))
+            f"LongCat-Video does not support mode '{mode}'. Use 't2v', 'i2v', "
+            f"'extend' (continuation), or 'at2v'/'ai2v' (audio-driven avatar)."))
 
     payload = {
         "prompt": request.prompt or "",
@@ -4147,6 +4150,35 @@ async def _generate_longcat(request: VideoGenerationRequest, model_name: str,
     if request.negative_prompt:
         payload["negative_prompt"] = request.negative_prompt
 
+    # The variant and its flags gate what can load at all: INT8 and the DMD
+    # distillation exist only for the avatar-1.5 weights. The service refuses an
+    # impossible combination rather than loading something else.
+    for _k in ("variant", "use_int8", "use_distill", "ref_img_index",
+               "mask_frame_range"):
+        if model_cfg.get(_k) not in (None, ""):
+            payload[_k] = model_cfg[_k]
+
+    # The avatar tasks need an audio track, and ai2v a reference image too.
+    if _task in ("at2v", "ai2v"):
+        # `audio_file` already means "existing audio, base64 or URL" on this request;
+        # `add_audio` is a BOOLEAN for the post-processing path and must not be used here.
+        _audio = request.audio_file
+        if not _audio:
+            raise HTTPException(status_code=400, detail=(
+                f"mode '{mode}' is audio-driven and needs an audio track in "
+                f"`audio_file` (base64 or URL)"))
+        payload["audio"] = _as_b64(_audio)
+        if getattr(request, "text_guidance_scale", None) is not None:
+            payload["text_guidance_scale"] = float(request.text_guidance_scale)
+        if getattr(request, "audio_guidance_scale", None) is not None:
+            payload["audio_guidance_scale"] = float(request.audio_guidance_scale)
+    if _task == "ai2v":
+        _ref = request.init_image or request.image
+        if not _ref:
+            raise HTTPException(status_code=400, detail=(
+                "mode 'ai2v' needs a reference image (init_image or image)"))
+        payload["image"] = _as_b64(_ref)
+
     # i2v takes one conditioning image; coderai accepts it as init_image or image.
     if _task == "i2v":
         _first = request.init_image or request.image

codai/pydantic/videorequest.py

@@ -102,6 +102,13 @@ class VideoGenerationRequest(BaseModel):
         "LongCat-Video: how many frames of the previous segment each new segment "
         "conditions on (default 13 of 93). More = stronger motion continuity, fewer "
         "new frames per segment."))
+    # The avatar pipeline takes TWO guidance scales where the base one takes a single
+    # guidance_scale: how hard to follow the prompt, and how hard to follow the audio.
+    text_guidance_scale: Optional[float] = Field(None, description=(
+        "LongCat-Video avatar: prompt adherence (default 4.0, or 1.0 distilled)."))
+    audio_guidance_scale: Optional[float] = Field(None, description=(
+        "LongCat-Video avatar: how closely the motion follows the audio "
+        "(default 4.0, or 1.0 distilled)."))
     quality: Optional[str] = Field(None, description=(
         "LongCat-Video quality preset: 'draft' (base stage only, fastest), 'fast' "
         "(distilled, 16 steps — the default), 'best' (base + 720p refinement). Ignored "

docs/longcat-video.md

@@ -134,10 +134,52 @@ building or downloading anything. `distribute` fans independent generations acro
 engines and nodes. A `video-longcat` RunPod capability image needs a custom Dockerfile
 (the generic one is python:3.12 + cu128, wrong on both axes) and is not built yet.
 
+## Variants and the avatar families
+
+The avatar weights are **a different pipeline** upstream
+(`LongCatVideoAvatarPipeline`), with its own methods and **two** guidance scales — how
+hard to follow the prompt, and how hard to follow the audio. Its distilled pass is also
+not the base model's: 8 steps against `dmd_lora`, where the base model's is 16 against
+`cfg_step_lora`.
+
+| | base | Avatar | Avatar-1.5 |
+|---|---|---|---|
+| audio encoder | — | chinese-wav2vec2-base | whisper-large-v3 |
+| modes | t2v, i2v, extend | at2v, ai2v | at2v, ai2v |
+| INT8 | no | no | **yes** (`base_model_int8/`) |
+| distilled | `cfg_step_lora`, 16 steps | no | **required**, `dmd_lora`, 8 steps |
+
+Unsupported combinations are **refused before loading** rather than quietly loading
+something else: INT8 or the DMD distillation on anything but avatar-1.5, avatar-1.5
+without distilled sampling, a base-model mode on avatar weights (or the reverse), and the
+GGUF variants, which need ComfyUI's gguf loader.
+
+FP8 loads the DiT as `float8_e4m3fn`. Community FP8 layouts differ between publishers
+(Kijai's scaled format is not a plain e4m3 dump), so if a repo needs its own loader the
+load fails with that reason instead of producing noise.
+
+Audio-driven generation:
+
+```jsonc
+POST /v1/video/generations
+{
+  "model": "longcat-avatar-15",
+  "mode": "ai2v",                 // at2v = audio+text; ai2v adds a reference image
+  "prompt": "a news anchor reading the headlines",
+  "audio_file": "<base64 or URL>",
+  "init_image": "<base64 or URL>",
+  "audio_guidance_scale": 4.0
+}
+```
+
+The audio is resampled to 16 kHz (the encoder's rate) and embedded by the family's
+encoder. `ref_img_index` (0–24; 30 reduces repeated actions) and `mask_frame_range` are
+set on the model entry.
+
 ## Not done yet
 
-- audio-driven avatar tasks; GGUF variants (they need the ComfyUI gguf loader)
+- GGUF variants (they need the ComfyUI gguf loader)
 - context-parallel multi-GPU (`cp_size` is accepted but not yet acted on)
 - the `video-longcat` pod image
 - LoRA/QLoRA training on top of it (feasible — upstream's own refinement expert is a LoRA)
-- **no generation has been verified against real weights**
+- **no generation has been verified against real weights**, avatar included

requirements-longcat.txt

@@ -53,6 +53,12 @@ imageio==2.37.0
 imageio-ffmpeg==0.6.0
 pyarrow==20.0.0
 
+# Audio-driven avatar generation: the audio track is resampled to 16 kHz with librosa
+# and embedded by the family's encoder (chinese-wav2vec2-base for Avatar, whisper-large-v3
+# for Avatar-1.5). Upstream installs librosa and ffmpeg alongside the pins above; ffmpeg
+# comes from the image, imageio-ffmpeg covers the muxing.
+librosa==0.10.2.post1
+
 # Attention backend. Upstream defaults to FlashAttention-2 but documents FA3 and xformers
 # as alternatives, and flash-attn==2.7.4.post1 is a SOURCE build needing ninja and a CUDA
 # toolchain — minutes of compile, and it fails outright on a host without nvcc wired up.

tests/test_longcat_variants.py

+"""Step 6: weight variants and the audio-driven avatar tasks.
+
+Read from upstream's avatar demo rather than assumed, because almost nothing carries over
+from the base model:
+
+- avatar is a DIFFERENT pipeline class, ``LongCatVideoAvatarPipeline``, with its own
+  methods ``generate_at2v`` / ``generate_ai2v`` / ``generate_avc``;
+- it takes TWO guidance scales (text and audio) where the base pipeline takes one;
+- its distilled pass is ``dmd_lora`` at **8** steps, not ``cfg_step_lora`` at 16;
+- ``use_int8`` is a separate ``load_quantized_dit(subfolder="base_model_int8")``, and both
+  it and the DMD LoRA exist only for avatar-1.5 — which also *requires* distilled sampling.
+
+Asking for an unsupported combination has to be an error, because the alternative is
+quietly loading something that is not what was asked for.
+"""
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+
+def _mod(name, rel):
+    spec = importlib.util.spec_from_file_location(name, ROOT / rel)
+    m = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(m)
+    return m
+
+
+LC = _mod("longcat_common", "tools/longcat_common.py")
+SVC = _mod("longcat_service", "tools/longcat_service.py")
+VIDEO = (ROOT / "codai/api/video.py").read_text()
+SERVICE_SRC = (ROOT / "tools/longcat_service.py").read_text()
+
+
+# ------------------------------------------------------------ family detection
+@pytest.mark.parametrize("path,family", [
+    ("meituan-longcat/LongCat-Video", ""),
+    ("/w/LongCat-Video-Avatar", "avatar"),
+    ("/w/LongCat-Video-Avatar-1.5", "avatar-1.5"),
+    ("/w/longcat_video_avatar_1.5", "avatar-1.5"),
+    ("szwagros/LongCat-Video-Avatar-1.5-fp8", "avatar-1.5"),
+])
+def test_the_family_is_read_from_the_path(path, family):
+    """The family IS the repo — three separate downloads with different audio encoders."""
+    assert LC.family_of(path) == family
+
+
+def test_each_family_has_its_own_audio_encoder():
+    assert LC.AVATAR_ENCODERS["avatar"] == "chinese-wav2vec2-base"
+    assert LC.AVATAR_ENCODERS["avatar-1.5"] == "whisper-large-v3"
+
+
+# ------------------------------------------------------------ variant gating
+def test_int8_is_refused_on_the_base_model():
+    probs = LC.variant_problems("int8", "")
+    assert probs and "avatar-1.5" in probs[0]
+
+
+def test_int8_is_accepted_on_avatar_15():
+    assert LC.variant_problems("int8", "avatar-1.5", use_int8=True) == []
+
+
+def test_the_dmd_distillation_is_refused_on_the_plain_avatar():
+    probs = LC.variant_problems("bf16", "avatar", use_distill=True)
+    assert any("DMD" in p for p in probs)
+
+
+def test_avatar_15_requires_distilled_sampling():
+    """Upstream states it; without the flag the 8-step DMD path is not taken."""
+    probs = LC.variant_problems("bf16", "avatar-1.5")
+    assert probs and "requires distilled sampling" in probs[0]
+    assert LC.variant_problems("bf16", "avatar-1.5", use_distill=True) == []
+
+
+def test_gguf_is_refused_with_the_reason():
+    """They are packaged for ComfyUI's gguf loader, which is not implemented."""
+    probs = LC.variant_problems("gguf", "")
+    assert probs and "ComfyUI" in probs[0]
+
+
+def test_an_unknown_variant_is_refused():
+    assert LC.variant_problems("int4", "")
+
+
+def test_bf16_on_the_base_model_needs_nothing():
+    assert LC.variant_problems("bf16", "") == []
+
+
+def test_fp8_is_allowed_everywhere_it_is_published():
+    assert LC.variant_problems("fp8", "") == []
+    assert LC.variant_problems("fp8", "avatar-1.5", use_distill=True) == []
+
+
+# ------------------------------------------------------------ avatar checkpoint checks
+def test_int8_needs_its_subfolder(tmp_path):
+    probs = LC.avatar_problems(str(tmp_path), "avatar-1.5", use_int8=True)
+    assert probs and LC.INT8_SUBDIR in probs[0]
+    (tmp_path / LC.INT8_SUBDIR).mkdir()
+    assert LC.avatar_problems(str(tmp_path), "avatar-1.5", use_int8=True) == []
+
+
+def test_the_dmd_lora_must_be_present(tmp_path):
+    probs = LC.avatar_problems(str(tmp_path), "avatar-1.5", use_distill=True)
+    assert probs and "dmd_lora" in probs[0]
+
+
+def test_a_non_avatar_checkpoint_cannot_serve_the_audio_tasks():
+    probs = LC.avatar_problems("/w/LongCat-Video", "")
+    assert probs and "audio-driven" in probs[0]
+
+
+# ------------------------------------------------------------ the avatar step table
+def test_the_avatar_distilled_pass_is_eight_steps_not_sixteen():
+    """The base model's distilled pass is 16 steps against cfg_step_lora; the avatar's is
+    8 against dmd_lora. Using the base numbers would run it wrong."""
+    assert LC.avatar_stage_params("distill")["num_inference_steps"] == 8
+    assert LC.stage_params("distill")["num_inference_steps"] == 16
+
+
+def test_the_avatar_pass_has_two_guidance_scales():
+    p = LC.avatar_stage_params("base")
+    assert "text_guidance_scale" in p and "audio_guidance_scale" in p
+    assert "guidance_scale" not in p
+    assert LC.avatar_stage_params("distill")["audio_guidance_scale"] == 1.0
+
+
+def test_the_dmd_lora_dimensions_are_pinned():
+    assert (LC.DMD_NETWORK_DIM, LC.DMD_NETWORK_ALPHA) == (128, 64)
+
+
+# ------------------------------------------------------------ the service calls
+class _AvatarPipe:
+    def __init__(self):
+        self.calls = []
+
+    def _out(self):
+        import numpy as np
+        return [np.zeros((4, 8, 8, 3), dtype="float32")]
+
+    def generate_at2v(self, **kw):
+        self.calls.append(("at2v", kw)); return self._out()
+
+    def generate_ai2v(self, **kw):
+        self.calls.append(("ai2v", kw)); return self._out()
+
+    def generate_avc(self, **kw):
+        self.calls.append(("avc", kw)); return self._out()
+
+
+def _ctx(**kw):
+    base = {"prompt": "speak", "negative_prompt": "", "height": 480, "width": 832,
+            "num_frames": 93, "num_cond_frames": 13, "audio_emb": "EMB",
+            "image": None, "generator": None, "use_distill": False,
+            "num_inference_steps": None, "text_guidance_scale": None,
+            "audio_guidance_scale": None, "offload_kv_cache": False,
+            "ref_img_index": None, "mask_frame_range": None, "stage": "base"}
+    base.update(kw)
+    return base
+
+
+def test_at2v_is_called_for_audio_only():
+    p = _AvatarPipe()
+    SVC._avatar_pass(p, "at2v", _ctx())
+    name, kw = p.calls[0]
+    assert name == "at2v"
+    assert kw["audio_emb"] == "EMB" and kw["output_type"] == "both"
+    assert "text_guidance_scale" in kw and "audio_guidance_scale" in kw
+    assert "guidance_scale" not in kw
+
+
+def test_ai2v_passes_the_reference_image():
+    p = _AvatarPipe()
+    SVC._avatar_pass(p, "ai2v", _ctx(image="REF"))
+    name, kw = p.calls[0]
+    assert name == "ai2v" and kw["image"] == "REF"
+
+
+def test_continuation_uses_avc_with_the_avatar_options():
+    p = _AvatarPipe()
+    SVC._avatar_pass(p, "at2v", _ctx(ref_img_index=4, mask_frame_range="0:10"),
+                     cond=["prev"])
+    name, kw = p.calls[0]
+    assert name == "avc"
+    assert kw["video"] == ["prev"] and kw["num_cond_frames"] == 13
+    assert kw["use_kv_cache"] is True
+    assert kw["ref_img_index"] == 4 and kw["mask_frame_range"] == "0:10"
+
+
+def test_the_distill_flag_is_forwarded():
+    p = _AvatarPipe()
+    SVC._avatar_pass(p, "at2v", _ctx(use_distill=True, stage="distill"))
+    _n, kw = p.calls[0]
+    assert kw["use_distill"] is True
+    assert kw["num_inference_steps"] == 8
+
+
+def test_avatar_tasks_route_through_one_pass():
+    p = _AvatarPipe()
+    SVC._one_pass(p, "at2v", "base", _ctx())
+    assert p.calls and p.calls[0][0] == "at2v"
+
+
+# ------------------------------------------------------------ loading
+def test_the_avatar_pipeline_class_is_used():
+    assert "LongCatVideoAvatarPipeline" in SERVICE_SRC
+    assert "pipeline_longcat_video_avatar" in SERVICE_SRC
+
+
+def test_int8_uses_the_repos_quantised_loader():
+    assert "load_quantized_dit" in SERVICE_SRC
+    assert "base_model_int8" in (LC.INT8_SUBDIR,)
+
+
+def test_fp8_loads_as_that_dtype_and_says_so():
+    """Community FP8 layouts differ between publishers, so this must fail with the real
+    reason rather than silently produce noise."""
+    seg = SERVICE_SRC[SERVICE_SRC.index('== "fp8"'):]
+    assert "float8_e4m3fn" in seg[:600]
+
+
+def test_the_audio_is_resampled_to_the_encoders_rate():
+    assert "sr=16000" in SERVICE_SRC
+    assert "get_audio_embedding" in SERVICE_SRC
+
+
+def test_librosa_is_a_declared_dependency():
+    """The house rule: a new pip dependency goes in the requirements file, not only in
+    the code that imports it."""
+    req = (ROOT / "requirements-longcat.txt").read_text()
+    assert "librosa" in req
+
+
+# ------------------------------------------------------------ dispatch
+def test_the_avatar_modes_are_mapped():
+    seg = VIDEO[VIDEO.index('_task = {"t2v"'):VIDEO.index("if _task is None")]
+    for mode, task in (("at2v", "at2v"), ("ai2v", "ai2v"), ("avatar", "at2v")):
+        assert f'"{mode}": "{task}"' in seg, mode
+
+
+def test_the_audio_comes_from_audio_file_not_the_add_audio_boolean():
+    """add_audio is a BOOLEAN for the post-processing path; using it would pass True
+    into the base64 encoder."""
+    fn = VIDEO[VIDEO.index("async def _generate_longcat"):]
+    fn = fn[:fn.index("\nasync def ")]
+    assert "request.audio_file" in fn
+    assert "add_audio" not in fn.split("_audio =")[1][:200]
+
+
+def test_an_avatar_request_without_audio_is_refused():
+    fn = VIDEO[VIDEO.index("async def _generate_longcat"):]
+    fn = fn[:fn.index("\nasync def ")]
+    seg = fn[fn.index('if _task in ("at2v"'):]
+    assert "status_code=400" in seg[:900]
+
+
+def test_the_variant_flags_are_forwarded_to_the_service():
+    fn = VIDEO[VIDEO.index("async def _generate_longcat"):]
+    fn = fn[:fn.index("\nasync def ")]
+    for key in ("variant", "use_int8", "use_distill", "ref_img_index",
+                "mask_frame_range"):
+        assert f'"{key}"' in fn, key
+
+
+def test_the_request_model_carries_the_avatar_scales():
+    from codai.pydantic.videorequest import VideoGenerationRequest as R
+    r = R(model="m", prompt="p", text_guidance_scale=4.0, audio_guidance_scale=3.0)
+    assert (r.text_guidance_scale, r.audio_guidance_scale) == (4.0, 3.0)

tools/longcat_common.py

@@ -62,6 +62,111 @@ STAGE_DEFAULTS = {
 }
 
 
+# ── variants ─────────────────────────────────────────────────────────────────
+# Which checkpoint family a model entry points at, and what each supports. Getting this
+# wrong is not a crash but a wrong answer: --use_int8 and the DMD distillation exist only
+# for avatar-v1.5, and asking for them elsewhere would silently load something else.
+VARIANTS = ("bf16", "fp8", "int8", "gguf")
+
+# The avatar families are separate HuggingFace repos with their own audio encoders.
+AVATAR_ENCODERS = {
+    "avatar": "chinese-wav2vec2-base",      # v1.0
+    "avatar-1.5": "whisper-large-v3",
+}
+
+# Only avatar-1.5 ships an INT8 DiT (under base_model_int8/) and the DMD LoRA, and its
+# distilled path is REQUIRED rather than optional.
+INT8_FAMILIES = ("avatar-1.5",)
+DMD_FAMILIES = ("avatar-1.5",)
+INT8_SUBDIR = "base_model_int8"
+DMD_LORA = "lora/dmd_lora.safetensors"
+DMD_NETWORK_DIM = 128
+DMD_NETWORK_ALPHA = 64
+
+# The avatar distilled pass is NOT the base model's: 8 steps against dmd_lora, where the
+# base model's is 16 against cfg_step_lora.
+AVATAR_STAGE_DEFAULTS = {
+    #            steps, text guidance, audio guidance
+    "base":        (50, 4.0, 4.0),
+    "distill":     (8, 1.0, 1.0),
+}
+
+AVATAR_TASKS = ("at2v", "ai2v")
+
+
+def family_of(model_path: str = "", variant: str = "") -> str:
+    """Which checkpoint family a path names: '', 'avatar' or 'avatar-1.5'.
+
+    Read from the path because the family IS the repo — meituan-longcat/LongCat-Video,
+    -Avatar and -Avatar-1.5 are three downloads with different audio encoders."""
+    h = str(model_path or "").lower().replace("_", "-")
+    if "avatar-1.5" in h or "avatar1.5" in h:
+        return "avatar-1.5"
+    if "avatar" in h:
+        return "avatar"
+    return ""
+
+
+def variant_problems(variant: str, family: str = "", use_int8: bool = False,
+                     use_distill: bool = False) -> list:
+    """Reasons this combination cannot be served, or [].
+
+    Checked before loading so an unsupported flag is an error the operator can act on,
+    not a silently different model."""
+    problems = []
+    v = (variant or "bf16").strip().lower()
+    if v not in VARIANTS:
+        problems.append(f"unknown variant {variant!r}; expected one of {VARIANTS}")
+    if v == "gguf":
+        problems.append(
+            "the GGUF variants are packaged for ComfyUI's gguf loader, which coderai "
+            "does not implement — use bf16, fp8, or the official int8 (avatar-1.5)")
+    if (v == "int8" or use_int8) and family not in INT8_FAMILIES:
+        problems.append(
+            f"INT8 weights exist only for {'/'.join(INT8_FAMILIES)}; this checkpoint "
+            f"reads as {family or 'the base model'}")
+    if use_distill and family and family not in DMD_FAMILIES:
+        problems.append(
+            f"the DMD distillation exists only for {'/'.join(DMD_FAMILIES)}; "
+            f"this checkpoint reads as {family}")
+    if family == "avatar-1.5" and not (use_distill or v == "int8"):
+        # Upstream states v1.5 requires distilled sampling.
+        problems.append(
+            "avatar-1.5 requires distilled sampling — set use_distill on the model "
+            "entry (its 8-step DMD pass is the supported path)")
+    return problems
+
+
+def avatar_stage_params(stage: str, steps=None, text_guidance=None,
+                        audio_guidance=None) -> dict:
+    """Steps and the TWO guidance scales the avatar pipeline takes."""
+    key = stage if stage in AVATAR_STAGE_DEFAULTS else "base"
+    base_steps, base_text, base_audio = AVATAR_STAGE_DEFAULTS[key]
+    return {
+        "num_inference_steps": int(steps) if steps else base_steps,
+        "text_guidance_scale": float(text_guidance) if text_guidance is not None
+                               else base_text,
+        "audio_guidance_scale": float(audio_guidance) if audio_guidance is not None
+                                else base_audio,
+    }
+
+
+def avatar_problems(checkpoint_dir: str, family: str, use_int8: bool = False,
+                    use_distill: bool = False) -> list:
+    """What an avatar checkpoint is missing for this configuration, or []."""
+    if not family:
+        return ["this checkpoint is not an avatar model (its path names no avatar "
+                "family) — the audio-driven tasks need LongCat-Video-Avatar or "
+                "-Avatar-1.5"]
+    root = os.path.expanduser(checkpoint_dir or "")
+    problems = []
+    if use_int8 and not os.path.isdir(os.path.join(root, INT8_SUBDIR)):
+        problems.append(f"use_int8 needs {INT8_SUBDIR}/ in {root}")
+    if use_distill and not os.path.isfile(os.path.join(root, DMD_LORA)):
+        problems.append(f"use_distill needs {DMD_LORA} in {root}")
+    return problems
+
+
 def resolve_stages(preset: str = "fast", stage: str = "") -> tuple:
     """Which stages one request runs.
 

tools/longcat_service.py

@@ -58,7 +58,8 @@ LC = _load_common()
 _lock = threading.RLock()
 _state = {
     "pipe": None, "model": "", "source": "", "dtype": "bfloat16",
-    "loaded_lora": None,          # which stage's LoRA is currently fused
+    "family": "", "variant": "bf16", "use_int8": False, "use_distill": False,
+    "cp_split_hw": None, "audio_encoder": None,
 }
 _progress = {"active": False, "stage": "", "segment": 0, "segments": 0,
              "step": 0, "steps": 0}
@@ -107,11 +108,69 @@ def load_pipeline(checkpoint_dir: str, source_dir: str, dtype: str = "bfloat16",
         vae = AutoencoderKLWan.from_pretrained(root, subfolder="vae", torch_dtype=td)
         scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
             root, subfolder="scheduler")
+        family = LC.family_of(root)
+        use_int8 = bool(_state.get("use_int8"))
+        use_distill_lora = bool(_state.get("use_distill"))
+
+        if use_int8:
+            # avatar-1.5 only: a pre-quantised DiT under base_model_int8/, loaded by the
+            # repo's own helper rather than from_pretrained.
+            from longcat_video.modules.longcat_video_dit import load_quantized_dit
+            dit = load_quantized_dit(root, subfolder=LC.INT8_SUBDIR,
+                                     cp_split_hw=_state.get("cp_split_hw"))
+            log(f"loaded the INT8 DiT from {LC.INT8_SUBDIR}/")
+        else:
+            kw = {"torch_dtype": td}
+            if str(_state.get("variant") or "").lower() == "fp8":
+                # Community FP8 weights. The layouts differ between publishers (Kijai's
+                # scaled format is not the same as a plain e4m3 dump), so this loads it
+                # as the dtype it is and says so rather than pretending to normalise
+                # them: if a repo needs its own loader, the load fails here with the
+                # real reason instead of producing noise.
+                try:
+                    kw["torch_dtype"] = torch.float8_e4m3fn
+                    log("loading the DiT as float8_e4m3fn (community FP8 weights)")
+                except AttributeError:
+                    log("WARNING: this torch has no float8_e4m3fn — loading in "
+                        f"{dtype} instead")
             dit = LongCatVideoTransformer3DModel.from_pretrained(
-            root, subfolder="dit", torch_dtype=td)
-
+                root, subfolder="dit", **kw)
+
+        if family:
+            # The avatar families are a DIFFERENT pipeline class with its own methods
+            # (generate_at2v / generate_ai2v / generate_avc) and two guidance scales.
+            from longcat_video.pipeline_longcat_video_avatar import (
+                LongCatVideoAvatarPipeline)
+            pipe = LongCatVideoAvatarPipeline(
+                tokenizer=tokenizer, text_encoder=text_encoder, vae=vae,
+                scheduler=scheduler, dit=dit)
+            _state["audio_encoder"] = _load_audio_encoder(root, family)
+            log(f"avatar family '{family}' "
+                f"(audio encoder: {LC.AVATAR_ENCODERS.get(family)})")
+        else:
             pipe = LongCatVideoPipeline(tokenizer=tokenizer, text_encoder=text_encoder,
                                         vae=vae, scheduler=scheduler, dit=dit)
+        _state["family"] = family
+
+        if use_distill_lora and family:
+            # The avatar distilled pass is dmd_lora at 8 steps — NOT the base model's
+            # cfg_step_lora at 16, which use_distill=True on the base pipeline applies
+            # internally.
+            try:
+                dit.load_lora(os.path.join(root, LC.DMD_LORA), name="dmd",
+                              lora_network_dim=LC.DMD_NETWORK_DIM,
+                              lora_network_alpha=LC.DMD_NETWORK_ALPHA)
+            except AttributeError:
+                # Older/newer builds name it differently; the enable step is what counts.
+                pass
+            if hasattr(dit, "enable_loras"):
+                dit.enable_loras(["dmd"])
+                log(f"enabled the DMD LoRA (dim={LC.DMD_NETWORK_DIM}, "
+                    f"alpha={LC.DMD_NETWORK_ALPHA})")
+            else:
+                raise RuntimeError(
+                    "use_distill needs the DiT's enable_loras() — this build of "
+                    "longcat_video does not expose it")
         # Offload is opt-in: the 13.6B DiT wants ~27 GB at bf16 and the reported peak for
         # a full profile is ~41.6 GB, so a smaller card needs it. Not every build of the
         # upstream pipeline exposes the diffusers hooks, hence the guarded calls.
@@ -130,17 +189,35 @@ def load_pipeline(checkpoint_dir: str, source_dir: str, dtype: str = "bfloat16",
                 pipe.to("cuda")
             else:
                 log("WARNING: no CUDA device visible — this will be extremely slow")
-        _state.update(pipe=pipe, model=root, source=src, dtype=dtype, loaded_lora=None)
+        _state.update(pipe=pipe, model=root, source=src, dtype=dtype)
         log("ready")
         return pipe
 
 
+def _load_audio_encoder(root: str, family: str):
+    """The family's audio encoder: chinese-wav2vec2-base for Avatar, whisper-large-v3 for
+    Avatar-1.5. Upstream ships it inside the checkpoint; fall back to the hub id."""
+    name = LC.AVATAR_ENCODERS.get(family)
+    if not name:
+        return None
+    local = os.path.join(os.path.expanduser(root), name)
+    path = local if os.path.isdir(local) else name
+    try:
+        from longcat_video.pipeline_longcat_video_avatar import get_audio_encoder
+        return get_audio_encoder(path, family)
+    except ImportError:
+        # The helper moved; the pipeline can still embed audio itself in that case.
+        log(f"WARNING: get_audio_encoder is not importable — relying on the "
+            f"pipeline's own audio embedding")
+        return None
+
+
 def unload(reason: str = ""):
     with _lock:
         if _state["pipe"] is None:
             return
         _state["pipe"] = None
-        _state["loaded_lora"] = None
+        _state["audio_encoder"] = None
         try:
             import torch
             import gc
@@ -212,6 +289,9 @@ def _one_pass(pipe, task: str, stage: str, ctx: dict, cond=None, log=print):
         # nothing); the 50-step passes do.
         common["negative_prompt"] = ctx["negative_prompt"]
 
+    if task in LC.AVATAR_TASKS:
+        return _avatar_pass(pipe, task, dict(ctx, stage=stage), cond=cond)
+
     if cond is not None:
         kw = dict(common)
         kw.update(video=cond, num_cond_frames=ctx["num_cond_frames"],
@@ -330,6 +410,69 @@ def _peak_vram_gb() -> float:
         return 0.0
 
 
+def _audio_embedding(pipe, audio_b64: str, fps: int, family: str):
+    """Decode the audio track and embed it the way the avatar pipeline expects.
+
+    Upstream: librosa.load(..., sr=16000) then pipe.get_audio_embedding(speech, fps=…,
+    sample_rate=…, model_type=…). The 16 kHz is the encoder's rate, not a preference."""
+    import tempfile
+    import librosa
+    raw = audio_b64.split(",", 1)[1] if str(audio_b64).startswith("data:") else audio_b64
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fh:
+        fh.write(base64.b64decode(raw))
+        path = fh.name
+    try:
+        speech, sr = librosa.load(path, sr=16000)
+        device = 0
+        try:
+            import torch
+            device = torch.cuda.current_device() if torch.cuda.is_available() else "cpu"
+        except Exception:
+            pass
+        return pipe.get_audio_embedding(speech, fps=fps, device=device,
+                                        sample_rate=sr, model_type=family)
+    finally:
+        try:
+            os.unlink(path)
+        except Exception:
+            pass
+
+
+def _avatar_pass(pipe, task: str, ctx: dict, cond=None):
+    """One avatar call. at2v / ai2v to start, avc to continue.
+
+    The avatar pipeline takes TWO guidance scales (text and audio) where the base one
+    takes a single guidance_scale, and output_type='both' is what upstream asks for."""
+    params = LC.avatar_stage_params(ctx.get("stage") or "base",
+                                    ctx.get("num_inference_steps"),
+                                    ctx.get("text_guidance_scale"),
+                                    ctx.get("audio_guidance_scale"))
+    kw = {
+        "prompt": ctx["prompt"], "height": ctx["height"], "width": ctx["width"],
+        "num_frames": ctx["num_frames"], "audio_emb": ctx["audio_emb"],
+        "output_type": "both", **params,
+    }
+    if ctx.get("negative_prompt"):
+        kw["negative_prompt"] = ctx["negative_prompt"]
+    if ctx.get("generator") is not None:
+        kw["generator"] = ctx["generator"]
+    if ctx.get("use_distill"):
+        kw["use_distill"] = True
+
+    if cond is not None:
+        kw.update(video=cond, num_cond_frames=ctx["num_cond_frames"],
+                  use_kv_cache=True,
+                  offload_kv_cache=bool(ctx.get("offload_kv_cache")),
+                  enhance_hf=False)
+        for opt in ("ref_img_index", "mask_frame_range"):
+            if ctx.get(opt) is not None:
+                kw[opt] = ctx[opt]
+        return pipe.generate_avc(**kw)[0]
+    if task == "ai2v":
+        return pipe.generate_ai2v(image=ctx["image"], **kw)[0]
+    return pipe.generate_at2v(**kw)[0]
+
+
 def generate(body: dict) -> dict:
     """One request: text-to-video, image-to-video or continuation, over N segments.
 
@@ -348,9 +491,28 @@ def generate(body: dict) -> dict:
         raise ValueError("; ".join(problems))
 
     task = (body.get("task") or "t2v").strip().lower()
-    if task not in ("t2v", "i2v", "vc"):
-        raise ValueError(f"unknown task {task!r}; expected t2v, i2v or vc")
+    if task not in ("t2v", "i2v", "vc") + LC.AVATAR_TASKS:
+        raise ValueError(f"unknown task {task!r}; expected t2v, i2v, vc, "
+                         f"{' or '.join(LC.AVATAR_TASKS)}")
+
+    # Variant and family gating BEFORE loading: INT8 and the DMD distillation exist only
+    # for avatar-1.5, and asking for them elsewhere would quietly load something else.
+    family = LC.family_of(checkpoint)
+    variant = str(body.get("variant") or "bf16").strip().lower()
+    use_int8 = bool(body.get("use_int8")) or variant == "int8"
+    use_distill_lora = bool(body.get("use_distill"))
+    problems = LC.variant_problems(variant, family, use_int8, use_distill_lora)
+    if task in LC.AVATAR_TASKS:
+        problems += LC.avatar_problems(checkpoint, family, use_int8, use_distill_lora)
+    elif family:
+        problems.append(
+            f"this is the {family} checkpoint, which serves the audio-driven tasks "
+            f"({'/'.join(LC.AVATAR_TASKS)}); use the base LongCat-Video weights for "
+            f"'{task}'")
+    if problems:
+        raise ValueError("; ".join(problems))
 
+    _state.update(variant=variant, use_int8=use_int8, use_distill=use_distill_lora)
     pipe = load_pipeline(checkpoint, body.get("source") or _state["source"],
                          body.get("dtype") or _state["dtype"],
                          body.get("offload") or "")
@@ -374,6 +536,16 @@ def generate(body: dict) -> dict:
         if not body.get("image"):
             raise ValueError("task 'i2v' needs an image")
         image = _decode_image(body["image"])
+    audio_emb = None
+    if task in LC.AVATAR_TASKS:
+        if not body.get("audio"):
+            raise ValueError(f"task {task!r} needs an audio track")
+        audio_emb = _audio_embedding(pipe, body["audio"], fps, family)
+    if task == "ai2v":
+        if not body.get("image"):
+            raise ValueError("task 'ai2v' needs a reference image")
+        image = _decode_image(body["image"])
+
     cond_video = None
     if task == "vc":
         if not body.get("cond_frames_b64"):
@@ -399,6 +571,11 @@ def generate(body: dict) -> dict:
         "guidance_scale": body.get("guidance_scale"),
         "offload_kv_cache": body.get("offload_kv_cache"),
         "image": image, "cond_video": cond_video, "generator": generator,
+        "audio_emb": audio_emb, "use_distill": use_distill_lora,
+        "text_guidance_scale": body.get("text_guidance_scale"),
+        "audio_guidance_scale": body.get("audio_guidance_scale"),
+        "ref_img_index": body.get("ref_img_index"),
+        "mask_frame_range": body.get("mask_frame_range"),
     }
 
     _yield_flag.clear()
@@ -481,7 +658,9 @@ def make_handler():
         def do_GET(self):
             if self.path.startswith("/health"):
                 self._json({"ok": True, "loaded": _state["pipe"] is not None,
-                            "model": _state["model"], "stages": list(LC.STAGES)})
+                            "model": _state["model"], "stages": list(LC.STAGES),
+                            "family": _state.get("family") or "",
+                            "variant": _state.get("variant") or "bf16"})
             elif self.path.startswith("/progress"):
                 with _lock:
                     self._json(dict(_progress))