longcat: weight variants and the audio-driven avatar tasks (v0.2.35)

Step 6. Read from upstream's avatar demo rather than assumed, because almost nothing
carries over from the base model:

- avatar is a DIFFERENT pipeline class, LongCatVideoAvatarPipeline, with its own methods
  generate_at2v / generate_ai2v / generate_avc;
- it takes TWO guidance scales — text and audio — where the base pipeline takes one;
- its distilled pass is dmd_lora at 8 steps with lora_network_dim=128/alpha=64 and
  dit.enable_loras(["dmd"]), NOT cfg_step_lora at 16;
- use_int8 is a separate load_quantized_dit(subfolder="base_model_int8"), not a dtype;
- both INT8 and the DMD LoRA exist only for avatar-1.5, which also REQUIRES distilled
  sampling;
- the audio is librosa.load(sr=16000) then pipe.get_audio_embedding(...), and the encoder
  differs per family (chinese-wav2vec2-base for Avatar, whisper-large-v3 for Avatar-1.5).

Unsupported combinations are refused BEFORE loading, because the alternative is quietly
serving something that is not what was asked for: INT8 or DMD on anything but
avatar-1.5, avatar-1.5 without distilled sampling, a base-model mode on avatar weights or
the reverse, and GGUF, which needs ComfyUI's gguf loader and says so.

FP8 loads the DiT as float8_e4m3fn. Community layouts differ between publishers (Kijai's
scaled format is not a plain e4m3 dump), so a repo that needs its own loader fails with
that reason rather than producing noise.

Also: librosa is a real new dependency and went into requirements-longcat.txt, not only
into the code that imports it. And the avatar audio comes from the request's `audio_file`
field — `add_audio` is a BOOLEAN for the post-processing path, and using it would have
passed True into the base64 encoder.

35 new tests, 188 across the six LongCat suites.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.34"
+__version__ = "0.2.35"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/video.py

@@ -4128,12 +4128,15 @@ async def _generate_longcat(request: VideoGenerationRequest, model_name: str,
     mode = (request.mode or "t2v").lower()
     _task = {"t2v": "t2v", "": "t2v",
              "i2v": "i2v", "ti2v": "i2v",
-             "extend": "vc", "vc": "vc", "v2v": "vc"}.get(mode)
+             "extend": "vc", "vc": "vc", "v2v": "vc",
+             # Audio-driven avatar: a different pipeline class upstream, with its own
+             # methods and two guidance scales. 'at2v' is audio+text, 'ai2v' adds a
+             # reference image.
+             "at2v": "at2v", "ai2v": "ai2v", "avatar": "at2v"}.get(mode)
     if _task is None:
         raise HTTPException(status_code=400, detail=(
-            f"LongCat-Video does not support mode '{mode}'. Use 't2v', 'i2v' or "
-            f"'extend' (continuation); the audio-driven avatar tasks are a separate "
-            f"increment."))
+            f"LongCat-Video does not support mode '{mode}'. Use 't2v', 'i2v', "
+            f"'extend' (continuation), or 'at2v'/'ai2v' (audio-driven avatar)."))
 
     payload = {
         "prompt": request.prompt or "",
@@ -4147,6 +4150,35 @@ async def _generate_longcat(request: VideoGenerationRequest, model_name: str,
     if request.negative_prompt:
         payload["negative_prompt"] = request.negative_prompt
 
+    # The variant and its flags gate what can load at all: INT8 and the DMD
+    # distillation exist only for the avatar-1.5 weights. The service refuses an
+    # impossible combination rather than loading something else.
+    for _k in ("variant", "use_int8", "use_distill", "ref_img_index",
+               "mask_frame_range"):
+        if model_cfg.get(_k) not in (None, ""):
+            payload[_k] = model_cfg[_k]
+
+    # The avatar tasks need an audio track, and ai2v a reference image too.
+    if _task in ("at2v", "ai2v"):
+        # `audio_file` already means "existing audio, base64 or URL" on this request;
+        # `add_audio` is a BOOLEAN for the post-processing path and must not be used here.
+        _audio = request.audio_file
+        if not _audio:
+            raise HTTPException(status_code=400, detail=(
+                f"mode '{mode}' is audio-driven and needs an audio track in "
+                f"`audio_file` (base64 or URL)"))
+        payload["audio"] = _as_b64(_audio)
+        if getattr(request, "text_guidance_scale", None) is not None:
+            payload["text_guidance_scale"] = float(request.text_guidance_scale)
+        if getattr(request, "audio_guidance_scale", None) is not None:
+            payload["audio_guidance_scale"] = float(request.audio_guidance_scale)
+    if _task == "ai2v":
+        _ref = request.init_image or request.image
+        if not _ref:
+            raise HTTPException(status_code=400, detail=(
+                "mode 'ai2v' needs a reference image (init_image or image)"))
+        payload["image"] = _as_b64(_ref)
+
     # i2v takes one conditioning image; coderai accepts it as init_image or image.
     if _task == "i2v":
         _first = request.init_image or request.image

codai/pydantic/videorequest.py

@@ -102,6 +102,13 @@ class VideoGenerationRequest(BaseModel):
         "LongCat-Video: how many frames of the previous segment each new segment "
         "conditions on (default 13 of 93). More = stronger motion continuity, fewer "
         "new frames per segment."))
+    # The avatar pipeline takes TWO guidance scales where the base one takes a single
+    # guidance_scale: how hard to follow the prompt, and how hard to follow the audio.
+    text_guidance_scale: Optional[float] = Field(None, description=(
+        "LongCat-Video avatar: prompt adherence (default 4.0, or 1.0 distilled)."))
+    audio_guidance_scale: Optional[float] = Field(None, description=(
+        "LongCat-Video avatar: how closely the motion follows the audio "
+        "(default 4.0, or 1.0 distilled)."))
     quality: Optional[str] = Field(None, description=(
         "LongCat-Video quality preset: 'draft' (base stage only, fastest), 'fast' "
         "(distilled, 16 steps — the default), 'best' (base + 720p refinement). Ignored "

docs/longcat-video.md

@@ -134,10 +134,52 @@ building or downloading anything. `distribute` fans independent generations acro
 engines and nodes. A `video-longcat` RunPod capability image needs a custom Dockerfile
 (the generic one is python:3.12 + cu128, wrong on both axes) and is not built yet.
 
+## Variants and the avatar families
+
+The avatar weights are **a different pipeline** upstream
+(`LongCatVideoAvatarPipeline`), with its own methods and **two** guidance scales — how
+hard to follow the prompt, and how hard to follow the audio. Its distilled pass is also
+not the base model's: 8 steps against `dmd_lora`, where the base model's is 16 against
+`cfg_step_lora`.
+
+| | base | Avatar | Avatar-1.5 |
+|---|---|---|---|
+| audio encoder | — | chinese-wav2vec2-base | whisper-large-v3 |
+| modes | t2v, i2v, extend | at2v, ai2v | at2v, ai2v |
+| INT8 | no | no | **yes** (`base_model_int8/`) |
+| distilled | `cfg_step_lora`, 16 steps | no | **required**, `dmd_lora`, 8 steps |
+
+Unsupported combinations are **refused before loading** rather than quietly loading
+something else: INT8 or the DMD distillation on anything but avatar-1.5, avatar-1.5
+without distilled sampling, a base-model mode on avatar weights (or the reverse), and the
+GGUF variants, which need ComfyUI's gguf loader.
+
+FP8 loads the DiT as `float8_e4m3fn`. Community FP8 layouts differ between publishers
+(Kijai's scaled format is not a plain e4m3 dump), so if a repo needs its own loader the
+load fails with that reason instead of producing noise.
+
+Audio-driven generation:
+
+```jsonc
+POST /v1/video/generations
+{
+  "model": "longcat-avatar-15",
+  "mode": "ai2v",                 // at2v = audio+text; ai2v adds a reference image
+  "prompt": "a news anchor reading the headlines",
+  "audio_file": "<base64 or URL>",
+  "init_image": "<base64 or URL>",
+  "audio_guidance_scale": 4.0
+}
+```
+
+The audio is resampled to 16 kHz (the encoder's rate) and embedded by the family's
+encoder. `ref_img_index` (0–24; 30 reduces repeated actions) and `mask_frame_range` are
+set on the model entry.
+
 ## Not done yet
 
-- audio-driven avatar tasks; GGUF variants (they need the ComfyUI gguf loader)
+- GGUF variants (they need the ComfyUI gguf loader)
 - context-parallel multi-GPU (`cp_size` is accepted but not yet acted on)
 - the `video-longcat` pod image
 - LoRA/QLoRA training on top of it (feasible — upstream's own refinement expert is a LoRA)
-- **no generation has been verified against real weights**
+- **no generation has been verified against real weights**, avatar included

requirements-longcat.txt

@@ -53,6 +53,12 @@ imageio==2.37.0
 imageio-ffmpeg==0.6.0
 pyarrow==20.0.0
 
+# Audio-driven avatar generation: the audio track is resampled to 16 kHz with librosa
+# and embedded by the family's encoder (chinese-wav2vec2-base for Avatar, whisper-large-v3
+# for Avatar-1.5). Upstream installs librosa and ffmpeg alongside the pins above; ffmpeg
+# comes from the image, imageio-ffmpeg covers the muxing.
+librosa==0.10.2.post1
+
 # Attention backend. Upstream defaults to FlashAttention-2 but documents FA3 and xformers
 # as alternatives, and flash-attn==2.7.4.post1 is a SOURCE build needing ninja and a CUDA
 # toolchain — minutes of compile, and it fails outright on a host without nvcc wired up.

tools/longcat_common.py

@@ -62,6 +62,111 @@ STAGE_DEFAULTS = {
 }
 
 
+# ── variants ─────────────────────────────────────────────────────────────────
+# Which checkpoint family a model entry points at, and what each supports. Getting this
+# wrong is not a crash but a wrong answer: --use_int8 and the DMD distillation exist only
+# for avatar-v1.5, and asking for them elsewhere would silently load something else.
+VARIANTS = ("bf16", "fp8", "int8", "gguf")
+
+# The avatar families are separate HuggingFace repos with their own audio encoders.
+AVATAR_ENCODERS = {
+    "avatar": "chinese-wav2vec2-base",      # v1.0
+    "avatar-1.5": "whisper-large-v3",
+}
+
+# Only avatar-1.5 ships an INT8 DiT (under base_model_int8/) and the DMD LoRA, and its
+# distilled path is REQUIRED rather than optional.
+INT8_FAMILIES = ("avatar-1.5",)
+DMD_FAMILIES = ("avatar-1.5",)
+INT8_SUBDIR = "base_model_int8"
+DMD_LORA = "lora/dmd_lora.safetensors"
+DMD_NETWORK_DIM = 128
+DMD_NETWORK_ALPHA = 64
+
+# The avatar distilled pass is NOT the base model's: 8 steps against dmd_lora, where the
+# base model's is 16 against cfg_step_lora.
+AVATAR_STAGE_DEFAULTS = {
+    #            steps, text guidance, audio guidance
+    "base":        (50, 4.0, 4.0),
+    "distill":     (8, 1.0, 1.0),
+}
+
+AVATAR_TASKS = ("at2v", "ai2v")
+
+
+def family_of(model_path: str = "", variant: str = "") -> str:
+    """Which checkpoint family a path names: '', 'avatar' or 'avatar-1.5'.
+
+    Read from the path because the family IS the repo — meituan-longcat/LongCat-Video,
+    -Avatar and -Avatar-1.5 are three downloads with different audio encoders."""
+    h = str(model_path or "").lower().replace("_", "-")
+    if "avatar-1.5" in h or "avatar1.5" in h:
+        return "avatar-1.5"
+    if "avatar" in h:
+        return "avatar"
+    return ""
+
+
+def variant_problems(variant: str, family: str = "", use_int8: bool = False,
+                     use_distill: bool = False) -> list:
+    """Reasons this combination cannot be served, or [].
+
+    Checked before loading so an unsupported flag is an error the operator can act on,
+    not a silently different model."""
+    problems = []
+    v = (variant or "bf16").strip().lower()
+    if v not in VARIANTS:
+        problems.append(f"unknown variant {variant!r}; expected one of {VARIANTS}")
+    if v == "gguf":
+        problems.append(
+            "the GGUF variants are packaged for ComfyUI's gguf loader, which coderai "
+            "does not implement — use bf16, fp8, or the official int8 (avatar-1.5)")
+    if (v == "int8" or use_int8) and family not in INT8_FAMILIES:
+        problems.append(
+            f"INT8 weights exist only for {'/'.join(INT8_FAMILIES)}; this checkpoint "
+            f"reads as {family or 'the base model'}")
+    if use_distill and family and family not in DMD_FAMILIES:
+        problems.append(
+            f"the DMD distillation exists only for {'/'.join(DMD_FAMILIES)}; "
+            f"this checkpoint reads as {family}")
+    if family == "avatar-1.5" and not (use_distill or v == "int8"):
+        # Upstream states v1.5 requires distilled sampling.
+        problems.append(
+            "avatar-1.5 requires distilled sampling — set use_distill on the model "
+            "entry (its 8-step DMD pass is the supported path)")
+    return problems
+
+
+def avatar_stage_params(stage: str, steps=None, text_guidance=None,
+                        audio_guidance=None) -> dict:
+    """Steps and the TWO guidance scales the avatar pipeline takes."""
+    key = stage if stage in AVATAR_STAGE_DEFAULTS else "base"
+    base_steps, base_text, base_audio = AVATAR_STAGE_DEFAULTS[key]
+    return {
+        "num_inference_steps": int(steps) if steps else base_steps,
+        "text_guidance_scale": float(text_guidance) if text_guidance is not None
+                               else base_text,
+        "audio_guidance_scale": float(audio_guidance) if audio_guidance is not None
+                                else base_audio,
+    }
+
+
+def avatar_problems(checkpoint_dir: str, family: str, use_int8: bool = False,
+                    use_distill: bool = False) -> list:
+    """What an avatar checkpoint is missing for this configuration, or []."""
+    if not family:
+        return ["this checkpoint is not an avatar model (its path names no avatar "
+                "family) — the audio-driven tasks need LongCat-Video-Avatar or "
+                "-Avatar-1.5"]
+    root = os.path.expanduser(checkpoint_dir or "")
+    problems = []
+    if use_int8 and not os.path.isdir(os.path.join(root, INT8_SUBDIR)):
+        problems.append(f"use_int8 needs {INT8_SUBDIR}/ in {root}")
+    if use_distill and not os.path.isfile(os.path.join(root, DMD_LORA)):
+        problems.append(f"use_distill needs {DMD_LORA} in {root}")
+    return problems
+
+
 def resolve_stages(preset: str = "fast", stage: str = "") -> tuple:
     """Which stages one request runs.