video: VACE frame-tail extend, cancellable downloads, MMA fight variety

Downloads: run each model download in a clean `python -m
codai.admin.download_worker` subprocess streaming JSON progress, so the
Stop button reliably cancels by terminating the process (HF parallel/Xet
chunk transfers ignore in-thread flags). Adds download-cancel-all. Avoids
multiprocessing spawn, which re-imports the server launcher as __main__.

VACE extension: detect WanVACEPipeline; new 'extend' mode + cond_frames
request field condition on the previous chained part's frame tail (real
motion -> forward continuation, fixing the single-frame boomerang).
_build_vace_conditioning builds the (video, mask) pair; _snap_wan_frames
enforces 4k+1; only the freshly generated frames are returned. VACE also
serves keyframe i2v / t2v via masking; i2v/t2v fallbacks skipped for it.
Township auto-uses extend for chained parts when the model is VACE.

Fight prompts: full-MMA system prompt + rotating per-clip action focus
(kicks/knees/elbows/takedowns/ground/submissions) and occasional blood,
rebalanced fallback templates, keyframe wardrobe enforcement.
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
parent 07b3be5c
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
"""Out-of-process model downloader.
Model downloads (``snapshot_download`` / ``hf_hub_download``) cannot be cancelled
from the calling thread: huggingface_hub fetches each file with several parallel
chunk connections (and, with Xet, an entirely separate transfer path), none of
which honour an in-thread "please stop" flag — so a daemon thread running the
download is effectively un-stoppable. Running the download in a *child process*
makes cancellation reliable: the supervisor simply ``terminate()``s the process,
which tears down every chunk connection at once.
This module is deliberately self-contained (stdlib + huggingface_hub +
codai.models.cache only, NO FastAPI / torch imports) so it loads fast and safely
under the ``spawn`` start method.
"""
import os
import time
_DISK_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MB safety margin
def _check_disk_space(path: str, needed_bytes: int = 0) -> None:
"""Raise RuntimeError if `path`'s filesystem lacks enough free space."""
import os as _os
import shutil
check_path = path
while check_path and not _os.path.exists(check_path):
parent = _os.path.dirname(check_path)
if parent == check_path:
break
check_path = parent
try:
free = shutil.disk_usage(check_path).free
except OSError:
return
required = needed_bytes + _DISK_MIN_FREE_BYTES
if free < required:
free_gb = free / 1e9
needed_gb = needed_bytes / 1e9
msg = (
f"Not enough disk space: {free_gb:.1f} GB free"
+ (f", ~{needed_gb:.1f} GB needed" if needed_bytes else "")
+ ". Free up space and try again."
)
raise RuntimeError(msg)
def _get_hf_expected_size(model_id: str, file_pattern: str) -> int:
"""Return expected download size in bytes for a HF model (best-effort, 0 on failure)."""
try:
import fnmatch
from huggingface_hub import model_info as _hf_model_info
info = _hf_model_info(model_id, files_metadata=True)
siblings = info.siblings or []
if file_pattern:
if file_pattern.startswith('.'):
pats = [f"*{file_pattern}"]
elif '/' in file_pattern:
pats = [file_pattern]
else:
pats = [f"*{file_pattern}"]
siblings = [s for s in siblings if any(fnmatch.fnmatch(s.rfilename, p) for p in pats)]
return sum(getattr(s, 'size', 0) or 0 for s in siblings)
except Exception:
return 0
def _make_tqdm_class(q, cache_dir=None):
"""tqdm-compatible class that forwards progress events to the multiprocessing
queue `q`. Cancellation is handled by the parent terminating this process, so
no in-band cancel flag is needed here."""
import time as _time
class _PQTqdm:
def __init__(self, iterable=None, desc=None, total=None, initial=0, **kwargs):
self.iterable = iterable
self.desc = str(desc or 'downloading')
self.total = int(total) if total else 0
self.n = int(initial) if initial else 0
self._start = _time.time()
self._update_count = 0
if self.total:
q.put({"type": "start", "filename": self.desc, "total": self.total})
def update(self, n=1):
self.n += n
self._update_count += 1
if cache_dir and self._update_count % 64 == 0:
_check_disk_space(cache_dir)
elapsed = (_time.time() - self._start) or 0.001
rate = self.n / elapsed
eta = (self.total - self.n) / rate if rate and self.total else None
pct = round(self.n / self.total * 100, 1) if self.total else 0
q.put({
"type": "progress",
"filename": self.desc,
"downloaded": self.n,
"total": self.total,
"percent": pct,
"rate": round(rate),
"eta": round(eta) if eta is not None else None,
})
def close(self): pass
def refresh(self, nolock=False, lock_args=None): pass
def clear(self, nolock=False): pass
def display(self, msg=None, pos=None): pass
def unpause(self): pass
def moveto(self, n): pass
def set_postfix(self, *a, **kw): pass
def set_description(self, desc=None, **kw):
if desc: self.desc = str(desc)
def set_postfix_str(self, *a, **kw): pass
def reset(self, total=None):
self.n = 0
self._start = _time.time()
if total is not None: self.total = int(total)
def __enter__(self): return self
def __exit__(self, *a): self.close()
def __iter__(self):
for obj in (self.iterable or []):
yield obj
def write(self, s, **kw):
q.put({"type": "info", "message": str(s)})
monitor_interval = 0
monitor = None
_lock = None
@classmethod
def get_lock(cls):
import threading
if cls._lock is None:
cls._lock = threading.RLock()
return cls._lock
@classmethod
def set_lock(cls, lock):
cls._lock = lock
return _PQTqdm
def run_download(model_id: str, file_pattern: str, q) -> None:
"""Child-process entry point: download `model_id` (optionally filtered by
`file_pattern`) and stream progress events into the multiprocessing queue `q`.
Terminating this process cancels the download cleanly."""
try:
from codai.models.cache import (
is_huggingface_model_id, get_model_cache_dir, get_hf_hub_cache_dir,
)
from huggingface_hub import snapshot_download
if is_huggingface_model_id(model_id):
is_gguf_download = file_pattern and '.gguf' in file_pattern.lower()
if is_gguf_download:
gguf_cache = get_model_cache_dir()
dl_cache_dir = gguf_cache
else:
dl_cache_dir = get_hf_hub_cache_dir()
expected_bytes = _get_hf_expected_size(model_id, file_pattern)
_check_disk_space(dl_cache_dir, expected_bytes)
tqdm_cls = _make_tqdm_class(q, cache_dir=dl_cache_dir)
if is_gguf_download:
import fnmatch as _fnmatch
import shutil as _shutil
from huggingface_hub import list_repo_files, hf_hub_download
_is_exact = ('*' not in file_pattern and '?' not in file_pattern
and file_pattern.lower().endswith('.gguf'))
if _is_exact:
matching = [file_pattern]
else:
if file_pattern.startswith('.'):
pat = f"*{file_pattern}"
elif '/' in file_pattern:
pat = file_pattern
else:
pat = f"*{file_pattern}"
all_repo_files = list(list_repo_files(model_id))
matching = [
f for f in all_repo_files
if _fnmatch.fnmatch(f, pat) or _fnmatch.fnmatch(os.path.basename(f), pat)
]
if not matching:
q.put({"type": "error", "message": f"No files matching {file_pattern!r} found in {model_id}"})
return
last_dest = gguf_cache
for hf_filename in matching:
basename = os.path.basename(hf_filename)
q.put({"type": "info", "message": f"Downloading {basename} from {model_id}…"})
dl_path = hf_hub_download(
repo_id=model_id,
filename=hf_filename,
local_dir=gguf_cache,
tqdm_class=tqdm_cls,
)
flat_dest = os.path.join(gguf_cache, basename)
if os.path.abspath(dl_path) != os.path.abspath(flat_dest) and os.path.isfile(dl_path):
_shutil.move(dl_path, flat_dest)
last_dest = flat_dest
path = last_dest
elif file_pattern:
if file_pattern.startswith('.'):
allow = [f"*{file_pattern}"]
elif '/' in file_pattern:
allow = [file_pattern]
else:
allow = [f"*{file_pattern}"]
q.put({"type": "info", "message": f"Downloading {allow[0]} from {model_id}…"})
path = snapshot_download(model_id, cache_dir=dl_cache_dir, allow_patterns=allow, tqdm_class=tqdm_cls)
else:
q.put({"type": "info", "message": f"Downloading full repository {model_id}…"})
path = snapshot_download(model_id, cache_dir=dl_cache_dir, tqdm_class=tqdm_cls)
else:
# Direct URL download (non-HF source)
import requests as _req
import hashlib
dl_cache_dir = get_model_cache_dir()
_check_disk_space(dl_cache_dir)
url_path = model_id.split('?')[0]
filename = os.path.basename(url_path) or "model.bin"
url_hash = hashlib.sha256(model_id.encode()).hexdigest()
dest = os.path.join(dl_cache_dir, f"{url_hash}_{filename}")
if os.path.exists(dest):
q.put({"type": "done", "path": dest})
return
resp = _req.get(model_id, stream=True, timeout=60, allow_redirects=True)
resp.raise_for_status()
total = int(resp.headers.get('content-length', 0))
if total:
_check_disk_space(dl_cache_dir, total)
q.put({"type": "start", "filename": filename, "total": total})
downloaded = 0
start_t = time.time()
last_evt = 0.0
with open(dest, 'wb') as f:
for chunk in resp.iter_content(chunk_size=524288):
if chunk:
if downloaded % (64 * 1024 * 1024) < len(chunk):
_check_disk_space(dl_cache_dir)
f.write(chunk)
downloaded += len(chunk)
now = time.time()
if now - last_evt >= 0.25:
last_evt = now
elapsed = (now - start_t) or 0.001
rate = downloaded / elapsed
eta = (total - downloaded) / rate if rate and total else None
q.put({
"type": "progress", "filename": filename,
"downloaded": downloaded, "total": total,
"percent": round(downloaded / total * 100, 1) if total else 0,
"rate": round(rate),
"eta": round(eta) if eta is not None else None,
})
path = dest
q.put({"type": "done", "path": str(path)})
except Exception as exc:
q.put({"type": "error", "message": str(exc)})
class _StdoutQueue:
"""Queue-compatible shim that emits each event as one JSON line on stdout.
Lets ``run_download`` (written against a ``.put(evt)`` queue) drive the CLI
entry point unchanged: the parent reads these lines back and relays them onto
the SSE stream."""
def put(self, evt):
import sys as _sys
import json as _json
try:
_sys.stdout.write(_json.dumps(evt) + "\n")
_sys.stdout.flush()
except Exception:
pass
def main(argv=None):
"""CLI entry point: ``python -m codai.admin.download_worker <model_id> [pattern]``.
Run as a *subprocess* (not multiprocessing) so the child's ``__main__`` is this
module — not the server's launcher — avoiding a costly/hanging re-import of the
whole server under the spawn start method. Cancellation = terminating this
process, which tears down every HF chunk connection at once."""
import argparse
ap = argparse.ArgumentParser(prog="codai.admin.download_worker")
ap.add_argument("model_id")
ap.add_argument("file_pattern", nargs="?", default="")
args = ap.parse_args(argv)
run_download(args.model_id, args.file_pattern or "", _StdoutQueue())
if __name__ == "__main__":
main()
......@@ -45,6 +45,7 @@ config_manager = None # set via set_config_manager()
_download_sessions: dict = {}
_download_status: dict = {} # session_id → latest progress state (survives SSE disconnect)
_download_cancelled: set = set() # session_ids the user has requested to cancel
_download_procs: dict = {} # session_id → multiprocessing.Process running the download
def get_active_download_model_ids() -> set:
......@@ -749,7 +750,9 @@ def _make_tqdm_class(pq, status=None, session_id=None, cache_dir=None):
def _run_download_thread(session_id: str, model_id: str, file_pattern: str, pq):
"""Background thread: download model via HF snapshot_download and stream progress events."""
"""Supervisor thread: spawn a child process that performs the download and
relay its progress events onto the SSE queue `pq`. Running the download out of
process is what makes it cancellable — see the inline note below."""
import time
import os
......@@ -776,149 +779,67 @@ def _run_download_thread(session_id: str, model_id: str, file_pattern: str, pq):
elif t == "info":
status["last_info"] = evt.get("message", "")
try:
from codai.models.cache import is_huggingface_model_id, get_model_cache_dir, get_hf_hub_cache_dir
from huggingface_hub import snapshot_download
if is_huggingface_model_id(model_id):
# GGUF files always land in the GGUF cache (flat); everything else
# (full repos, transformers checkpoints, diffusers, …) goes in the HF cache.
is_gguf_download = file_pattern and '.gguf' in file_pattern.lower()
if is_gguf_download:
gguf_cache = get_model_cache_dir()
dl_cache_dir = gguf_cache
else:
dl_cache_dir = get_hf_hub_cache_dir()
# Pre-check disk space using HF file-size metadata
expected_bytes = _get_hf_expected_size(model_id, file_pattern)
_check_disk_space(dl_cache_dir, expected_bytes)
tqdm_cls = _make_tqdm_class(pq, status=status, session_id=session_id, cache_dir=dl_cache_dir)
# Run the actual download in a SEPARATE SUBPROCESS so it can be cancelled
# reliably. huggingface_hub fetches each file over several parallel chunk
# connections (and, with Xet, a separate transfer path) that ignore any
# in-thread stop flag, so a daemon thread can't be interrupted — but
# terminating a process tears every connection down at once. We launch a clean
# `python -m codai.admin.download_worker` (NOT multiprocessing: the spawn start
# method re-imports the parent's __main__, i.e. the server launcher, which
# hangs re-initialising the whole server). The child streams progress events as
# JSON lines on stdout, which we relay onto this session's SSE queue.
import subprocess as _sp
import sys as _sys
proc = _sp.Popen(
[_sys.executable, "-m", "codai.admin.download_worker", model_id, file_pattern or ""],
stdout=_sp.PIPE, stderr=_sp.STDOUT, text=True, bufsize=1,
)
_download_procs[session_id] = proc
if is_gguf_download:
import fnmatch as _fnmatch
import shutil as _shutil
from huggingface_hub import list_repo_files, hf_hub_download
terminal = None # set to "done"/"error" once the child reports a final event
try:
for line in proc.stdout:
line = line.strip()
if not line:
continue
try:
evt = _j.loads(line)
except Exception:
# Non-JSON output (warnings / tracebacks) → surface as info.
push({"type": "info", "message": line})
continue
etype = evt.get("type")
push(evt)
if etype in ("done", "error"):
terminal = etype
except Exception as exc:
push({"type": "error", "message": str(exc)})
finally:
# Ensure the child is gone (cancel, crash, or normal exit).
if proc.poll() is None:
try:
proc.terminate()
proc.wait(timeout=10)
except Exception:
pass
if proc.poll() is None:
try:
proc.kill()
except Exception:
pass
_download_procs.pop(session_id, None)
# If pattern has no wildcards and looks like an exact filename, skip listing.
_is_exact = ('*' not in file_pattern and '?' not in file_pattern
and file_pattern.lower().endswith('.gguf'))
if _is_exact:
matching = [file_pattern]
else:
# Resolve the pattern to actual filenames in the repo
if file_pattern.startswith('.'):
pat = f"*{file_pattern}"
elif '/' in file_pattern:
pat = file_pattern
else:
pat = f"*{file_pattern}"
all_repo_files = list(list_repo_files(model_id))
matching = [
f for f in all_repo_files
if _fnmatch.fnmatch(f, pat) or _fnmatch.fnmatch(os.path.basename(f), pat)
]
if not matching:
push({"type": "error", "message": f"No files matching {file_pattern!r} found in {model_id}"})
return
last_dest = gguf_cache
for hf_filename in matching:
basename = os.path.basename(hf_filename)
push({"type": "info", "message": f"Downloading {basename} from {model_id}…"})
dl_path = hf_hub_download(
repo_id=model_id,
filename=hf_filename,
local_dir=gguf_cache,
tqdm_class=tqdm_cls,
)
# hf_hub_download preserves subfolder structure; flatten to cache root
flat_dest = os.path.join(gguf_cache, basename)
if os.path.abspath(dl_path) != os.path.abspath(flat_dest) and os.path.isfile(dl_path):
_shutil.move(dl_path, flat_dest)
last_dest = flat_dest
path = last_dest
elif file_pattern:
# Non-GGUF pattern — use snapshot into HF cache
if file_pattern.startswith('.'):
allow = [f"*{file_pattern}"]
elif '/' in file_pattern:
allow = [file_pattern]
else:
allow = [f"*{file_pattern}"]
push({"type": "info", "message": f"Downloading {allow[0]} from {model_id}…"})
path = snapshot_download(model_id, cache_dir=dl_cache_dir, allow_patterns=allow, tqdm_class=tqdm_cls)
if terminal is None:
# Child ended without a done/error event → cancelled or died.
if session_id in _download_cancelled:
pq.put({"type": "cancelled", "message": "Download cancelled by user"})
_download_status.get(session_id, {}).update({"status": "cancelled"})
else:
push({"type": "info", "message": f"Downloading full repository {model_id}…"})
path = snapshot_download(model_id, cache_dir=dl_cache_dir, tqdm_class=tqdm_cls)
else:
# Direct URL download (non-HF source)
import requests as _req
import hashlib
dl_cache_dir = get_model_cache_dir()
_check_disk_space(dl_cache_dir) # basic free-space sanity check before connecting
url_path = model_id.split('?')[0]
filename = os.path.basename(url_path) or "model.bin"
url_hash = hashlib.sha256(model_id.encode()).hexdigest()
dest = os.path.join(dl_cache_dir, f"{url_hash}_{filename}")
if os.path.exists(dest):
push({"type": "done", "path": dest})
return
resp = _req.get(model_id, stream=True, timeout=60, allow_redirects=True)
resp.raise_for_status()
total = int(resp.headers.get('content-length', 0))
if total:
_check_disk_space(dl_cache_dir, total)
push({"type": "start", "filename": filename, "total": total})
tqdm_cls = _make_tqdm_class(pq, status=status, session_id=session_id, cache_dir=dl_cache_dir)
downloaded = 0
start_t = time.time()
last_evt = 0.0
with open(dest, 'wb') as f:
for chunk in resp.iter_content(chunk_size=524288):
if chunk:
if session_id in _download_cancelled:
raise RuntimeError("Download cancelled by user")
# Check disk space roughly every 64 MB
if downloaded % (64 * 1024 * 1024) < len(chunk):
_check_disk_space(dl_cache_dir)
f.write(chunk)
downloaded += len(chunk)
now = time.time()
if now - last_evt >= 0.25:
last_evt = now
elapsed = (now - start_t) or 0.001
rate = downloaded / elapsed
eta = (total - downloaded) / rate if rate and total else None
push({
"type": "progress", "filename": filename,
"downloaded": downloaded, "total": total,
"percent": round(downloaded / total * 100, 1) if total else 0,
"rate": round(rate),
"eta": round(eta) if eta is not None else None,
})
path = dest
push({"type": "done", "path": str(path)})
push({"type": "error", "message": "Download process exited unexpectedly"})
except Exception as exc:
if session_id in _download_cancelled:
pq.put({"type": "cancelled", "message": "Download cancelled by user"})
_download_status.get(session_id, {}).update({"status": "cancelled"})
else:
push({"type": "error", "message": str(exc)})
finally:
_download_cancelled.discard(session_id)
def _gc():
time.sleep(300)
_download_sessions.pop(session_id, None)
......@@ -971,7 +892,7 @@ async def api_download_stream(
try:
evt = await loop.run_in_executor(None, lambda: pq.get(timeout=2))
yield f"data: {_j.dumps(evt)}\n\n"
if evt.get("type") in ("done", "error"):
if evt.get("type") in ("done", "error", "cancelled"):
break
except _q.Empty:
yield 'data: {"type":"keepalive"}\n\n'
......@@ -1036,13 +957,38 @@ async def api_list_downloads(username: str = Depends(require_admin)):
@router.post("/admin/api/download-cancel/{session_id}", summary="Cancel a download")
async def api_cancel_download(session_id: str, username: str = Depends(require_admin)):
"""Request cancellation of an active download session."""
"""Cancel an active download by terminating its worker process immediately.
Flagging the session (so the supervisor classifies it as cancelled, not
failed) and killing the child process tears down every HF chunk connection at
once — the supervisor's relay loop then exits cleanly."""
if session_id not in _download_sessions and session_id not in _download_status:
raise HTTPException(status_code=404, detail="Download session not found")
_download_cancelled.add(session_id)
proc = _download_procs.get(session_id)
if proc is not None and proc.poll() is None:
try:
proc.terminate()
except Exception:
pass
return {"success": True}
@router.post("/admin/api/download-cancel-all", summary="Cancel all active downloads")
async def api_cancel_all_downloads(username: str = Depends(require_admin)):
"""Cancel every active download at once (terminates all worker processes)."""
sessions = list(_download_procs.keys())
for sid in sessions:
_download_cancelled.add(sid)
proc = _download_procs.get(sid)
if proc is not None and proc.poll() is None:
try:
proc.terminate()
except Exception:
pass
return {"success": True, "cancelled": len(sessions)}
@router.post("/admin/api/model-upload", summary="Upload a model file")
async def api_model_upload(request: Request, username: str = Depends(require_admin)):
"""Upload a GGUF model file in chunks."""
......
......@@ -327,6 +327,16 @@ def _detect_pipeline_class(model_name: str, mode: str):
if 'animatediff' in n or 'animateddiff' in n:
return AnimateDiffPipeline
if 'wan' in n:
# VACE (incl. Wan2.2-VACE-Fun) is an all-in-one control model loaded by
# its own pipeline — used for frame-tail continuation ('extend'), and it
# also serves plain t2v/i2v via masked conditioning. It must win over the
# t2v/i2v pipeline classes regardless of the requested mode.
if 'vace' in n:
try:
from diffusers import WanVACEPipeline
return WanVACEPipeline
except ImportError:
pass
# Wan ships separate t2v and i2v transformers; pick the i2v pipeline
# for the keyframe bridge when an init image is supplied.
if mode in ('i2v', 'ti2v'):
......@@ -1658,6 +1668,36 @@ def _sync_video_loras(pipe, loras) -> None:
pipe._coderai_active_loras = desired
def _snap_wan_frames(n: int) -> int:
"""Snap a frame count to the Wan VAE's temporal grid (4k+1). The VACE
video/mask lists must be exactly num_frames long, and the pipeline expects a
4k+1 count, so we round to the nearest valid value (min 5)."""
n = max(5, int(n))
k = round((n - 1) / 4)
return max(5, k * 4 + 1)
def _build_vace_conditioning(cond_frames, num_frames: int, width: int, height: int):
"""Build the (video, mask) lists a WanVACE pipeline expects.
`cond_frames` are the leading conditioning frames (the previous clip's tail for
'extend', or a single keyframe for i2v): the model is conditioned on them and
generates the remaining frames forward. Per the VACE convention the mask is an
'L' image per frame — BLACK (0) = condition on this frame, WHITE (255) =
generate it — and unknown frames are gray placeholders in `video`.
"""
from PIL import Image as _Image
cond = [f.convert('RGB').resize((width, height)) for f in cond_frames]
k = len(cond)
k = min(k, num_frames)
gray = _Image.new('RGB', (width, height), (128, 128, 128))
black = _Image.new('L', (width, height), 0) # condition
white = _Image.new('L', (width, height), 255) # generate
video = cond[:k] + [gray] * (num_frames - k)
mask = [black] * k + [white] * (num_frames - k)
return video, mask
def _run_pipeline(pipe, kw: dict):
result = pipe(**kw)
# NB: `getattr(result, 'frames', None) or result[0]` is WRONG — when .frames
......@@ -1851,7 +1891,39 @@ def _generate_video(pipe, request: VideoGenerationRequest):
init_src = request.init_image or request.image
if mode == 'i2v' and init_src:
# VACE pipelines (e.g. Wan2.2-VACE-Fun) condition via a (video, mask) pair, not
# an `image` kwarg. They power 'extend' (frame-tail continuation — the real fix
# for the chained-clip boomerang, since the model sees several real frames of
# motion and carries it forward) and also serve i2v/ti2v/t2v via masking.
is_vace = type(pipe).__name__ == 'WanVACEPipeline'
_vace_trim = 0 # frames to drop from the output start (the re-rendered tail)
if is_vace:
w = int(kw.get('width') or 512)
h = int(kw.get('height') or 512)
cond_b64 = list(getattr(request, 'cond_frames', None) or [])
new_frames = int(kw.get('num_frames') or 16)
if mode == 'extend' and cond_b64:
cond = [_pil_from_b64(x) for x in cond_b64]
k = min(len(cond), max(1, new_frames - 1))
cond = cond[-k:]
total = _snap_wan_frames(k + new_frames)
kw['video'], kw['mask'] = _build_vace_conditioning(cond, total, w, h)
kw['num_frames'] = total
_vace_trim = k # caller keeps only the freshly generated continuation
elif init_src:
# Keyframe-anchored (i2v/ti2v): condition on the single init frame.
total = _snap_wan_frames(new_frames)
kw['video'], kw['mask'] = _build_vace_conditioning(
[_pil_from_b64(init_src)], total, w, h)
kw['num_frames'] = total
else:
# Pure t2v on VACE: snap frame count, leave video/mask unset (free gen).
kw['num_frames'] = _snap_wan_frames(new_frames)
kw.pop('image', None)
kw.pop('image_end', None)
elif mode == 'i2v' and init_src:
kw['image'] = _pil_from_b64(init_src)
kw.pop('prompt', None) # SVD doesn't take text
......@@ -1878,8 +1950,12 @@ def _generate_video(pipe, request: VideoGenerationRequest):
# * t2v request on an i2v model (36-ch) → run i2v with a neutral seed frame.
# Both rebuild a sibling pipeline that REUSES the same components, so fused
# acceleration and per-request LoRAs on the shared transformer carry over.
pipe, mode = _maybe_t2v_fallback(pipe, kw, mode)
pipe, mode = _maybe_i2v_fallback(pipe, kw, mode)
# The t2v/i2v fallbacks rebuild sibling Wan *I2V/T2V* pipelines (by transformer
# in-channels); they don't apply to a VACE pipeline, which conditions via the
# (video, mask) pair instead of an `image` kwarg.
if not is_vace:
pipe, mode = _maybe_t2v_fallback(pipe, kw, mode)
pipe, mode = _maybe_i2v_fallback(pipe, kw, mode)
# Per-request LoRA adapters (e.g. per-character identity LoRAs). Sync the
# pipeline's adapters to this request's set, REUSING them if identical to the
......@@ -1897,6 +1973,13 @@ def _generate_video(pipe, request: VideoGenerationRequest):
raise
_vid_progress_done()
task_registry.finish(_tid, "done")
# VACE 'extend' re-renders the conditioning tail as the first _vace_trim frames;
# drop them so only the fresh forward continuation is returned.
if _vace_trim:
try:
frames = frames[_vace_trim:]
except Exception:
pass
return frames, fps
......
......@@ -72,9 +72,10 @@ class VideoGenerationRequest(BaseModel):
mode: Optional[str] = Field("t2v", description=(
"Generation mode: 't2v' text-to-video; 'i2v' image-to-video (init_image required, "
"prompt dropped); 'ti2v' text+init image (prompt is primary driver); 'v2v' "
"video-to-video (video required); 'interp' frame interpolation (init_image+end_image). "
"The server gracefully falls back between Wan t2v/i2v pipelines when a model only "
"supports one."))
"video-to-video (video required); 'interp' frame interpolation (init_image+end_image); "
"'extend' VACE frame-tail continuation (cond_frames = the previous clip's last frames; "
"requires a VACE model). The server gracefully falls back between Wan t2v/i2v pipelines "
"when a model only supports one."))
# Input media (base64 or URL)
image: Optional[str] = Field(None, description="Alias for init_image (base64 or URL).")
......@@ -82,6 +83,10 @@ class VideoGenerationRequest(BaseModel):
end_image: Optional[str] = Field(None, description="Last frame, for 'interp' mode (base64 or URL).")
video: Optional[str] = Field(None, description="Input video for v2v / audio manipulation (base64 or URL).")
strength: Optional[float] = Field(None, description="Denoising strength for v2v (0–1).")
cond_frames: Optional[List[str]] = Field(None, description=(
"Ordered conditioning frames (base64/URL) for 'extend' mode on a VACE model: the tail "
"of the previous clip. The model conditions on these (their real motion gives velocity) "
"and generates num_frames NEW frames continuing forward; only the new frames are returned."))
# Camera motion hint
camera_motion: Optional[str] = None # zoom-in | zoom-out | pan-left | pan-right | tilt-up | tilt-down | rotate
......
......@@ -374,8 +374,9 @@ ENVIRONMENT_POOL = [
# especially when anchored to a keyframe. Kept here so the planner and the per-match
# Re-plan stay in sync.
FIGHT_PROMPT_SUFFIX = ("African township free fight, fast-paced, rapid explosive "
"motion, dynamic action, cinematic, consistent characters, "
"wardrobe and setting")
"motion, dynamic action, continuous forward motion that never "
"reverses, rewinds or loops back, cinematic, consistent "
"characters, wardrobe and setting")
def _continuity_clause(env_name: str) -> str:
......@@ -391,21 +392,36 @@ def _continuity_clause(env_name: str) -> str:
"background, surfaces and crowd in every shot")
FIGHT_SHOT_TEMPLATES = [
"exchanging heavy blows at close range, both fighters connecting, crowd erupting",
"delivering a powerful uppercut, opponent's head snapping back on impact",
"grappling and clinching against the ropes, sweat flying, crowd pushing forward",
"dodging a hook and countering with a fast body shot, fluid athletic movement",
"thrown back against the ropes, covering up desperately, absorbing punishment",
"landing an explosive four-punch combination, each blow landing clean",
"circling cautiously, both fighters reading each other, tension building",
"throwing a spinning heel kick, opponent barely ducking under it",
"ground-and-pound sequence, dominant position, crowd on their feet",
"catching an overhand right, knees buckling, clutching for a clinch to survive",
"breaking from a clinch, both fighters throwing wild hooks simultaneously",
"landing a clean liver shot, opponent visibly hurt, doubling over",
"referee warning both fighters, tempers flaring, crowd booing and cheering",
"slipping inside a jab and returning a sharp elbow, street-fight style",
"both fighters bloodied and exhausted, still throwing hard in the final seconds",
"landing a thudding low leg kick that buckles the opponent's lead leg, crowd erupting",
"launching a spinning back-kick into the body, opponent folding over the impact",
"exploding forward with a flying knee to the jaw, blood spraying from the nose",
"shooting in for a double-leg takedown and slamming the opponent to the dirt",
"raining down ground-and-pound from full mount, opponent's face bloodied and covering up",
"throwing a sharp elbow in the clinch, opening a cut over the opponent's eyebrow",
"snapping a high head kick that grazes the ducking opponent, sweat flying",
"driving knees into the body against the fence, opponent grunting and giving ground",
"catching a kick and countering with a takedown into side control, crowd on their feet",
"delivering a powerful uppercut, opponent's head snapping back, blood on the lip",
"sprawling out of a takedown and scrambling back up to throwing a knee",
"locking in a tight guillotine choke as the opponent thrashes to escape",
"stuffing a shot and landing a brutal knee to the head, opponent staggering",
"exchanging heavy hooks and overhands at close range, both fighters connecting",
"slipping inside and returning a slashing elbow, both fighters bloodied and still swinging",
]
# Rotating technique focus passed one-per-clip to the prompt writer so a match
# doesn't collapse into "all punches". The planner cycles through these (shuffled)
# so consecutive clips emphasise different MMA disciplines — the strongest lever
# against boxing-only monotony since the LLM otherwise gravitates to punches.
FIGHT_ACTION_FOCUS = [
"a kicking exchange (low leg kicks, body kicks, high kicks or push kicks)",
"knees and elbows in a tight clinch against the wall or fence",
"a takedown, slam or throw driving the fight to the ground",
"ground-and-pound or a scramble for position on the floor",
"a submission attempt (choke, armbar or guillotine) and the escape",
"a spinning or flying technique (spinning back-kick, spinning elbow, flying knee)",
"fast boxing combinations mixed with head movement and counters",
"a defensive sequence — sprawl, slip and counter back to offence",
]
WIN_SHOT_TEMPLATES = {
......@@ -872,10 +888,18 @@ class CoderAIClient:
width: int = 832, height: int = 480,
seed: int = None,
init_image: bytes = None,
loras: list = None) -> bytes:
# When a keyframe image is supplied, drive the model as text+image→video
# (ti2v) so the first frame already shows the right fighters.
mode = "ti2v" if init_image else "t2v"
loras: list = None,
cond_frames: list = None) -> bytes:
# `cond_frames` (a list of PNG byte tails from the previous chained part)
# drives VACE 'extend' continuation: the model sees real motion and carries
# it FORWARD, fixing the forward-then-rewind boomerang of single-frame
# seeding. Otherwise, a keyframe image drives text+image→video (ti2v).
if cond_frames:
mode = "extend"
elif init_image:
mode = "ti2v"
else:
mode = "t2v"
body = {
"model": model, "prompt": prompt,
"num_frames": num_frames, "fps": fps,
......@@ -890,6 +914,11 @@ class CoderAIClient:
body["seed"] = seed
if init_image:
body["init_image"] = "data:image/png;base64," + base64.b64encode(init_image).decode()
if cond_frames:
body["cond_frames"] = [
"data:image/png;base64," + base64.b64encode(f).decode()
for f in cond_frames
]
if loras:
body["loras"] = loras
......@@ -916,10 +945,20 @@ class CoderAIClient:
_LLM_SYSTEM = """\
You are a creative director writing vivid video-generation prompts for African street fighting scenes.
Each prompt must be ONE sentence, 15-35 words, cinematic and specific.
Emphasize FAST, continuous, explosive motion — rapid strikes, quick footwork, dynamic momentum and \
follow-through; describe action mid-movement, never static, posed, or slow-motion.
Emphasize FAST, continuous, explosive motion — describe action mid-movement, never static, posed, or \
slow-motion. The motion must PROGRESS FORWARD in one direction through the clip: no reversing, no \
rewinding, no looping back to the starting pose, no boomerang or back-and-forth motion.
This is a full MMA / no-rules street fight, NOT boxing — do NOT default to only punches. Across clips \
draw widely from the WHOLE arsenal: head and body punches, but also high kicks, low leg kicks, push \
kicks, spinning back-kicks, flying knees, knees in the clinch, elbow strikes, takedowns and slams, \
sprawls, ground-and-pound, mount and guard scrambles, chokes and submission attempts, throws, \
shoulder charges and dirty street-fighting. Each prompt should center on a DIFFERENT technique than \
the recent ones — favour kicks, knees, elbows, grappling and ground work over plain punches.
Make it gritty and visceral: it is fine and encouraged to SOMETIMES (not every clip) show blood — a \
bloodied nose or lip, a cut over the eye, blood spray or sweat-and-blood on the face — when a hard \
blow lands.
Vary camera angles (close-up, wide, low angle, over-shoulder), lighting (dusk, generator light, \
noon sun, spotlight), and action (strikes, clinch, footwork, takedown, ground work, crowd reaction).
noon sun, spotlight).
Always refer to each fighter by their NAME (given in the user message), not only by their description.
WARDROBE CONTINUITY (critical): every clip of a match — and every chained part within a clip, plus the \
outcome clips — must show each fighter in the IDENTICAL outfit: the same garments, exact same colours, \
......@@ -959,7 +998,7 @@ class PromptGenerator:
self._used_outcome: dict[str, list[str]] = {}
def fight_shot(self, f1: str, f2: str, env_desc: str, match_context: str = "",
avoid: list = None) -> str:
avoid: list = None, action_focus: str = "") -> str:
"""Generate a unique fight shot prompt.
`avoid` is an explicit list of actions to steer away from — used to pass
......@@ -982,6 +1021,8 @@ class PromptGenerator:
try:
used_hint = (f"\nAvoid repeating these actions: {'; '.join(avoid_set)}."
if avoid_set else "")
focus_hint = (f"\nThis clip should focus on {action_focus}."
if action_focus else "")
_d1 = self.char_descriptions.get(f1, "")
_d2 = self.char_descriptions.get(f2, "")
f1_desc = f"{f1} ({_d1})" if _d1 else f1
......@@ -992,7 +1033,7 @@ class PromptGenerator:
user=(
f"Fighter 1: {f1_desc}. Fighter 2: {f2_desc}. "
f"Location: {env_desc}. "
f"{match_context}{used_hint}\n"
f"{match_context}{used_hint}{focus_hint}\n"
"Write one fight action shot prompt. Refer to each "
"fighter by their NAME (not just their description)."
),
......@@ -1158,6 +1199,35 @@ def _last_frame_png(mp4_path: str) -> Optional[bytes]:
except Exception: pass
def _last_frames_png(mp4_path: str, k: int) -> list:
"""Extract the LAST `k` frames of a clip as a list of PNG bytes, in order
(oldest → newest). Used to seed a VACE 'extend' continuation with real motion
so the join carries velocity forward instead of boomeranging. Returns [] on
failure (caller falls back to single-frame seeding)."""
if k <= 1:
one = _last_frame_png(mp4_path)
return [one] if one else []
tmpd = None
try:
tmpd = tempfile.mkdtemp(prefix="twtail_")
# Pull the last ~2s, write each decoded frame, then keep the final k.
subprocess.run(
["ffmpeg", "-y", "-sseof", "-2", "-i", mp4_path,
"-q:v", "2", os.path.join(tmpd, "f_%04d.png")],
check=True, capture_output=True,
)
files = sorted(Path(tmpd).glob("f_*.png"))
files = files[-k:]
frames = [p.read_bytes() for p in files]
return [f for f in frames if f]
except Exception:
return []
finally:
if tmpd:
import shutil as _sh
_sh.rmtree(tmpd, ignore_errors=True)
# Wan2.2-A14B is trained for clips up to ~81 frames; beyond that temporal
# coherence breaks down (the frames visibly "jump") in a SINGLE generation.
MODEL_MAX_FRAMES = 81
......@@ -1170,6 +1240,11 @@ MODEL_MAX_FRAMES = 81
SINGLE_CLIP_MAX_FRAMES = 50 # max frames in ONE model generation (≤ MODEL_MAX_FRAMES)
MAX_PLANNED_FRAMES = 480 # ceiling for a whole (possibly chained) clip
# Number of trailing frames of the previous chained part fed to a VACE model as
# 'extend' conditioning. More frames = stronger motion-continuity (kills the
# forward/rewind boomerang) but more conditioning cost; ~5 carries velocity well.
VACE_TAIL_FRAMES = 5
# Per fight-clip frame budget. Frame count (not seconds) is the real control: it's
# the model's motion budget and is fps-independent, so a clip is CLIP_*_FRAMES
# frames played at the chosen fps → duration = frames / fps (e.g. 50-70 frames at
......@@ -1404,6 +1479,60 @@ def _build_char_descriptions(out_dir: Path) -> dict:
return desc
# Clothing nouns to lift the fighter's fixed outfit out of their profile prompt,
# so every keyframe can state it explicitly (the LoRA/IP-Adapter alone drift).
_CLOTHING_WORDS = (
"shorts", "trunks", "singlet", "vest", "gloves", "wraps", "hand wraps",
"sports bra", "bra", "tank top", "tank", "t-shirt", "shirt", "hoodie",
"boots", "headgear", "headband", "bandana", "jersey", "tracksuit", "kit",
"trousers", "pants", "leggings", "belt", "gi", "kimono", "sneakers",
)
def _extract_outfit(prompt_text: str) -> str:
"""Pull the wardrobe phrase(s) out of a profile prompt (e.g. 'worn boxing
shorts', 'boxing singlet', 'sports bra and MMA shorts'). Returns '' when none
are found. Comma/semicolon/period clauses that mention a clothing noun are
kept (up to two), so the colour/material adjectives stay attached."""
import re as _re
out, seen = [], set()
for part in _re.split(r"[,.;]", prompt_text or ""):
p = part.strip()
pl = p.lower()
if not p:
continue
if any(w in pl for w in _CLOTHING_WORDS):
key = pl
if key not in seen:
seen.add(key)
out.append(p)
return ", ".join(out[:2])
def _build_char_outfits(out_dir: Path) -> dict:
"""Return {name: outfit_phrase} from each fighter's profile PROMPT (the
`prompt` field carries clothing; `description` usually doesn't). Merges
FIGHTER_POOL + locally saved meta.json so user-created fighters are covered."""
outfits = {}
for f in FIGHTER_POOL:
o = _extract_outfit(f.get("prompt", ""))
if o:
outfits[f["name"]] = o
chars_dir = out_dir / "characters"
if chars_dir.exists():
for d in chars_dir.iterdir():
meta_path = d / "meta.json"
if meta_path.exists():
try:
meta = json.loads(meta_path.read_text())
o = _extract_outfit(meta.get("prompt", "") or meta.get("description", ""))
if o:
outfits[d.name] = o
except Exception:
pass
return outfits
_VALID_CONSISTENCY = {"prompt", "ipadapter", "keyframe", "lora"}
......@@ -1933,6 +2062,12 @@ def _generate_keyframes(client: CoderAIClient, image_model: str, keyframe_dir: P
keyframe_dir.mkdir(parents=True, exist_ok=True)
use_ip = "ipadapter" in consistency or "keyframe" in consistency
use_lora = "lora" in consistency
# Each fighter's fixed outfit (lifted from their profile prompt). Stated
# EXPLICITLY in every keyframe prompt so the image model paints the right
# clothes instead of drifting — the keyframe then anchors the I2V clip, so
# this is the strongest lever for wardrobe consistency.
_out_dir = keyframe_dir.parent.parent
_outfits = _build_char_outfits(_out_dir)
# Map match_name -> [f1, f2] so an outcome keyframe attaches BOTH match
# fighters' LoRAs (+ env), like the clips do — not just the single fighter
......@@ -1977,6 +2112,15 @@ def _generate_keyframes(client: CoderAIClient, image_model: str, keyframe_dir: P
loras = (_lora_specs_for(fighters, lora_map, lora_weight)
+ _env_lora_specs_for(env, env_lora_map, env_lora_weight)) or None
kf_prompt = prompt
# Force each fighter's exact outfit up front so the keyframe paints the
# right clothes (e.g. "khumalo wearing boxing singlet; dlamini wearing
# worn boxing shorts"). The wardrobe lead beats a trailing mention.
_wardrobe = "; ".join(f"{n} wearing {_outfits[n]}"
for n in fighters if _outfits.get(n))
if _wardrobe:
kf_prompt = (f"{_wardrobe}. {kf_prompt} "
f"— each fighter in their exact same outfit, same colours, "
f"consistent wardrobe")
if env:
kf_prompt = f"[{env} location] " + kf_prompt
try:
......@@ -2234,12 +2378,17 @@ def stage_videos(client: CoderAIClient, video_model: str, out_dir: Path,
# shots already written for the SAME match, so a 12-clip fight stays
# varied throughout (not just within the global recent-5 window).
match_avoid = []
for c in m["clips"]:
# Shuffled technique cycle so consecutive clips emphasise different
# disciplines (kicks, clinch, ground, submissions…) instead of all punches.
_focus_cycle = list(FIGHT_ACTION_FOCUS)
random.shuffle(_focus_cycle)
for _ci, c in enumerate(m["clips"]):
_pidx += 1
shot = prompter.fight_shot(
m["f1"], m["f2"], m["env_desc"],
match_context=f"Match stage: {c['intensity']}. ",
avoid=match_avoid)
avoid=match_avoid,
action_focus=_focus_cycle[_ci % len(_focus_cycle)])
c["shot"] = shot
f1_hint = _fighter_desc_hint(m['f1'], char_descriptions)
f2_hint = _fighter_desc_hint(m['f2'], char_descriptions)
......@@ -2409,11 +2558,13 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
MODEL_MAX_FRAMES))
def _render_once(label, prompt, profiles, env, nf, out_path,
fighters=None, init_override=None, step_cb=None):
fighters=None, init_override=None, step_cb=None,
cond_frames=None):
"""One model generation → out_path. `init_override` (PNG bytes) wins over
any keyframe; pass it to chain a sub-render onto the previous one's last
frame. Returns (ok, duration_or_None, fatal)."""
init_image = init_override
frame. `cond_frames` (list of PNG byte tails) instead drives VACE 'extend'
continuation. Returns (ok, duration_or_None, fatal)."""
init_image = None if cond_frames else init_override
loras = None
if use_lora:
# Video-DiT LoRAs trained for THIS video model (image LoRAs can't apply
......@@ -2442,7 +2593,7 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
character_profiles=profiles, environment_name=env,
num_frames=nf, fps=fps, seed=random.randint(0, 2**31),
width=_vw, height=_vh,
init_image=init_image, loras=loras,
init_image=init_image, loras=loras, cond_frames=cond_frames,
poll_fn=client.video_progress, step_cb=step_cb,
)
Path(out_path).write_bytes(mp4)
......@@ -2486,29 +2637,49 @@ def _stage_videos_render(client, video_model, video_dir, fight_plan, outcome_pla
# stray files for _scan_matches to mis-parse; only the concatenated result
# lands at out_path.
import shutil as _sh
# A VACE model continues a chained part from the previous part's FRAME TAIL
# (real motion → carries velocity forward), the proper fix for the
# single-frame "boomerang". Non-VACE models fall back to single last-frame
# seeding + the forward-motion prompt nudge.
_vace = "vace" in (video_model or "").lower()
_log(f" ↪ {nf}f > {_chunk_max}f/render — chaining {len(budget)} parts "
f"{budget} into one shot")
f"{budget} into one shot" + (" [VACE frame-tail extend]" if _vace else ""))
tmpd = tempfile.mkdtemp(prefix="twshot_")
parts, prev_last = [], None
parts, prev_last, prev_tail = [], None, None
try:
for pi, pn in enumerate(budget):
part_path = os.path.join(tmpd, f"part{pi:02d}.mp4")
seed_img = keyframe if pi == 0 else (prev_last or keyframe)
# Tag each part's step updates with part N/total so the UI can show
# "concatenating shot — part 2/3" alongside the diffusion step.
_pcb = ((lambda prog, _p=pi + 1, _n=len(budget):
step_cb({**(prog or {}), "part": _p, "parts": _n}))
if step_cb else None)
# Seeding for this part:
# • part 0 → the clip keyframe (single image).
# • VACE → the previous part's frame tail (motion continuation).
# • else → the previous part's last frame (single image) + the
# forward-motion prompt nudge to discourage rewinding.
seed_img = keyframe if pi == 0 else (prev_last or keyframe)
cond_frames = prev_tail if (_vace and pi > 0) else None
if cond_frames:
seed_img = None # VACE conditions via the tail, not an init frame
part_prompt = prompt if pi == 0 else (
"Continuing seamlessly from the previous moment, the fight keeps "
"moving FORWARD into the next action — new strikes and movement that "
"advance the exchange. " + prompt)
ok, _dur, is_fatal = _render_once(
f"{label} [part {pi+1}/{len(budget)}, {pn}f]",
prompt, profiles, env, pn, part_path,
fighters=fighters, init_override=seed_img, step_cb=_pcb)
part_prompt, profiles, env, pn, part_path,
fighters=fighters, init_override=seed_img, step_cb=_pcb,
cond_frames=cond_frames)
if not ok:
return False, None, is_fatal
parts.append(part_path)
# Prepare seeds for the NEXT part.
prev_last = _last_frame_png(part_path)
if pi < len(budget) - 1 and prev_last is None:
_log(" ⚠ could not read part's last frame — next part falls "
prev_tail = _last_frames_png(part_path, VACE_TAIL_FRAMES) if _vace else None
if pi < len(budget) - 1 and not prev_last and not prev_tail:
_log(" ⚠ could not read part's tail — next part falls "
"back to the clip keyframe (possible visible seam)")
# Re-encode the join: stream-copying the parts makes players freeze on
# each part's first frame for its duration (static-first-half bug). The
......@@ -3341,11 +3512,14 @@ def launch_web_ui(default_args):
f1_hint = _fighter_desc_hint(m["f1"], char_descriptions)
f2_hint = _fighter_desc_hint(m["f2"], char_descriptions)
match_avoid = []
_focus_cycle = list(FIGHT_ACTION_FOCUS)
random.shuffle(_focus_cycle)
for i, c in enumerate(new_clips):
shot = prompter.fight_shot(
m["f1"], m["f2"], m["env_desc"],
match_context=f"Match stage: {c['intensity']}. ",
avoid=match_avoid)
avoid=match_avoid,
action_focus=_focus_cycle[i % len(_focus_cycle)])
c["shot"] = shot
c["prompt"] = (f"{f1_hint} vs {f2_hint} — {shot} "
f"— {_continuity_clause(m.get('env'))} "
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment