Add character/environment profiles, 3D conversion, archive, multi-character dialog

- Character profiles: extract from images/video or generate from text prompt;
  up to 6 profiles per image/video generation via IP-Adapter conditioning
- Environment profiles: same multi-slot system for scene/background conditioning
- Multi-character dialog for video: per-character TTS synthesis with voice profile
  support, sequential or manual timing, ffmpeg amix, Wav2Lip/SadTalker lip sync
- 2D3D conversion: image/video to stereo/anaglyph/depth/mesh and back;
  text/image to 3D model (GLB); turntable video rendering
- Generation archive: auto-save all outputs with configurable retention and
  hourly cleanup; browse/delete from web UI and API
- Voice profile enhancements: extract from video, PATCH update, GET by name;
  TTS endpoint now accepts voice_profile for F5-TTS cloning
- Image generation progress tracking via GET /v1/images/progress
- Web Studio: Profiles tab (Characters, Environments, Voices) with full managers;
  multi-slot character/environment selectors in all panels; character dialog
  section in all video panels; archive settings in Settings page
- Fix: Profiles tab sub-tab switching now correctly updates the content panel
Co-Authored-By: 's avatarClaude Sonnet 4.6 <noreply@anthropic.com>
parent 19e28d3d
This diff is collapsed.
......@@ -1556,6 +1556,11 @@ async def api_get_settings(username: str = Depends(require_admin)):
"device_id": c.vulkan.device_id,
"single_gpu": c.vulkan.single_gpu,
},
"archive": {
"enabled": c.archive.enabled,
"directory": c.archive.directory,
"retention": c.archive.retention,
},
"system_prompt": c.system_prompt,
"tools_closer_prompt": c.tools_closer_prompt,
"grammar_guided": c.grammar_guided,
......@@ -1632,8 +1637,95 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
if "parser" in data:
c.parser = data["parser"]
if "archive" in data:
import os as _os
from codai.api.archive import archive_manager, RETENTION_OPTIONS
arc = data["archive"]
c.archive.enabled = bool(arc.get("enabled", c.archive.enabled))
raw_dir = (arc.get("directory") or "").strip()
c.archive.directory = raw_dir # store as-is (empty = default)
ret = arc.get("retention", c.archive.retention)
c.archive.retention = ret if ret in RETENTION_OPTIONS else "never"
# Resolve for live reconfiguration
cfg_dir = str(config_manager.config_dir)
resolved = raw_dir if raw_dir and _os.path.isabs(raw_dir) else _os.path.join(cfg_dir, raw_dir or "archive")
archive_manager.configure(c.archive.enabled, resolved, c.archive.retention)
config_manager.save_config()
return {"success": True}
# =============================================================================
# Archive management
# =============================================================================
@router.get("/admin/archive", response_class=HTMLResponse)
async def archive_page(request: Request, username: str = Depends(require_admin)):
return templates.TemplateResponse(request, "archive.html", {"username": username, "is_admin": True})
@router.get("/admin/api/archive")
async def api_archive_list(
limit: int = 50,
offset: int = 0,
username: str = Depends(require_admin),
):
from codai.api.archive import archive_manager
entries = archive_manager.list_entries(limit=limit, offset=offset)
total = archive_manager.count_entries()
return {"entries": entries, "total": total}
@router.get("/admin/api/archive/{gen_id}")
async def api_archive_get(gen_id: str, username: str = Depends(require_admin)):
from codai.api.archive import archive_manager
entry = archive_manager.get_entry(gen_id)
if entry is None:
raise HTTPException(status_code=404, detail="Entry not found")
return entry
@router.delete("/admin/api/archive/{gen_id}")
async def api_archive_delete(gen_id: str, username: str = Depends(require_admin)):
from codai.api.archive import archive_manager
if not archive_manager.delete_entry(gen_id):
raise HTTPException(status_code=404, detail="Entry not found")
return {"success": True}
@router.get("/admin/api/archive/{gen_id}/files/{filename}")
async def api_archive_file(
gen_id: str,
filename: str,
username: str = Depends(require_auth),
):
from fastapi.responses import FileResponse
from codai.api.archive import archive_manager
path = archive_manager.get_file_path(gen_id, filename)
if path is None:
raise HTTPException(status_code=404, detail="File not found")
import mimetypes
media_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
return FileResponse(path, media_type=media_type)
@router.get("/admin/api/archive-settings")
async def api_archive_settings_get(username: str = Depends(require_admin)):
if config_manager is None or config_manager.config is None:
raise HTTPException(status_code=503, detail="Config not ready")
import os as _os
c = config_manager.config.archive
from codai.api.archive import RETENTION_OPTIONS
default_dir = _os.path.join(str(config_manager.config_dir), "archive")
return {
"enabled": c.enabled,
"directory": c.directory,
"default_directory": default_dir,
"retention": c.retention,
"retention_options": RETENTION_OPTIONS,
}
# --- HuggingFace model search proxy ---
import re as _re
......@@ -1794,6 +1886,110 @@ async def api_hf_model_files(model_id: str, username: str = Depends(require_admi
return files
# =============================================================================
# Character profile management proxy (admin UI)
# =============================================================================
@router.get("/admin/api/characters")
async def api_list_characters(username: str = Depends(require_auth)):
from codai.api.characters import _list_characters
return {"characters": _list_characters()}
@router.get("/admin/api/characters/{name}")
async def api_get_character(name: str, username: str = Depends(require_auth)):
from codai.api.characters import _load_character_meta, _load_character_images
meta = _load_character_meta(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Character '{name}' not found")
images = _load_character_images(name)
return {
"name": meta['name'],
"description": meta.get('description', ''),
"image_count": meta['image_count'],
"created_at": meta['created_at'],
"images": [img.model_dump() for img in images],
}
@router.delete("/admin/api/characters/{name}")
async def api_delete_character(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
from codai.api.characters import _char_dir
cdir = _char_dir(name)
if not _os.path.isdir(cdir):
raise HTTPException(status_code=404, detail=f"Character '{name}' not found")
shutil.rmtree(cdir)
return {"ok": True, "name": name}
# =============================================================================
# Environment profile management proxy (admin UI)
# =============================================================================
@router.get("/admin/api/environments")
async def api_list_environments(username: str = Depends(require_auth)):
from codai.api.environments import _list_environments
return {"environments": _list_environments()}
@router.get("/admin/api/environments/{name}")
async def api_get_environment(name: str, username: str = Depends(require_auth)):
from codai.api.environments import _load_environment_meta, _load_environment_images
meta = _load_environment_meta(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Environment '{name}' not found")
images = _load_environment_images(name)
return {
"name": meta['name'],
"description": meta.get('description', ''),
"image_count": meta['image_count'],
"created_at": meta['created_at'],
"images": [img.model_dump() for img in images],
}
@router.delete("/admin/api/environments/{name}")
async def api_delete_environment(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
from codai.api.environments import _env_dir
edir = _env_dir(name)
if not _os.path.isdir(edir):
raise HTTPException(status_code=404, detail=f"Environment '{name}' not found")
shutil.rmtree(edir)
return {"ok": True, "name": name}
# =============================================================================
# Voice profile management proxy (admin UI)
# =============================================================================
@router.get("/admin/api/voices")
async def api_list_voices(username: str = Depends(require_auth)):
from codai.api.voice_clone import _list_voices
return {"voices": _list_voices()}
@router.get("/admin/api/voices/{name}")
async def api_get_voice(name: str, username: str = Depends(require_auth)):
from codai.api.voice_clone import _load_voice
meta = _load_voice(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
return {"voice": meta}
@router.delete("/admin/api/voices/{name}")
async def api_delete_voice(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
from codai.api.voice_clone import _voice_path
vdir = _voice_path(name)
if not _os.path.exists(vdir):
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
shutil.rmtree(vdir)
return {"deleted": True, "name": name}
@router.get("/admin/api/hf-model-info")
async def api_hf_model_info(model_id: str, username: str = Depends(require_admin)):
"""Full metadata for a single HuggingFace model repo."""
......
This diff is collapsed.
......@@ -27,6 +27,7 @@
<a href="/admin/models" class="nav-link {% if '/models' in request.url.path %}active{% endif %}">Models</a>
<a href="/admin/tokens" class="nav-link {% if '/tokens' in request.url.path %}active{% endif %}">Tokens</a>
<a href="/admin/users" class="nav-link {% if '/users' in request.url.path %}active{% endif %}">Users</a>
<a href="/admin/archive" class="nav-link {% if '/archive' in request.url.path %}active{% endif %}">Archive</a>
<a href="/admin/settings" class="nav-link {% if '/settings' in request.url.path %}active{% endif %}">Settings</a>
{% endif %}
<button class="nav-link nav-donate-btn" onclick="document.getElementById('donateModal').classList.add('show')">Donate &#9829;</button>
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -69,6 +69,38 @@
<span class="form-hint">Models will inherit this as default when configured</span>
</div>
</div>
<!-- Archive -->
<div class="card mb-0" style="margin-top:1rem">
<div class="card-title">Generation Archive</div>
<div class="form-row">
<label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
<input type="checkbox" id="s-arc-enabled">
<span style="font-size:13px;font-weight:500">Enable generation archive</span>
</label>
<span class="form-hint">When enabled, every generation (image, video, audio, text) is logged with its prompt, model, and output files.</span>
</div>
<div class="form-row">
<label class="form-label">Archive directory <span class="muted" id="s-arc-dir-hint"></span></label>
<input type="text" id="s-arc-dir" class="form-input" placeholder="(default)">
<span class="form-hint">Absolute path or relative to the config directory. Leave blank to use the default.</span>
</div>
<div class="form-row" style="margin:0">
<label class="form-label">Retention period</label>
<select id="s-arc-retention" class="form-input" style="max-width:200px">
<option value="1h">1 hour</option>
<option value="1d">1 day</option>
<option value="2d">2 days</option>
<option value="1w">1 week</option>
<option value="1m">1 month</option>
<option value="3m">3 months</option>
<option value="6m">6 months</option>
<option value="1y">1 year</option>
<option value="never">Never (keep forever)</option>
</select>
<span class="form-hint">Archived entries older than this are automatically deleted. Takes effect immediately on save.</span>
</div>
</div>
{% endblock %}
{% block scripts %}
......@@ -99,6 +131,18 @@ async function loadSettings(){
document.getElementById('s-gguf-cache').value = d.models?.gguf_cache_dir ?? '';
document.getElementById('s-offload-dir').value = d.offload?.directory ?? './offload';
toggleHttps();
// Archive
const arc = d.archive || {};
document.getElementById('s-arc-enabled').checked = !!arc.enabled;
document.getElementById('s-arc-dir').value = arc.directory ?? '';
document.getElementById('s-arc-retention').value = arc.retention ?? 'never';
// Show effective default dir as placeholder hint
try {
const as = await fetch('/admin/api/archive-settings').then(r=>r.json());
const hint = document.getElementById('s-arc-dir-hint');
if (hint && as.default_directory) hint.textContent = `(default: ${as.default_directory})`;
document.getElementById('s-arc-dir').placeholder = as.default_directory || '(default)';
} catch(_){}
}catch(e){ showAlert('error','Failed to load settings: '+e.message); }
}
......@@ -119,14 +163,19 @@ async function saveSettings(){
},
offload:{
directory: document.getElementById('s-offload-dir').value.trim() || './offload',
}
},
archive:{
enabled: document.getElementById('s-arc-enabled').checked,
directory: document.getElementById('s-arc-dir').value.trim(),
retention: document.getElementById('s-arc-retention').value,
},
};
try{
const r = await fetch('/admin/api/settings',{
method:'POST', headers:{'Content-Type':'application/json'},
body: JSON.stringify(data)
});
if(r.ok) showAlert('info','Settings saved. Restart CoderAI to apply.');
if(r.ok) showAlert('info','Settings saved. Archive changes take effect immediately; restart CoderAI for other changes.');
else{ const e=await r.json(); showAlert('error', e.detail||'Save failed'); }
}catch(e){ showAlert('error','Error: '+e.message); }
}
......
......@@ -66,8 +66,33 @@ def set_global_file_path_wrapper(path: str):
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Lifespan context manager for startup/shutdown."""
# Startup
import asyncio as _asyncio
async def _archive_cleanup_loop():
from codai.api.archive import archive_manager
while True:
await _asyncio.sleep(3600)
try:
archive_manager.run_cleanup()
except Exception:
pass
cleanup_task = _asyncio.create_task(_archive_cleanup_loop())
# Run initial cleanup on startup
try:
from codai.api.archive import archive_manager
archive_manager.run_cleanup()
except Exception:
pass
yield
cleanup_task.cancel()
try:
await cleanup_task
except _asyncio.CancelledError:
pass
# Shutdown
multi_model_manager.cleanup()
model_manager.cleanup()
......@@ -101,6 +126,8 @@ from codai.api.voice_clone import router as voice_clone_router
from codai.api.voice_convert import router as voice_convert_router
from codai.api.faceswap import router as faceswap_router
from codai.api.characters import router as characters_router
from codai.api.spatial import router as spatial_router
from codai.api.environments import router as environments_router
from codai.admin.routes import router as admin_router
# Import and add middleware
......@@ -133,6 +160,8 @@ app.include_router(voice_clone_router)
app.include_router(voice_convert_router)
app.include_router(faceswap_router)
app.include_router(characters_router)
app.include_router(environments_router)
app.include_router(spatial_router)
app.include_router(admin_router)
......@@ -166,4 +195,59 @@ async def get_file(filename: str):
raise HTTPException(status_code=403, detail="Access denied")
if not os.path.isfile(candidate):
raise HTTPException(status_code=404, detail="File not found")
return FileResponse(candidate)
\ No newline at end of file
return FileResponse(candidate)
_IMAGE_EXTS = {'.png', '.jpg', '.jpeg', '.webp', '.gif'}
_VIDEO_EXTS = {'.mp4', '.webm', '.avi', '.mov'}
_AUDIO_EXTS = {'.wav', '.mp3', '.ogg', '.flac', '.aac', '.m4a'}
@app.get("/v1/archive")
async def list_archive():
"""List all generated files in the output directory."""
if not global_file_path or not os.path.isdir(global_file_path):
return {"files": []}
files = []
try:
names = os.listdir(global_file_path)
except OSError:
return {"files": []}
for fname in names:
fpath = os.path.join(global_file_path, fname)
if not os.path.isfile(fpath):
continue
ext = os.path.splitext(fname)[1].lower()
if ext in _IMAGE_EXTS:
ftype = 'image'
elif ext in _VIDEO_EXTS:
ftype = 'video'
elif ext in _AUDIO_EXTS:
ftype = 'audio'
else:
continue
stat = os.stat(fpath)
files.append({
'filename': fname,
'type': ftype,
'size': stat.st_size,
'created': stat.st_mtime,
'url': f'/v1/files/{fname}',
})
files.sort(key=lambda f: f['created'], reverse=True)
return {"files": files}
@app.delete("/v1/archive/{filename}")
async def delete_archive_file(filename: str):
"""Delete a generated file from the output directory."""
if not global_file_path:
raise HTTPException(status_code=404, detail="File not found")
safe_base = os.path.realpath(global_file_path)
candidate = os.path.realpath(os.path.join(global_file_path, filename))
if not (candidate == safe_base or candidate.startswith(safe_base + os.sep)):
raise HTTPException(status_code=403, detail="Access denied")
if not os.path.isfile(candidate):
raise HTTPException(status_code=404, detail="File not found")
os.remove(candidate)
return {"deleted": filename}
\ No newline at end of file
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Generation archive — persists all generated artifacts with metadata on disk."""
import io
import json
import logging
import os
import shutil
import threading
import time
import uuid
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
_log = logging.getLogger(__name__)
RETENTION_OPTIONS = ["1h", "1d", "2d", "1w", "1m", "3m", "6m", "1y", "never"]
_RETENTION_SECONDS: Dict[str, Optional[int]] = {
"1h": 3_600,
"1d": 86_400,
"2d": 172_800,
"1w": 604_800,
"1m": 2_592_000, # 30 days
"3m": 7_776_000, # 90 days
"6m": 15_552_000, # 180 days
"1y": 31_536_000, # 365 days
"never": None,
}
def _safe_id(s: str) -> bool:
return bool(s) and "/" not in s and ".." not in s and "\\" not in s
def _safe_filename(s: str) -> bool:
return bool(s) and "/" not in s and ".." not in s and "\\" not in s
class ArchiveManager:
"""Manages the on-disk generation archive."""
def __init__(self):
self.enabled: bool = True
self.directory: str = os.path.abspath("./archive")
self.retention: str = "never"
self._lock = threading.Lock()
def configure(self, enabled: bool, directory: str, retention: str):
self.enabled = enabled
self.directory = os.path.abspath(directory or "./archive")
self.retention = retention if retention in _RETENTION_SECONDS else "never"
def _gen_dir(self, gen_id: str) -> str:
return os.path.join(self.directory, gen_id)
# ------------------------------------------------------------------
# Save
# ------------------------------------------------------------------
def save_generation(
self,
gen_type: str,
endpoint: str,
model: Optional[str],
prompt: Optional[str],
params: Dict[str, Any],
artifacts: List[Tuple[bytes, str]], # [(raw_bytes, file_ext), ...]
) -> Optional[str]:
"""Save a generation to archive. Returns gen_id on success, None otherwise."""
if not self.enabled:
return None
try:
gen_id = f"{int(time.time())}_{uuid.uuid4().hex[:8]}"
gen_dir = self._gen_dir(gen_id)
os.makedirs(gen_dir, exist_ok=True)
saved_files: List[str] = []
preview_file: Optional[str] = None
for i, (data, ext) in enumerate(artifacts):
if not data:
continue
fname = f"output_{i}.{ext}"
with open(os.path.join(gen_dir, fname), "wb") as fh:
fh.write(data)
saved_files.append(fname)
if i == 0 and preview_file is None:
try:
preview_file = self._make_preview(data, ext, gen_dir)
except Exception as e:
_log.debug("Archive preview failed: %s", e)
meta: Dict[str, Any] = {
"id": gen_id,
"type": gen_type,
"endpoint": endpoint,
"model": model or "",
"prompt": prompt or "",
"parameters": params,
"files": saved_files,
"preview": preview_file or (saved_files[0] if saved_files else None),
"created_at": int(time.time()),
"created_at_iso": datetime.utcnow().isoformat() + "Z",
}
with open(os.path.join(gen_dir, "metadata.json"), "w") as fh:
json.dump(meta, fh, indent=2)
return gen_id
except Exception as e:
_log.warning("Archive save failed: %s", e)
return None
# ------------------------------------------------------------------
# Query
# ------------------------------------------------------------------
def list_entries(self, limit: int = 50, offset: int = 0) -> List[Dict]:
"""Return metadata dicts, newest first."""
if not os.path.isdir(self.directory):
return []
entries: List[Dict] = []
try:
for name in os.listdir(self.directory):
meta_path = os.path.join(self.directory, name, "metadata.json")
if not os.path.isfile(meta_path):
continue
try:
with open(meta_path) as fh:
entries.append(json.load(fh))
except Exception:
pass
except Exception:
pass
entries.sort(key=lambda x: x.get("created_at", 0), reverse=True)
return entries[offset: offset + limit]
def count_entries(self) -> int:
if not os.path.isdir(self.directory):
return 0
count = 0
try:
for name in os.listdir(self.directory):
if os.path.isfile(os.path.join(self.directory, name, "metadata.json")):
count += 1
except Exception:
pass
return count
def get_entry(self, gen_id: str) -> Optional[Dict]:
if not _safe_id(gen_id):
return None
meta_path = os.path.join(self._gen_dir(gen_id), "metadata.json")
if not os.path.isfile(meta_path):
return None
try:
with open(meta_path) as fh:
return json.load(fh)
except Exception:
return None
def delete_entry(self, gen_id: str) -> bool:
if not _safe_id(gen_id):
return False
gen_dir = self._gen_dir(gen_id)
if not os.path.isdir(gen_dir):
return False
try:
shutil.rmtree(gen_dir)
return True
except Exception:
return False
def get_file_path(self, gen_id: str, filename: str) -> Optional[str]:
if not _safe_id(gen_id) or not _safe_filename(filename):
return None
path = os.path.join(self._gen_dir(gen_id), filename)
return path if os.path.isfile(path) else None
# ------------------------------------------------------------------
# Cleanup
# ------------------------------------------------------------------
def run_cleanup(self):
"""Remove entries older than the configured retention window."""
max_age = _RETENTION_SECONDS.get(self.retention)
if max_age is None:
return
if not os.path.isdir(self.directory):
return
cutoff = time.time() - max_age
deleted = 0
try:
for name in list(os.listdir(self.directory)):
meta_path = os.path.join(self.directory, name, "metadata.json")
try:
with open(meta_path) as fh:
meta = json.load(fh)
if meta.get("created_at", time.time()) < cutoff:
shutil.rmtree(os.path.join(self.directory, name), ignore_errors=True)
deleted += 1
except Exception:
pass
except Exception:
pass
if deleted:
_log.info("Archive cleanup: removed %d entry/entries", deleted)
# ------------------------------------------------------------------
# Preview generation
# ------------------------------------------------------------------
def _make_preview(self, data: bytes, ext: str, gen_dir: str) -> Optional[str]:
ext = ext.lower()
if ext in ("png", "jpg", "jpeg", "webp", "bmp", "gif"):
from PIL import Image
img = Image.open(io.BytesIO(data)).convert("RGB")
img.thumbnail((512, 512))
dst = os.path.join(gen_dir, "preview.webp")
img.save(dst, "WEBP", quality=75)
return "preview.webp"
if ext in ("mp4", "webm", "avi", "mov"):
import subprocess
import tempfile
with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as tmp:
tmp.write(data)
tmp_path = tmp.name
frame_path = os.path.join(gen_dir, "_frame_tmp.jpg")
try:
r = subprocess.run(
["ffmpeg", "-y", "-i", tmp_path, "-vframes", "1", "-q:v", "3", frame_path],
capture_output=True, timeout=15,
)
if r.returncode == 0 and os.path.exists(frame_path):
from PIL import Image
img = Image.open(frame_path).convert("RGB")
img.thumbnail((512, 512))
dst = os.path.join(gen_dir, "preview.webp")
img.save(dst, "WEBP", quality=75)
return "preview.webp"
except Exception:
pass
finally:
try:
os.unlink(tmp_path)
except Exception:
pass
try:
os.unlink(frame_path)
except Exception:
pass
# Audio or unrecognised — no visual preview
return None
# Module-level singleton
archive_manager = ArchiveManager()
......@@ -199,4 +199,24 @@ async def audio_generate(request: AudioGenerationRequest, http_request: Request
raise HTTPException(status_code=500, detail=f"Audio generation failed: {e}")
result = _save_audio_response(audio_bytes, ext, http_request)
try:
from codai.api.archive import archive_manager
asyncio.get_event_loop().create_task(asyncio.to_thread(
archive_manager.save_generation,
"audio", "/v1/audio/generate",
model_name,
request.prompt,
{
"duration": request.duration,
"top_k": request.top_k,
"top_p": request.top_p,
"temperature": request.temperature,
"cfg_coef": request.cfg_coef,
},
[(audio_bytes, ext)],
))
except Exception:
pass
return AudioGenerationResponse(created=int(time.time()), data=[result])
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -34,9 +34,11 @@ from starlette.middleware.base import BaseHTTPMiddleware
class BearerAuthMiddleware(BaseHTTPMiddleware):
"""Reject /v1/ API requests that lack a valid Bearer token or active web session."""
_EXEMPT_PATHS = {"/v1/images/progress"}
async def dispatch(self, request: Request, call_next):
path = request.url.path
if not path.startswith("/v1/"):
if not path.startswith("/v1/") or path in self._EXEMPT_PATHS:
return await call_next(request)
from codai.admin import routes as _admin_routes
......@@ -112,14 +114,20 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
return prefix
return ""
# Lightweight polling endpoints that must never be rate-limited
_EXEMPT_PATHS = {"/v1/images/progress"}
async def dispatch(self, request: Request, call_next):
if not RATE_LIMITING_ENABLED:
return await call_next(request)
path = request.url.path
# Queue-size enforcement for authenticated API requests
if any(path.startswith(p) for p in _QUEUED_PREFIXES):
if path in self._EXEMPT_PATHS:
return await call_next(request)
# Queue-size enforcement for authenticated API requests (not for status polls)
if path not in self._EXEMPT_PATHS and any(path.startswith(p) for p in _QUEUED_PREFIXES):
from codai.queue.manager import queue_manager
if await queue_manager.is_full():
return JSONResponse(
......
This diff is collapsed.
......@@ -18,10 +18,12 @@
Text-to-speech endpoints for the codai API.
"""
import asyncio
import base64
import os
from typing import Optional
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, ConfigDict
# Import from codai modules
......@@ -51,7 +53,8 @@ class TTSRequest(BaseModel):
voice: str = "af_sarah"
response_format: str = "mp3"
speed: float = 1.0
voice_profile: Optional[str] = None # saved voice profile name (uses F5-TTS cloning)
model_config = ConfigDict(extra="allow")
......@@ -62,13 +65,39 @@ class TTSResponse(BaseModel):
@router.post("/v1/audio/speech")
async def create_speech(request: TTSRequest):
async def create_speech(request: TTSRequest, http_request: Request = None):
"""
Text-to-speech endpoint (OpenAI-compatible).
Supports:
- Kokoro TTS models (when --tts-model is specified)
"""
# If a voice profile is requested, delegate to voice cloning (F5-TTS)
if request.voice_profile:
from codai.api.voice_clone import _load_voice, _f5tts_clone
meta = _load_voice(request.voice_profile)
if not meta:
raise HTTPException(status_code=404,
detail=f"Voice profile '{request.voice_profile}' not found")
ref_audio_path = meta['audio_file']
ref_text = meta.get('transcript', '')
if not ref_text:
raise HTTPException(status_code=400,
detail="Voice profile has no transcript; update it with PATCH /v1/audio/voices/{name}")
try:
audio_bytes = await asyncio.get_event_loop().run_in_executor(
None, _f5tts_clone,
ref_audio_path, ref_text, request.input,
request.speed or 1.0, None,
)
except ImportError:
raise HTTPException(status_code=501,
detail="f5-tts not installed. Run: pip install f5-tts")
except Exception as e:
raise HTTPException(status_code=500, detail=f"Voice cloning failed: {e}")
audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
return {"audio": audio_base64}
# Use the manager to resolve the model and manage VRAM
model_info = multi_model_manager.request_model(
requested_model=request.model,
......@@ -119,10 +148,23 @@ async def create_speech(request: TTSRequest):
speed = request.speed or 1.0
audio_bytes = kokoro_model.generate(request.input, voice=voice, speed=speed)
try:
from codai.api.archive import archive_manager
asyncio.get_event_loop().create_task(asyncio.to_thread(
archive_manager.save_generation,
"tts", "/v1/audio/speech",
model_name,
request.input,
{"voice": voice, "speed": speed, "response_format": request.response_format},
[(audio_bytes, request.response_format or "mp3")],
))
except Exception:
pass
# Convert to base64
audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
return {
"audio": audio_base64
}
......
This diff is collapsed.
"""
Voice cloning endpoints.
POST /v1/audio/clone — synthesize speech in a cloned voice
GET /v1/audio/voices — list saved voice profiles
POST /v1/audio/voices — save a named voice profile (ref audio + transcript)
DELETE /v1/audio/voices/{name} — delete a voice profile
POST /v1/audio/clone — synthesize speech in a cloned voice
GET /v1/audio/voices — list saved voice profiles
POST /v1/audio/voices — save a named voice profile (ref audio + transcript)
POST /v1/audio/voices/extract — extract voice profile from audio or video file
PATCH /v1/audio/voices/{name} — update description / replace reference audio
DELETE /v1/audio/voices/{name} — delete a voice profile
"""
import asyncio
......@@ -12,6 +14,7 @@ import base64
import io
import json
import os
import subprocess
import tempfile
import time
from typing import Optional
......@@ -103,6 +106,17 @@ def _decode_audio(data: str) -> tuple[bytes, str]:
return base64.b64decode(data), '.wav'
def _decode_b64_or_url(data: str) -> bytes:
if data.startswith('data:'):
_, b64 = data.split(',', 1)
return base64.b64decode(b64)
if data.startswith('http://') or data.startswith('https://'):
import urllib.request
with urllib.request.urlopen(data, timeout=60) as r:
return r.read()
return base64.b64decode(data)
def _f5tts_clone(ref_audio_path: str, ref_text: str, gen_text: str,
speed: float = 1.0, seed: Optional[int] = None) -> bytes:
"""Run F5-TTS voice cloning, return WAV bytes."""
......@@ -152,6 +166,26 @@ def _save_audio_response(audio_bytes: bytes, http_request: Request) -> dict:
return {"b64_wav": base64.b64encode(audio_bytes).decode()}
# ---------------------------------------------------------------------------
# Pydantic models
# ---------------------------------------------------------------------------
class VoiceExtractRequest(BaseModel):
name: str
description: str = ''
audio: Optional[str] = None # base64/URL audio file
video: Optional[str] = None # base64/URL video (audio track extracted)
transcript: Optional[str] = '' # optional; auto-transcribed if omitted
model_config = ConfigDict(extra="allow")
class VoicePatchRequest(BaseModel):
description: Optional[str] = None
transcript: Optional[str] = None
audio: Optional[str] = None # replace reference audio (base64)
model_config = ConfigDict(extra="allow")
# ---------------------------------------------------------------------------
# Voice profile management
# ---------------------------------------------------------------------------
......@@ -198,6 +232,122 @@ async def delete_voice(name: str):
return {"deleted": True, "name": name}
@router.patch("/v1/audio/voices/{name}")
async def patch_voice(name: str, req: VoicePatchRequest):
"""Update description, transcript, or reference audio of a saved voice profile."""
meta = _load_voice(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
vdir = _voice_path(name)
if req.description is not None:
meta['description'] = req.description
if req.transcript is not None:
meta['transcript'] = req.transcript
if req.audio:
audio_bytes, ext = _decode_audio(req.audio)
audio_file = os.path.join(vdir, f'ref{ext}')
# Remove old audio file(s)
for f in os.listdir(vdir):
if f.startswith('ref.') or f.startswith('ref'):
try:
os.unlink(os.path.join(vdir, f))
except Exception:
pass
with open(audio_file, 'wb') as f:
f.write(audio_bytes)
meta['audio_file'] = audio_file
meta['audio_ext'] = ext
with open(os.path.join(vdir, 'meta.json'), 'w') as f:
json.dump(meta, f)
return {"updated": True, "voice": meta}
@router.get("/v1/audio/voices/{name}")
async def get_voice(name: str):
"""Get a single voice profile metadata."""
meta = _load_voice(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
return {"voice": meta}
@router.post("/v1/audio/voices/extract")
async def extract_voice(req: VoiceExtractRequest):
"""
Extract a voice profile from a source audio or video file.
- audio: base64/URL audio file (wav, mp3, flac, …)
- video: base64/URL video file — audio track is extracted automatically
- transcript: optional; auto-transcribed with Whisper when omitted
"""
if not req.name.replace('-', '').replace('_', '').isalnum():
raise HTTPException(status_code=400, detail="Voice name must be alphanumeric (hyphens/underscores allowed)")
if not req.audio and not req.video:
raise HTTPException(status_code=400, detail="Provide audio or video")
temps = []
try:
audio_bytes: Optional[bytes] = None
audio_ext = '.wav'
if req.video:
# Extract audio track from video with ffmpeg
raw = _decode_b64_or_url(req.video)
in_tmp = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False)
in_tmp.write(raw)
in_tmp.close()
temps.append(in_tmp.name)
out_tmp = tempfile.NamedTemporaryFile(suffix='.wav', delete=False)
out_tmp.close()
temps.append(out_tmp.name)
r = subprocess.run(
['ffmpeg', '-y', '-i', in_tmp.name, '-vn', '-acodec', 'pcm_s16le',
'-ar', '22050', '-ac', '1', out_tmp.name],
capture_output=True, timeout=120,
)
if r.returncode != 0:
raise HTTPException(status_code=422,
detail=f"Audio extraction from video failed: {r.stderr.decode(errors='replace')[:200]}")
with open(out_tmp.name, 'rb') as f:
audio_bytes = f.read()
else:
audio_bytes, audio_ext = _decode_audio(req.audio)
# Auto-transcribe if transcript not provided
transcript = req.transcript or ''
if not transcript:
try:
import whisper
in_tmp2 = tempfile.NamedTemporaryFile(suffix=audio_ext, delete=False)
in_tmp2.write(audio_bytes)
in_tmp2.close()
temps.append(in_tmp2.name)
model = whisper.load_model('base')
result = model.transcribe(in_tmp2.name)
transcript = result.get('text', '').strip()
except Exception:
pass
# Validate audio
try:
import soundfile as sf
sf.info(io.BytesIO(audio_bytes))
except Exception as e:
raise HTTPException(status_code=400, detail=f"Invalid audio: {e}")
meta = _save_voice(req.name, audio_bytes, audio_ext, transcript, req.description)
return {"created": True, "voice": meta}
finally:
for t in temps:
try:
os.unlink(t)
except Exception:
pass
# ---------------------------------------------------------------------------
# Voice cloning TTS
# ---------------------------------------------------------------------------
......
......@@ -98,6 +98,14 @@ class WhisperConfig:
server_port: int = 8744
@dataclass
class ArchiveConfig:
"""Generation archive configuration."""
enabled: bool = True
directory: str = "" # empty = <config_dir>/archive; relative paths resolve from config_dir
retention: str = "never" # one of: 1h 1d 2d 1w 1m 3m 6m 1y never
@dataclass
class Config:
"""Main configuration class."""
......@@ -109,6 +117,7 @@ class Config:
vulkan: VulkanConfig = field(default_factory=VulkanConfig)
image: ImageConfig = field(default_factory=ImageConfig)
whisper: WhisperConfig = field(default_factory=WhisperConfig)
archive: ArchiveConfig = field(default_factory=ArchiveConfig)
system_prompt: Optional[str] = None
tools_closer_prompt: bool = False
grammar_guided: bool = False
......@@ -244,6 +253,7 @@ class ConfigManager:
vulkan=VulkanConfig(**config_data.get("vulkan", {})),
image=ImageConfig(**config_data.get("image", {})),
whisper=WhisperConfig(**config_data.get("whisper", {})),
archive=ArchiveConfig(**config_data.get("archive", {})),
system_prompt=config_data.get("system_prompt"),
tools_closer_prompt=config_data.get("tools_closer_prompt", False),
grammar_guided=config_data.get("grammar_guided", False),
......@@ -347,6 +357,11 @@ class ConfigManager:
"vae_tiling": self.config.image.vae_tiling,
"clip_on_cpu": self.config.image.clip_on_cpu
},
"archive": {
"enabled": self.config.archive.enabled,
"directory": self.config.archive.directory,
"retention": self.config.archive.retention,
},
"system_prompt": self.config.system_prompt,
"tools_closer_prompt": self.config.tools_closer_prompt,
"grammar_guided": self.config.grammar_guided,
......
......@@ -22,6 +22,7 @@ import os
from codai.cli import parse_args
from codai.config import ConfigManager
from codai.admin.routes import init_session_manager, set_config_manager
from codai.api.archive import archive_manager
def main():
......@@ -55,6 +56,18 @@ def main():
if config.models.gguf_cache_dir:
os.environ['CODERAI_CACHE_DIR'] = config.models.gguf_cache_dir
# Configure generation archive
_arc_dir = config.archive.directory
if not _arc_dir:
_arc_dir = os.path.join(config_dir, "archive")
elif not os.path.isabs(_arc_dir):
_arc_dir = os.path.join(config_dir, _arc_dir)
archive_manager.configure(
enabled=config.archive.enabled,
directory=_arc_dir,
retention=config.archive.retention,
)
# Initialize admin session manager and expose config to admin routes
from pathlib import Path
init_session_manager(Path(config_dir))
......@@ -364,6 +377,8 @@ def main():
multi_model_manager.model_backend_types[mid] = (
m.get("backend", "auto") if isinstance(m, dict) else "auto"
)
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Audio models
audio_models = models_config.get("audio_models", [])
......@@ -400,6 +415,8 @@ def main():
mid = _model_id(m)
if mid:
multi_model_manager.set_image_model(mid, config=_model_cfg(m, "image"))
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Vision models
vision_models = models_config.get("vision_models", [])
......@@ -407,6 +424,8 @@ def main():
mid = _model_id(m)
if mid:
multi_model_manager.set_vision_model(mid, config=_model_cfg(m, "vision"))
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# TTS models
tts_models = models_config.get("tts_models", [])
......@@ -414,6 +433,8 @@ def main():
mid = _model_id(m)
if mid:
multi_model_manager.set_tts_model(mid, config=_model_cfg(m, "tts") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Video generation models
video_models = models_config.get("video_models", [])
......@@ -421,6 +442,8 @@ def main():
mid = _model_id(m)
if mid:
multi_model_manager.set_video_model(mid, config=_model_cfg(m, "video") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Audio generation models (MusicGen, AudioLDM2, …)
audio_gen_models = models_config.get("audio_gen_models", [])
......@@ -428,6 +451,8 @@ def main():
mid = _model_id(m)
if mid:
multi_model_manager.set_audio_gen_model(mid, config=_model_cfg(m, "audio_gen") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Embedding models
embedding_models = models_config.get("embedding_models", [])
......@@ -435,6 +460,8 @@ def main():
mid = _model_id(m)
if mid:
multi_model_manager.set_embedding_model(mid, config=_model_cfg(m, "embedding") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Register aliases
aliases = models_config.get("aliases", {})
......@@ -623,10 +650,20 @@ def main():
if global_file_path:
set_fs_file_path(global_file_path)
# Set spatial (3D) module global args
from codai.api.spatial import set_global_args as set_spatial_global_args, set_global_file_path as set_spatial_file_path
set_spatial_global_args(global_args)
if global_file_path:
set_spatial_file_path(global_file_path)
# Set character profiles module global args
from codai.api.characters import set_global_args as set_chars_global_args
set_chars_global_args(global_args)
# Set environment profiles module global args
from codai.api.environments import set_global_args as set_envs_global_args
set_envs_global_args(global_args)
# Set embeddings module global args
from codai.api.embeddings import set_global_args as set_embed_global_args
set_embed_global_args(global_args)
......
......@@ -62,6 +62,12 @@ class ModelCapabilities:
lip_sync: bool = False # Wav2Lip, SadTalker
video_dubbing: bool = False # Translation + TTS + lip sync
# 3D generation and conversion
image_to_3d: bool = False # 2D image → stereo / anaglyph / depth / mesh
video_to_3d: bool = False # 2D video → stereo / anaglyph / depth video
model_3d_generation: bool = False # text / image → 3D model (GLB)
model_3d_to_image: bool = False # 3D model → rendered 2D image / video
def to_list(self) -> List[str]:
out = []
for name, val in self.__dataclass_fields__.items():
......@@ -85,6 +91,15 @@ def detect_model_capabilities(model_name: str) -> ModelCapabilities:
n = model_name.lower()
# ── 3D generation ────────────────────────────────────────────────────────
if any(x in n for x in ['triposr', 'tsr', 'shap-e', 'shape-e', 'point-e', 'pointe',
'zero123', 'wonder3d', 'instant3d', 'one-2-3-45',
'mvdiffusion', 'syncdreamer', 'dreamgaussian']):
caps.model_3d_generation = True
caps.image_to_3d = True
caps.model_3d_to_image = True
return caps
# ── Video generation ─────────────────────────────────────────────────────
if any(x in n for x in ['cogvideox', 'cogvideo', 'ltx-video', 'ltxvideo',
'hunyuan-video', 'mochi-1', 'dynamicrafter',
......
......@@ -1632,6 +1632,23 @@ class MultiModelManager:
'error': f"Model '{resolved_name}' is not available. Use one of: {', '.join(allowed_ids)}",
}
# Step 1c: Resolve short-name to canonical registered name (e.g. "Foo" → "org/Foo")
_type_registry = {
"image": self.image_models,
"audio": self.audio_models,
"tts": [self.tts_model] if self.tts_model else [],
"vision": self.vision_models,
"video": self.video_models,
"audio_gen": self.audio_gen_models,
"embedding": self.embedding_models,
}
_candidates = _type_registry.get(model_type, []) if model_type else []
if resolved_name not in _candidates:
for _m in _candidates:
if "/" in _m and _m.split("/")[-1] == resolved_name:
resolved_name = _m
break
# Step 2: Build the model key (prefixed with type)
if model_type and model_type != "text":
model_key = f"{model_type}:{resolved_name}"
......@@ -1641,6 +1658,19 @@ class MultiModelManager:
# Step 3: Check if already loaded in self.models
existing_model = self.models.get(model_key)
# Fuzzy fallback: if not found by exact key, check for org-prefix
# variations (e.g. "image:Model" vs "image:org/Model").
if existing_model is None:
type_prefix = f"{model_type}:" if model_type and model_type != "text" else ""
short = resolved_name.split("/")[-1]
for k, v in self.models.items():
k_name = k[len(type_prefix):] if type_prefix and k.startswith(type_prefix) else k
if k_name == resolved_name or k_name.split("/")[-1] == short:
model_key = k
resolved_name = k_name
existing_model = v
break
# =====================================================================
# PER-MODEL LOAD MODE: Check per-model config first.
# Per-model "load" = pre-loaded (treat as loadall for this model).
......
......@@ -31,6 +31,8 @@ class AudioGenerationRequest(BaseModel):
seed: Optional[int] = None
# Reference audio for melody conditioning (MusicGen Melody)
melody: Optional[str] = None # base64/URL
# Voice profile for singing/speech conditioning
voice_profile: Optional[str] = None # saved voice profile name
# Output
response_format: Optional[str] = "url" # url | b64_wav | b64_mp3
user: Optional[str] = None
......
......@@ -34,6 +34,12 @@ class ImageGenerationRequest(BaseModel):
seed: Optional[int] = None
user: Optional[str] = None
disable_safety_checker: Optional[bool] = False
negative_prompt: Optional[str] = None
# Character consistency
character_profiles: Optional[List[str]] = None # saved profile names
character_references: Optional[List[str]] = None # inline base64 images
character_strength: Optional[float] = 0.6 # IP-Adapter scale
model_config = ConfigDict(extra="allow")
......
......@@ -20,6 +20,18 @@ from typing import Dict, List, Optional
from pydantic import BaseModel, ConfigDict
class CharacterDialogLine(BaseModel):
"""One spoken line in a multi-character dialog sequence."""
character: Optional[str] = None # character profile name (used for lip-sync face)
voice: Optional[str] = None # voice profile name OR TTS voice id
text: str # dialog text to synthesise
start_time: Optional[float] = None # explicit offset in seconds (None = sequential)
lip_sync: Optional[bool] = True # apply lip sync for this line
lang: Optional[str] = None # language hint for TTS
speed: Optional[float] = 1.0
model_config = ConfigDict(extra="allow")
class VideoGenerationRequest(BaseModel):
model: str
prompt: str = ""
......@@ -78,6 +90,9 @@ class VideoGenerationRequest(BaseModel):
lip_sync: Optional[bool] = False # warp mouth to match audio
lip_sync_method: Optional[str] = "wav2lip" # wav2lip | sadtalker
# ── Multi-character dialog ────────────────────────────────────────────
dialogs: Optional[List[CharacterDialogLine]] = None
# ── Subtitles ────────────────────────────────────────────────────────
generate_subtitles: Optional[bool] = False
burn_subtitles: Optional[bool] = False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment