Add character/environment profiles, 3D conversion, archive, multi-character dialog

- Character profiles: extract from images/video or generate from text prompt;
  up to 6 profiles per image/video generation via IP-Adapter conditioning
- Environment profiles: same multi-slot system for scene/background conditioning
- Multi-character dialog for video: per-character TTS synthesis with voice profile
  support, sequential or manual timing, ffmpeg amix, Wav2Lip/SadTalker lip sync
- 2D3D conversion: image/video to stereo/anaglyph/depth/mesh and back;
  text/image to 3D model (GLB); turntable video rendering
- Generation archive: auto-save all outputs with configurable retention and
  hourly cleanup; browse/delete from web UI and API
- Voice profile enhancements: extract from video, PATCH update, GET by name;
  TTS endpoint now accepts voice_profile for F5-TTS cloning
- Image generation progress tracking via GET /v1/images/progress
- Web Studio: Profiles tab (Characters, Environments, Voices) with full managers;
  multi-slot character/environment selectors in all panels; character dialog
  section in all video panels; archive settings in Settings page
- Fix: Profiles tab sub-tab switching now correctly updates the content panel
Co-Authored-By: 's avatarClaude Sonnet 4.6 <noreply@anthropic.com>
parent 19e28d3d
This diff is collapsed.
...@@ -1556,6 +1556,11 @@ async def api_get_settings(username: str = Depends(require_admin)): ...@@ -1556,6 +1556,11 @@ async def api_get_settings(username: str = Depends(require_admin)):
"device_id": c.vulkan.device_id, "device_id": c.vulkan.device_id,
"single_gpu": c.vulkan.single_gpu, "single_gpu": c.vulkan.single_gpu,
}, },
"archive": {
"enabled": c.archive.enabled,
"directory": c.archive.directory,
"retention": c.archive.retention,
},
"system_prompt": c.system_prompt, "system_prompt": c.system_prompt,
"tools_closer_prompt": c.tools_closer_prompt, "tools_closer_prompt": c.tools_closer_prompt,
"grammar_guided": c.grammar_guided, "grammar_guided": c.grammar_guided,
...@@ -1632,8 +1637,95 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad ...@@ -1632,8 +1637,95 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
if "parser" in data: if "parser" in data:
c.parser = data["parser"] c.parser = data["parser"]
if "archive" in data:
import os as _os
from codai.api.archive import archive_manager, RETENTION_OPTIONS
arc = data["archive"]
c.archive.enabled = bool(arc.get("enabled", c.archive.enabled))
raw_dir = (arc.get("directory") or "").strip()
c.archive.directory = raw_dir # store as-is (empty = default)
ret = arc.get("retention", c.archive.retention)
c.archive.retention = ret if ret in RETENTION_OPTIONS else "never"
# Resolve for live reconfiguration
cfg_dir = str(config_manager.config_dir)
resolved = raw_dir if raw_dir and _os.path.isabs(raw_dir) else _os.path.join(cfg_dir, raw_dir or "archive")
archive_manager.configure(c.archive.enabled, resolved, c.archive.retention)
config_manager.save_config() config_manager.save_config()
return {"success": True} return {"success": True}
# =============================================================================
# Archive management
# =============================================================================
@router.get("/admin/archive", response_class=HTMLResponse)
async def archive_page(request: Request, username: str = Depends(require_admin)):
return templates.TemplateResponse(request, "archive.html", {"username": username, "is_admin": True})
@router.get("/admin/api/archive")
async def api_archive_list(
limit: int = 50,
offset: int = 0,
username: str = Depends(require_admin),
):
from codai.api.archive import archive_manager
entries = archive_manager.list_entries(limit=limit, offset=offset)
total = archive_manager.count_entries()
return {"entries": entries, "total": total}
@router.get("/admin/api/archive/{gen_id}")
async def api_archive_get(gen_id: str, username: str = Depends(require_admin)):
from codai.api.archive import archive_manager
entry = archive_manager.get_entry(gen_id)
if entry is None:
raise HTTPException(status_code=404, detail="Entry not found")
return entry
@router.delete("/admin/api/archive/{gen_id}")
async def api_archive_delete(gen_id: str, username: str = Depends(require_admin)):
from codai.api.archive import archive_manager
if not archive_manager.delete_entry(gen_id):
raise HTTPException(status_code=404, detail="Entry not found")
return {"success": True}
@router.get("/admin/api/archive/{gen_id}/files/{filename}")
async def api_archive_file(
gen_id: str,
filename: str,
username: str = Depends(require_auth),
):
from fastapi.responses import FileResponse
from codai.api.archive import archive_manager
path = archive_manager.get_file_path(gen_id, filename)
if path is None:
raise HTTPException(status_code=404, detail="File not found")
import mimetypes
media_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
return FileResponse(path, media_type=media_type)
@router.get("/admin/api/archive-settings")
async def api_archive_settings_get(username: str = Depends(require_admin)):
if config_manager is None or config_manager.config is None:
raise HTTPException(status_code=503, detail="Config not ready")
import os as _os
c = config_manager.config.archive
from codai.api.archive import RETENTION_OPTIONS
default_dir = _os.path.join(str(config_manager.config_dir), "archive")
return {
"enabled": c.enabled,
"directory": c.directory,
"default_directory": default_dir,
"retention": c.retention,
"retention_options": RETENTION_OPTIONS,
}
# --- HuggingFace model search proxy --- # --- HuggingFace model search proxy ---
import re as _re import re as _re
...@@ -1794,6 +1886,110 @@ async def api_hf_model_files(model_id: str, username: str = Depends(require_admi ...@@ -1794,6 +1886,110 @@ async def api_hf_model_files(model_id: str, username: str = Depends(require_admi
return files return files
# =============================================================================
# Character profile management proxy (admin UI)
# =============================================================================
@router.get("/admin/api/characters")
async def api_list_characters(username: str = Depends(require_auth)):
from codai.api.characters import _list_characters
return {"characters": _list_characters()}
@router.get("/admin/api/characters/{name}")
async def api_get_character(name: str, username: str = Depends(require_auth)):
from codai.api.characters import _load_character_meta, _load_character_images
meta = _load_character_meta(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Character '{name}' not found")
images = _load_character_images(name)
return {
"name": meta['name'],
"description": meta.get('description', ''),
"image_count": meta['image_count'],
"created_at": meta['created_at'],
"images": [img.model_dump() for img in images],
}
@router.delete("/admin/api/characters/{name}")
async def api_delete_character(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
from codai.api.characters import _char_dir
cdir = _char_dir(name)
if not _os.path.isdir(cdir):
raise HTTPException(status_code=404, detail=f"Character '{name}' not found")
shutil.rmtree(cdir)
return {"ok": True, "name": name}
# =============================================================================
# Environment profile management proxy (admin UI)
# =============================================================================
@router.get("/admin/api/environments")
async def api_list_environments(username: str = Depends(require_auth)):
from codai.api.environments import _list_environments
return {"environments": _list_environments()}
@router.get("/admin/api/environments/{name}")
async def api_get_environment(name: str, username: str = Depends(require_auth)):
from codai.api.environments import _load_environment_meta, _load_environment_images
meta = _load_environment_meta(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Environment '{name}' not found")
images = _load_environment_images(name)
return {
"name": meta['name'],
"description": meta.get('description', ''),
"image_count": meta['image_count'],
"created_at": meta['created_at'],
"images": [img.model_dump() for img in images],
}
@router.delete("/admin/api/environments/{name}")
async def api_delete_environment(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
from codai.api.environments import _env_dir
edir = _env_dir(name)
if not _os.path.isdir(edir):
raise HTTPException(status_code=404, detail=f"Environment '{name}' not found")
shutil.rmtree(edir)
return {"ok": True, "name": name}
# =============================================================================
# Voice profile management proxy (admin UI)
# =============================================================================
@router.get("/admin/api/voices")
async def api_list_voices(username: str = Depends(require_auth)):
from codai.api.voice_clone import _list_voices
return {"voices": _list_voices()}
@router.get("/admin/api/voices/{name}")
async def api_get_voice(name: str, username: str = Depends(require_auth)):
from codai.api.voice_clone import _load_voice
meta = _load_voice(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
return {"voice": meta}
@router.delete("/admin/api/voices/{name}")
async def api_delete_voice(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
from codai.api.voice_clone import _voice_path
vdir = _voice_path(name)
if not _os.path.exists(vdir):
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
shutil.rmtree(vdir)
return {"deleted": True, "name": name}
@router.get("/admin/api/hf-model-info") @router.get("/admin/api/hf-model-info")
async def api_hf_model_info(model_id: str, username: str = Depends(require_admin)): async def api_hf_model_info(model_id: str, username: str = Depends(require_admin)):
"""Full metadata for a single HuggingFace model repo.""" """Full metadata for a single HuggingFace model repo."""
......
This diff is collapsed.
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
<a href="/admin/models" class="nav-link {% if '/models' in request.url.path %}active{% endif %}">Models</a> <a href="/admin/models" class="nav-link {% if '/models' in request.url.path %}active{% endif %}">Models</a>
<a href="/admin/tokens" class="nav-link {% if '/tokens' in request.url.path %}active{% endif %}">Tokens</a> <a href="/admin/tokens" class="nav-link {% if '/tokens' in request.url.path %}active{% endif %}">Tokens</a>
<a href="/admin/users" class="nav-link {% if '/users' in request.url.path %}active{% endif %}">Users</a> <a href="/admin/users" class="nav-link {% if '/users' in request.url.path %}active{% endif %}">Users</a>
<a href="/admin/archive" class="nav-link {% if '/archive' in request.url.path %}active{% endif %}">Archive</a>
<a href="/admin/settings" class="nav-link {% if '/settings' in request.url.path %}active{% endif %}">Settings</a> <a href="/admin/settings" class="nav-link {% if '/settings' in request.url.path %}active{% endif %}">Settings</a>
{% endif %} {% endif %}
<button class="nav-link nav-donate-btn" onclick="document.getElementById('donateModal').classList.add('show')">Donate &#9829;</button> <button class="nav-link nav-donate-btn" onclick="document.getElementById('donateModal').classList.add('show')">Donate &#9829;</button>
......
This source diff could not be displayed because it is too large. You can view the blob instead.
...@@ -69,6 +69,38 @@ ...@@ -69,6 +69,38 @@
<span class="form-hint">Models will inherit this as default when configured</span> <span class="form-hint">Models will inherit this as default when configured</span>
</div> </div>
</div> </div>
<!-- Archive -->
<div class="card mb-0" style="margin-top:1rem">
<div class="card-title">Generation Archive</div>
<div class="form-row">
<label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
<input type="checkbox" id="s-arc-enabled">
<span style="font-size:13px;font-weight:500">Enable generation archive</span>
</label>
<span class="form-hint">When enabled, every generation (image, video, audio, text) is logged with its prompt, model, and output files.</span>
</div>
<div class="form-row">
<label class="form-label">Archive directory <span class="muted" id="s-arc-dir-hint"></span></label>
<input type="text" id="s-arc-dir" class="form-input" placeholder="(default)">
<span class="form-hint">Absolute path or relative to the config directory. Leave blank to use the default.</span>
</div>
<div class="form-row" style="margin:0">
<label class="form-label">Retention period</label>
<select id="s-arc-retention" class="form-input" style="max-width:200px">
<option value="1h">1 hour</option>
<option value="1d">1 day</option>
<option value="2d">2 days</option>
<option value="1w">1 week</option>
<option value="1m">1 month</option>
<option value="3m">3 months</option>
<option value="6m">6 months</option>
<option value="1y">1 year</option>
<option value="never">Never (keep forever)</option>
</select>
<span class="form-hint">Archived entries older than this are automatically deleted. Takes effect immediately on save.</span>
</div>
</div>
{% endblock %} {% endblock %}
{% block scripts %} {% block scripts %}
...@@ -99,6 +131,18 @@ async function loadSettings(){ ...@@ -99,6 +131,18 @@ async function loadSettings(){
document.getElementById('s-gguf-cache').value = d.models?.gguf_cache_dir ?? ''; document.getElementById('s-gguf-cache').value = d.models?.gguf_cache_dir ?? '';
document.getElementById('s-offload-dir').value = d.offload?.directory ?? './offload'; document.getElementById('s-offload-dir').value = d.offload?.directory ?? './offload';
toggleHttps(); toggleHttps();
// Archive
const arc = d.archive || {};
document.getElementById('s-arc-enabled').checked = !!arc.enabled;
document.getElementById('s-arc-dir').value = arc.directory ?? '';
document.getElementById('s-arc-retention').value = arc.retention ?? 'never';
// Show effective default dir as placeholder hint
try {
const as = await fetch('/admin/api/archive-settings').then(r=>r.json());
const hint = document.getElementById('s-arc-dir-hint');
if (hint && as.default_directory) hint.textContent = `(default: ${as.default_directory})`;
document.getElementById('s-arc-dir').placeholder = as.default_directory || '(default)';
} catch(_){}
}catch(e){ showAlert('error','Failed to load settings: '+e.message); } }catch(e){ showAlert('error','Failed to load settings: '+e.message); }
} }
...@@ -119,14 +163,19 @@ async function saveSettings(){ ...@@ -119,14 +163,19 @@ async function saveSettings(){
}, },
offload:{ offload:{
directory: document.getElementById('s-offload-dir').value.trim() || './offload', directory: document.getElementById('s-offload-dir').value.trim() || './offload',
} },
archive:{
enabled: document.getElementById('s-arc-enabled').checked,
directory: document.getElementById('s-arc-dir').value.trim(),
retention: document.getElementById('s-arc-retention').value,
},
}; };
try{ try{
const r = await fetch('/admin/api/settings',{ const r = await fetch('/admin/api/settings',{
method:'POST', headers:{'Content-Type':'application/json'}, method:'POST', headers:{'Content-Type':'application/json'},
body: JSON.stringify(data) body: JSON.stringify(data)
}); });
if(r.ok) showAlert('info','Settings saved. Restart CoderAI to apply.'); if(r.ok) showAlert('info','Settings saved. Archive changes take effect immediately; restart CoderAI for other changes.');
else{ const e=await r.json(); showAlert('error', e.detail||'Save failed'); } else{ const e=await r.json(); showAlert('error', e.detail||'Save failed'); }
}catch(e){ showAlert('error','Error: '+e.message); } }catch(e){ showAlert('error','Error: '+e.message); }
} }
......
...@@ -66,8 +66,33 @@ def set_global_file_path_wrapper(path: str): ...@@ -66,8 +66,33 @@ def set_global_file_path_wrapper(path: str):
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
"""Lifespan context manager for startup/shutdown.""" """Lifespan context manager for startup/shutdown."""
# Startup import asyncio as _asyncio
async def _archive_cleanup_loop():
from codai.api.archive import archive_manager
while True:
await _asyncio.sleep(3600)
try:
archive_manager.run_cleanup()
except Exception:
pass
cleanup_task = _asyncio.create_task(_archive_cleanup_loop())
# Run initial cleanup on startup
try:
from codai.api.archive import archive_manager
archive_manager.run_cleanup()
except Exception:
pass
yield yield
cleanup_task.cancel()
try:
await cleanup_task
except _asyncio.CancelledError:
pass
# Shutdown # Shutdown
multi_model_manager.cleanup() multi_model_manager.cleanup()
model_manager.cleanup() model_manager.cleanup()
...@@ -101,6 +126,8 @@ from codai.api.voice_clone import router as voice_clone_router ...@@ -101,6 +126,8 @@ from codai.api.voice_clone import router as voice_clone_router
from codai.api.voice_convert import router as voice_convert_router from codai.api.voice_convert import router as voice_convert_router
from codai.api.faceswap import router as faceswap_router from codai.api.faceswap import router as faceswap_router
from codai.api.characters import router as characters_router from codai.api.characters import router as characters_router
from codai.api.spatial import router as spatial_router
from codai.api.environments import router as environments_router
from codai.admin.routes import router as admin_router from codai.admin.routes import router as admin_router
# Import and add middleware # Import and add middleware
...@@ -133,6 +160,8 @@ app.include_router(voice_clone_router) ...@@ -133,6 +160,8 @@ app.include_router(voice_clone_router)
app.include_router(voice_convert_router) app.include_router(voice_convert_router)
app.include_router(faceswap_router) app.include_router(faceswap_router)
app.include_router(characters_router) app.include_router(characters_router)
app.include_router(environments_router)
app.include_router(spatial_router)
app.include_router(admin_router) app.include_router(admin_router)
...@@ -166,4 +195,59 @@ async def get_file(filename: str): ...@@ -166,4 +195,59 @@ async def get_file(filename: str):
raise HTTPException(status_code=403, detail="Access denied") raise HTTPException(status_code=403, detail="Access denied")
if not os.path.isfile(candidate): if not os.path.isfile(candidate):
raise HTTPException(status_code=404, detail="File not found") raise HTTPException(status_code=404, detail="File not found")
return FileResponse(candidate) return FileResponse(candidate)
\ No newline at end of file
_IMAGE_EXTS = {'.png', '.jpg', '.jpeg', '.webp', '.gif'}
_VIDEO_EXTS = {'.mp4', '.webm', '.avi', '.mov'}
_AUDIO_EXTS = {'.wav', '.mp3', '.ogg', '.flac', '.aac', '.m4a'}
@app.get("/v1/archive")
async def list_archive():
"""List all generated files in the output directory."""
if not global_file_path or not os.path.isdir(global_file_path):
return {"files": []}
files = []
try:
names = os.listdir(global_file_path)
except OSError:
return {"files": []}
for fname in names:
fpath = os.path.join(global_file_path, fname)
if not os.path.isfile(fpath):
continue
ext = os.path.splitext(fname)[1].lower()
if ext in _IMAGE_EXTS:
ftype = 'image'
elif ext in _VIDEO_EXTS:
ftype = 'video'
elif ext in _AUDIO_EXTS:
ftype = 'audio'
else:
continue
stat = os.stat(fpath)
files.append({
'filename': fname,
'type': ftype,
'size': stat.st_size,
'created': stat.st_mtime,
'url': f'/v1/files/{fname}',
})
files.sort(key=lambda f: f['created'], reverse=True)
return {"files": files}
@app.delete("/v1/archive/{filename}")
async def delete_archive_file(filename: str):
"""Delete a generated file from the output directory."""
if not global_file_path:
raise HTTPException(status_code=404, detail="File not found")
safe_base = os.path.realpath(global_file_path)
candidate = os.path.realpath(os.path.join(global_file_path, filename))
if not (candidate == safe_base or candidate.startswith(safe_base + os.sep)):
raise HTTPException(status_code=403, detail="Access denied")
if not os.path.isfile(candidate):
raise HTTPException(status_code=404, detail="File not found")
os.remove(candidate)
return {"deleted": filename}
\ No newline at end of file
# CoderAI - OpenAI-compatible API server
# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Generation archive — persists all generated artifacts with metadata on disk."""
import io
import json
import logging
import os
import shutil
import threading
import time
import uuid
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
_log = logging.getLogger(__name__)
RETENTION_OPTIONS = ["1h", "1d", "2d", "1w", "1m", "3m", "6m", "1y", "never"]
_RETENTION_SECONDS: Dict[str, Optional[int]] = {
"1h": 3_600,
"1d": 86_400,
"2d": 172_800,
"1w": 604_800,
"1m": 2_592_000, # 30 days
"3m": 7_776_000, # 90 days
"6m": 15_552_000, # 180 days
"1y": 31_536_000, # 365 days
"never": None,
}
def _safe_id(s: str) -> bool:
return bool(s) and "/" not in s and ".." not in s and "\\" not in s
def _safe_filename(s: str) -> bool:
return bool(s) and "/" not in s and ".." not in s and "\\" not in s
class ArchiveManager:
"""Manages the on-disk generation archive."""
def __init__(self):
self.enabled: bool = True
self.directory: str = os.path.abspath("./archive")
self.retention: str = "never"
self._lock = threading.Lock()
def configure(self, enabled: bool, directory: str, retention: str):
self.enabled = enabled
self.directory = os.path.abspath(directory or "./archive")
self.retention = retention if retention in _RETENTION_SECONDS else "never"
def _gen_dir(self, gen_id: str) -> str:
return os.path.join(self.directory, gen_id)
# ------------------------------------------------------------------
# Save
# ------------------------------------------------------------------
def save_generation(
self,
gen_type: str,
endpoint: str,
model: Optional[str],
prompt: Optional[str],
params: Dict[str, Any],
artifacts: List[Tuple[bytes, str]], # [(raw_bytes, file_ext), ...]
) -> Optional[str]:
"""Save a generation to archive. Returns gen_id on success, None otherwise."""
if not self.enabled:
return None
try:
gen_id = f"{int(time.time())}_{uuid.uuid4().hex[:8]}"
gen_dir = self._gen_dir(gen_id)
os.makedirs(gen_dir, exist_ok=True)
saved_files: List[str] = []
preview_file: Optional[str] = None
for i, (data, ext) in enumerate(artifacts):
if not data:
continue
fname = f"output_{i}.{ext}"
with open(os.path.join(gen_dir, fname), "wb") as fh:
fh.write(data)
saved_files.append(fname)
if i == 0 and preview_file is None:
try:
preview_file = self._make_preview(data, ext, gen_dir)
except Exception as e:
_log.debug("Archive preview failed: %s", e)
meta: Dict[str, Any] = {
"id": gen_id,
"type": gen_type,
"endpoint": endpoint,
"model": model or "",
"prompt": prompt or "",
"parameters": params,
"files": saved_files,
"preview": preview_file or (saved_files[0] if saved_files else None),
"created_at": int(time.time()),
"created_at_iso": datetime.utcnow().isoformat() + "Z",
}
with open(os.path.join(gen_dir, "metadata.json"), "w") as fh:
json.dump(meta, fh, indent=2)
return gen_id
except Exception as e:
_log.warning("Archive save failed: %s", e)
return None
# ------------------------------------------------------------------
# Query
# ------------------------------------------------------------------
def list_entries(self, limit: int = 50, offset: int = 0) -> List[Dict]:
"""Return metadata dicts, newest first."""
if not os.path.isdir(self.directory):
return []
entries: List[Dict] = []
try:
for name in os.listdir(self.directory):
meta_path = os.path.join(self.directory, name, "metadata.json")
if not os.path.isfile(meta_path):
continue
try:
with open(meta_path) as fh:
entries.append(json.load(fh))
except Exception:
pass
except Exception:
pass
entries.sort(key=lambda x: x.get("created_at", 0), reverse=True)
return entries[offset: offset + limit]
def count_entries(self) -> int:
if not os.path.isdir(self.directory):
return 0
count = 0
try:
for name in os.listdir(self.directory):
if os.path.isfile(os.path.join(self.directory, name, "metadata.json")):
count += 1
except Exception:
pass
return count
def get_entry(self, gen_id: str) -> Optional[Dict]:
if not _safe_id(gen_id):
return None
meta_path = os.path.join(self._gen_dir(gen_id), "metadata.json")
if not os.path.isfile(meta_path):
return None
try:
with open(meta_path) as fh:
return json.load(fh)
except Exception:
return None
def delete_entry(self, gen_id: str) -> bool:
if not _safe_id(gen_id):
return False
gen_dir = self._gen_dir(gen_id)
if not os.path.isdir(gen_dir):
return False
try:
shutil.rmtree(gen_dir)
return True
except Exception:
return False
def get_file_path(self, gen_id: str, filename: str) -> Optional[str]:
if not _safe_id(gen_id) or not _safe_filename(filename):
return None
path = os.path.join(self._gen_dir(gen_id), filename)
return path if os.path.isfile(path) else None
# ------------------------------------------------------------------
# Cleanup
# ------------------------------------------------------------------
def run_cleanup(self):
"""Remove entries older than the configured retention window."""
max_age = _RETENTION_SECONDS.get(self.retention)
if max_age is None:
return
if not os.path.isdir(self.directory):
return
cutoff = time.time() - max_age
deleted = 0
try:
for name in list(os.listdir(self.directory)):
meta_path = os.path.join(self.directory, name, "metadata.json")
try:
with open(meta_path) as fh:
meta = json.load(fh)
if meta.get("created_at", time.time()) < cutoff:
shutil.rmtree(os.path.join(self.directory, name), ignore_errors=True)
deleted += 1
except Exception:
pass
except Exception:
pass
if deleted:
_log.info("Archive cleanup: removed %d entry/entries", deleted)
# ------------------------------------------------------------------
# Preview generation
# ------------------------------------------------------------------
def _make_preview(self, data: bytes, ext: str, gen_dir: str) -> Optional[str]:
ext = ext.lower()
if ext in ("png", "jpg", "jpeg", "webp", "bmp", "gif"):
from PIL import Image
img = Image.open(io.BytesIO(data)).convert("RGB")
img.thumbnail((512, 512))
dst = os.path.join(gen_dir, "preview.webp")
img.save(dst, "WEBP", quality=75)
return "preview.webp"
if ext in ("mp4", "webm", "avi", "mov"):
import subprocess
import tempfile
with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as tmp:
tmp.write(data)
tmp_path = tmp.name
frame_path = os.path.join(gen_dir, "_frame_tmp.jpg")
try:
r = subprocess.run(
["ffmpeg", "-y", "-i", tmp_path, "-vframes", "1", "-q:v", "3", frame_path],
capture_output=True, timeout=15,
)
if r.returncode == 0 and os.path.exists(frame_path):
from PIL import Image
img = Image.open(frame_path).convert("RGB")
img.thumbnail((512, 512))
dst = os.path.join(gen_dir, "preview.webp")
img.save(dst, "WEBP", quality=75)
return "preview.webp"
except Exception:
pass
finally:
try:
os.unlink(tmp_path)
except Exception:
pass
try:
os.unlink(frame_path)
except Exception:
pass
# Audio or unrecognised — no visual preview
return None
# Module-level singleton
archive_manager = ArchiveManager()
...@@ -199,4 +199,24 @@ async def audio_generate(request: AudioGenerationRequest, http_request: Request ...@@ -199,4 +199,24 @@ async def audio_generate(request: AudioGenerationRequest, http_request: Request
raise HTTPException(status_code=500, detail=f"Audio generation failed: {e}") raise HTTPException(status_code=500, detail=f"Audio generation failed: {e}")
result = _save_audio_response(audio_bytes, ext, http_request) result = _save_audio_response(audio_bytes, ext, http_request)
try:
from codai.api.archive import archive_manager
asyncio.get_event_loop().create_task(asyncio.to_thread(
archive_manager.save_generation,
"audio", "/v1/audio/generate",
model_name,
request.prompt,
{
"duration": request.duration,
"top_k": request.top_k,
"top_p": request.top_p,
"temperature": request.temperature,
"cfg_coef": request.cfg_coef,
},
[(audio_bytes, ext)],
))
except Exception:
pass
return AudioGenerationResponse(created=int(time.time()), data=[result]) return AudioGenerationResponse(created=int(time.time()), data=[result])
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -34,9 +34,11 @@ from starlette.middleware.base import BaseHTTPMiddleware ...@@ -34,9 +34,11 @@ from starlette.middleware.base import BaseHTTPMiddleware
class BearerAuthMiddleware(BaseHTTPMiddleware): class BearerAuthMiddleware(BaseHTTPMiddleware):
"""Reject /v1/ API requests that lack a valid Bearer token or active web session.""" """Reject /v1/ API requests that lack a valid Bearer token or active web session."""
_EXEMPT_PATHS = {"/v1/images/progress"}
async def dispatch(self, request: Request, call_next): async def dispatch(self, request: Request, call_next):
path = request.url.path path = request.url.path
if not path.startswith("/v1/"): if not path.startswith("/v1/") or path in self._EXEMPT_PATHS:
return await call_next(request) return await call_next(request)
from codai.admin import routes as _admin_routes from codai.admin import routes as _admin_routes
...@@ -112,14 +114,20 @@ class RateLimitMiddleware(BaseHTTPMiddleware): ...@@ -112,14 +114,20 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
return prefix return prefix
return "" return ""
# Lightweight polling endpoints that must never be rate-limited
_EXEMPT_PATHS = {"/v1/images/progress"}
async def dispatch(self, request: Request, call_next): async def dispatch(self, request: Request, call_next):
if not RATE_LIMITING_ENABLED: if not RATE_LIMITING_ENABLED:
return await call_next(request) return await call_next(request)
path = request.url.path path = request.url.path
# Queue-size enforcement for authenticated API requests if path in self._EXEMPT_PATHS:
if any(path.startswith(p) for p in _QUEUED_PREFIXES): return await call_next(request)
# Queue-size enforcement for authenticated API requests (not for status polls)
if path not in self._EXEMPT_PATHS and any(path.startswith(p) for p in _QUEUED_PREFIXES):
from codai.queue.manager import queue_manager from codai.queue.manager import queue_manager
if await queue_manager.is_full(): if await queue_manager.is_full():
return JSONResponse( return JSONResponse(
......
This diff is collapsed.
...@@ -18,10 +18,12 @@ ...@@ -18,10 +18,12 @@
Text-to-speech endpoints for the codai API. Text-to-speech endpoints for the codai API.
""" """
import asyncio
import base64 import base64
import os import os
from typing import Optional
from fastapi import APIRouter, HTTPException from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, ConfigDict from pydantic import BaseModel, ConfigDict
# Import from codai modules # Import from codai modules
...@@ -51,7 +53,8 @@ class TTSRequest(BaseModel): ...@@ -51,7 +53,8 @@ class TTSRequest(BaseModel):
voice: str = "af_sarah" voice: str = "af_sarah"
response_format: str = "mp3" response_format: str = "mp3"
speed: float = 1.0 speed: float = 1.0
voice_profile: Optional[str] = None # saved voice profile name (uses F5-TTS cloning)
model_config = ConfigDict(extra="allow") model_config = ConfigDict(extra="allow")
...@@ -62,13 +65,39 @@ class TTSResponse(BaseModel): ...@@ -62,13 +65,39 @@ class TTSResponse(BaseModel):
@router.post("/v1/audio/speech") @router.post("/v1/audio/speech")
async def create_speech(request: TTSRequest): async def create_speech(request: TTSRequest, http_request: Request = None):
""" """
Text-to-speech endpoint (OpenAI-compatible). Text-to-speech endpoint (OpenAI-compatible).
Supports: Supports:
- Kokoro TTS models (when --tts-model is specified) - Kokoro TTS models (when --tts-model is specified)
""" """
# If a voice profile is requested, delegate to voice cloning (F5-TTS)
if request.voice_profile:
from codai.api.voice_clone import _load_voice, _f5tts_clone
meta = _load_voice(request.voice_profile)
if not meta:
raise HTTPException(status_code=404,
detail=f"Voice profile '{request.voice_profile}' not found")
ref_audio_path = meta['audio_file']
ref_text = meta.get('transcript', '')
if not ref_text:
raise HTTPException(status_code=400,
detail="Voice profile has no transcript; update it with PATCH /v1/audio/voices/{name}")
try:
audio_bytes = await asyncio.get_event_loop().run_in_executor(
None, _f5tts_clone,
ref_audio_path, ref_text, request.input,
request.speed or 1.0, None,
)
except ImportError:
raise HTTPException(status_code=501,
detail="f5-tts not installed. Run: pip install f5-tts")
except Exception as e:
raise HTTPException(status_code=500, detail=f"Voice cloning failed: {e}")
audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
return {"audio": audio_base64}
# Use the manager to resolve the model and manage VRAM # Use the manager to resolve the model and manage VRAM
model_info = multi_model_manager.request_model( model_info = multi_model_manager.request_model(
requested_model=request.model, requested_model=request.model,
...@@ -119,10 +148,23 @@ async def create_speech(request: TTSRequest): ...@@ -119,10 +148,23 @@ async def create_speech(request: TTSRequest):
speed = request.speed or 1.0 speed = request.speed or 1.0
audio_bytes = kokoro_model.generate(request.input, voice=voice, speed=speed) audio_bytes = kokoro_model.generate(request.input, voice=voice, speed=speed)
try:
from codai.api.archive import archive_manager
asyncio.get_event_loop().create_task(asyncio.to_thread(
archive_manager.save_generation,
"tts", "/v1/audio/speech",
model_name,
request.input,
{"voice": voice, "speed": speed, "response_format": request.response_format},
[(audio_bytes, request.response_format or "mp3")],
))
except Exception:
pass
# Convert to base64 # Convert to base64
audio_base64 = base64.b64encode(audio_bytes).decode('utf-8') audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
return { return {
"audio": audio_base64 "audio": audio_base64
} }
......
This diff is collapsed.
""" """
Voice cloning endpoints. Voice cloning endpoints.
POST /v1/audio/clone — synthesize speech in a cloned voice POST /v1/audio/clone — synthesize speech in a cloned voice
GET /v1/audio/voices — list saved voice profiles GET /v1/audio/voices — list saved voice profiles
POST /v1/audio/voices — save a named voice profile (ref audio + transcript) POST /v1/audio/voices — save a named voice profile (ref audio + transcript)
DELETE /v1/audio/voices/{name} — delete a voice profile POST /v1/audio/voices/extract — extract voice profile from audio or video file
PATCH /v1/audio/voices/{name} — update description / replace reference audio
DELETE /v1/audio/voices/{name} — delete a voice profile
""" """
import asyncio import asyncio
...@@ -12,6 +14,7 @@ import base64 ...@@ -12,6 +14,7 @@ import base64
import io import io
import json import json
import os import os
import subprocess
import tempfile import tempfile
import time import time
from typing import Optional from typing import Optional
...@@ -103,6 +106,17 @@ def _decode_audio(data: str) -> tuple[bytes, str]: ...@@ -103,6 +106,17 @@ def _decode_audio(data: str) -> tuple[bytes, str]:
return base64.b64decode(data), '.wav' return base64.b64decode(data), '.wav'
def _decode_b64_or_url(data: str) -> bytes:
if data.startswith('data:'):
_, b64 = data.split(',', 1)
return base64.b64decode(b64)
if data.startswith('http://') or data.startswith('https://'):
import urllib.request
with urllib.request.urlopen(data, timeout=60) as r:
return r.read()
return base64.b64decode(data)
def _f5tts_clone(ref_audio_path: str, ref_text: str, gen_text: str, def _f5tts_clone(ref_audio_path: str, ref_text: str, gen_text: str,
speed: float = 1.0, seed: Optional[int] = None) -> bytes: speed: float = 1.0, seed: Optional[int] = None) -> bytes:
"""Run F5-TTS voice cloning, return WAV bytes.""" """Run F5-TTS voice cloning, return WAV bytes."""
...@@ -152,6 +166,26 @@ def _save_audio_response(audio_bytes: bytes, http_request: Request) -> dict: ...@@ -152,6 +166,26 @@ def _save_audio_response(audio_bytes: bytes, http_request: Request) -> dict:
return {"b64_wav": base64.b64encode(audio_bytes).decode()} return {"b64_wav": base64.b64encode(audio_bytes).decode()}
# ---------------------------------------------------------------------------
# Pydantic models
# ---------------------------------------------------------------------------
class VoiceExtractRequest(BaseModel):
name: str
description: str = ''
audio: Optional[str] = None # base64/URL audio file
video: Optional[str] = None # base64/URL video (audio track extracted)
transcript: Optional[str] = '' # optional; auto-transcribed if omitted
model_config = ConfigDict(extra="allow")
class VoicePatchRequest(BaseModel):
description: Optional[str] = None
transcript: Optional[str] = None
audio: Optional[str] = None # replace reference audio (base64)
model_config = ConfigDict(extra="allow")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Voice profile management # Voice profile management
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
...@@ -198,6 +232,122 @@ async def delete_voice(name: str): ...@@ -198,6 +232,122 @@ async def delete_voice(name: str):
return {"deleted": True, "name": name} return {"deleted": True, "name": name}
@router.patch("/v1/audio/voices/{name}")
async def patch_voice(name: str, req: VoicePatchRequest):
"""Update description, transcript, or reference audio of a saved voice profile."""
meta = _load_voice(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
vdir = _voice_path(name)
if req.description is not None:
meta['description'] = req.description
if req.transcript is not None:
meta['transcript'] = req.transcript
if req.audio:
audio_bytes, ext = _decode_audio(req.audio)
audio_file = os.path.join(vdir, f'ref{ext}')
# Remove old audio file(s)
for f in os.listdir(vdir):
if f.startswith('ref.') or f.startswith('ref'):
try:
os.unlink(os.path.join(vdir, f))
except Exception:
pass
with open(audio_file, 'wb') as f:
f.write(audio_bytes)
meta['audio_file'] = audio_file
meta['audio_ext'] = ext
with open(os.path.join(vdir, 'meta.json'), 'w') as f:
json.dump(meta, f)
return {"updated": True, "voice": meta}
@router.get("/v1/audio/voices/{name}")
async def get_voice(name: str):
"""Get a single voice profile metadata."""
meta = _load_voice(name)
if not meta:
raise HTTPException(status_code=404, detail=f"Voice '{name}' not found")
return {"voice": meta}
@router.post("/v1/audio/voices/extract")
async def extract_voice(req: VoiceExtractRequest):
"""
Extract a voice profile from a source audio or video file.
- audio: base64/URL audio file (wav, mp3, flac, …)
- video: base64/URL video file — audio track is extracted automatically
- transcript: optional; auto-transcribed with Whisper when omitted
"""
if not req.name.replace('-', '').replace('_', '').isalnum():
raise HTTPException(status_code=400, detail="Voice name must be alphanumeric (hyphens/underscores allowed)")
if not req.audio and not req.video:
raise HTTPException(status_code=400, detail="Provide audio or video")
temps = []
try:
audio_bytes: Optional[bytes] = None
audio_ext = '.wav'
if req.video:
# Extract audio track from video with ffmpeg
raw = _decode_b64_or_url(req.video)
in_tmp = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False)
in_tmp.write(raw)
in_tmp.close()
temps.append(in_tmp.name)
out_tmp = tempfile.NamedTemporaryFile(suffix='.wav', delete=False)
out_tmp.close()
temps.append(out_tmp.name)
r = subprocess.run(
['ffmpeg', '-y', '-i', in_tmp.name, '-vn', '-acodec', 'pcm_s16le',
'-ar', '22050', '-ac', '1', out_tmp.name],
capture_output=True, timeout=120,
)
if r.returncode != 0:
raise HTTPException(status_code=422,
detail=f"Audio extraction from video failed: {r.stderr.decode(errors='replace')[:200]}")
with open(out_tmp.name, 'rb') as f:
audio_bytes = f.read()
else:
audio_bytes, audio_ext = _decode_audio(req.audio)
# Auto-transcribe if transcript not provided
transcript = req.transcript or ''
if not transcript:
try:
import whisper
in_tmp2 = tempfile.NamedTemporaryFile(suffix=audio_ext, delete=False)
in_tmp2.write(audio_bytes)
in_tmp2.close()
temps.append(in_tmp2.name)
model = whisper.load_model('base')
result = model.transcribe(in_tmp2.name)
transcript = result.get('text', '').strip()
except Exception:
pass
# Validate audio
try:
import soundfile as sf
sf.info(io.BytesIO(audio_bytes))
except Exception as e:
raise HTTPException(status_code=400, detail=f"Invalid audio: {e}")
meta = _save_voice(req.name, audio_bytes, audio_ext, transcript, req.description)
return {"created": True, "voice": meta}
finally:
for t in temps:
try:
os.unlink(t)
except Exception:
pass
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Voice cloning TTS # Voice cloning TTS
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
......
...@@ -98,6 +98,14 @@ class WhisperConfig: ...@@ -98,6 +98,14 @@ class WhisperConfig:
server_port: int = 8744 server_port: int = 8744
@dataclass
class ArchiveConfig:
"""Generation archive configuration."""
enabled: bool = True
directory: str = "" # empty = <config_dir>/archive; relative paths resolve from config_dir
retention: str = "never" # one of: 1h 1d 2d 1w 1m 3m 6m 1y never
@dataclass @dataclass
class Config: class Config:
"""Main configuration class.""" """Main configuration class."""
...@@ -109,6 +117,7 @@ class Config: ...@@ -109,6 +117,7 @@ class Config:
vulkan: VulkanConfig = field(default_factory=VulkanConfig) vulkan: VulkanConfig = field(default_factory=VulkanConfig)
image: ImageConfig = field(default_factory=ImageConfig) image: ImageConfig = field(default_factory=ImageConfig)
whisper: WhisperConfig = field(default_factory=WhisperConfig) whisper: WhisperConfig = field(default_factory=WhisperConfig)
archive: ArchiveConfig = field(default_factory=ArchiveConfig)
system_prompt: Optional[str] = None system_prompt: Optional[str] = None
tools_closer_prompt: bool = False tools_closer_prompt: bool = False
grammar_guided: bool = False grammar_guided: bool = False
...@@ -244,6 +253,7 @@ class ConfigManager: ...@@ -244,6 +253,7 @@ class ConfigManager:
vulkan=VulkanConfig(**config_data.get("vulkan", {})), vulkan=VulkanConfig(**config_data.get("vulkan", {})),
image=ImageConfig(**config_data.get("image", {})), image=ImageConfig(**config_data.get("image", {})),
whisper=WhisperConfig(**config_data.get("whisper", {})), whisper=WhisperConfig(**config_data.get("whisper", {})),
archive=ArchiveConfig(**config_data.get("archive", {})),
system_prompt=config_data.get("system_prompt"), system_prompt=config_data.get("system_prompt"),
tools_closer_prompt=config_data.get("tools_closer_prompt", False), tools_closer_prompt=config_data.get("tools_closer_prompt", False),
grammar_guided=config_data.get("grammar_guided", False), grammar_guided=config_data.get("grammar_guided", False),
...@@ -347,6 +357,11 @@ class ConfigManager: ...@@ -347,6 +357,11 @@ class ConfigManager:
"vae_tiling": self.config.image.vae_tiling, "vae_tiling": self.config.image.vae_tiling,
"clip_on_cpu": self.config.image.clip_on_cpu "clip_on_cpu": self.config.image.clip_on_cpu
}, },
"archive": {
"enabled": self.config.archive.enabled,
"directory": self.config.archive.directory,
"retention": self.config.archive.retention,
},
"system_prompt": self.config.system_prompt, "system_prompt": self.config.system_prompt,
"tools_closer_prompt": self.config.tools_closer_prompt, "tools_closer_prompt": self.config.tools_closer_prompt,
"grammar_guided": self.config.grammar_guided, "grammar_guided": self.config.grammar_guided,
......
...@@ -22,6 +22,7 @@ import os ...@@ -22,6 +22,7 @@ import os
from codai.cli import parse_args from codai.cli import parse_args
from codai.config import ConfigManager from codai.config import ConfigManager
from codai.admin.routes import init_session_manager, set_config_manager from codai.admin.routes import init_session_manager, set_config_manager
from codai.api.archive import archive_manager
def main(): def main():
...@@ -55,6 +56,18 @@ def main(): ...@@ -55,6 +56,18 @@ def main():
if config.models.gguf_cache_dir: if config.models.gguf_cache_dir:
os.environ['CODERAI_CACHE_DIR'] = config.models.gguf_cache_dir os.environ['CODERAI_CACHE_DIR'] = config.models.gguf_cache_dir
# Configure generation archive
_arc_dir = config.archive.directory
if not _arc_dir:
_arc_dir = os.path.join(config_dir, "archive")
elif not os.path.isabs(_arc_dir):
_arc_dir = os.path.join(config_dir, _arc_dir)
archive_manager.configure(
enabled=config.archive.enabled,
directory=_arc_dir,
retention=config.archive.retention,
)
# Initialize admin session manager and expose config to admin routes # Initialize admin session manager and expose config to admin routes
from pathlib import Path from pathlib import Path
init_session_manager(Path(config_dir)) init_session_manager(Path(config_dir))
...@@ -364,6 +377,8 @@ def main(): ...@@ -364,6 +377,8 @@ def main():
multi_model_manager.model_backend_types[mid] = ( multi_model_manager.model_backend_types[mid] = (
m.get("backend", "auto") if isinstance(m, dict) else "auto" m.get("backend", "auto") if isinstance(m, dict) else "auto"
) )
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Audio models # Audio models
audio_models = models_config.get("audio_models", []) audio_models = models_config.get("audio_models", [])
...@@ -400,6 +415,8 @@ def main(): ...@@ -400,6 +415,8 @@ def main():
mid = _model_id(m) mid = _model_id(m)
if mid: if mid:
multi_model_manager.set_image_model(mid, config=_model_cfg(m, "image")) multi_model_manager.set_image_model(mid, config=_model_cfg(m, "image"))
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Vision models # Vision models
vision_models = models_config.get("vision_models", []) vision_models = models_config.get("vision_models", [])
...@@ -407,6 +424,8 @@ def main(): ...@@ -407,6 +424,8 @@ def main():
mid = _model_id(m) mid = _model_id(m)
if mid: if mid:
multi_model_manager.set_vision_model(mid, config=_model_cfg(m, "vision")) multi_model_manager.set_vision_model(mid, config=_model_cfg(m, "vision"))
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# TTS models # TTS models
tts_models = models_config.get("tts_models", []) tts_models = models_config.get("tts_models", [])
...@@ -414,6 +433,8 @@ def main(): ...@@ -414,6 +433,8 @@ def main():
mid = _model_id(m) mid = _model_id(m)
if mid: if mid:
multi_model_manager.set_tts_model(mid, config=_model_cfg(m, "tts") if isinstance(m, dict) else {}) multi_model_manager.set_tts_model(mid, config=_model_cfg(m, "tts") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Video generation models # Video generation models
video_models = models_config.get("video_models", []) video_models = models_config.get("video_models", [])
...@@ -421,6 +442,8 @@ def main(): ...@@ -421,6 +442,8 @@ def main():
mid = _model_id(m) mid = _model_id(m)
if mid: if mid:
multi_model_manager.set_video_model(mid, config=_model_cfg(m, "video") if isinstance(m, dict) else {}) multi_model_manager.set_video_model(mid, config=_model_cfg(m, "video") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Audio generation models (MusicGen, AudioLDM2, …) # Audio generation models (MusicGen, AudioLDM2, …)
audio_gen_models = models_config.get("audio_gen_models", []) audio_gen_models = models_config.get("audio_gen_models", [])
...@@ -428,6 +451,8 @@ def main(): ...@@ -428,6 +451,8 @@ def main():
mid = _model_id(m) mid = _model_id(m)
if mid: if mid:
multi_model_manager.set_audio_gen_model(mid, config=_model_cfg(m, "audio_gen") if isinstance(m, dict) else {}) multi_model_manager.set_audio_gen_model(mid, config=_model_cfg(m, "audio_gen") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Embedding models # Embedding models
embedding_models = models_config.get("embedding_models", []) embedding_models = models_config.get("embedding_models", [])
...@@ -435,6 +460,8 @@ def main(): ...@@ -435,6 +460,8 @@ def main():
mid = _model_id(m) mid = _model_id(m)
if mid: if mid:
multi_model_manager.set_embedding_model(mid, config=_model_cfg(m, "embedding") if isinstance(m, dict) else {}) multi_model_manager.set_embedding_model(mid, config=_model_cfg(m, "embedding") if isinstance(m, dict) else {})
if isinstance(m, dict) and m.get("alias"):
multi_model_manager.set_model_alias(m["alias"], mid)
# Register aliases # Register aliases
aliases = models_config.get("aliases", {}) aliases = models_config.get("aliases", {})
...@@ -623,10 +650,20 @@ def main(): ...@@ -623,10 +650,20 @@ def main():
if global_file_path: if global_file_path:
set_fs_file_path(global_file_path) set_fs_file_path(global_file_path)
# Set spatial (3D) module global args
from codai.api.spatial import set_global_args as set_spatial_global_args, set_global_file_path as set_spatial_file_path
set_spatial_global_args(global_args)
if global_file_path:
set_spatial_file_path(global_file_path)
# Set character profiles module global args # Set character profiles module global args
from codai.api.characters import set_global_args as set_chars_global_args from codai.api.characters import set_global_args as set_chars_global_args
set_chars_global_args(global_args) set_chars_global_args(global_args)
# Set environment profiles module global args
from codai.api.environments import set_global_args as set_envs_global_args
set_envs_global_args(global_args)
# Set embeddings module global args # Set embeddings module global args
from codai.api.embeddings import set_global_args as set_embed_global_args from codai.api.embeddings import set_global_args as set_embed_global_args
set_embed_global_args(global_args) set_embed_global_args(global_args)
......
...@@ -62,6 +62,12 @@ class ModelCapabilities: ...@@ -62,6 +62,12 @@ class ModelCapabilities:
lip_sync: bool = False # Wav2Lip, SadTalker lip_sync: bool = False # Wav2Lip, SadTalker
video_dubbing: bool = False # Translation + TTS + lip sync video_dubbing: bool = False # Translation + TTS + lip sync
# 3D generation and conversion
image_to_3d: bool = False # 2D image → stereo / anaglyph / depth / mesh
video_to_3d: bool = False # 2D video → stereo / anaglyph / depth video
model_3d_generation: bool = False # text / image → 3D model (GLB)
model_3d_to_image: bool = False # 3D model → rendered 2D image / video
def to_list(self) -> List[str]: def to_list(self) -> List[str]:
out = [] out = []
for name, val in self.__dataclass_fields__.items(): for name, val in self.__dataclass_fields__.items():
...@@ -85,6 +91,15 @@ def detect_model_capabilities(model_name: str) -> ModelCapabilities: ...@@ -85,6 +91,15 @@ def detect_model_capabilities(model_name: str) -> ModelCapabilities:
n = model_name.lower() n = model_name.lower()
# ── 3D generation ────────────────────────────────────────────────────────
if any(x in n for x in ['triposr', 'tsr', 'shap-e', 'shape-e', 'point-e', 'pointe',
'zero123', 'wonder3d', 'instant3d', 'one-2-3-45',
'mvdiffusion', 'syncdreamer', 'dreamgaussian']):
caps.model_3d_generation = True
caps.image_to_3d = True
caps.model_3d_to_image = True
return caps
# ── Video generation ───────────────────────────────────────────────────── # ── Video generation ─────────────────────────────────────────────────────
if any(x in n for x in ['cogvideox', 'cogvideo', 'ltx-video', 'ltxvideo', if any(x in n for x in ['cogvideox', 'cogvideo', 'ltx-video', 'ltxvideo',
'hunyuan-video', 'mochi-1', 'dynamicrafter', 'hunyuan-video', 'mochi-1', 'dynamicrafter',
......
...@@ -1632,6 +1632,23 @@ class MultiModelManager: ...@@ -1632,6 +1632,23 @@ class MultiModelManager:
'error': f"Model '{resolved_name}' is not available. Use one of: {', '.join(allowed_ids)}", 'error': f"Model '{resolved_name}' is not available. Use one of: {', '.join(allowed_ids)}",
} }
# Step 1c: Resolve short-name to canonical registered name (e.g. "Foo" → "org/Foo")
_type_registry = {
"image": self.image_models,
"audio": self.audio_models,
"tts": [self.tts_model] if self.tts_model else [],
"vision": self.vision_models,
"video": self.video_models,
"audio_gen": self.audio_gen_models,
"embedding": self.embedding_models,
}
_candidates = _type_registry.get(model_type, []) if model_type else []
if resolved_name not in _candidates:
for _m in _candidates:
if "/" in _m and _m.split("/")[-1] == resolved_name:
resolved_name = _m
break
# Step 2: Build the model key (prefixed with type) # Step 2: Build the model key (prefixed with type)
if model_type and model_type != "text": if model_type and model_type != "text":
model_key = f"{model_type}:{resolved_name}" model_key = f"{model_type}:{resolved_name}"
...@@ -1641,6 +1658,19 @@ class MultiModelManager: ...@@ -1641,6 +1658,19 @@ class MultiModelManager:
# Step 3: Check if already loaded in self.models # Step 3: Check if already loaded in self.models
existing_model = self.models.get(model_key) existing_model = self.models.get(model_key)
# Fuzzy fallback: if not found by exact key, check for org-prefix
# variations (e.g. "image:Model" vs "image:org/Model").
if existing_model is None:
type_prefix = f"{model_type}:" if model_type and model_type != "text" else ""
short = resolved_name.split("/")[-1]
for k, v in self.models.items():
k_name = k[len(type_prefix):] if type_prefix and k.startswith(type_prefix) else k
if k_name == resolved_name or k_name.split("/")[-1] == short:
model_key = k
resolved_name = k_name
existing_model = v
break
# ===================================================================== # =====================================================================
# PER-MODEL LOAD MODE: Check per-model config first. # PER-MODEL LOAD MODE: Check per-model config first.
# Per-model "load" = pre-loaded (treat as loadall for this model). # Per-model "load" = pre-loaded (treat as loadall for this model).
......
...@@ -31,6 +31,8 @@ class AudioGenerationRequest(BaseModel): ...@@ -31,6 +31,8 @@ class AudioGenerationRequest(BaseModel):
seed: Optional[int] = None seed: Optional[int] = None
# Reference audio for melody conditioning (MusicGen Melody) # Reference audio for melody conditioning (MusicGen Melody)
melody: Optional[str] = None # base64/URL melody: Optional[str] = None # base64/URL
# Voice profile for singing/speech conditioning
voice_profile: Optional[str] = None # saved voice profile name
# Output # Output
response_format: Optional[str] = "url" # url | b64_wav | b64_mp3 response_format: Optional[str] = "url" # url | b64_wav | b64_mp3
user: Optional[str] = None user: Optional[str] = None
......
...@@ -34,6 +34,12 @@ class ImageGenerationRequest(BaseModel): ...@@ -34,6 +34,12 @@ class ImageGenerationRequest(BaseModel):
seed: Optional[int] = None seed: Optional[int] = None
user: Optional[str] = None user: Optional[str] = None
disable_safety_checker: Optional[bool] = False disable_safety_checker: Optional[bool] = False
negative_prompt: Optional[str] = None
# Character consistency
character_profiles: Optional[List[str]] = None # saved profile names
character_references: Optional[List[str]] = None # inline base64 images
character_strength: Optional[float] = 0.6 # IP-Adapter scale
model_config = ConfigDict(extra="allow") model_config = ConfigDict(extra="allow")
......
...@@ -20,6 +20,18 @@ from typing import Dict, List, Optional ...@@ -20,6 +20,18 @@ from typing import Dict, List, Optional
from pydantic import BaseModel, ConfigDict from pydantic import BaseModel, ConfigDict
class CharacterDialogLine(BaseModel):
"""One spoken line in a multi-character dialog sequence."""
character: Optional[str] = None # character profile name (used for lip-sync face)
voice: Optional[str] = None # voice profile name OR TTS voice id
text: str # dialog text to synthesise
start_time: Optional[float] = None # explicit offset in seconds (None = sequential)
lip_sync: Optional[bool] = True # apply lip sync for this line
lang: Optional[str] = None # language hint for TTS
speed: Optional[float] = 1.0
model_config = ConfigDict(extra="allow")
class VideoGenerationRequest(BaseModel): class VideoGenerationRequest(BaseModel):
model: str model: str
prompt: str = "" prompt: str = ""
...@@ -78,6 +90,9 @@ class VideoGenerationRequest(BaseModel): ...@@ -78,6 +90,9 @@ class VideoGenerationRequest(BaseModel):
lip_sync: Optional[bool] = False # warp mouth to match audio lip_sync: Optional[bool] = False # warp mouth to match audio
lip_sync_method: Optional[str] = "wav2lip" # wav2lip | sadtalker lip_sync_method: Optional[str] = "wav2lip" # wav2lip | sadtalker
# ── Multi-character dialog ────────────────────────────────────────────
dialogs: Optional[List[CharacterDialogLine]] = None
# ── Subtitles ──────────────────────────────────────────────────────── # ── Subtitles ────────────────────────────────────────────────────────
generate_subtitles: Optional[bool] = False generate_subtitles: Optional[bool] = False
burn_subtitles: Optional[bool] = False burn_subtitles: Optional[bool] = False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment