Skip to content

C
coderai
Commit 227c3dbd authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

Small interface details

parent 0ce4d5af
Hide whitespace changes
Inline Side-by-side
Showing with 594 additions and 165 deletions
codai/admin/routes.py
View file @ 227c3dbd
......@@ -912,6 +912,7 @@ def _scan_caches() -> dict:
"model_type": cfg[1] if cfg[1] and cfg[1] != "gguf_models" else "text_models",
"settings": cfg[0] if isinstance(cfg[0], dict) else {},
"capabilities": caps.to_list(),
"source_repo": repo.repo_id,
})
continue # skip adding to hf list
......@@ -1275,9 +1276,24 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
if result.get("already_loaded"):
return {"success": True, "already_loaded": True}
# Proactive VRAM check: evict loaded models if free VRAM is insufficient.
# request_model handles this for "on-request" load mode; cover remaining modes here.
# HuggingFace repo IDs return needed_gb==0 (size unknown); handle that case too.
model_key = result.get("model_key") or path
needed_gb = multi_model_manager._get_model_used_vram_gb(model_key, path)
free_gb = multi_model_manager._get_free_vram_gb()
if needed_gb > 0 and free_gb < needed_gb:
print(f"Admin model-load: need {needed_gb:.1f} GB VRAM, have {free_gb:.1f} GB free — evicting models")
multi_model_manager._evict_models_for_vram(needed_gb)
elif needed_gb == 0 and multi_model_manager.models and free_gb < 4.0:
# Unknown model size but VRAM nearly full — evict everything to avoid OOM on first attempt
print(f"Admin model-load: unknown model size, only {free_gb:.1f} GB free — evicting models proactively")
multi_model_manager.unload_all_models()
# Not loaded yet — trigger actual load
try:
if model_type == "text":
# _load_model_by_name already records the VRAM delta internally
mm = multi_model_manager._load_model_by_name(result["model_name"] or path)
if mm is None:
raise RuntimeError("Model failed to load")
......@@ -1286,6 +1302,7 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
elif model_type == "audio":
wsm = multi_model_manager.whisper_servers.get(path)
if wsm is not None:
_snap = multi_model_manager.vram_before_load()
started = wsm.start(getattr(wsm, "_model_path", None), gpu_device=getattr(wsm, "_gpu_device", 0))
if not wsm.is_running():
raise RuntimeError("whisper-server failed to start")
......@@ -1293,12 +1310,14 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
multi_model_manager.models[model_key] = wsm
multi_model_manager.active_in_vram = model_key
multi_model_manager.models_in_vram.add(model_key)
multi_model_manager.record_vram_delta(model_key, _snap)
return {"success": True, "already_loaded": False, "started_model": started}
elif model_type == "image":
from codai.api.images import _load_diffusers_pipeline, _is_gguf_model, _load_sdcpp_model
from codai.api.state import get_global_args
global_args = get_global_args()
model_key = f"image:{path}"
_snap = multi_model_manager.vram_before_load()
if _is_gguf_model(path):
resolved = multi_model_manager.load_model(path)
import os as _os
......@@ -1306,10 +1325,12 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
sd_model = _load_sdcpp_model(resolved, global_args)
if sd_model:
multi_model_manager.add_model(model_key, sd_model)
multi_model_manager.record_vram_delta(model_key, _snap)
else:
pipeline = _load_diffusers_pipeline(path, global_args)
if pipeline:
multi_model_manager.add_model(model_key, pipeline)
multi_model_manager.record_vram_delta(model_key, _snap)
return {"success": True, "already_loaded": False}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
......@@ -1453,10 +1474,16 @@ async def api_model_configure(request: Request, username: str = Depends(require_
used_vram_gb = data.get("used_vram_gb")
if used_vram_gb is None:
import os
from codai.models.cache import is_huggingface_model_id
if os.path.isfile(path):
size_bytes = os.path.getsize(path)
multiplier = 1.1 if path.endswith(".gguf") else 1.2
used_vram_gb = round(size_bytes / 1e9 * multiplier, 2)
elif is_huggingface_model_id(path):
from codai.models.manager import MultiModelManager
size_bytes = MultiModelManager._hf_cached_model_size_bytes(path)
if size_bytes > 0:
used_vram_gb = round(size_bytes / 1e9 * 1.2, 2)
# Build settings entry
entry: dict = {"path": path, "model_type": model_types[0], "model_types": model_types}
......@@ -1912,6 +1939,24 @@ async def api_get_character(name: str, username: str = Depends(require_auth)):
}
@router.get("/admin/api/characters/{name}/thumbnail")
async def api_character_thumbnail(name: str, username: str = Depends(require_auth)):
import os as _os
from codai.api.characters import _char_dir, _load_character_meta
from fastapi.responses import FileResponse
meta = _load_character_meta(name)
if not meta:
raise HTTPException(status_code=404)
cdir = _char_dir(name)
for img_info in meta.get('images', []):
fpath = _os.path.join(cdir, img_info['file'])
if _os.path.exists(fpath):
ext = img_info['file'].rsplit('.', 1)[-1].lower()
media_type = 'image/png' if ext == 'png' else 'image/jpeg'
return FileResponse(fpath, media_type=media_type)
raise HTTPException(status_code=404)
@router.delete("/admin/api/characters/{name}")
async def api_delete_character(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
......@@ -1949,6 +1994,24 @@ async def api_get_environment(name: str, username: str = Depends(require_auth)):
}
@router.get("/admin/api/environments/{name}/thumbnail")
async def api_environment_thumbnail(name: str, username: str = Depends(require_auth)):
import os as _os
from codai.api.environments import _env_dir, _load_environment_meta
from fastapi.responses import FileResponse
meta = _load_environment_meta(name)
if not meta:
raise HTTPException(status_code=404)
edir = _env_dir(name)
for img_info in meta.get('images', []):
fpath = _os.path.join(edir, img_info['file'])
if _os.path.exists(fpath):
ext = img_info['file'].rsplit('.', 1)[-1].lower()
media_type = 'image/png' if ext == 'png' else 'image/jpeg'
return FileResponse(fpath, media_type=media_type)
raise HTTPException(status_code=404)
@router.delete("/admin/api/environments/{name}")
async def api_delete_environment(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
......
codai/admin/templates/base.html
View file @ 227c3dbd
......@@ -23,6 +23,7 @@
<div class="nav-links">
<a href="/admin" class="nav-link {% if request.url.path == '/admin' %}active{% endif %}">Overview</a>
<a href="/chat" class="nav-link {% if request.url.path == '/chat' %}active{% endif %}">Studio</a>
<a href="/docs" class="nav-link" target="_blank">API Docs</a>
{% if is_admin|default(false) %}
<a href="/admin/models" class="nav-link {% if '/models' in request.url.path %}active{% endif %}">Models</a>
<a href="/admin/tokens" class="nav-link {% if '/tokens' in request.url.path %}active{% endif %}">Tokens</a>
......
codai/admin/templates/chat.html
View file @ 227c3dbd
......@@ -208,6 +208,9 @@ a.dl { display:inline-block; margin-top:.4rem; }
.cap-chip.dim { opacity:.72; }
.cap-missing, .cap-note { font-size:12px; color:var(--text-2); }
.cap-note strong, .cap-missing strong { color:var(--text-1); }
.cap-find-link { display:inline-flex; align-items:center; gap:2px; text-decoration:none; border-radius:4px; transition:opacity .15s; }
.cap-find-link:hover { opacity:.75; }
.cap-find-icon { font-size:10px; color:var(--text-3); }
.cap-output-note {
width:100%; max-width:960px; border:1px solid rgba(245,158,11,.35); background:rgba(58,37,16,.85);
color:#f6d08a; border-radius:8px; padding:.85rem 1rem; display:flex; flex-direction:column; gap:.4rem;
......@@ -335,6 +338,24 @@ a.dl { display:inline-block; margin-top:.4rem; }
.arch-btn:hover { background:var(--surface-2); color:var(--text-1); }
.arch-btn.del:hover { background:#3a1010; color:#f07070; border-color:#5a2020; }
.arch-empty { color:var(--text-3); text-align:center; padding:2rem 1rem; grid-column:1/-1; font-size:13px; }
/* ── Profile panels ───────────────────────────────────────────── */
.prof-gen-out { align-items:flex-start; justify-content:flex-start; flex-direction:column; padding:0; }
.prof-out-hd { display:flex; align-items:center; justify-content:space-between; padding:.6rem 1rem .55rem; flex-shrink:0; border-bottom:1px solid var(--border); width:100%; box-sizing:border-box; }
.prof-out-hd-title { font-size:13px; font-weight:600; color:var(--text-1); }
.prof-out-body { flex:1; overflow-y:auto; padding:.8rem; width:100%; box-sizing:border-box; }
.prof-card-info { padding:.6rem .75rem .75rem; display:flex; flex-direction:column; gap:.15rem; flex:1; }
.prof-card-name { font-weight:600; font-size:13px; color:var(--text-1); overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.prof-card-desc { font-size:11.5px; color:var(--text-2); overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.prof-card-meta { font-size:11px; color:var(--text-3); }
.prof-card-actions { display:flex; gap:.4rem; margin-top:.45rem; }
.prof-voice-list { display:flex; flex-direction:column; gap:.55rem; }
.prof-voice-card { display:flex; align-items:flex-start; gap:.85rem; background:var(--surface-1); border:1px solid var(--border); border-radius:8px; padding:.75rem .9rem; }
.prof-voice-icon { width:40px; height:40px; border-radius:8px; background:var(--surface-3); display:flex; align-items:center; justify-content:center; font-size:1.3rem; flex-shrink:0; }
.prof-voice-info { flex:1; min-width:0; }
.prof-voice-name { font-weight:600; font-size:13px; color:var(--text-1); }
.prof-voice-meta { font-size:11px; color:var(--text-3); margin-top:.15rem; }
.prof-voice-quote { font-style:italic; color:var(--text-3); font-size:11.5px; margin-top:.15rem; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.prof-voice-actions { display:flex; gap:.4rem; margin-top:.5rem; }
/* ── Role-picker popup ────────────────────────────────────────── */
.role-picker-popup { position:fixed; z-index:9999; background:var(--surface-1); border:1px solid var(--border); border-radius:8px; padding:.7rem; box-shadow:0 8px 24px rgba(0,0,0,.5); min-width:200px; max-width:280px; }
.role-picker-header { font-size:12px; color:var(--text-2); margin-bottom:.5rem; }
......@@ -1734,34 +1755,23 @@ a.dl { display:inline-block; margin-top:.4rem; }
<!-- ═══════════════ PROFILES — CHARACTER ═══════════════ -->
<div class="panel" id="panel-prof-char">
<div class="gen-wrap" style="max-width:860px">
<div class="gen-hd">Character Profiles</div>
<!-- Toolbar -->
<div class="frow" style="gap:.5rem;margin-bottom:.6rem">
<button class="btn btn-primary btn-sm" onclick="profCharLoad()">Refresh</button>
<button class="btn btn-ghost btn-sm" onclick="profCharShowCreate()">+ New Profile</button>
</div>
<!-- Create / Edit form (hidden by default) -->
<div id="prof-char-form" style="display:none;background:var(--surface-2);border-radius:8px;padding:1rem;margin-bottom:1rem">
<div class="gen-hd" style="font-size:.85rem;margin-bottom:.5rem" id="prof-char-form-title">New Character Profile</div>
<div class="gen-wrap">
<div class="gen-ctrl">
<div class="gen-hd">New Character</div>
<input type="hidden" id="pc-edit-name">
<div class="frow"><label class="fl">Name*</label><input class="fi" id="pc-name" placeholder="alice"></div>
<div class="frow"><label class="fl">Description</label><input class="fi" id="pc-desc" placeholder="optional"></div>
<!-- Mode selector -->
<div class="frow" style="align-items:center;gap:1rem;margin-bottom:.2rem">
<label class="fl">Mode</label>
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer">
<input type="radio" name="pc-mode" value="extract" checked onchange="pcModeChange()"> Extract from media
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer;font-size:12px">
<input type="radio" name="pc-mode" value="extract" checked onchange="pcModeChange()"> Extract
</label>
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer">
<input type="radio" name="pc-mode" value="generate" onchange="pcModeChange()"> Generate from prompt
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer;font-size:12px">
<input type="radio" name="pc-mode" value="generate" onchange="pcModeChange()"> Generate
</label>
</div>
<!-- Extract-from-media fields -->
<div id="pc-extract-fields">
<div class="frow"><label class="fl">Source images</label>
<div style="flex:1">
......@@ -1777,15 +1787,12 @@ a.dl { display:inline-block; margin-top:.4rem; }
</div>
</div>
<!-- Generate-from-prompt fields -->
<div id="pc-generate-fields" style="display:none">
<div class="frow" style="align-items:flex-start"><label class="fl" style="padding-top:.3rem">Visual prompt*</label>
<div class="frow" style="align-items:flex-start"><label class="fl" style="padding-top:.3rem">Prompt*</label>
<textarea class="fi" id="pc-gen-prompt" rows="3" placeholder="e.g. a tall woman with red hair and blue eyes, realistic, studio lighting" style="flex:1;resize:vertical"></textarea>
</div>
<div class="frow"><label class="fl">Image model</label>
<select class="fi" id="pc-gen-model" style="flex:1">
<option value="">Default image model</option>
</select>
<select class="fi" id="pc-gen-model" style="flex:1"><option value="">Default image model</option></select>
</div>
<div class="frow"><label class="fl">Ref images (n)</label>
<input type="number" class="fi" id="pc-gen-n" value="3" min="1" max="8" style="width:100px">
......@@ -1806,49 +1813,45 @@ a.dl { display:inline-block; margin-top:.4rem; }
</details>
</div>
<div style="display:flex;gap:.5rem;margin-top:.6rem">
<div style="display:flex;gap:.5rem;margin-top:.7rem">
<button class="btn btn-primary btn-sm" id="pc-submit-btn" onclick="profCharSubmit()">Extract &amp; Save</button>
<button class="btn btn-ghost btn-sm" onclick="profCharHideForm()">Cancel</button>
<button class="btn btn-ghost btn-sm" onclick="profCharShowCreate()">Reset</button>
</div>
<div id="pc-form-status" class="small muted" style="margin-top:.4rem"></div>
</div>
<!-- Profile list -->
<div id="prof-char-list" class="archive-grid" style="margin-top:.6rem">
<div class="arch-empty">Loading…</div>
<div class="gen-out prof-gen-out">
<div class="prof-out-hd">
<span class="prof-out-hd-title">Character Profiles</span>
<button class="btn btn-ghost btn-sm" onclick="profCharLoad()">Refresh</button>
</div>
<div class="prof-out-body">
<div id="prof-char-list" class="archive-grid">
<div class="arch-empty">Loading…</div>
</div>
</div>
</div>
</div>
</div>
<!-- ═══════════════ PROFILES — ENVIRONMENT ═══════════════ -->
<div class="panel" id="panel-prof-env">
<div class="gen-wrap" style="max-width:860px">
<div class="gen-hd">Environment Profiles</div>
<!-- Toolbar -->
<div class="frow" style="gap:.5rem;margin-bottom:.6rem">
<button class="btn btn-primary btn-sm" onclick="profEnvLoad()">Refresh</button>
<button class="btn btn-ghost btn-sm" onclick="profEnvShowCreate()">+ New Profile</button>
</div>
<!-- Create / Edit form (hidden by default) -->
<div id="prof-env-form" style="display:none;background:var(--surface-2);border-radius:8px;padding:1rem;margin-bottom:1rem">
<div class="gen-hd" style="font-size:.85rem;margin-bottom:.5rem">New Environment Profile</div>
<div class="gen-wrap">
<div class="gen-ctrl">
<div class="gen-hd">New Environment</div>
<div class="frow"><label class="fl">Name*</label><input class="fi" id="pe-name" placeholder="living-room"></div>
<div class="frow"><label class="fl">Description</label><input class="fi" id="pe-desc" placeholder="optional"></div>
<!-- Mode selector -->
<div class="frow" style="align-items:center;gap:1rem;margin-bottom:.2rem">
<label class="fl">Mode</label>
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer">
<input type="radio" name="pe-mode" value="extract" checked onchange="peModeChange()"> Extract from media
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer;font-size:12px">
<input type="radio" name="pe-mode" value="extract" checked onchange="peModeChange()"> Extract
</label>
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer">
<input type="radio" name="pe-mode" value="generate" onchange="peModeChange()"> Generate from prompt
<label style="display:flex;align-items:center;gap:.3rem;cursor:pointer;font-size:12px">
<input type="radio" name="pe-mode" value="generate" onchange="peModeChange()"> Generate
</label>
</div>
<!-- Extract-from-media fields -->
<div id="pe-extract-fields">
<div class="frow"><label class="fl">Source images</label>
<div style="flex:1">
......@@ -1864,15 +1867,12 @@ a.dl { display:inline-block; margin-top:.4rem; }
</div>
</div>
<!-- Generate-from-prompt fields -->
<div id="pe-generate-fields" style="display:none">
<div class="frow" style="align-items:flex-start"><label class="fl" style="padding-top:.3rem">Visual prompt*</label>
<div class="frow" style="align-items:flex-start"><label class="fl" style="padding-top:.3rem">Prompt*</label>
<textarea class="fi" id="pe-gen-prompt" rows="3" placeholder="e.g. a cozy living room with wooden floors, warm lighting, bookshelves" style="flex:1;resize:vertical"></textarea>
</div>
<div class="frow"><label class="fl">Image model</label>
<select class="fi" id="pe-gen-model" style="flex:1">
<option value="">Default image model</option>
</select>
<select class="fi" id="pe-gen-model" style="flex:1"><option value="">Default image model</option></select>
</div>
<div class="frow"><label class="fl">Ref images (n)</label>
<input type="number" class="fi" id="pe-gen-n" value="3" min="1" max="8" style="width:100px">
......@@ -1893,52 +1893,57 @@ a.dl { display:inline-block; margin-top:.4rem; }
</details>
</div>
<div style="display:flex;gap:.5rem;margin-top:.6rem">
<div style="display:flex;gap:.5rem;margin-top:.7rem">
<button class="btn btn-primary btn-sm" id="pe-submit-btn" onclick="profEnvSubmit()">Extract &amp; Save</button>
<button class="btn btn-ghost btn-sm" onclick="profEnvHideForm()">Cancel</button>
<button class="btn btn-ghost btn-sm" onclick="profEnvShowCreate()">Reset</button>
</div>
<div id="pe-form-status" class="small muted" style="margin-top:.4rem"></div>
</div>
<!-- Profile list -->
<div id="prof-env-list" class="archive-grid" style="margin-top:.6rem">
<div class="arch-empty">Loading…</div>
<div class="gen-out prof-gen-out">
<div class="prof-out-hd">
<span class="prof-out-hd-title">Environment Profiles</span>
<button class="btn btn-ghost btn-sm" onclick="profEnvLoad()">Refresh</button>
</div>
<div class="prof-out-body">
<div id="prof-env-list" class="archive-grid">
<div class="arch-empty">Loading…</div>
</div>
</div>
</div>
</div>
</div>
<!-- ═══════════════ PROFILES — VOICE ═══════════════ -->
<div class="panel" id="panel-prof-voice">
<div class="gen-wrap" style="max-width:860px">
<div class="gen-hd">Voice Profiles</div>
<!-- Toolbar -->
<div class="frow" style="gap:.5rem;margin-bottom:.6rem">
<button class="btn btn-primary btn-sm" onclick="profVoiceLoad()">Refresh</button>
<button class="btn btn-ghost btn-sm" onclick="profVoiceShowCreate()">+ New Profile</button>
</div>
<!-- Create / Edit form -->
<div id="prof-voice-form" style="display:none;background:var(--surface-2);border-radius:8px;padding:1rem;margin-bottom:1rem">
<div class="gen-hd" style="font-size:.85rem;margin-bottom:.5rem">Extract Voice Profile</div>
<div class="gen-wrap">
<div class="gen-ctrl">
<div class="gen-hd">Extract Voice</div>
<div class="frow"><label class="fl">Name*</label><input class="fi" id="pv-name" placeholder="john-doe"></div>
<div class="frow"><label class="fl">Description</label><input class="fi" id="pv-desc" placeholder="optional"></div>
<div class="frow"><label class="fl">Audio or Video file*</label>
<div class="frow"><label class="fl">Audio or Video*</label>
<input type="file" id="pv-file" accept="audio/*,video/*" style="flex:1">
</div>
<div class="frow"><label class="fl">Transcript (optional)</label>
<textarea class="fi fs" id="pv-transcript" rows="2" placeholder="Leave empty to auto-transcribe with Whisper"></textarea>
<div class="frow" style="align-items:flex-start"><label class="fl" style="padding-top:.3rem">Transcript</label>
<textarea class="fi fs" id="pv-transcript" rows="3" placeholder="Leave empty to auto-transcribe with Whisper" style="resize:vertical"></textarea>
</div>
<div style="display:flex;gap:.5rem;margin-top:.6rem">
<div style="display:flex;gap:.5rem;margin-top:.7rem">
<button class="btn btn-primary btn-sm" onclick="profVoiceSave()">Extract &amp; Save</button>
<button class="btn btn-ghost btn-sm" onclick="profVoiceHideForm()">Cancel</button>
<button class="btn btn-ghost btn-sm" onclick="profVoiceShowCreate()">Reset</button>
</div>
<div id="pv-form-status" class="small muted" style="margin-top:.4rem"></div>
</div>
<!-- Voice list -->
<div id="prof-voice-list" style="display:flex;flex-direction:column;gap:.6rem;margin-top:.6rem">
<div class="arch-empty">Loading…</div>
<div class="gen-out prof-gen-out">
<div class="prof-out-hd">
<span class="prof-out-hd-title">Voice Profiles</span>
<button class="btn btn-ghost btn-sm" onclick="profVoiceLoad()">Refresh</button>
</div>
<div class="prof-out-body">
<div id="prof-voice-list" class="prof-voice-list">
<div class="arch-empty">Loading…</div>
</div>
</div>
</div>
</div>
</div>
......@@ -2284,6 +2289,48 @@ const STUDIO_CAPABILITIES = {
io:'Input: noisy audio. Output: cleaned audio artifact plus applied-operation metadata.'
}
};
// Maps a capability token to the best HuggingFace search parameters
const CAP_TO_HF_SEARCH = {
'image_generation': { pipeline:'text-to-image', q:'', gguf:'no-gguf' },
'image_to_image': { pipeline:'image-to-image', q:'', gguf:'no-gguf' },
'inpainting': { pipeline:'image-to-image', q:'inpainting', gguf:'no-gguf' },
'image_upscaling': { pipeline:'image-to-image', q:'upscale', gguf:'no-gguf' },
'depth_estimation': { pipeline:'depth-estimation', q:'', gguf:'no-gguf' },
'image_segmentation': { pipeline:'image-segmentation', q:'', gguf:'no-gguf' },
'video_generation': { pipeline:'text-to-video', q:'', gguf:'no-gguf' },
'image_to_video': { pipeline:'image-to-video', q:'', gguf:'no-gguf' },
'video_to_video': { pipeline:'', q:'video to video', gguf:'no-gguf' },
'video_interpolation': { pipeline:'', q:'frame interpolation',gguf:'no-gguf' },
'video_upscaling': { pipeline:'', q:'video upscaling', gguf:'no-gguf' },
'speech_to_text': { pipeline:'automatic-speech-recognition',q:'', gguf:'gguf' },
'text_to_speech': { pipeline:'text-to-speech', q:'', gguf:'no-gguf' },
'audio_generation': { pipeline:'text-to-audio', q:'', gguf:'no-gguf' },
'text_generation': { pipeline:'text-generation', q:'', gguf:'gguf' },
'audio_to_audio': { pipeline:'audio-to-audio', q:'voice conversion', gguf:'no-gguf' },
'subtitle_generation': { pipeline:'automatic-speech-recognition',q:'', gguf:'gguf' },
'image_to_3d': { pipeline:'', q:'image to 3d', gguf:'no-gguf' },
'video_to_3d': { pipeline:'', q:'video depth 3d', gguf:'no-gguf' },
'model_3d_generation': { pipeline:'', q:'3d generation', gguf:'no-gguf' },
'model_3d_to_image': { pipeline:'', q:'3d rendering', gguf:'no-gguf' },
};
function capSearchUrl(cap) {
const s = CAP_TO_HF_SEARCH[cap];
if (!s) return null;
const p = new URLSearchParams({ tab:'search', q: s.q, pipeline: s.pipeline, gguf: s.gguf });
return '/admin/models?' + p.toString();
}
function capMissingHtml(caps, label) {
if (!caps.length) return '';
const links = caps.map(cap => {
const url = capSearchUrl(cap);
const chip = `<span class="cap-chip dim">${cap.replace(/_/g,' ')}</span>`;
return url
? `<a href="${url}" target="_blank" class="cap-find-link" title="Find ${cap.replace(/_/g,' ')} model on HuggingFace">${chip}<span class="cap-find-icon">↗</span></a>`
: chip;
}).join(' ');
return `<div class="cap-missing"><strong>${label}:</strong> ${links}</div>`;
}
const SUB_CAPABILITY_RULES = {
'img-gen': { category:'image', requiresAny:['image_generation'] },
'img-edit': { category:'image', requiresAny:['image_to_image'] },
......@@ -2323,7 +2370,7 @@ const SUB_CAPABILITY_RULES = {
'aud-stems': { category:'audio', optional:['audio_generation'], fallbackTypes:['audio'] },
'aud-clean': { category:'audio', optional:['speech_to_text'], fallbackTypes:['audio'] },
};
const CATEGORY_TABS = ['chat', 'image', 'video', 'audio', '3d', 'pipe', 'embed', 'archive'];
const CATEGORY_TABS = ['chat', 'image', 'video', 'audio', '3d', 'profiles', 'pipe', 'embed', 'archive'];
const CATEGORY_SUBS = {
image: Object.keys(SUB_CAPABILITY_RULES).filter(sub => SUB_CAT[sub] === 'image'),
video: Object.keys(SUB_CAPABILITY_RULES).filter(sub => SUB_CAT[sub] === 'video'),
......@@ -2445,6 +2492,7 @@ function evaluateCategoryState(cat, subStates, caps, type) {
}
if (cat === 'pipe') return 'none';
if (cat === 'archive') return 'none';
if (cat === 'profiles') return 'none';
if (cat === 'embed') return caps.has('embeddings') || type === 'embedding' ? 'available' : 'unavailable';
if (cat === '3d') return 'none'; // always show 3D tab (no required capability)
const states = (CATEGORY_SUBS[cat] || []).map(sub => subStates[sub]).filter(Boolean);
......@@ -2514,8 +2562,8 @@ function renderCapabilityCard(sub) {
const availabilityLabel = details.availability === 'available' ? 'Ready' : details.availability === 'partial' ? 'Partial' : 'Unavailable';
const availabilityClass = details.availability === 'available' ? ' ok' : details.availability === 'partial' ? ' warn' : ' dim';
const missingBits = [];
if (details.missingRequired.length) missingBits.push(`<div class="cap-missing"><strong>Missing required:</strong> ${details.missingRequired.join(', ')}</div>`);
if (details.missingOptional.length) missingBits.push(`<div class="cap-missing"><strong>Limited without:</strong> ${details.missingOptional.join(', ')}</div>`);
if (details.missingRequired.length) missingBits.push(capMissingHtml(details.missingRequired, 'Missing required'));
if (details.missingOptional.length) missingBits.push(capMissingHtml(details.missingOptional, 'Limited without'));
const notes = (details.notes || []).map(note => `<div class="cap-note">${note}</div>`).join('');
shell.innerHTML = `
<div class="cap-card-top">
......@@ -2565,8 +2613,8 @@ function renderCapabilityCards() {
const missingOptional = (rule.optional || []).filter(c => !allCaps.has(c));
const label = document.querySelector(`.t2btn[data-sub="${sub}"]`)?.childNodes[0]?.textContent?.trim() || sub;
const missingBits = [];
if (missingRequired.length) missingBits.push(`<div class="cap-missing"><strong>Missing required:</strong> ${missingRequired.join(', ')}</div>`);
if (missingOptional.length) missingBits.push(`<div class="cap-missing"><strong>Limited without:</strong> ${missingOptional.join(', ')}</div>`);
if (missingRequired.length) missingBits.push(capMissingHtml(missingRequired, 'Missing required'));
if (missingOptional.length) missingBits.push(capMissingHtml(missingOptional, 'Limited without'));
const availabilityLabel = state === 'partial' ? 'Partial' : 'Unavailable';
const availabilityClass = state === 'partial' ? ' warn' : ' dim';
shell.innerHTML = `
......@@ -2589,13 +2637,25 @@ function renderCapabilityOutputNote(sub) {
if (!panel) return;
panel.querySelector(`#${noteId}`)?.remove();
if (!details || details.availability === 'available') return;
const missing = [];
if (details.missingRequired.length) missing.push(...details.missingRequired.map(item => `<strong>Required:</strong> ${item}`));
if (details.missingOptional.length) missing.push(...details.missingOptional.map(item => `<strong>Optional:</strong> ${item}`));
if (!missing.length && Array.isArray(details.notes)) missing.push(...details.notes);
const missingItems = [];
details.missingRequired.forEach(cap => {
const url = capSearchUrl(cap);
const label = cap.replace(/_/g,' ');
missingItems.push(url
? `<strong>Required:</strong> <a href="${url}" target="_blank" class="cap-find-link">${label}<span class="cap-find-icon">↗</span></a>`
: `<strong>Required:</strong> ${label}`);
});
details.missingOptional.forEach(cap => {
const url = capSearchUrl(cap);
const label = cap.replace(/_/g,' ');
missingItems.push(url
? `<strong>Optional:</strong> <a href="${url}" target="_blank" class="cap-find-link">${label}<span class="cap-find-icon">↗</span></a>`
: `<strong>Optional:</strong> ${label}`);
});
if (!missingItems.length && Array.isArray(details.notes)) missingItems.push(...details.notes);
const title = details.availability === 'unavailable' ? '⚠ Feature unavailable' : '⚠ Feature partially available';
const detailHtml = missing.length
? `<ul>${missing.map(item => `<li>${item}</li>`).join('')}</ul>`
const detailHtml = missingItems.length
? `<ul>${missingItems.map(item => `<li>${item}</li>`).join('')}</ul>`
: '<div>Some required pieces are missing for this surface.</div>';
const target = panel.querySelector('.gen-out') || panel.querySelector('.gen-wrap') || panel;
target.insertAdjacentHTML('afterbegin', `
......@@ -3146,10 +3206,42 @@ function initRequestPreviews() {
// ─────────────────────────────────────────────────────────────────
// Boot
// ─────────────────────────────────────────────────────────────────
function deduplicateModels(raw) {
const TYPE_PREFIXES = /^(audio|tts|image|vision|video|audio_gen|embedding):/;
// Group entries sharing the same alias → keep only the entry whose id === alias
const aliasGroups = {};
const noAlias = [];
for (const m of raw) {
if (m.alias) (aliasGroups[m.alias] = aliasGroups[m.alias] || []).push(m);
else noAlias.push(m);
}
const result = [];
for (const alias in aliasGroups) {
const g = aliasGroups[alias];
result.push(g.find(m => m.id === alias) || g[0]);
}
// For alias-less entries, skip bare basenames that are short forms of a full path
const fullIds = new Set(noAlias.map(m => m.id));
for (const m of noAlias) {
if (m.id.includes('/')) {
result.push(m);
} else if (![...fullIds].some(id => id.endsWith('/' + m.id))) {
result.push(m);
}
}
// Drop "type:modelname" entries when "modelname" already appears in the result
const resultIds = new Set(result.map(m => m.id));
return result.filter(m => {
const match = m.id.match(TYPE_PREFIXES);
return !match || !resultIds.has(m.id.slice(match[0].length));
});
}
async function loadModels() {
try {
const d = await fetch('/v1/models').then(r => r.json());
models = d.data || [];
models = deduplicateModels(d.data || []);
renderSidebar();
if (models.length) selectModel(models[0]);
} catch(e) {
......@@ -5406,14 +5498,11 @@ function profCharShowCreate() {
$('pc-gen-n').value = '3';
$('pc-gen-steps').value = '';
$('pc-gen-w').value = '512'; $('pc-gen-h').value = '512';
// Reset to extract mode
document.querySelectorAll('input[name="pc-mode"]').forEach(r => r.checked = r.value === 'extract');
pcModeChange();
// Populate image model selector
pcPopulateModelSelect();
$('prof-char-form').style.display = 'block';
}
function profCharHideForm() { $('prof-char-form').style.display = 'none'; }
function profCharHideForm() { profCharShowCreate(); }
function pcModeChange() {
const mode = (document.querySelector('input[name="pc-mode"]:checked') || {}).value || 'extract';
......@@ -5538,16 +5627,26 @@ function renderCharList() {
const el = $('prof-char-list');
if (!el) return;
if (!_charProfiles.length) { el.innerHTML = '<div class="arch-empty">No character profiles yet.</div>'; return; }
el.innerHTML = _charProfiles.map(p => `
<div class="arch-card" style="position:relative;padding:1rem">
<div style="font-weight:600">${escapeHtml(p.name)}</div>
${p.description ? `<div class="small muted">${escapeHtml(p.description)}</div>` : ''}
<div class="small muted">${p.image_count} reference image${p.image_count!==1?'s':''} · ${new Date(p.created_at*1000).toLocaleDateString()}</div>
<div style="display:flex;gap:.4rem;margin-top:.5rem">
<button class="btn btn-ghost btn-sm" onclick="profCharView('${escapeHtml(p.name)}')">View</button>
<button class="btn btn-danger btn-sm" onclick="profCharDelete('${escapeHtml(p.name)}')">Delete</button>
el.innerHTML = _charProfiles.map(p => {
const n = escapeHtml(p.name);
const thumb = p.image_count > 0
? `<img class="arch-thumb" src="/admin/api/characters/${encodeURIComponent(p.name)}/thumbnail" loading="lazy" onerror="this.outerHTML='<div class=arch-thumb-ph>\u{1F464}</div>'" onclick="profCharView('${n}')">`
: `<div class="arch-thumb-ph" onclick="profCharView('${n}')">\u{1F464}</div>`;
const date = new Date(p.created_at * 1000).toLocaleDateString();
return `
<div class="arch-card">
${thumb}
<div class="prof-card-info">
<div class="prof-card-name">${n}</div>
${p.description ? `<div class="prof-card-desc">${escapeHtml(p.description)}</div>` : ''}
<div class="prof-card-meta">${p.image_count} ref image${p.image_count!==1?'s':''} · ${date}</div>
<div class="prof-card-actions">
<button class="btn btn-ghost btn-sm" onclick="profCharView('${n}')">View</button>
<button class="btn btn-danger btn-sm" onclick="profCharDelete('${n}')">Delete</button>
</div>
</div>
</div>`).join('');
</div>`;
}).join('');
}
async function profCharView(name) {
......@@ -5572,9 +5671,8 @@ async function profCharDelete(name) {
function profVoiceShowCreate() {
$('pv-name').value=''; $('pv-desc').value=''; $('pv-transcript').value='';
$('pv-form-status').textContent='';
$('prof-voice-form').style.display='block';
}
function profVoiceHideForm() { $('prof-voice-form').style.display='none'; }
function profVoiceHideForm() { profVoiceShowCreate(); }
async function profVoiceSave() {
const name = ($('pv-name').value||'').trim();
......@@ -5619,16 +5717,24 @@ function renderVoiceList(voices) {
const el = $('prof-voice-list');
if (!el) return;
if (!voices.length) { el.innerHTML = '<div class="arch-empty">No voice profiles yet.</div>'; return; }
el.innerHTML = voices.map(v => `
<div style="background:var(--surface-2);border-radius:8px;padding:.8rem 1rem;display:flex;align-items:center;gap:.8rem">
<div style="flex:1">
<div style="font-weight:600">${escapeHtml(v.name)}</div>
${v.description ? `<div class="small muted">${escapeHtml(v.description)}</div>` : ''}
${v.transcript ? `<div class="small muted" style="font-style:italic">"${escapeHtml(v.transcript.slice(0,80))}${v.transcript.length>80?'…':''}"</div>` : ''}
<div class="small muted">${new Date(v.created_at*1000).toLocaleDateString()}</div>
el.innerHTML = voices.map(v => {
const n = escapeHtml(v.name);
const date = new Date(v.created_at * 1000).toLocaleDateString();
const quote = v.transcript ? `<div class="prof-voice-quote">&ldquo;${escapeHtml(v.transcript.slice(0,90))}${v.transcript.length>90?'&hellip;':''}&rdquo;</div>` : '';
return `
<div class="prof-voice-card">
<div class="prof-voice-icon">&#127908;</div>
<div class="prof-voice-info">
<div class="prof-voice-name">${n}</div>
${v.description ? `<div class="prof-voice-meta">${escapeHtml(v.description)}</div>` : ''}
${quote}
<div class="prof-voice-meta">${date}</div>
<div class="prof-voice-actions">
<button class="btn btn-danger btn-sm" onclick="profVoiceDelete('${n}')">Delete</button>
</div>
</div>
<button class="btn btn-danger btn-sm" onclick="profVoiceDelete('${escapeHtml(v.name)}')">Delete</button>
</div>`).join('');
</div>`;
}).join('');
}
async function profVoiceDelete(name) {
......@@ -5654,9 +5760,8 @@ function profEnvShowCreate() {
document.querySelectorAll('input[name="pe-mode"]').forEach(r => r.checked = r.value === 'extract');
peModeChange();
pePopulateModelSelect();
$('prof-env-form').style.display = 'block';
}
function profEnvHideForm() { $('prof-env-form').style.display = 'none'; }
function profEnvHideForm() { profEnvShowCreate(); }
function peModeChange() {
const mode = (document.querySelector('input[name="pe-mode"]:checked') || {}).value || 'extract';
......@@ -5766,16 +5871,26 @@ function renderEnvList() {
const el = $('prof-env-list');
if (!el) return;
if (!_envProfiles.length) { el.innerHTML = '<div class="arch-empty">No environment profiles yet.</div>'; return; }
el.innerHTML = _envProfiles.map(p => `
<div class="arch-card" style="position:relative;padding:1rem">
<div style="font-weight:600">${escapeHtml(p.name)}</div>
${p.description ? `<div class="small muted">${escapeHtml(p.description)}</div>` : ''}
<div class="small muted">${p.image_count} reference image${p.image_count!==1?'s':''} · ${new Date(p.created_at*1000).toLocaleDateString()}</div>
<div style="display:flex;gap:.4rem;margin-top:.5rem">
<button class="btn btn-ghost btn-sm" onclick="profEnvView('${escapeHtml(p.name)}')">View</button>
<button class="btn btn-danger btn-sm" onclick="profEnvDelete('${escapeHtml(p.name)}')">Delete</button>
el.innerHTML = _envProfiles.map(p => {
const n = escapeHtml(p.name);
const thumb = p.image_count > 0
? `<img class="arch-thumb" src="/admin/api/environments/${encodeURIComponent(p.name)}/thumbnail" loading="lazy" onerror="this.outerHTML='<div class=arch-thumb-ph>\u{1F304}</div>'" onclick="profEnvView('${n}')">`
: `<div class="arch-thumb-ph" onclick="profEnvView('${n}')">\u{1F304}</div>`;
const date = new Date(p.created_at * 1000).toLocaleDateString();
return `
<div class="arch-card">
${thumb}
<div class="prof-card-info">
<div class="prof-card-name">${n}</div>
${p.description ? `<div class="prof-card-desc">${escapeHtml(p.description)}</div>` : ''}
<div class="prof-card-meta">${p.image_count} ref image${p.image_count!==1?'s':''} · ${date}</div>
<div class="prof-card-actions">
<button class="btn btn-ghost btn-sm" onclick="profEnvView('${n}')">View</button>
<button class="btn btn-danger btn-sm" onclick="profEnvDelete('${n}')">Delete</button>
</div>
</div>
</div>`).join('');
</div>`;
}).join('');
}
async function profEnvView(name) {
......
codai/admin/templates/models.html
View file @ 227c3dbd
......@@ -142,8 +142,8 @@
<div style="display:flex;align-items:center;gap:.35rem">
<span class="fl">Format</span>
<div class="tog-grp">
<button class="tog-btn on" data-val="gguf">GGUF</button>
<button class="tog-btn" data-val="all">All</button>
<button class="tog-btn" data-val="gguf">GGUF</button>
<button class="tog-btn on" data-val="all">All</button>
<button class="tog-btn" data-val="no-gguf">No GGUF</button>
</div>
</div>
......@@ -652,7 +652,7 @@ async function loadGlobalSettings(){
}
/* ── GGUF format toggle ──────────────────────────────── */
let _ggufMode = 'gguf';
let _ggufMode = 'all';
document.querySelectorAll('.tog-btn').forEach(btn=>{
btn.addEventListener('click',()=>{
document.querySelectorAll('.tog-btn').forEach(b=>b.classList.remove('on'));
......@@ -671,6 +671,7 @@ function getChips(id){return[...document.querySelectorAll('#'+id+' .chip.on')].m
let _results = [];
let _filesCache = {};
let _activeQuants = new Set();
let _cachedSearchIds = new Set(); // HF repo IDs (and GGUF source_repos) cached locally
function estimateModelSize(modelId){
const id = modelId.toLowerCase();
......@@ -735,12 +736,14 @@ async function doSearch(){
vramDot = `<span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:${color};margin-right:.35rem" title="Est. ${estSize}GB / ${vramAvail}GB available"></span>`;
}
const capBadges = fmtCapabilities(m.capabilities||[]);
const isDownloaded = _cachedSearchIds.has(m.id);
const downloadedBadge = isDownloaded ? `<span class="badge badge-ok" style="font-size:10px;padding:.1rem .35rem;margin-left:.4rem;vertical-align:middle">✓ local</span>` : '';
return `
<div style="padding:.75rem 0;border-bottom:1px solid var(--border)">
<div style="display:flex;align-items:flex-start;justify-content:space-between;gap:.5rem">
<div style="min-width:0;flex:1">
<div style="font-weight:500;font-size:13px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;display:flex;align-items:center"
title="${esc(m.id)}">${vramDot}${esc(m.id)}</div>
title="${esc(m.id)}">${vramDot}${esc(m.id)}${downloadedBadge}</div>
<div style="font-size:11px;color:var(--text-3);margin-top:.25rem;display:flex;align-items:center;gap:.5rem;flex-wrap:wrap">
${m.pipeline_tag?`<span class="badge badge-user">${esc(m.pipeline_tag)}</span>`:''}
${capBadges}
......@@ -1300,6 +1303,11 @@ async function loadCachedModels(){
'<th style="text-align:center;padding:.3rem .25rem;font-weight:700">Config</th>'+
'<th></th></tr></thead><tbody>'+rows.join('')+'</tbody></table>';
}
// Rebuild set of locally cached model IDs for search results indicator
_cachedSearchIds.clear();
(d.hf||[]).forEach(m => _cachedSearchIds.add(m.id));
(d.gguf||[]).forEach(f => { if (f.source_repo) _cachedSearchIds.add(f.source_repo); });
// Remove any previously rendered whisper-server card before inserting the new one
document.querySelectorAll('#ws-rendered-card').forEach(el=>el.remove());
const wsHtml = _renderWhisperServerRows(whisperModels);
......@@ -1374,6 +1382,31 @@ async function refreshLocal(){
loadGlobalSettings();
refreshLocal();
// ── Deep-link from Studio: /admin/models?tab=search&q=...&pipeline=...&gguf=...
(function applyDeepLink(){
const p = new URLSearchParams(location.search);
if (p.get('tab') !== 'search') return;
// Switch to the HF search tab
const tabBtn = document.querySelector('.tab[onclick*="search"]');
if (tabBtn) switchTab('search', tabBtn);
// Fill query
const q = p.get('q') || '';
const searchQ = document.getElementById('search-q');
if (searchQ) searchQ.value = q;
// Set pipeline filter
const pipeline = p.get('pipeline') || '';
const pipelineEl = document.getElementById('filter-pipeline');
if (pipelineEl && pipeline) pipelineEl.value = pipeline;
// Set gguf mode toggle
const gguf = p.get('gguf') || 'gguf';
document.querySelectorAll('.tog-btn').forEach(btn => {
btn.classList.toggle('on', btn.dataset.val === gguf);
if (btn.dataset.val === gguf) btn.click(); // fires the existing toggle handler
});
// Trigger search automatically
doSearch();
})();
async function clearCacheConfirm(type){
const labels = {hf:'HuggingFace', gguf:'GGUF', all:'ALL'};
if(!confirm(`Delete ${labels[type]} model cache? This cannot be undone.`)) return;
......@@ -1401,6 +1434,16 @@ async function deleteModelConfirm(idx){
}
/* ── type checkbox helpers ─────────────────────────────── */
function _autoDetectParser(path) {
const n = (path || '').toLowerCase();
if (n.includes('llama') || n.includes('codellama')) return 'llama';
if (n.includes('mistral') || n.includes('mixtral')) return 'mistral';
if (/phi[-_.]?\d|phi.*instruct/i.test(n)) return 'phi';
if (n.includes('gemma')) return 'gemma';
if (n.includes('qwen')) return 'qwen';
return 'auto';
}
function _capabilitiesToTypes(caps) {
const categories = new Set(), subs = new Set();
if (!caps || !caps.length) return {categories, subs};
......@@ -1628,7 +1671,7 @@ function openCfgModal(idx){
document.getElementById('cfg-offload-strategy').value = s.offload_strategy || 'auto';
document.getElementById('cfg-offload-dir').value = s.offload_dir || _defaultOffloadDir;
document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
document.getElementById('cfg-parser').value = s.parser || 'auto';
document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
document.getElementById('cfg-tools').checked = !!s.tools_closer_prompt;
document.getElementById('cfg-grammar').checked = !!s.grammar_guided;
openModal('cfg-modal');
......
codai/api/images.py
View file @ 227c3dbd
......@@ -414,6 +414,24 @@ def _load_diffusers_pipeline(model_name: str, global_args):
f"The model may be too large for available VRAM. Error: {e}"
)
# =====================================================================
# Proactive VRAM eviction before first load attempt
# =====================================================================
# HF repo IDs have no local file, so their size is unknown; we can't
# pre-compute needed_gb. Instead, evict if other models are loaded and
# free VRAM is below 15% of total (almost certainly an OOM on attempt 1).
if torch.cuda.is_available():
try:
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
_free, _total = torch.cuda.mem_get_info()
if _total > 0 and (_free / _total) < 0.15:
print(f"Low VRAM ({_free/1e9:.1f} GB free of {_total/1e9:.1f} GB) with "
f"{len(_mmm.models)} model(s) loaded — evicting before load attempt")
_mmm.unload_all_models()
except Exception:
pass
# =====================================================================
# Standard loading path (with OOM fallback)
# =====================================================================
......@@ -467,6 +485,16 @@ def _load_diffusers_pipeline(model_name: str, global_args):
if is_oom and load_attempt < max_attempts:
print(f"OOM during model loading: {load_error}")
# Evict other loaded models from VRAM before retrying
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
print(f"Evicting {len(_mmm.models)} loaded model(s) to free VRAM for retry...")
_mmm.unload_all_models()
else:
import gc as _gc
_gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
print(f"Retrying with more aggressive memory optimization...")
pipeline = None # Reset for retry
else:
......@@ -1156,9 +1184,14 @@ def _load_img2img_pipeline(model_name: str, global_args):
except RuntimeError as e:
if 'out of memory' in str(e).lower() and attempt < 2:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
print(f"OOM loading img2img — evicting {len(_mmm.models)} model(s) to free VRAM...")
_mmm.unload_all_models()
else:
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
continue
raise
......@@ -1289,9 +1322,14 @@ def _load_inpaint_pipeline(model_name: str, global_args):
except RuntimeError as e:
if 'out of memory' in str(e).lower() and attempt < 2:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
print(f"OOM loading inpaint — evicting {len(_mmm.models)} model(s) to free VRAM...")
_mmm.unload_all_models()
else:
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
continue
raise
......
codai/api/log.py
View file @ 227c3dbd
......@@ -80,21 +80,22 @@ async def log_requests(request: Request, call_next):
body = b""
body_str = ""
model = "—"
try:
body = await request.body()
body_str = body.decode('utf-8')
parsed = json.loads(body_str)
model = parsed.get("model", "—")
if global_debug:
print(f"\n{'='*80}")
print(f"=== FULL REQUEST DEBUG ===")
print(f"Method: {request.method} URL: {request.url}")
print(json.dumps(parsed, indent=2))
print(f"{'='*80}\n")
except Exception as e:
if global_debug:
print(f"Error reading request body: {e}")
if request.method in ("POST", "PUT", "PATCH"):
try:
body = await request.body()
body_str = body.decode('utf-8')
parsed = json.loads(body_str)
model = parsed.get("model", "—")
if global_debug:
print(f"\n{'='*80}")
print(f"=== FULL REQUEST DEBUG ===")
print(f"Method: {request.method} URL: {request.url}")
print(json.dumps(parsed, indent=2))
print(f"{'='*80}\n")
except Exception as e:
if global_debug:
print(f"Error reading request body: {e}")
t0 = time.time()
response = await call_next(request)
......
codai/models/manager.py
View file @ 227c3dbd
......@@ -513,6 +513,7 @@ class MultiModelManager:
self.model_pools: Dict[str, ModelInstancePool] = {} # per-key instance pools
self._pending_new_instance: set = set() # keys awaiting a second+ instance load
self._global_max_instances: int = 1 # set from config at startup
self._measured_vram_gb: Dict[str, float] = {} # actual measured VRAM delta per model key
@property
def image_model(self) -> Optional[str]:
......@@ -681,8 +682,10 @@ class MultiModelManager:
kwargs['max_gpu_percent'] = global_args.max_gpu_percent
print(f"Loading default model on demand: {self.default_model}")
_snap = self.vram_before_load()
model_manager.load_model(self.default_model, backend_type=backend_type, **kwargs)
self.add_model(self.default_model, model_manager)
self.record_vram_delta(self.default_model, _snap)
self.current_model_key = self.default_model
print(f"Model loaded successfully: {self.default_model}")
return model_manager
......@@ -739,8 +742,10 @@ class MultiModelManager:
inst_num = pool.count + 1 if pool else 1
print(f"Loading model on demand: {model_name}"
+ (f" (instance {inst_num})" if inst_num > 1 else ""))
_snap = self.vram_before_load()
model_manager.load_model(model_name, backend_type=backend_type, **kwargs)
self.add_model(model_name, model_manager)
self.record_vram_delta(model_name, _snap)
self.current_model_key = model_name
print(f"Model loaded successfully: {model_name}"
+ (f" (instance {inst_num})" if inst_num > 1 else ""))
......@@ -748,7 +753,7 @@ class MultiModelManager:
except Exception as e:
print(f"Error loading model {model_name}: {e}")
return None
def set_audio_model(self, model_name: str, config: Dict = None):
"""Add an audio transcription model and download/cache it if needed."""
if model_name not in self.audio_models:
......@@ -1401,28 +1406,169 @@ class MultiModelManager:
pass
return 999.0 # Unknown — assume enough
@staticmethod
def _free_vram_snapshot() -> float:
"""Return current free VRAM in bytes (CUDA) or -1 if unavailable."""
try:
import torch
if torch.cuda.is_available():
free, _ = torch.cuda.mem_get_info()
return float(free)
except Exception:
pass
return -1.0
def vram_before_load(self) -> float:
"""Call immediately before loading a model; returns a snapshot for delta measurement."""
return self._free_vram_snapshot()
def record_vram_delta(self, model_key: str, free_before: float) -> None:
"""Call immediately after a model finishes loading to record actual VRAM consumed.
If the measured value exceeds the stored estimate by more than 10%, the real
value is written back into the model config and persisted to models.json so
future eviction decisions use the accurate figure.
"""
if free_before < 0:
return
free_after = self._free_vram_snapshot()
if free_after < 0:
return
delta_gb = (free_before - free_after) / 1e9
if delta_gb <= 0:
return
measured = round(delta_gb, 3)
# Read the pre-existing estimate BEFORE storing the measurement so that
# _get_model_used_vram_gb doesn't return the measured value as the baseline.
estimated = self._get_model_used_vram_gb(model_key)
self._measured_vram_gb[model_key] = measured
print(f"Measured VRAM for '{model_key}': {measured:.2f} GB")
# Persist the real value when it's meaningfully larger than the current estimate
try:
if estimated <= 0 or measured > estimated * 1.10:
label = f"{estimated:.2f} GB estimated" if estimated > 0 else "no estimate"
print(f" Updating used_vram_gb for '{model_key}': {label} → {measured:.2f} GB (real)")
# Update in-memory config
cfg = self.config.get(model_key, {})
cfg["used_vram_gb"] = measured
self.config[model_key] = cfg
# Persist to models.json via config_manager
try:
from codai.admin.routes import config_manager
if config_manager is not None:
# Strip type prefix to get the bare path/id used in models_data
bare = model_key.split(":", 1)[1] if ":" in model_key else model_key
for cat in ("text_models", "image_models", "audio_models", "tts_models",
"vision_models", "video_models", "audio_gen_models",
"embedding_models"):
for entry in config_manager.models_data.get(cat, []):
if not isinstance(entry, dict):
continue
epath = entry.get("path") or entry.get("id") or ""
if epath == bare or epath.split("/")[-1] == bare.split("/")[-1]:
entry["used_vram_gb"] = measured
config_manager.save_models()
except Exception as e:
print(f" Warning: could not persist used_vram_gb: {e}")
except Exception as e:
print(f" Warning: VRAM update check failed: {e}")
# In-process cache: model_id -> size_bytes (never stale within a run)
_hf_size_cache: Dict[str, int] = {}
@staticmethod
def _hf_cached_model_size_bytes(model_id: str) -> int:
"""Return total weight-file bytes for a HuggingFace model.
Tries in order:
1. In-process memory cache (no I/O).
2. Local HuggingFace hub cache scan.
3. HuggingFace API (network, one call per model per process lifetime).
Returns 0 on any failure.
"""
if model_id in MultiModelManager._hf_size_cache:
return MultiModelManager._hf_size_cache[model_id]
weight_exts = {'.safetensors', '.bin', '.gguf', '.ggml', '.pt'}
# --- Try local HF hub cache first (no network) ---
try:
from huggingface_hub import scan_cache_dir
from codai.models.cache import get_all_cache_dirs
hf_dir = get_all_cache_dirs().get("huggingface")
if hf_dir:
info = scan_cache_dir(hf_dir)
for repo in info.repos:
if repo.repo_id != model_id:
continue
revs = sorted(repo.revisions, key=lambda r: r.last_modified, reverse=True)
if revs:
total = sum(
f.size_on_disk
for f in revs[0].files
if os.path.splitext(f.file_name)[1].lower() in weight_exts
)
if total > 0:
MultiModelManager._hf_size_cache[model_id] = total
return total
except Exception:
pass
# --- Fallback: HuggingFace API (one network call, result cached) ---
try:
import urllib.request, urllib.parse, json as _json
safe_id = urllib.parse.quote(model_id, safe="/")
url = f"https://huggingface.co/api/models/{safe_id}"
req = urllib.request.Request(url, headers={"User-Agent": "coderai/1.0"})
with urllib.request.urlopen(req, timeout=10) as resp:
data = _json.loads(resp.read())
total = 0
for sib in data.get("siblings", []):
name = sib.get("rfilename", "")
if os.path.splitext(name)[1].lower() not in weight_exts:
continue
lfs = sib.get("lfs") or {}
size = lfs.get("size") or sib.get("size") or 0
total += size
if total > 0:
MultiModelManager._hf_size_cache[model_id] = total
return total
except Exception:
pass
# Cache the miss too so we don't hammer the API on every call
MultiModelManager._hf_size_cache[model_id] = 0
return 0
def _get_model_used_vram_gb(self, model_key: str, resolved_name: str = None) -> float:
"""Return VRAM requirement in GB for a model.
Uses configured used_vram_gb when set; otherwise estimates from file size
for local model files (GGUF, GGML, whisper .bin, etc.) — weights + KV cache
+ 15% buffer. Returns 0 when the requirement cannot be determined.
Priority:
1. Explicit ``used_vram_gb`` in the model's config entry.
2. File size of a local weight file (GGUF/GGML/bin/safetensors).
3. File size of matching weight files in the HuggingFace hub cache.
Returns 0 when the requirement cannot be determined.
"""
cfg = self.config.get(model_key, {})
explicit = cfg.get("used_vram_gb")
if explicit:
return float(explicit)
# Build a list of candidate paths to check
# Measured actual VRAM delta from when this model was loaded
measured = self._measured_vram_gb.get(model_key)
if measured:
return measured
# Build a list of candidate paths/IDs to check
candidates = []
if resolved_name:
candidates.append(resolved_name)
# model_key is often the path for text GGUF models
candidates.append(model_key)
# Strip type prefix (e.g. "audio:/path/to/model.bin")
if ":" in model_key:
candidates.append(model_key.split(":", 1)[1])
# Config may store the path explicitly
for field in ("path", "model_path", "model"):
v = cfg.get(field)
if v:
......@@ -1430,6 +1576,8 @@ class MultiModelManager:
import os
local_exts = {'.gguf', '.ggml', '.bin', '.pt', '.safetensors'}
# --- Pass 1: local file with a known extension ---
for path in candidates:
if not path:
continue
......@@ -1439,18 +1587,28 @@ class MultiModelManager:
try:
size_bytes = os.path.getsize(path)
weights_gb = size_bytes / 1e9
# KV cache: 2 bytes × 2 (K+V) × n_layers × n_ctx × head_dim
# We don't know the architecture, so estimate from n_ctx alone.
# Empirically ~0.5 MB per 1 K tokens covers most small models.
n_ctx = cfg.get("n_ctx") or 2048
kv_cache_gb = (n_ctx / 1024) * 0.5 / 1024 # 0.5 MB per 1 K ctx → GB
# 15% overhead for activations, graph buffers, scratch space
return weights_gb * 1.15 + kv_cache_gb
except OSError:
continue
# --- Pass 2: HuggingFace repo ID → scan local HF hub cache ---
from codai.models.cache import is_huggingface_model_id
for candidate in candidates:
if not candidate or not is_huggingface_model_id(candidate):
continue
# Strip any type prefix that survived (e.g. "image:org/repo")
repo_id = candidate.split(":", 1)[1] if ":" in candidate else candidate
if not is_huggingface_model_id(repo_id):
continue
size_bytes = self._hf_cached_model_size_bytes(repo_id)
if size_bytes > 0:
weights_gb = size_bytes / 1e9
# Diffusers / transformers models don't have a KV-cache term;
# use 20% overhead for activations, optimizer states, scratch buffers.
return weights_gb * 1.2
return 0
def _evict_models_for_vram(self, needed_gb: float):
......@@ -1477,18 +1635,28 @@ class MultiModelManager:
except Exception:
pass
def _size_label(key: str) -> str:
m = self._measured_vram_gb.get(key)
e = self._get_model_used_vram_gb(key)
if m:
return f"measured {m:.2f} GB"
if e:
return f"estimated {e:.2f} GB"
return "size unknown"
# First pass: evict non-active models in LRU order
for key in list(self.models.keys()):
if key == self.active_in_vram:
continue
if self._get_free_vram_gb() >= needed_gb:
break
print(f"On-request VRAM eviction: unloading '{key}' to free VRAM")
print(f"On-request VRAM eviction: unloading '{key}' ({_size_label(key)}) to free VRAM")
_evict_key(key)
# Second pass: evict active model if still not enough
if self._get_free_vram_gb() < needed_gb and self.active_in_vram and self.active_in_vram in self.models:
print(f"On-request VRAM eviction: unloading active model '{self.active_in_vram}' to free VRAM")
print(f"On-request VRAM eviction: unloading active model '{self.active_in_vram}' "
f"({_size_label(self.active_in_vram)}) to free VRAM")
_evict_key(self.active_in_vram)
self.active_in_vram = None
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment