Skip to content

C
coderai
Commit 227c3dbd authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

Small interface details

parent 0ce4d5af
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 594 additions and 165 deletions
codai/admin/routes.py
View file @ 227c3dbd
......@@ -912,6 +912,7 @@ def _scan_caches() -> dict:
"model_type": cfg[1] if cfg[1] and cfg[1] != "gguf_models" else "text_models",
"settings": cfg[0] if isinstance(cfg[0], dict) else {},
"capabilities": caps.to_list(),
"source_repo": repo.repo_id,
})
continue # skip adding to hf list
......@@ -1275,9 +1276,24 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
if result.get("already_loaded"):
return {"success": True, "already_loaded": True}
# Proactive VRAM check: evict loaded models if free VRAM is insufficient.
# request_model handles this for "on-request" load mode; cover remaining modes here.
# HuggingFace repo IDs return needed_gb==0 (size unknown); handle that case too.
model_key = result.get("model_key") or path
needed_gb = multi_model_manager._get_model_used_vram_gb(model_key, path)
free_gb = multi_model_manager._get_free_vram_gb()
if needed_gb > 0 and free_gb < needed_gb:
print(f"Admin model-load: need {needed_gb:.1f} GB VRAM, have {free_gb:.1f} GB free — evicting models")
multi_model_manager._evict_models_for_vram(needed_gb)
elif needed_gb == 0 and multi_model_manager.models and free_gb < 4.0:
# Unknown model size but VRAM nearly full — evict everything to avoid OOM on first attempt
print(f"Admin model-load: unknown model size, only {free_gb:.1f} GB free — evicting models proactively")
multi_model_manager.unload_all_models()
# Not loaded yet — trigger actual load
try:
if model_type == "text":
# _load_model_by_name already records the VRAM delta internally
mm = multi_model_manager._load_model_by_name(result["model_name"] or path)
if mm is None:
raise RuntimeError("Model failed to load")
......@@ -1286,6 +1302,7 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
elif model_type == "audio":
wsm = multi_model_manager.whisper_servers.get(path)
if wsm is not None:
_snap = multi_model_manager.vram_before_load()
started = wsm.start(getattr(wsm, "_model_path", None), gpu_device=getattr(wsm, "_gpu_device", 0))
if not wsm.is_running():
raise RuntimeError("whisper-server failed to start")
......@@ -1293,12 +1310,14 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
multi_model_manager.models[model_key] = wsm
multi_model_manager.active_in_vram = model_key
multi_model_manager.models_in_vram.add(model_key)
multi_model_manager.record_vram_delta(model_key, _snap)
return {"success": True, "already_loaded": False, "started_model": started}
elif model_type == "image":
from codai.api.images import _load_diffusers_pipeline, _is_gguf_model, _load_sdcpp_model
from codai.api.state import get_global_args
global_args = get_global_args()
model_key = f"image:{path}"
_snap = multi_model_manager.vram_before_load()
if _is_gguf_model(path):
resolved = multi_model_manager.load_model(path)
import os as _os
......@@ -1306,10 +1325,12 @@ async def api_model_load(request: Request, username: str = Depends(require_admin
sd_model = _load_sdcpp_model(resolved, global_args)
if sd_model:
multi_model_manager.add_model(model_key, sd_model)
multi_model_manager.record_vram_delta(model_key, _snap)
else:
pipeline = _load_diffusers_pipeline(path, global_args)
if pipeline:
multi_model_manager.add_model(model_key, pipeline)
multi_model_manager.record_vram_delta(model_key, _snap)
return {"success": True, "already_loaded": False}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
......@@ -1453,10 +1474,16 @@ async def api_model_configure(request: Request, username: str = Depends(require_
used_vram_gb = data.get("used_vram_gb")
if used_vram_gb is None:
import os
from codai.models.cache import is_huggingface_model_id
if os.path.isfile(path):
size_bytes = os.path.getsize(path)
multiplier = 1.1 if path.endswith(".gguf") else 1.2
used_vram_gb = round(size_bytes / 1e9 * multiplier, 2)
elif is_huggingface_model_id(path):
from codai.models.manager import MultiModelManager
size_bytes = MultiModelManager._hf_cached_model_size_bytes(path)
if size_bytes > 0:
used_vram_gb = round(size_bytes / 1e9 * 1.2, 2)
# Build settings entry
entry: dict = {"path": path, "model_type": model_types[0], "model_types": model_types}
......@@ -1912,6 +1939,24 @@ async def api_get_character(name: str, username: str = Depends(require_auth)):
}
@router.get("/admin/api/characters/{name}/thumbnail")
async def api_character_thumbnail(name: str, username: str = Depends(require_auth)):
import os as _os
from codai.api.characters import _char_dir, _load_character_meta
from fastapi.responses import FileResponse
meta = _load_character_meta(name)
if not meta:
raise HTTPException(status_code=404)
cdir = _char_dir(name)
for img_info in meta.get('images', []):
fpath = _os.path.join(cdir, img_info['file'])
if _os.path.exists(fpath):
ext = img_info['file'].rsplit('.', 1)[-1].lower()
media_type = 'image/png' if ext == 'png' else 'image/jpeg'
return FileResponse(fpath, media_type=media_type)
raise HTTPException(status_code=404)
@router.delete("/admin/api/characters/{name}")
async def api_delete_character(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
......@@ -1949,6 +1994,24 @@ async def api_get_environment(name: str, username: str = Depends(require_auth)):
}
@router.get("/admin/api/environments/{name}/thumbnail")
async def api_environment_thumbnail(name: str, username: str = Depends(require_auth)):
import os as _os
from codai.api.environments import _env_dir, _load_environment_meta
from fastapi.responses import FileResponse
meta = _load_environment_meta(name)
if not meta:
raise HTTPException(status_code=404)
edir = _env_dir(name)
for img_info in meta.get('images', []):
fpath = _os.path.join(edir, img_info['file'])
if _os.path.exists(fpath):
ext = img_info['file'].rsplit('.', 1)[-1].lower()
media_type = 'image/png' if ext == 'png' else 'image/jpeg'
return FileResponse(fpath, media_type=media_type)
raise HTTPException(status_code=404)
@router.delete("/admin/api/environments/{name}")
async def api_delete_environment(name: str, username: str = Depends(require_auth)):
import os as _os, shutil
......
codai/admin/templates/base.html
View file @ 227c3dbd
......@@ -23,6 +23,7 @@
<div class="nav-links">
<a href="/admin" class="nav-link {% if request.url.path == '/admin' %}active{% endif %}">Overview</a>
<a href="/chat" class="nav-link {% if request.url.path == '/chat' %}active{% endif %}">Studio</a>
<a href="/docs" class="nav-link" target="_blank">API Docs</a>
{% if is_admin|default(false) %}
<a href="/admin/models" class="nav-link {% if '/models' in request.url.path %}active{% endif %}">Models</a>
<a href="/admin/tokens" class="nav-link {% if '/tokens' in request.url.path %}active{% endif %}">Tokens</a>
......
codai/admin/templates/chat.html
View file @ 227c3dbd
This diff is collapsed.
codai/admin/templates/models.html
View file @ 227c3dbd
......@@ -142,8 +142,8 @@
<div style="display:flex;align-items:center;gap:.35rem">
<span class="fl">Format</span>
<div class="tog-grp">
<button class="tog-btn on" data-val="gguf">GGUF</button>
<button class="tog-btn" data-val="all">All</button>
<button class="tog-btn" data-val="gguf">GGUF</button>
<button class="tog-btn on" data-val="all">All</button>
<button class="tog-btn" data-val="no-gguf">No GGUF</button>
</div>
</div>
......@@ -652,7 +652,7 @@ async function loadGlobalSettings(){
}
/* ── GGUF format toggle ──────────────────────────────── */
let _ggufMode = 'gguf';
let _ggufMode = 'all';
document.querySelectorAll('.tog-btn').forEach(btn=>{
btn.addEventListener('click',()=>{
document.querySelectorAll('.tog-btn').forEach(b=>b.classList.remove('on'));
......@@ -671,6 +671,7 @@ function getChips(id){return[...document.querySelectorAll('#'+id+' .chip.on')].m
let _results = [];
let _filesCache = {};
let _activeQuants = new Set();
let _cachedSearchIds = new Set(); // HF repo IDs (and GGUF source_repos) cached locally
function estimateModelSize(modelId){
const id = modelId.toLowerCase();
......@@ -735,12 +736,14 @@ async function doSearch(){
vramDot = `<span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:${color};margin-right:.35rem" title="Est. ${estSize}GB / ${vramAvail}GB available"></span>`;
}
const capBadges = fmtCapabilities(m.capabilities||[]);
const isDownloaded = _cachedSearchIds.has(m.id);
const downloadedBadge = isDownloaded ? `<span class="badge badge-ok" style="font-size:10px;padding:.1rem .35rem;margin-left:.4rem;vertical-align:middle">✓ local</span>` : '';
return `
<div style="padding:.75rem 0;border-bottom:1px solid var(--border)">
<div style="display:flex;align-items:flex-start;justify-content:space-between;gap:.5rem">
<div style="min-width:0;flex:1">
<div style="font-weight:500;font-size:13px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;display:flex;align-items:center"
title="${esc(m.id)}">${vramDot}${esc(m.id)}</div>
title="${esc(m.id)}">${vramDot}${esc(m.id)}${downloadedBadge}</div>
<div style="font-size:11px;color:var(--text-3);margin-top:.25rem;display:flex;align-items:center;gap:.5rem;flex-wrap:wrap">
${m.pipeline_tag?`<span class="badge badge-user">${esc(m.pipeline_tag)}</span>`:''}
${capBadges}
......@@ -1300,6 +1303,11 @@ async function loadCachedModels(){
'<th style="text-align:center;padding:.3rem .25rem;font-weight:700">Config</th>'+
'<th></th></tr></thead><tbody>'+rows.join('')+'</tbody></table>';
}
// Rebuild set of locally cached model IDs for search results indicator
_cachedSearchIds.clear();
(d.hf||[]).forEach(m => _cachedSearchIds.add(m.id));
(d.gguf||[]).forEach(f => { if (f.source_repo) _cachedSearchIds.add(f.source_repo); });
// Remove any previously rendered whisper-server card before inserting the new one
document.querySelectorAll('#ws-rendered-card').forEach(el=>el.remove());
const wsHtml = _renderWhisperServerRows(whisperModels);
......@@ -1374,6 +1382,31 @@ async function refreshLocal(){
loadGlobalSettings();
refreshLocal();
// ── Deep-link from Studio: /admin/models?tab=search&q=...&pipeline=...&gguf=...
(function applyDeepLink(){
const p = new URLSearchParams(location.search);
if (p.get('tab') !== 'search') return;
// Switch to the HF search tab
const tabBtn = document.querySelector('.tab[onclick*="search"]');
if (tabBtn) switchTab('search', tabBtn);
// Fill query
const q = p.get('q') || '';
const searchQ = document.getElementById('search-q');
if (searchQ) searchQ.value = q;
// Set pipeline filter
const pipeline = p.get('pipeline') || '';
const pipelineEl = document.getElementById('filter-pipeline');
if (pipelineEl && pipeline) pipelineEl.value = pipeline;
// Set gguf mode toggle
const gguf = p.get('gguf') || 'gguf';
document.querySelectorAll('.tog-btn').forEach(btn => {
btn.classList.toggle('on', btn.dataset.val === gguf);
if (btn.dataset.val === gguf) btn.click(); // fires the existing toggle handler
});
// Trigger search automatically
doSearch();
})();
async function clearCacheConfirm(type){
const labels = {hf:'HuggingFace', gguf:'GGUF', all:'ALL'};
if(!confirm(`Delete ${labels[type]} model cache? This cannot be undone.`)) return;
......@@ -1401,6 +1434,16 @@ async function deleteModelConfirm(idx){
}
/* ── type checkbox helpers ─────────────────────────────── */
function _autoDetectParser(path) {
const n = (path || '').toLowerCase();
if (n.includes('llama') || n.includes('codellama')) return 'llama';
if (n.includes('mistral') || n.includes('mixtral')) return 'mistral';
if (/phi[-_.]?\d|phi.*instruct/i.test(n)) return 'phi';
if (n.includes('gemma')) return 'gemma';
if (n.includes('qwen')) return 'qwen';
return 'auto';
}
function _capabilitiesToTypes(caps) {
const categories = new Set(), subs = new Set();
if (!caps || !caps.length) return {categories, subs};
......@@ -1628,7 +1671,7 @@ function openCfgModal(idx){
document.getElementById('cfg-offload-strategy').value = s.offload_strategy || 'auto';
document.getElementById('cfg-offload-dir').value = s.offload_dir || _defaultOffloadDir;
document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
document.getElementById('cfg-parser').value = s.parser || 'auto';
document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
document.getElementById('cfg-tools').checked = !!s.tools_closer_prompt;
document.getElementById('cfg-grammar').checked = !!s.grammar_guided;
openModal('cfg-modal');
......
codai/api/images.py
View file @ 227c3dbd
......@@ -414,6 +414,24 @@ def _load_diffusers_pipeline(model_name: str, global_args):
f"The model may be too large for available VRAM. Error: {e}"
)
# =====================================================================
# Proactive VRAM eviction before first load attempt
# =====================================================================
# HF repo IDs have no local file, so their size is unknown; we can't
# pre-compute needed_gb. Instead, evict if other models are loaded and
# free VRAM is below 15% of total (almost certainly an OOM on attempt 1).
if torch.cuda.is_available():
try:
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
_free, _total = torch.cuda.mem_get_info()
if _total > 0 and (_free / _total) < 0.15:
print(f"Low VRAM ({_free/1e9:.1f} GB free of {_total/1e9:.1f} GB) with "
f"{len(_mmm.models)} model(s) loaded — evicting before load attempt")
_mmm.unload_all_models()
except Exception:
pass
# =====================================================================
# Standard loading path (with OOM fallback)
# =====================================================================
......@@ -467,6 +485,16 @@ def _load_diffusers_pipeline(model_name: str, global_args):
if is_oom and load_attempt < max_attempts:
print(f"OOM during model loading: {load_error}")
# Evict other loaded models from VRAM before retrying
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
print(f"Evicting {len(_mmm.models)} loaded model(s) to free VRAM for retry...")
_mmm.unload_all_models()
else:
import gc as _gc
_gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
print(f"Retrying with more aggressive memory optimization...")
pipeline = None # Reset for retry
else:
......@@ -1156,9 +1184,14 @@ def _load_img2img_pipeline(model_name: str, global_args):
except RuntimeError as e:
if 'out of memory' in str(e).lower() and attempt < 2:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
print(f"OOM loading img2img — evicting {len(_mmm.models)} model(s) to free VRAM...")
_mmm.unload_all_models()
else:
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
continue
raise
......@@ -1289,9 +1322,14 @@ def _load_inpaint_pipeline(model_name: str, global_args):
except RuntimeError as e:
if 'out of memory' in str(e).lower() and attempt < 2:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
from codai.models.manager import multi_model_manager as _mmm
if _mmm.models:
print(f"OOM loading inpaint — evicting {len(_mmm.models)} model(s) to free VRAM...")
_mmm.unload_all_models()
else:
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
continue
raise
......
codai/api/log.py
View file @ 227c3dbd
......@@ -80,21 +80,22 @@ async def log_requests(request: Request, call_next):
body = b""
body_str = ""
model = "—"
try:
body = await request.body()
body_str = body.decode('utf-8')
parsed = json.loads(body_str)
model = parsed.get("model", "—")
if global_debug:
print(f"\n{'='*80}")
print(f"=== FULL REQUEST DEBUG ===")
print(f"Method: {request.method} URL: {request.url}")
print(json.dumps(parsed, indent=2))
print(f"{'='*80}\n")
except Exception as e:
if global_debug:
print(f"Error reading request body: {e}")
if request.method in ("POST", "PUT", "PATCH"):
try:
body = await request.body()
body_str = body.decode('utf-8')
parsed = json.loads(body_str)
model = parsed.get("model", "—")
if global_debug:
print(f"\n{'='*80}")
print(f"=== FULL REQUEST DEBUG ===")
print(f"Method: {request.method} URL: {request.url}")
print(json.dumps(parsed, indent=2))
print(f"{'='*80}\n")
except Exception as e:
if global_debug:
print(f"Error reading request body: {e}")
t0 = time.time()
response = await call_next(request)
......
codai/models/manager.py
View file @ 227c3dbd
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment