Skip to content

C
coderai
Commit f524118d authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

Parallel loading

parent 0cf7c3c7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 505 additions and 162 deletions
CoderAI.gif 0 → 100644
View file @ f524118d
This diff is collapsed.
README.md
View file @ f524118d
# CoderAI
![CoderAI](CoderAI.gif)
An OpenAI-compatible API server with web administration dashboard, supporting multiple GPU backends: NVIDIA (CUDA), AMD (Vulkan), and Intel (Vulkan). Configuration-driven architecture with per-model settings and full multi-modal support.
## Features
......@@ -72,6 +74,20 @@ Built-in multi-step pipelines callable from the API or web UI:
---
## Quick Start
```bash
git clone git@git.nexlab.net:nexlab/coderai.git
cd coderai
./build.sh all # build all backends (recommended)
source venv_all/bin/activate
python coderai # starts on http://127.0.0.1:8776
```
That's it. Open `http://127.0.0.1:8776/admin` and log in with `admin` / `admin`.
---
## Installation
### Prerequisites
......@@ -81,7 +97,7 @@ Built-in multi-step pipelines callable from the API or web UI:
- For AMD/Intel GPUs (Vulkan): Vulkan drivers and SDK
- For CPU-only: No additional requirements
### Quick Install with Build Script
### Build Script
```bash
git clone git@git.nexlab.net:nexlab/coderai.git
......@@ -159,16 +175,16 @@ python coderai --config /path/to/cfg # Custom config directory
python coderai --debug # Debug mode
```
Server starts on `http://0.0.0.0:8000`.
Server starts on `http://127.0.0.1:8776` by default.
### Access Points
| URL | Description |
|---|---|
| `http://localhost:8000/admin` | Admin dashboard |
| `http://localhost:8000/chat` | Web Studio (generation UI) |
| `http://localhost:8000/v1/*` | OpenAI-compatible API |
| `http://localhost:8000/docs` | Interactive API docs |
| `http://127.0.0.1:8776/admin` | Admin dashboard |
| `http://127.0.0.1:8776/chat` | Web Studio (generation UI) |
| `http://127.0.0.1:8776/v1/*` | OpenAI-compatible API |
| `http://127.0.0.1:8776/docs` | Interactive API docs |
Default credentials: `admin` / `admin` (prompted to change on first login).
......@@ -191,7 +207,7 @@ Config files live in `~/.coderai/` (or `--config` path):
```json
{
"server": { "host": "0.0.0.0", "port": 8000 },
"server": { "host": "127.0.0.1", "port": 8776 },
"backend": { "type": "auto" },
"models": { "default_load_mode": "ondemand" },
"offload": { "load_in_4bit": false, "flash_attention": false },
......@@ -399,10 +415,23 @@ MAX_JOBS=4 pip install flash-attn --no-build-isolation
---
## Developer
**Stefy Lanza** <stefy@nexlab.net>
## License
GNU General Public License v3.0 — see [LICENSE.md](LICENSE.md).
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
## Donations
If you find CoderAI useful, consider supporting development:
- **Bitcoin (BTC):** `bc1qcpt2uutqkz4456j5r78rjm3gwq03h5fpwmcc5u`
- **Ethereum (ETH):** `0xdA6dAb526515b5cb556d20269207D43fcc760E51`
## Contributing
Merge requests welcome.
......
codai/admin/routes.py
View file @ f524118d
......@@ -1188,15 +1188,22 @@ async def api_model_disable(request: Request, username: str = Depends(require_ad
"""Remove a model from models.json (keeps it cached locally)."""
if config_manager is None:
raise HTTPException(status_code=503, detail="Config manager not initialized")
import os as _os
data = await request.json()
path = data.get("path") or data.get("model_id", "")
# Also match by bare filename so entries stored without full path are caught
fname = _os.path.basename(path) if (_os.sep in path or "/" in path) else ""
def _matches(m_entry) -> bool:
key = m_entry if isinstance(m_entry, str) else m_entry.get("path", m_entry.get("id", ""))
return key == path or (fname and _os.path.basename(key) == fname)
changed = False
for cat in ("text_models", "image_models", "audio_models",
"gguf_models", "tts_models", "vision_models", "video_models",
"audio_gen_models", "embedding_models"):
lst = config_manager.models_data.get(cat, [])
new_lst = [m for m in lst
if (m if isinstance(m, str) else m.get("path", m.get("id", ""))) != path]
new_lst = [m for m in lst if not _matches(m)]
if len(new_lst) != len(lst):
config_manager.models_data[cat] = new_lst
changed = True
......@@ -1391,13 +1398,23 @@ async def api_model_configure(request: Request, username: str = Depends(require_
if not model_types:
model_types = ["text_models"]
# Remove from all categories (handles type changes)
# Remove from all categories (handles type changes and quant switches)
# paths_to_remove: the new path + the original path before a quant change
import os as _os
paths_to_remove = {path}
orig_path = (data.get("orig_path") or "").strip()
if orig_path and orig_path != path:
paths_to_remove.add(orig_path)
# Also match by bare filename so entries stored without full path are caught
fnames_to_remove = {_os.path.basename(p) for p in paths_to_remove if _os.sep in p or "/" in p}
def _should_remove(m_entry) -> bool:
key = m_entry if isinstance(m_entry, str) else m_entry.get("path", m_entry.get("id", ""))
return key in paths_to_remove or (fnames_to_remove and _os.path.basename(key) in fnames_to_remove)
for cat in valid | {"gguf_models"}:
lst = config_manager.models_data.get(cat, [])
config_manager.models_data[cat] = [
m for m in lst
if (m if isinstance(m, str) else m.get("path", m.get("id", ""))) != path
]
config_manager.models_data[cat] = [m for m in lst if not _should_remove(m)]
# Auto-estimate used_vram_gb from file size if not provided
used_vram_gb = data.get("used_vram_gb")
......@@ -1418,7 +1435,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
for key in ("alias", "backend", "load_mode", "n_gpu_layers", "n_ctx",
"max_gpu_percent", "manual_ram_gb", "load_in_4bit", "load_in_8bit",
"flash_attention", "no_ram", "offload_strategy", "offload_dir",
"system_prompt", "parser", "tools_closer_prompt", "grammar_guided"):
"system_prompt", "parser", "tools_closer_prompt", "grammar_guided",
"max_instances", "preload_all_instances"):
if key in data:
entry[key] = data[key]
......
codai/admin/templates/models.html
View file @ f524118d
......@@ -338,6 +338,7 @@
</div>
<div class="modal-body">
<input type="hidden" id="cfg-path">
<input type="hidden" id="cfg-orig-path">
<input type="hidden" id="cfg-orig-type">
<!-- identity -->
......@@ -386,7 +387,7 @@
</div>
<div class="form-row" style="margin:0">
<label class="form-label">Load mode</label>
<select id="cfg-load-mode" class="form-input">
<select id="cfg-load-mode" class="form-input" onchange="_updatePreloadAllVisibility()">
<option value="load">Load (pre-load in VRAM)</option>
<option value="on-request">On-request (load when needed)</option>
</select>
......@@ -398,6 +399,20 @@
<input type="number" id="cfg-used-vram" class="form-input" min="0" step="0.1" placeholder="auto-estimate">
<span class="form-hint" id="cfg-used-vram-hint" style="font-size:11px;color:var(--text-3)"></span>
</div>
<div class="form-row" style="margin:0">
<label class="form-label">Max instances
<span class="muted" title="How many copies of this model can be loaded simultaneously in VRAM to serve parallel requests. Requires enough free VRAM for each additional copy."> (?)</span>
</label>
<input type="number" id="cfg-max-instances" class="form-input" min="1" step="1" value="1" oninput="_updatePreloadAllVisibility()">
<span style="font-size:11px;color:var(--text-3)">Parallel copies in VRAM</span>
</div>
</div>
<div id="cfg-preload-all-row" style="display:none;margin-top:.5rem">
<label style="display:flex;align-items:center;gap:.5rem;font-size:13px;cursor:pointer">
<input type="checkbox" id="cfg-preload-all-instances">
Load all instances at boot
<span class="muted" style="font-size:11px" title="Pre-load all N copies of this model into VRAM at server startup instead of loading each copy on demand.">(load all ${max_instances} copies on startup)</span>
</label>
</div>
<!-- inference -->
......@@ -1354,6 +1369,7 @@ function openCfgModal(idx){
document.getElementById('cfg-modal-title').textContent = m.in_config ? 'Configure model' : 'Add to CoderAI';
document.getElementById('cfg-id-label').textContent = m.label;
document.getElementById('cfg-path').value = m.path;
document.getElementById('cfg-orig-path').value = m.path; // frozen at open time
document.getElementById('cfg-orig-type').value = m.defaultType;
// Quantization selector for grouped GGUF models
......@@ -1416,11 +1432,15 @@ function openCfgModal(idx){
// Used VRAM
const usedVram = s.used_vram_gb != null ? s.used_vram_gb : null;
document.getElementById('cfg-used-vram').value = usedVram != null ? usedVram : '';
// Show estimate hint from file size (GGUF: ~1.1x file size; HF: from size_gb)
const estVram = _estimateVram(m);
// Show estimate hint (weights × 1.15 + KV cache ~0.5 MB per 1 K ctx)
const nCtxForEst = s.n_ctx || 2048;
const estVram = _estimateVram(m, nCtxForEst);
document.getElementById('cfg-used-vram-hint').textContent = estVram ? `Estimated: ~${estVram.toFixed(1)} GB` : '';
document.getElementById('cfg-gpu-layers').value = s.n_gpu_layers !== undefined ? s.n_gpu_layers : -1;
document.getElementById('cfg-n-ctx').value = s.n_ctx || 2048;
document.getElementById('cfg-n-ctx').value = nCtxForEst;
document.getElementById('cfg-max-instances').value = s.max_instances != null ? s.max_instances : 1;
document.getElementById('cfg-preload-all-instances').checked = !!s.preload_all_instances;
_updatePreloadAllVisibility();
document.getElementById('cfg-max-gpu').value = s.max_gpu_percent != null ? s.max_gpu_percent : '';
document.getElementById('cfg-ram-gb').value = s.manual_ram_gb != null ? s.manual_ram_gb : '';
document.getElementById('cfg-4bit').checked = !!s.load_in_4bit;
......@@ -1439,10 +1459,19 @@ function openCfgModal(idx){
openModal('cfg-modal');
}
function _estimateVram(m) {
// Estimate VRAM from file size: GGUF ~1.1x, HF safetensors ~1.2x
if (m.size_gb) return m.size_gb * (m.cacheType === 'gguf' ? 1.1 : 1.2);
return null;
function _updatePreloadAllVisibility() {
const loadMode = document.getElementById('cfg-load-mode').value;
const maxInst = parseInt(document.getElementById('cfg-max-instances').value) || 1;
const row = document.getElementById('cfg-preload-all-row');
row.style.display = (loadMode === 'load' && maxInst > 1) ? '' : 'none';
}
function _estimateVram(m, nCtx) {
// Mirror the server-side formula: weights × 1.15 + KV cache (0.5 MB per 1 K ctx)
if (!m.size_gb) return null;
const weights = m.size_gb * (m.cacheType === 'gguf' ? 1.0 : 1.1);
const kvCacheGb = ((nCtx || 2048) / 1024) * 0.5 / 1024;
return weights * 1.15 + kvCacheGb;
}
async function saveModelConfig(){
......@@ -1451,8 +1480,10 @@ async function saveModelConfig(){
const ramGb = parseFloat(document.getElementById('cfg-ram-gb').value);
const usedVram = parseFloat(document.getElementById('cfg-used-vram').value);
const {primaryType, model_types, video_subtypes} = _getCheckedTypes();
const origPath = document.getElementById('cfg-orig-path').value;
const data = {
path,
orig_path: (origPath && origPath !== path) ? origPath : undefined,
model_type: primaryType,
model_types: model_types,
video_subtypes: video_subtypes.length ? video_subtypes : undefined,
......@@ -1460,6 +1491,8 @@ async function saveModelConfig(){
backend: document.getElementById('cfg-backend').value,
load_mode: document.getElementById('cfg-load-mode').value,
used_vram_gb: isNaN(usedVram) ? null : usedVram,
max_instances: parseInt(document.getElementById('cfg-max-instances').value) || 1,
preload_all_instances: document.getElementById('cfg-preload-all-instances').checked,
n_gpu_layers: parseInt(document.getElementById('cfg-gpu-layers').value) || -1,
n_ctx: parseInt(document.getElementById('cfg-n-ctx').value) || 2048,
max_gpu_percent: isNaN(maxGpu) ? null : maxGpu,
......@@ -1557,14 +1590,20 @@ async function unloadModel(idx){
async function disableModel(idx){
const m = _localModels[idx];
if(!confirm('Remove this model from CoderAI config? It will stay in the local cache.')) return;
// Collect all paths that are marked in_config for this model (covers quant groups)
const paths = m.ggufGroup
? m.ggufGroup.filter(f=>f.in_config).map(f=>f.path)
: (m.in_config ? [m.path] : []);
if(!paths.length) paths.push(m.path); // fallback
try{
const r = await fetch('/admin/api/model-disable',{
method:'POST', headers:{'Content-Type':'application/json'},
body: JSON.stringify({path: m.path})
});
const d = await r.json();
if(d.success) loadCachedModels();
else alert('Error: '+(d.detail||'Unknown'));
// Disable each configured path in the group
for(const p of paths){
await fetch('/admin/api/model-disable',{
method:'POST', headers:{'Content-Type':'application/json'},
body: JSON.stringify({path: p})
});
}
loadCachedModels();
}catch(e){ alert('Error: '+e.message); }
}
......
codai/api/text.py
View file @ f524118d
......@@ -293,19 +293,28 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
if model_info.get('error'):
raise HTTPException(status_code=404, detail=model_info['error'])
# Try to get the appropriate model (request_model handles VRAM cleanup)
mm = multi_model_manager.get_model_for_request(requested_model)
# Acquire the least-busy instance (increments ref-count; released on response completion)
_model_key = model_info.get('model_key')
_instance_idx = None
_acq = multi_model_manager.acquire_model_instance(_model_key) if _model_key else None
if _acq:
_instance_idx, mm = _acq
else:
mm = multi_model_manager.get_model_for_request(requested_model)
def _release_instance():
if _instance_idx is not None and _model_key:
multi_model_manager.release_model_instance(_model_key, _instance_idx)
if mm is None:
# Model not loaded - try to use default
_release_instance()
if model_manager.backend is not None:
# Fallback to legacy model_manager
current_manager = model_manager
else:
raise HTTPException(status_code=503, detail="Model not loaded")
else:
current_manager = mm
# Inject system prompt if --system-prompt flag was provided
messages = request.messages
global_system_prompt = get_global_system_prompt()
......@@ -816,9 +825,16 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
# No tool calls, yield the content as usual
yield f"data: {json.dumps({'choices': [{'delta': {'content': second_pass_result}, 'finish_reason': 'stop'}]})}\n\n"
yield "data: [DONE]\n\n"
async def _raw_stream_with_release():
try:
async for chunk in raw_stream_generate():
yield chunk
finally:
_release_instance()
from fastapi.responses import StreamingResponse
return StreamingResponse(raw_stream_generate(), media_type="text/event-stream")
return StreamingResponse(_raw_stream_with_release(), media_type="text/event-stream")
# Non-streaming path (already implemented above)
# First pass: generate until reasoning close tag
......@@ -1127,9 +1143,29 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
return JSONResponse(content=formatted_response, headers=headers)
if request.stream:
async def _managed_stream():
try:
async for chunk in stream_chat_response(
messages_dict,
response_model_name,
request.max_tokens,
request.temperature,
request.top_p,
stop_sequences,
tools_dict,
current_manager,
tool_parser,
request.response_format,
):
yield chunk
finally:
_release_instance()
from fastapi.responses import StreamingResponse
return StreamingResponse(
stream_chat_response(
return StreamingResponse(_managed_stream(), media_type="text/event-stream")
else:
try:
return await generate_chat_response(
messages_dict,
response_model_name,
request.max_tokens,
......@@ -1140,23 +1176,10 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
current_manager,
tool_parser,
request.response_format,
),
media_type="text/event-stream",
)
else:
return await generate_chat_response(
messages_dict,
response_model_name,
request.max_tokens,
request.temperature,
request.top_p,
stop_sequences,
tools_dict,
current_manager,
tool_parser,
request.response_format,
force_reasoning_args,
)
force_reasoning_args,
)
finally:
_release_instance()
async def stream_chat_response(
messages: List[Dict],
......@@ -1701,28 +1724,54 @@ async def completions(request: CompletionRequest):
if model_info.get('error'):
raise HTTPException(status_code=404, detail=model_info['error'])
# Try to get the appropriate model (request_model handles VRAM cleanup)
mm = multi_model_manager.get_model_for_request(requested_model)
# Acquire the least-busy instance (increments ref-count; released on response completion)
_model_key = model_info.get('model_key')
_instance_idx = None
_acq = multi_model_manager.acquire_model_instance(_model_key) if _model_key else None
if _acq:
_instance_idx, mm = _acq
else:
mm = multi_model_manager.get_model_for_request(requested_model)
def _release_instance():
if _instance_idx is not None and _model_key:
multi_model_manager.release_model_instance(_model_key, _instance_idx)
if mm is None:
# Model not loaded - try to use default
_release_instance()
if model_manager.backend is not None:
# Fallback to legacy model_manager
current_manager = model_manager
else:
raise HTTPException(status_code=503, detail="Model not loaded")
else:
current_manager = mm
prompts = request.prompt if isinstance(request.prompt, list) else [request.prompt]
stop_sequences = []
if request.stop:
stop_sequences = [request.stop] if isinstance(request.stop, str) else request.stop
if request.stream:
async def _managed_completion_stream():
try:
async for chunk in stream_completion_response(
prompts[0],
request.model,
request.max_tokens,
request.temperature,
request.top_p,
stop_sequences,
current_manager,
):
yield chunk
finally:
_release_instance()
from fastapi.responses import StreamingResponse
return StreamingResponse(
stream_completion_response(
return StreamingResponse(_managed_completion_stream(), media_type="text/event-stream")
else:
try:
return await generate_completion_response(
prompts[0],
request.model,
request.max_tokens,
......@@ -1730,19 +1779,9 @@ async def completions(request: CompletionRequest):
request.top_p,
stop_sequences,
current_manager,
),
media_type="text/event-stream",
)
else:
return await generate_completion_response(
prompts[0],
request.model,
request.max_tokens,
request.temperature,
request.top_p,
stop_sequences,
current_manager,
)
)
finally:
_release_instance()
async def stream_completion_response(
prompt: str,
......
codai/config.py
View file @ f524118d
......@@ -25,8 +25,8 @@ from dataclasses import dataclass, field
@dataclass
class ServerConfig:
"""Server configuration."""
host: str = "0.0.0.0"
port: int = 8000
host: str = "127.0.0.1"
port: int = 8776
https: bool = False
https_key_path: Optional[str] = None
https_cert_path: Optional[str] = None
......@@ -49,6 +49,7 @@ class ModelsConfig:
default_load_mode: str = "ondemand"
hf_cache_dir: Optional[str] = None
gguf_cache_dir: Optional[str] = None
max_model_instances: int = 1 # max concurrent instances per model (global default; overridable per-model via "max_instances")
@dataclass
......@@ -150,8 +151,8 @@ class ConfigManager:
default_config = {
"version": "1.0",
"server": {
"host": "0.0.0.0",
"port": 8000,
"host": "127.0.0.1",
"port": 8776,
"https": False,
"https_key_path": None,
"https_cert_path": None
......
codai/main.py
View file @ f524118d
......@@ -234,6 +234,7 @@ def main():
load_mode = config.models.default_load_mode or "ondemand"
set_load_mode(load_mode)
multi_model_manager.set_load_mode(load_mode)
multi_model_manager._global_max_instances = config.models.max_model_instances
print(f"\nLoad mode: {load_mode}")
if load_mode == "ondemand":
......@@ -463,6 +464,18 @@ def main():
if mm is not None and mm.backend is not None:
multi_model_manager.active_in_vram = mid
print(f" Loaded: {mid}")
# Load additional instances if preload_all_instances is set
if isinstance(m, dict) and m.get("preload_all_instances"):
max_inst = m.get("max_instances", 1)
if isinstance(max_inst, int) and max_inst > 1:
for i in range(max_inst - 1):
multi_model_manager._pending_new_instance.add(mid)
mm2 = multi_model_manager._load_model_by_name(mid)
if mm2 is not None and mm2.backend is not None:
print(f" Loaded extra instance {i+2}/{max_inst}: {mid}")
else:
print(f" Warning: extra instance {i+2}/{max_inst} of '{mid}' failed to load")
break
else:
print(f" Warning: {mid} failed to load")
elif mtype == "audio" and mid in multi_model_manager.whisper_servers:
......
codai/models/manager.py
View file @ f524118d
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment