Parallel loading

README.md

 # CoderAI
 
+![CoderAI](CoderAI.gif)
+
 An OpenAI-compatible API server with web administration dashboard, supporting multiple GPU backends: NVIDIA (CUDA), AMD (Vulkan), and Intel (Vulkan). Configuration-driven architecture with per-model settings and full multi-modal support.
 
 ## Features
@@ -72,6 +74,20 @@ Built-in multi-step pipelines callable from the API or web UI:
 
 ---
 
+## Quick Start
+
+```bash
+git clone git@git.nexlab.net:nexlab/coderai.git
+cd coderai
+./build.sh all          # build all backends (recommended)
+source venv_all/bin/activate
+python coderai          # starts on http://127.0.0.1:8776
+```
+
+That's it. Open `http://127.0.0.1:8776/admin` and log in with `admin` / `admin`.
+
+---
+
 ## Installation
 
 ### Prerequisites
@@ -81,7 +97,7 @@ Built-in multi-step pipelines callable from the API or web UI:
 - For AMD/Intel GPUs (Vulkan): Vulkan drivers and SDK
 - For CPU-only: No additional requirements
 
-### Quick Install with Build Script
+### Build Script
 
 ```bash
 git clone git@git.nexlab.net:nexlab/coderai.git
@@ -159,16 +175,16 @@ python coderai --config /path/to/cfg   # Custom config directory
 python coderai --debug                 # Debug mode
 ```
 
-Server starts on `http://0.0.0.0:8000`.
+Server starts on `http://127.0.0.1:8776` by default.
 
 ### Access Points
 
 | URL | Description |
 |---|---|
-| `http://localhost:8000/admin` | Admin dashboard |
-| `http://localhost:8000/chat` | Web Studio (generation UI) |
-| `http://localhost:8000/v1/*` | OpenAI-compatible API |
-| `http://localhost:8000/docs` | Interactive API docs |
+| `http://127.0.0.1:8776/admin` | Admin dashboard |
+| `http://127.0.0.1:8776/chat` | Web Studio (generation UI) |
+| `http://127.0.0.1:8776/v1/*` | OpenAI-compatible API |
+| `http://127.0.0.1:8776/docs` | Interactive API docs |
 
 Default credentials: `admin` / `admin` (prompted to change on first login).
 
@@ -191,7 +207,7 @@ Config files live in `~/.coderai/` (or `--config` path):
 
 ```json
 {
-  "server": { "host": "0.0.0.0", "port": 8000 },
+  "server": { "host": "127.0.0.1", "port": 8776 },
   "backend": { "type": "auto" },
   "models": { "default_load_mode": "ondemand" },
   "offload": { "load_in_4bit": false, "flash_attention": false },
@@ -399,10 +415,23 @@ MAX_JOBS=4 pip install flash-attn --no-build-isolation
 
 ---
 
+## Developer
+
+**Stefy Lanza** <stefy@nexlab.net>
+
 ## License
 
 GNU General Public License v3.0 — see [LICENSE.md](LICENSE.md).
 
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+## Donations
+
+If you find CoderAI useful, consider supporting development:
+
+- **Bitcoin (BTC):** `bc1qcpt2uutqkz4456j5r78rjm3gwq03h5fpwmcc5u`
+- **Ethereum (ETH):** `0xdA6dAb526515b5cb556d20269207D43fcc760E51`
+
 ## Contributing
 
 Merge requests welcome.

codai/admin/routes.py

@@ -1188,15 +1188,22 @@ async def api_model_disable(request: Request, username: str = Depends(require_ad
     """Remove a model from models.json (keeps it cached locally)."""
     if config_manager is None:
         raise HTTPException(status_code=503, detail="Config manager not initialized")
+    import os as _os
     data = await request.json()
     path = data.get("path") or data.get("model_id", "")
+    # Also match by bare filename so entries stored without full path are caught
+    fname = _os.path.basename(path) if (_os.sep in path or "/" in path) else ""
+
+    def _matches(m_entry) -> bool:
+        key = m_entry if isinstance(m_entry, str) else m_entry.get("path", m_entry.get("id", ""))
+        return key == path or (fname and _os.path.basename(key) == fname)
+
     changed = False
     for cat in ("text_models", "image_models", "audio_models",
                 "gguf_models", "tts_models", "vision_models", "video_models",
                 "audio_gen_models", "embedding_models"):
         lst = config_manager.models_data.get(cat, [])
-        new_lst = [m for m in lst
-                   if (m if isinstance(m, str) else m.get("path", m.get("id", ""))) != path]
+        new_lst = [m for m in lst if not _matches(m)]
         if len(new_lst) != len(lst):
             config_manager.models_data[cat] = new_lst
             changed = True
@@ -1391,13 +1398,23 @@ async def api_model_configure(request: Request, username: str = Depends(require_
     if not model_types:
         model_types = ["text_models"]
 
-    # Remove from all categories (handles type changes)
+    # Remove from all categories (handles type changes and quant switches)
+    # paths_to_remove: the new path + the original path before a quant change
+    import os as _os
+    paths_to_remove = {path}
+    orig_path = (data.get("orig_path") or "").strip()
+    if orig_path and orig_path != path:
+        paths_to_remove.add(orig_path)
+    # Also match by bare filename so entries stored without full path are caught
+    fnames_to_remove = {_os.path.basename(p) for p in paths_to_remove if _os.sep in p or "/" in p}
+
+    def _should_remove(m_entry) -> bool:
+        key = m_entry if isinstance(m_entry, str) else m_entry.get("path", m_entry.get("id", ""))
+        return key in paths_to_remove or (fnames_to_remove and _os.path.basename(key) in fnames_to_remove)
+
     for cat in valid | {"gguf_models"}:
         lst = config_manager.models_data.get(cat, [])
-        config_manager.models_data[cat] = [
-            m for m in lst
-            if (m if isinstance(m, str) else m.get("path", m.get("id", ""))) != path
-        ]
+        config_manager.models_data[cat] = [m for m in lst if not _should_remove(m)]
 
     # Auto-estimate used_vram_gb from file size if not provided
     used_vram_gb = data.get("used_vram_gb")
@@ -1418,7 +1435,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
     for key in ("alias", "backend", "load_mode", "n_gpu_layers", "n_ctx",
                 "max_gpu_percent", "manual_ram_gb", "load_in_4bit", "load_in_8bit",
                 "flash_attention", "no_ram", "offload_strategy", "offload_dir",
-                "system_prompt", "parser", "tools_closer_prompt", "grammar_guided"):
+                "system_prompt", "parser", "tools_closer_prompt", "grammar_guided",
+                "max_instances", "preload_all_instances"):
         if key in data:
             entry[key] = data[key]
 

codai/admin/templates/models.html

@@ -338,6 +338,7 @@
     </div>
     <div class="modal-body">
       <input type="hidden" id="cfg-path">
+      <input type="hidden" id="cfg-orig-path">
       <input type="hidden" id="cfg-orig-type">
 
       <!-- identity -->
@@ -386,7 +387,7 @@
         </div>
         <div class="form-row" style="margin:0">
           <label class="form-label">Load mode</label>
-          <select id="cfg-load-mode" class="form-input">
+          <select id="cfg-load-mode" class="form-input" onchange="_updatePreloadAllVisibility()">
             <option value="load">Load (pre-load in VRAM)</option>
             <option value="on-request">On-request (load when needed)</option>
           </select>
@@ -398,6 +399,20 @@
           <input type="number" id="cfg-used-vram" class="form-input" min="0" step="0.1" placeholder="auto-estimate">
           <span class="form-hint" id="cfg-used-vram-hint" style="font-size:11px;color:var(--text-3)"></span>
         </div>
+        <div class="form-row" style="margin:0">
+          <label class="form-label">Max instances
+            <span class="muted" title="How many copies of this model can be loaded simultaneously in VRAM to serve parallel requests. Requires enough free VRAM for each additional copy."> (?)</span>
+          </label>
+          <input type="number" id="cfg-max-instances" class="form-input" min="1" step="1" value="1" oninput="_updatePreloadAllVisibility()">
+          <span style="font-size:11px;color:var(--text-3)">Parallel copies in VRAM</span>
+        </div>
+      </div>
+      <div id="cfg-preload-all-row" style="display:none;margin-top:.5rem">
+        <label style="display:flex;align-items:center;gap:.5rem;font-size:13px;cursor:pointer">
+          <input type="checkbox" id="cfg-preload-all-instances">
+          Load all instances at boot
+          <span class="muted" style="font-size:11px" title="Pre-load all N copies of this model into VRAM at server startup instead of loading each copy on demand.">(load all ${max_instances} copies on startup)</span>
+        </label>
       </div>
 
       <!-- inference -->
@@ -1354,6 +1369,7 @@ function openCfgModal(idx){
   document.getElementById('cfg-modal-title').textContent = m.in_config ? 'Configure model' : 'Add to CoderAI';
   document.getElementById('cfg-id-label').textContent = m.label;
   document.getElementById('cfg-path').value = m.path;
+  document.getElementById('cfg-orig-path').value = m.path;   // frozen at open time
   document.getElementById('cfg-orig-type').value = m.defaultType;
 
   // Quantization selector for grouped GGUF models
@@ -1416,11 +1432,15 @@ function openCfgModal(idx){
   // Used VRAM
   const usedVram = s.used_vram_gb != null ? s.used_vram_gb : null;
   document.getElementById('cfg-used-vram').value = usedVram != null ? usedVram : '';
-  // Show estimate hint from file size (GGUF: ~1.1x file size; HF: from size_gb)
-  const estVram = _estimateVram(m);
+  // Show estimate hint (weights × 1.15 + KV cache ~0.5 MB per 1 K ctx)
+  const nCtxForEst = s.n_ctx || 2048;
+  const estVram = _estimateVram(m, nCtxForEst);
   document.getElementById('cfg-used-vram-hint').textContent = estVram ? `Estimated: ~${estVram.toFixed(1)} GB` : '';
   document.getElementById('cfg-gpu-layers').value = s.n_gpu_layers !== undefined ? s.n_gpu_layers : -1;
-  document.getElementById('cfg-n-ctx').value = s.n_ctx || 2048;
+  document.getElementById('cfg-n-ctx').value = nCtxForEst;
+  document.getElementById('cfg-max-instances').value = s.max_instances != null ? s.max_instances : 1;
+  document.getElementById('cfg-preload-all-instances').checked = !!s.preload_all_instances;
+  _updatePreloadAllVisibility();
   document.getElementById('cfg-max-gpu').value = s.max_gpu_percent != null ? s.max_gpu_percent : '';
   document.getElementById('cfg-ram-gb').value = s.manual_ram_gb != null ? s.manual_ram_gb : '';
   document.getElementById('cfg-4bit').checked = !!s.load_in_4bit;
@@ -1439,10 +1459,19 @@ function openCfgModal(idx){
   openModal('cfg-modal');
 }
 
-function _estimateVram(m) {
-  // Estimate VRAM from file size: GGUF ~1.1x, HF safetensors ~1.2x
-  if (m.size_gb) return m.size_gb * (m.cacheType === 'gguf' ? 1.1 : 1.2);
-  return null;
+function _updatePreloadAllVisibility() {
+  const loadMode = document.getElementById('cfg-load-mode').value;
+  const maxInst = parseInt(document.getElementById('cfg-max-instances').value) || 1;
+  const row = document.getElementById('cfg-preload-all-row');
+  row.style.display = (loadMode === 'load' && maxInst > 1) ? '' : 'none';
+}
+
+function _estimateVram(m, nCtx) {
+  // Mirror the server-side formula: weights × 1.15 + KV cache (0.5 MB per 1 K ctx)
+  if (!m.size_gb) return null;
+  const weights = m.size_gb * (m.cacheType === 'gguf' ? 1.0 : 1.1);
+  const kvCacheGb = ((nCtx || 2048) / 1024) * 0.5 / 1024;
+  return weights * 1.15 + kvCacheGb;
 }
 
 async function saveModelConfig(){
@@ -1451,8 +1480,10 @@ async function saveModelConfig(){
   const ramGb  = parseFloat(document.getElementById('cfg-ram-gb').value);
   const usedVram = parseFloat(document.getElementById('cfg-used-vram').value);
   const {primaryType, model_types, video_subtypes} = _getCheckedTypes();
+  const origPath = document.getElementById('cfg-orig-path').value;
   const data = {
     path,
+    orig_path:         (origPath && origPath !== path) ? origPath : undefined,
     model_type:        primaryType,
     model_types:       model_types,
     video_subtypes:    video_subtypes.length ? video_subtypes : undefined,
@@ -1460,6 +1491,8 @@ async function saveModelConfig(){
     backend:           document.getElementById('cfg-backend').value,
     load_mode:         document.getElementById('cfg-load-mode').value,
     used_vram_gb:      isNaN(usedVram) ? null : usedVram,
+    max_instances:     parseInt(document.getElementById('cfg-max-instances').value) || 1,
+    preload_all_instances: document.getElementById('cfg-preload-all-instances').checked,
     n_gpu_layers:      parseInt(document.getElementById('cfg-gpu-layers').value) || -1,
     n_ctx:             parseInt(document.getElementById('cfg-n-ctx').value) || 2048,
     max_gpu_percent:   isNaN(maxGpu) ? null : maxGpu,
@@ -1557,14 +1590,20 @@ async function unloadModel(idx){
 async function disableModel(idx){
   const m = _localModels[idx];
   if(!confirm('Remove this model from CoderAI config? It will stay in the local cache.')) return;
+  // Collect all paths that are marked in_config for this model (covers quant groups)
+  const paths = m.ggufGroup
+    ? m.ggufGroup.filter(f=>f.in_config).map(f=>f.path)
+    : (m.in_config ? [m.path] : []);
+  if(!paths.length) paths.push(m.path);  // fallback
   try{
-    const r = await fetch('/admin/api/model-disable',{
-      method:'POST', headers:{'Content-Type':'application/json'},
-      body: JSON.stringify({path: m.path})
-    });
-    const d = await r.json();
-    if(d.success) loadCachedModels();
-    else alert('Error: '+(d.detail||'Unknown'));
+    // Disable each configured path in the group
+    for(const p of paths){
+      await fetch('/admin/api/model-disable',{
+        method:'POST', headers:{'Content-Type':'application/json'},
+        body: JSON.stringify({path: p})
+      });
+    }
+    loadCachedModels();
   }catch(e){ alert('Error: '+e.message); }
 }
 

codai/api/text.py

@@ -293,19 +293,28 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
     if model_info.get('error'):
         raise HTTPException(status_code=404, detail=model_info['error'])
     
-    # Try to get the appropriate model (request_model handles VRAM cleanup)
-    mm = multi_model_manager.get_model_for_request(requested_model)
-    
+    # Acquire the least-busy instance (increments ref-count; released on response completion)
+    _model_key = model_info.get('model_key')
+    _instance_idx = None
+    _acq = multi_model_manager.acquire_model_instance(_model_key) if _model_key else None
+    if _acq:
+        _instance_idx, mm = _acq
+    else:
+        mm = multi_model_manager.get_model_for_request(requested_model)
+
+    def _release_instance():
+        if _instance_idx is not None and _model_key:
+            multi_model_manager.release_model_instance(_model_key, _instance_idx)
+
     if mm is None:
-        # Model not loaded - try to use default
+        _release_instance()
         if model_manager.backend is not None:
-            # Fallback to legacy model_manager
             current_manager = model_manager
         else:
             raise HTTPException(status_code=503, detail="Model not loaded")
     else:
         current_manager = mm
-    
+
     # Inject system prompt if --system-prompt flag was provided
     messages = request.messages
     global_system_prompt = get_global_system_prompt()
@@ -816,9 +825,16 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
                     # No tool calls, yield the content as usual
                     yield f"data: {json.dumps({'choices': [{'delta': {'content': second_pass_result}, 'finish_reason': 'stop'}]})}\n\n"
                 yield "data: [DONE]\n\n"
-            
+
+            async def _raw_stream_with_release():
+                try:
+                    async for chunk in raw_stream_generate():
+                        yield chunk
+                finally:
+                    _release_instance()
+
             from fastapi.responses import StreamingResponse
-            return StreamingResponse(raw_stream_generate(), media_type="text/event-stream")
+            return StreamingResponse(_raw_stream_with_release(), media_type="text/event-stream")
         
         # Non-streaming path (already implemented above)
         # First pass: generate until reasoning close tag
@@ -1127,9 +1143,29 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
         return JSONResponse(content=formatted_response, headers=headers)
     
     if request.stream:
+        async def _managed_stream():
+            try:
+                async for chunk in stream_chat_response(
+                    messages_dict,
+                    response_model_name,
+                    request.max_tokens,
+                    request.temperature,
+                    request.top_p,
+                    stop_sequences,
+                    tools_dict,
+                    current_manager,
+                    tool_parser,
+                    request.response_format,
+                ):
+                    yield chunk
+            finally:
+                _release_instance()
+
         from fastapi.responses import StreamingResponse
-        return StreamingResponse(
-            stream_chat_response(
+        return StreamingResponse(_managed_stream(), media_type="text/event-stream")
+    else:
+        try:
+            return await generate_chat_response(
                 messages_dict,
                 response_model_name,
                 request.max_tokens,
@@ -1140,23 +1176,10 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
                 current_manager,
                 tool_parser,
                 request.response_format,
-            ),
-            media_type="text/event-stream",
-        )
-    else:
-        return await generate_chat_response(
-            messages_dict,
-            response_model_name,
-            request.max_tokens,
-            request.temperature,
-            request.top_p,
-            stop_sequences,
-            tools_dict,
-            current_manager,
-            tool_parser,
-            request.response_format,
-            force_reasoning_args,
-        )
+                force_reasoning_args,
+            )
+        finally:
+            _release_instance()
 
 async def stream_chat_response(
     messages: List[Dict],
@@ -1701,28 +1724,54 @@ async def completions(request: CompletionRequest):
     if model_info.get('error'):
         raise HTTPException(status_code=404, detail=model_info['error'])
     
-    # Try to get the appropriate model (request_model handles VRAM cleanup)
-    mm = multi_model_manager.get_model_for_request(requested_model)
-    
+    # Acquire the least-busy instance (increments ref-count; released on response completion)
+    _model_key = model_info.get('model_key')
+    _instance_idx = None
+    _acq = multi_model_manager.acquire_model_instance(_model_key) if _model_key else None
+    if _acq:
+        _instance_idx, mm = _acq
+    else:
+        mm = multi_model_manager.get_model_for_request(requested_model)
+
+    def _release_instance():
+        if _instance_idx is not None and _model_key:
+            multi_model_manager.release_model_instance(_model_key, _instance_idx)
+
     if mm is None:
-        # Model not loaded - try to use default
+        _release_instance()
         if model_manager.backend is not None:
-            # Fallback to legacy model_manager
             current_manager = model_manager
         else:
             raise HTTPException(status_code=503, detail="Model not loaded")
     else:
         current_manager = mm
-    
+
     prompts = request.prompt if isinstance(request.prompt, list) else [request.prompt]
     stop_sequences = []
     if request.stop:
         stop_sequences = [request.stop] if isinstance(request.stop, str) else request.stop
-    
+
     if request.stream:
+        async def _managed_completion_stream():
+            try:
+                async for chunk in stream_completion_response(
+                    prompts[0],
+                    request.model,
+                    request.max_tokens,
+                    request.temperature,
+                    request.top_p,
+                    stop_sequences,
+                    current_manager,
+                ):
+                    yield chunk
+            finally:
+                _release_instance()
+
         from fastapi.responses import StreamingResponse
-        return StreamingResponse(
-            stream_completion_response(
+        return StreamingResponse(_managed_completion_stream(), media_type="text/event-stream")
+    else:
+        try:
+            return await generate_completion_response(
                 prompts[0],
                 request.model,
                 request.max_tokens,
@@ -1730,19 +1779,9 @@ async def completions(request: CompletionRequest):
                 request.top_p,
                 stop_sequences,
                 current_manager,
-            ),
-            media_type="text/event-stream",
-        )
-    else:
-        return await generate_completion_response(
-            prompts[0],
-            request.model,
-            request.max_tokens,
-            request.temperature,
-            request.top_p,
-            stop_sequences,
-            current_manager,
-        )
+            )
+        finally:
+            _release_instance()
 
 async def stream_completion_response(
     prompt: str,

codai/config.py

@@ -25,8 +25,8 @@ from dataclasses import dataclass, field
 @dataclass
 class ServerConfig:
     """Server configuration."""
-    host: str = "0.0.0.0"
-    port: int = 8000
+    host: str = "127.0.0.1"
+    port: int = 8776
     https: bool = False
     https_key_path: Optional[str] = None
     https_cert_path: Optional[str] = None
@@ -49,6 +49,7 @@ class ModelsConfig:
     default_load_mode: str = "ondemand"
     hf_cache_dir: Optional[str] = None
     gguf_cache_dir: Optional[str] = None
+    max_model_instances: int = 1  # max concurrent instances per model (global default; overridable per-model via "max_instances")
 
 
 @dataclass
@@ -150,8 +151,8 @@ class ConfigManager:
             default_config = {
                 "version": "1.0",
                 "server": {
-                    "host": "0.0.0.0",
-                    "port": 8000,
+                    "host": "127.0.0.1",
+                    "port": 8776,
                     "https": False,
                     "https_key_path": None,
                     "https_cert_path": None

codai/main.py

@@ -234,6 +234,7 @@ def main():
     load_mode = config.models.default_load_mode or "ondemand"
     set_load_mode(load_mode)
     multi_model_manager.set_load_mode(load_mode)
+    multi_model_manager._global_max_instances = config.models.max_model_instances
     
     print(f"\nLoad mode: {load_mode}")
     if load_mode == "ondemand":
@@ -463,6 +464,18 @@ def main():
                 if mm is not None and mm.backend is not None:
                     multi_model_manager.active_in_vram = mid
                     print(f"  Loaded: {mid}")
+                    # Load additional instances if preload_all_instances is set
+                    if isinstance(m, dict) and m.get("preload_all_instances"):
+                        max_inst = m.get("max_instances", 1)
+                        if isinstance(max_inst, int) and max_inst > 1:
+                            for i in range(max_inst - 1):
+                                multi_model_manager._pending_new_instance.add(mid)
+                                mm2 = multi_model_manager._load_model_by_name(mid)
+                                if mm2 is not None and mm2.backend is not None:
+                                    print(f"  Loaded extra instance {i+2}/{max_inst}: {mid}")
+                                else:
+                                    print(f"  Warning: extra instance {i+2}/{max_inst} of '{mid}' failed to load")
+                                    break
                 else:
                     print(f"  Warning: {mid} failed to load")
             elif mtype == "audio" and mid in multi_model_manager.whisper_servers: