front: per-model engine_fallback option for an unavailable pin

By default a per-model engine pin is a hard constraint: if the pinned
engine is down/incompatible the request fails (no duplicate on another
card). Add an `engine_fallback` model-config flag (admin form checkbox +
persisted in models.json) that opts into the old behaviour — fall back to
a compatible engine when the pin can't be honoured. A pinned engine
that's merely busy-but-alive is still routed to (queues) in both modes;
fallback only applies when it's actually down or can't serve the model.

Threads pin_fallback through pick_engine; the front reads engine_fallback
via _load_pins/_model_info.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codai/admin/routes.py

@@ -2680,7 +2680,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                 "max_vram", "sdcpp_flash_attn", "sdcpp_diffusion_flash_attn", "vae_tiling",
                 "component_quantization", "output_crf", "force_vram_update",
                 "balanced_gpu_percent", "acceleration",
-                "cache_type_k", "cache_type_v", "turboquant", "engine",
+                "cache_type_k", "cache_type_v", "turboquant", "engine", "engine_fallback",
                 "quant_backend", "kv_cache_budget_mb", "kv_cache_slots", "mmproj",
                 "auto_compact", "auto_compact_pct", "auto_compact_strategy"):
         if key in data:

codai/admin/templates/models.html

@@ -567,6 +567,11 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
           <option value="">Default (auto — by capability)</option>
         </select>
         <span class="form-hint" style="font-size:11px">Pin this model to a specific engine/card. Overrides the default engine. Only shown when multiple engines are running.</span>
+        <label style="display:flex;align-items:center;gap:.4rem;margin-top:.4rem;font-size:12px">
+          <input type="checkbox" id="cfg-engine-fallback" style="width:auto;margin:0">
+          Fall back to another engine if the pinned one is unavailable
+        </label>
+        <span class="form-hint" style="font-size:11px">When off (default), a request fails if the pinned engine is down/busy-unreachable instead of running on a different card.</span>
       </div>
       <div style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem;margin-top:.75rem">
         <div class="form-row" style="margin:0">
@@ -3154,6 +3159,7 @@ function openCfgModal(idx, cfgIdx){
   document.getElementById('cfg-offload-dir').value = s.offload_dir || _defaultOffloadDir;
   _applyBackendCfgVisibility(m);
   _populateEnginePin(s.engine || '');
+  document.getElementById('cfg-engine-fallback').checked = !!s.engine_fallback;
   document.getElementById('cfg-sysprompt').value = s.system_prompt || '';
   document.getElementById('cfg-parser').value = s.parser || (!m.in_config ? _autoDetectParser(m.path) : 'auto');
   document.getElementById('cfg-tools').checked = !!s.tools_closer_prompt;
@@ -3507,6 +3513,7 @@ async function saveModelConfig(){
     offload_strategy:  document.getElementById('cfg-offload-strategy').value,
     offload_dir:       document.getElementById('cfg-offload-dir').value.trim() || './offload',
     engine:            document.getElementById('cfg-engine').value.trim() || null,
+    engine_fallback:   document.getElementById('cfg-engine-fallback').checked,
     system_prompt:     document.getElementById('cfg-sysprompt').value.trim() || null,
     parser:            document.getElementById('cfg-parser').value,
     tools_closer_prompt: document.getElementById('cfg-tools').checked,

codai/frontproxy/app.py

@@ -200,7 +200,8 @@ class FrontProxy:
         engine = _router.pick_engine(
             self.registry, path, method, model,
             required_cap=self._required_cap(path, model),
-            default_engine=self.default_engine, pinned=self._pin_for(model))
+            default_engine=self.default_engine, pinned=self._pin_for(model),
+            pin_fallback=bool(self._model_info(model).get("engine_fallback")))
         if engine is None:
             return {"status_code": 503, "headers": {"content-type": "application/json"},
                     "body": b'{"error":"No engine is ready yet."}'}
@@ -260,7 +261,8 @@ class FrontProxy:
                     continue
                 rec = {"engine": (m.get("engine") or "").strip() or None,
                        "backend": (m.get("backend") or "").strip() or None,
-                       "path": (m.get("path") or m.get("id") or "").strip() or None}
+                       "path": (m.get("path") or m.get("id") or "").strip() or None,
+                       "engine_fallback": bool(m.get("engine_fallback"))}
                 for field_ in (m.get("path"), m.get("id"), m.get("alias")):
                     if not field_:
                         continue
@@ -544,7 +546,8 @@ class FrontProxy:
         engine = _router.pick_engine(
             self.registry, path, method, model,
             required_cap=self._required_cap(path, model),
-            default_engine=self.default_engine, pinned=self._pin_for(model))
+            default_engine=self.default_engine, pinned=self._pin_for(model),
+            pin_fallback=bool(self._model_info(model).get("engine_fallback")))
         if engine is None:
             return JSONResponse(
                 {"error": "No engine is ready yet (still starting/loading)."},

codai/frontproxy/router.py

@@ -51,20 +51,24 @@ def is_admin_path(path: str) -> bool:
 _warned_pins: set = set()
 
 
-def _warn_bad_pin(model, pinned, cap, engine) -> None:
-    key = (model, pinned)
+def _warn_bad_pin(model, pinned, cap, engine, fallback: bool = False) -> None:
+    key = (model, pinned, fallback)
     if key in _warned_pins:
         return
     _warned_pins.add(key)
     if engine is None:
         reason = f"no engine named/backed '{pinned}' is declared"
-    elif not engine.healthy:
-        reason = f"engine '{pinned}' is not healthy"
-    else:
+    elif not engine.is_alive():
+        reason = f"engine '{pinned}' is down"
+    elif not engine.can_serve(cap):
         reason = (f"engine '{pinned}' (backend '{engine.backend}') can't serve a "
                   f"'{cap}' model (capabilities: {sorted(engine.capabilities)})")
+    else:
+        reason = f"engine '{pinned}' is unavailable"
+    tail = ("falling back to another compatible engine (engine_fallback)."
+            if fallback else "failing the request (the pin is a hard constraint).")
     print(f"[front] WARNING: model '{model}' is pinned to '{pinned}' but {reason}; "
-          f"falling back to a compatible engine.", flush=True)
+          f"{tail}", flush=True)
 
 
 def required_capability(model: Optional[str], path: Optional[str] = None,
@@ -100,7 +104,8 @@ def required_capability(model: Optional[str], path: Optional[str] = None,
 def pick_engine(registry: EngineRegistry, path: str, method: str,
                 model: Optional[str], required_cap: Optional[str] = None,
                 default_engine: Optional[str] = None,
-                pinned: Optional[str] = None) -> Optional[Engine]:
+                pinned: Optional[str] = None,
+                pin_fallback: bool = False) -> Optional[Engine]:
     """Return the engine to proxy this request to, or None if none are ready.
 
     Precedence for inference: per-model pin → engine already holding the model →
@@ -112,30 +117,31 @@ def pick_engine(registry: EngineRegistry, path: str, method: str,
     if method.upper() == "POST" and is_inference_path(path):
         cap = required_cap
 
-        # 0. The front's precomputed assignment is authoritative — it already folds
-        # in the pin, the default engine, and balanced auto-selection, and is what
-        # keeps a model on exactly one engine. Honour it first when it's compatible.
+        # 0. Per-model pin (models.json "engine") is a HARD constraint: the model
+        # runs on that engine or not at all. Route there if it can serve and its
+        # process is alive — even when it's busy (mid-generation → failing health
+        # polls), so the request queues on its gen-lock rather than spawning a
+        # duplicate elsewhere with the wrong settings. If the pinned engine is down
+        # or incompatible, fail (return None → 503) instead of falling back to a
+        # different engine.
+        if pinned:
+            e = registry.by_name(pinned)
+            if e is not None and e.can_serve(cap) and e.is_alive():
+                return e
+            _warn_bad_pin(model, pinned, cap, e, fallback=pin_fallback)
+            # Default: a hard pin fails rather than running elsewhere. When the
+            # model opts in (engine_fallback), fall through to pick another engine.
+            if not pin_fallback:
+                return None
+
+        # 1. The front's precomputed assignment is authoritative — it already folds
+        # in the default engine and balanced auto-selection, and keeps a model on
+        # exactly one engine. Honour it first when it's compatible.
         if model:
             owner = registry.engine_for_assigned(model)
             if owner is not None and owner.can_serve(cap):
                 return owner
 
-        # 1. Per-model pin (models.json "engine") — only honoured if compatible.
-        if pinned:
-            e = registry.by_name(pinned)
-            if e and e.can_serve(cap):
-                if e.healthy:
-                    return e
-                # Pinned engine is busy (mid-generation → failing health polls) but
-                # its process is alive: route here anyway so the request queues on
-                # its gen-lock, instead of duplicating a pinned model on another
-                # engine (which also ignores its configured n_ctx etc.).
-                if e.is_alive():
-                    return e
-            # Pin can't be honoured (engine down or incompatible) — say why (once
-            # per model+engine) instead of silently falling back.
-            _warn_bad_pin(model, pinned, cap, e)
-
         # 2. Engine that already has the model resident.
         if model:
             e = registry.engine_for_model(model, cap)