feat: add DINOv2-SALAD VPR (8448-d) as an isolated-subprocess embedding model

Second VPR option for REQUEST-vpr-embeddings.md, alongside the in-process
EigenPlaces. SALAD is near-SOTA for place recognition but its deps
(pytorch_lightning, pytorch_metric_learning) are heavy and version-sensitive, so
it runs in a SEPARATE process with those deps quarantined in a
--system-site-packages venv (shares the main torch, adds only the extras). The
main engine venv never imports them — verified clean.

- codai/api/vpr_salad_server.py: stdin image-path -> stdout JSON embedding server
  (same protocol as dinov2-embed), GPU-preferred with CPU fallback, announces its
  true width (8448) on ready.
- embeddings.py: 'salad'/'dinov2-salad' -> 'vpr-server' backend that spawns the
  server via /cache/salad-venv/bin/python; reuses the dinov2cpp reader/cleanup.
- Self-healing: _EmbeddingModel gains a respawn hook. If the manager kills the
  child under VRAM pressure but keeps the cached model, the embed path now
  respawns the server in place instead of failing forever with a stale handle
  (this was causing persistent "process died" 500s once salad was evicted).
- packaging/setup-salad-venv.sh: idempotent creator for the isolated venv.

Both VPR models pass the decisive ordering test on same-place/different-place
image sets (zero overlap between same-place min and different-place max); SALAD's
margin (0.516 vs 0.150) is wider than EigenPlaces' (0.479 vs 0.349). Verified:
8448-d L2=1.0, deterministic, ~0.2s warm / ~8s cold, GPU, subprocess isolation
intact. Config: models.json 'salad' + alias 'dinov2-salad', engine nvidia.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LoSpEthysqmseCc6Geizty

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.66"
+__version__ = "0.1.67"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/api/embeddings.py

@@ -58,14 +58,19 @@ class _EmbeddingModel:
     those branches and its VRAM is only reclaimed implicitly by gc.
     """
 
-    __slots__ = ("backend", "model", "lock")
+    __slots__ = ("backend", "model", "lock", "respawn")
 
-    def __init__(self, backend, model):
+    def __init__(self, backend, model, respawn=None):
         self.backend = backend
         self.model = model
+        # Optional zero-arg callable that returns a fresh, ready backend handle for
+        # subprocess models — lets the embed path self-heal a dead server in place
+        # instead of failing forever with a stale handle (the manager may have
+        # killed the child under VRAM pressure without dropping the cached model).
+        self.respawn = respawn
         # llama.cpp contexts are NOT thread-safe: two executor threads calling
         # embed() on one ctx segfault the engine. Serialize per model instance.
-        if backend in ('llama', 'llama-vl', 'dinov2cpp'):
+        if backend in ('llama', 'llama-vl', 'dinov2cpp', 'vpr-server'):
             import threading
             self.lock = threading.Lock()
         else:
@@ -92,7 +97,7 @@ class _EmbeddingModel:
 
     def _cleanup_locked(self):
         try:
-            if self.backend == 'dinov2cpp':
+            if self.backend in ('dinov2cpp', 'vpr-server'):
                 # persistent embed server subprocess — terminating it frees
                 # its (V)RAM; eviction restarts it on the next request.
                 try:
@@ -396,6 +401,81 @@ def _vpr_embed_images(model, pil_images, dimensions=None):
     return _truncate_dims(results, dimensions)
 
 
+_SALAD_IDS = {'salad', 'dinov2-salad'}
+
+
+def _is_salad(model_name) -> bool:
+    return str(model_name).strip().lower().replace('_', '-') in _SALAD_IDS
+
+
+def _load_salad(model_name, device, model_config=None):
+    """Spawn DINOv2-SALAD (near-SOTA VPR, 8448-d) as an ISOLATED subprocess so its
+    pytorch_lightning / pytorch_metric_learning deps stay out of the main engine
+    venv. Same stdin/stdout JSON protocol as dinov2-embed, so the 'vpr-server'
+    embed path reuses that reader. GPU-preferred (the child picks cuda), CPU only
+    as fallback."""
+    import os as _os
+    import subprocess
+    cfg = model_config or {}
+    raw = cfg.get('_raw_cfg') if isinstance(cfg.get('_raw_cfg'), dict) else {}
+    venv_py = (cfg.get('salad_venv_python') or raw.get('salad_venv_python')
+               or _os.environ.get('SALAD_VENV_PY') or '/cache/salad-venv/bin/python')
+    if not _os.path.isfile(venv_py):
+        raise RuntimeError(
+            f"SALAD venv python not found at {venv_py} — create it with "
+            "`python -m venv --system-site-packages /cache/salad-venv` then "
+            "`/cache/salad-venv/bin/pip install pytorch_lightning pytorch_metric_learning`")
+    server = _os.path.join(_os.path.dirname(__file__), 'vpr_salad_server.py')
+    env = dict(_os.environ)
+    env.setdefault('TORCH_HOME',
+                   _os.environ.get('CODERAI_TORCH_HOME', '/cache/torchhub'))
+    # GPU preferred; force CPU only when the model is pinned off-GPU.
+    _ngl = cfg.get('n_gpu_layers', raw.get('n_gpu_layers'))
+    if str(device) == 'cpu' or _ngl == 0:
+        env['SALAD_FORCE_CPU'] = '1'
+    _sz = cfg.get('vpr_image_size') or raw.get('vpr_image_size')
+    if _sz:
+        env['SALAD_IMAGE_SIZE'] = str(_sz)
+
+    def _die_with_parent():
+        try:
+            import ctypes
+            ctypes.CDLL('libc.so.6').prctl(1, 9)  # PR_SET_PDEATHSIG, SIGKILL
+        except Exception:
+            pass
+
+    import json as _json
+    import time as _time
+
+    def _spawn():
+        # Reap an orphaned/dead server for this script first.
+        try:
+            subprocess.run(['pkill', '-f', 'vpr_salad_server.py'], timeout=10)
+        except Exception:
+            pass
+        p = subprocess.Popen(
+            [venv_py, server],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL, env=env, text=True, bufsize=1,
+            preexec_fn=_die_with_parent)
+        _deadline = _time.time() + 600   # DINOv2 backbone load can be slow cold
+        while _time.time() < _deadline:
+            line = p.stdout.readline()
+            if not line:
+                raise RuntimeError("salad server exited during load")
+            line = line.strip()
+            if line.startswith('{'):
+                try:
+                    if _json.loads(line).get('ready'):
+                        return p
+                except Exception:
+                    pass
+        p.kill()
+        raise RuntimeError("salad server load timed out")
+
+    return _EmbeddingModel('vpr-server', _spawn(), respawn=_spawn)
+
+
 def _load_embedding_model(model_name: str, device: str, model_config: dict = None):
     from codai.models.hf_loading import build_from_pretrained_kwargs
     trust = _trust_remote_code(model_config)
@@ -405,10 +485,14 @@ def _load_embedding_model(model_name: str, device: str, model_config: dict = Non
     if _is_geoclip(model_name):
         return _load_geoclip(model_name, device)
 
-    # Visual place recognition (EigenPlaces). Logical id, not a repo/file.
+    # Visual place recognition (EigenPlaces, in-process). Logical id, not a repo.
     if _is_vpr(model_name):
         return _load_vpr(model_name, device, model_config)
 
+    # DINOv2-SALAD VPR — isolated subprocess (pytorch_lightning dep quarantine).
+    if _is_salad(model_name):
+        return _load_salad(model_name, device, model_config)
+
     # GGUF file → llama.cpp in embedding mode (works on whatever backend this
     # build targets: Vulkan on the radeon engine, CUDA on nvidia). Text-only —
     # llama.cpp's embedding path has no image tower wired here.
@@ -953,6 +1037,9 @@ def _embed_texts(model_obj, texts: List[str], dimensions=None) -> List[List[floa
         # VPR is image-only; the contract sends the image data URI in 'input'.
         return _vpr_embed_images(
             model, [_decode_image(t) for t in texts], dimensions)
+    elif backend == 'vpr-server':
+        # SALAD subprocess is image-only; 'input' carries image data URIs.
+        return _embed_images(model_obj, texts, dimensions)
     elif backend == 'llama':
         # llama.cpp embedding mode. With a pooling type baked into the GGUF the
         # result is one vector per input; without, per-token vectors — mean-pool
@@ -1095,9 +1182,9 @@ def _embed_images(model_obj, images: List[str], dimensions=None) -> List[List[fl
         return _geoclip_embed_images(model, pil_images, dimensions)
     elif backend == 'vpr':
         return _vpr_embed_images(model, pil_images, dimensions)
-    elif backend == 'dinov2cpp':
-        # dinov2-embed subprocess: temp-file the PIL images, send paths, read
-        # JSON lines back; normalize (the binary emits raw CLS values).
+    elif backend in ('dinov2cpp', 'vpr-server'):
+        # persistent embed subprocess (dinov2-embed / salad server): temp-file the
+        # PIL images, send paths, read JSON lines back; normalize.
         import contextlib
         import json as _json
         import math
@@ -1108,7 +1195,16 @@ def _embed_images(model_obj, images: List[str], dimensions=None) -> List[List[fl
         _mlock = getattr(model_obj, 'lock', None)
         with (_mlock if _mlock is not None else contextlib.nullcontext()):
             if proc.poll() is not None:
-                raise RuntimeError("dinov2-embed process died — retry (it will reload)")
+                # The server died (e.g. the manager killed the child under VRAM
+                # pressure without dropping this cached model). Self-heal: respawn
+                # in place if we know how, so we don't fail forever on a stale
+                # handle; only give up if no respawn hook is available.
+                _rs = getattr(model_obj, 'respawn', None)
+                if _rs is None:
+                    raise RuntimeError(
+                        "embed server process died — retry (it will reload)")
+                proc = _rs()
+                model_obj.model = proc
             for im in pil_images:
                 with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as f:
                     tmp = f.name

codai/api/vpr_salad_server.py

+#!/usr/bin/env python3
+# CoderAI - DINOv2-SALAD visual-place-recognition embedding server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+# GPLv3 (see the main project LICENSE).
+"""Run DINOv2-SALAD as an ISOLATED subprocess so its heavy deps
+(pytorch_lightning, pytorch_metric_learning) never enter the main engine venv —
+they live only in a --system-site-packages venv this script is launched with.
+
+Protocol (identical to the dinov2-embed server, so the embeddings backend reuses
+the same reader): read one image FILE PATH per line on stdin, write one JSON line
+per input on stdout —
+    {"ready": true, "device": "cuda", "dim": 8448}   # once, after the model loads
+    {"embedding": [ ... floats ... ]}                 # per image, L2-normalised
+    {"error": "..."}                                  # per image, on failure
+
+GPU is preferred; it falls back to CPU only when CUDA is unavailable or the GPU
+load fails (set SALAD_FORCE_CPU=1 to force CPU).
+"""
+import json
+import os
+import sys
+
+
+def main():
+    import torch
+    import torch.nn.functional as F
+    from PIL import Image
+    import torchvision.transforms as T
+
+    os.environ.setdefault(
+        'TORCH_HOME', os.environ.get('CODERAI_TORCH_HOME', '/cache/torchhub'))
+    force_cpu = os.environ.get('SALAD_FORCE_CPU') == '1'
+    want_gpu = torch.cuda.is_available() and not force_cpu
+
+    def _load(dev):
+        return torch.hub.load('serizba/salad', 'dinov2_salad').eval().to(dev)
+
+    device = 'cuda' if want_gpu else 'cpu'
+    try:
+        model = _load(device)
+    except Exception:
+        # GPU is preferred, CPU is the fallback when GPU load isn't possible.
+        if device != 'cpu':
+            device = 'cpu'
+            model = _load(device)
+        else:
+            raise
+
+    # DINOv2 backbone: side must be a multiple of the patch size (14). 322 = 14*23
+    # is the SALAD reference eval resolution. ImageNet normalisation.
+    size = int(os.environ.get('SALAD_IMAGE_SIZE', '322'))
+    tfm = T.Compose([
+        T.Resize((size, size)),
+        T.ToTensor(),
+        T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+
+    # Announce readiness (and the true descriptor width) so the parent can proceed.
+    with torch.no_grad():
+        _probe = model(torch.zeros(1, 3, size, size, device=device))
+    dim = int(_probe.shape[-1])
+    sys.stdout.write(json.dumps({"ready": True, "device": device, "dim": dim}) + "\n")
+    sys.stdout.flush()
+
+    for line in sys.stdin:
+        path = line.strip()
+        if not path:
+            continue
+        try:
+            im = Image.open(path).convert('RGB')
+            x = tfm(im).unsqueeze(0).to(device)
+            with torch.no_grad():
+                v = F.normalize(model(x), dim=-1)[0]
+            sys.stdout.write(json.dumps({"embedding": v.detach().cpu().tolist()}) + "\n")
+        except Exception as e:  # noqa: BLE001 — report, keep serving
+            sys.stdout.write(json.dumps({"error": str(e)}) + "\n")
+        sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    main()

packaging/setup-salad-venv.sh

+#!/bin/bash
+# Create the isolated venv for the DINOv2-SALAD VPR embedding server
+# (codai/api/vpr_salad_server.py). It SHARES the main coderai torch/torchvision
+# (--system-site-packages) and adds ONLY the heavy VPR-specific deps
+# (pytorch_lightning, pytorch_metric_learning) so they never enter — and cannot
+# break — the main engine venv. Idempotent; safe to re-run.
+#
+# The venv lives on a persistent volume (/cache) so it survives container
+# restarts. On a fresh machine/image, run this once; the 'salad'/'dinov2-salad'
+# embedding model errors with a clear message until it exists (EigenPlaces VPR
+# needs no venv and works regardless).
+set -e
+MAIN_PY="${CODERAI_PYTHON:-/opt/coderai/python/bin/python3}"
+VENV="${SALAD_VENV:-/cache/salad-venv}"
+
+echo "== creating SALAD venv at $VENV (shares $MAIN_PY site-packages) =="
+"$MAIN_PY" -m venv --system-site-packages "$VENV"
+"$VENV/bin/pip" install --no-input --disable-pip-version-check \
+    pytorch_lightning pytorch_metric_learning
+echo "== salad-venv ready: $VENV =="
+"$VENV/bin/python" -c "import pytorch_lightning, pytorch_metric_learning; print('  deps OK')"