embeddings: native GGUF Qwen2-VL image embeddings via llama.cpp mtmd

A GGUF embedder with a configured `mmproj` now loads as a vision-capable
'llama-vl' backend: the mtmd image tower feeds an embeddings context
(pooling LAST) under the GME chat prompt — the same scheme as the HF
qwenvl backend, so both produce the same vector space (~0.89 cosine
agreement at Q4). Runs natively on whatever the llama.cpp build targets
(Vulkan on radeon). Text goes through the same prompt+last-token path
so both modalities share the GME space; PIL-decoded images are passed
as raw RGB bitmaps (mtmd's stb can't read AVIF). Serialized on the
per-model lock; cleanup frees the mtmd ctx.

Reverts the front's image-reroute-to-HF-sibling: a model pinned to an
engine must be served there, not silently moved.

The gme mmproj was generated from the local HF checkpoint with
llama.cpp's convert_hf_to_gguf.py --mmproj (F16, 1.3 GB).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014S8VtAvG499SsCbeESRK7V

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.42"
+__version__ = "0.1.43"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/api/embeddings.py

@@ -65,7 +65,7 @@ class _EmbeddingModel:
         self.model = model
         # llama.cpp contexts are NOT thread-safe: two executor threads calling
         # embed() on one ctx segfault the engine. Serialize per model instance.
-        if backend == 'llama':
+        if backend in ('llama', 'llama-vl'):
             import threading
             self.lock = threading.Lock()
         else:
@@ -77,7 +77,17 @@ class _EmbeddingModel:
 
     def cleanup(self):
         try:
-            if self.backend == 'llama':
+            if self.backend == 'llama-vl':
+                llm, mctx = self.model
+                try:
+                    from llama_cpp import mtmd_cpp as _M
+                    if mctx:
+                        _M.mtmd_free(mctx)
+                except Exception:
+                    pass
+                if hasattr(llm, 'close'):
+                    llm.close()
+            elif self.backend == 'llama':
                 # llama.cpp model: close() frees the ctx + weights (VRAM incl.)
                 if hasattr(self.model, 'close'):
                     self.model.close()
@@ -178,11 +188,40 @@ def _load_embedding_model(model_name: str, device: str, model_config: dict = Non
     # llama.cpp's embedding path has no image tower wired here.
     if str(model_name).lower().endswith('.gguf'):
         try:
+            import os as _os
+            import llama_cpp as _L
             from llama_cpp import Llama
             cfg = model_config or {}
             raw = cfg.get('_raw_cfg') if isinstance(cfg.get('_raw_cfg'), dict) else {}
             n_ctx = int(cfg.get('n_ctx') or raw.get('n_ctx') or 2048)
             n_gpu_layers = cfg.get('n_gpu_layers', raw.get('n_gpu_layers', -1))
+            n_gpu_layers = int(n_gpu_layers if n_gpu_layers is not None else -1)
+            # A configured multimodal projector (mmproj) upgrades this to a
+            # VISION-capable embedder: the llama.cpp mtmd image tower feeds an
+            # embeddings context (pooling LAST), reproducing the GME scheme —
+            # last-token hidden state under the GME chat prompt — natively on
+            # whatever backend this llama.cpp build targets (Vulkan/CUDA/CPU).
+            _mmproj = cfg.get('mmproj') or raw.get('mmproj')
+            if _mmproj and _os.path.isfile(str(_mmproj)):
+                from llama_cpp import mtmd_cpp as _M
+                llm = Llama(
+                    model_path=model_name,
+                    embedding=True,
+                    pooling_type=_L.LLAMA_POOLING_TYPE_LAST,
+                    n_ctx=n_ctx, n_batch=n_ctx, n_ubatch=n_ctx,
+                    n_gpu_layers=n_gpu_layers,
+                    verbose=False,
+                )
+                mp = _M.mtmd_context_params_default()
+                mp.use_gpu = (n_gpu_layers != 0)
+                mp.print_timings = False
+                mp.n_threads = 8
+                mctx = _M.mtmd_init_from_file(
+                    str(_mmproj).encode(), llm._model.model, mp)
+                if not mctx:
+                    llm.close()
+                    raise RuntimeError(f"mtmd projector load failed: {_mmproj}")
+                return _EmbeddingModel('llama-vl', (llm, mctx))
             model = Llama(
                 model_path=model_name,
                 embedding=True,
@@ -194,7 +233,7 @@ def _load_embedding_model(model_name: str, device: str, model_config: dict = Non
                 # _embed_texts truncates inputs to n_ctx to close the loop.
                 n_batch=n_ctx,
                 n_ubatch=n_ctx,
-                n_gpu_layers=int(n_gpu_layers if n_gpu_layers is not None else -1),
+                n_gpu_layers=n_gpu_layers,
                 verbose=False,
             )
             return _EmbeddingModel('llama', model)
@@ -298,7 +337,7 @@ def _supports_images(model_obj) -> bool:
     """True if this loaded model can embed images (shared space for clip/ST
     multimodal; image-only space for the 'vision' backend)."""
     backend, model = model_obj
-    if backend in ('clip', 'vision', 'qwenvl'):
+    if backend in ('clip', 'vision', 'qwenvl', 'llama-vl'):
         return True
     if backend != 'sentence_transformers':
         return False
@@ -425,6 +464,116 @@ def _clip_feats(raw):
     raise RuntimeError("CLIP model returned no usable feature tensor")
 
 
+def _llama_vl_embed(model_obj, items, dimensions=None):
+    """GME-style embedding on a GGUF Qwen2-VL via llama.cpp mtmd: last-token
+    hidden state (ctx pooling LAST) under the GME chat prompt — the same scheme
+    as the HF `_qwenvl_embed`, so both backends produce the SAME vector space
+    (verified ~0.89 cosine agreement at Q4). `items` are {'text': str} /
+    {'image': PIL.Image} dicts. Caller need not hold the model lock."""
+    import ctypes
+    import math
+    import llama_cpp as _L
+    from llama_cpp import mtmd_cpp as _M
+    llm, mctx = model_obj.model
+    ctx = llm._ctx.ctx
+    n_embd = llm.n_embd()
+    try:
+        n_ctx = llm.n_ctx()
+    except Exception:
+        n_ctx = 2048
+    marker = _M.mtmd_default_marker()
+    if isinstance(marker, bytes):
+        marker = marker.decode()
+
+    def _prompt(body):
+        return (f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+                f"<|im_start|>user\n{body}<|im_end|>\n"
+                f"<|im_start|>assistant\n<|endoftext|>")
+
+    def _seq_embedding():
+        e = _L.llama_get_embeddings_seq(ctx, 0)
+        if not e:
+            e = _L.llama_get_embeddings_ith(ctx, -1)
+        if not e:
+            raise RuntimeError("llama.cpp returned no embedding")
+        v = [e[i] for i in range(n_embd)]
+        n = math.sqrt(sum(x * x for x in v)) or 1.0
+        return [x / n for x in v]
+
+    def _clear_kv():
+        try:
+            _L.llama_memory_clear(_L.llama_get_memory(ctx), True)
+        except Exception:
+            try:
+                llm.reset()
+            except Exception:
+                pass
+
+    out = []
+    _mlock = getattr(model_obj, 'lock', None)
+    import contextlib
+    with (_mlock if _mlock is not None else contextlib.nullcontext()):
+        for it in items:
+            _clear_kv()
+            img = it.get('image')
+            if img is not None:
+                # PIL image → raw RGB bitmap straight into mtmd (its own stb
+                # decoder can't read AVIF/WebP variants PIL already handled).
+                rgb = img.convert('RGB')
+                raw = rgb.tobytes()
+                buf = (ctypes.c_uint8 * len(raw)).from_buffer_copy(raw)
+                bmp = _M.mtmd_bitmap_init(rgb.width, rgb.height, buf)
+                if not bmp:
+                    raise ValueError("mtmd bitmap init failed")
+                chunks = _M.mtmd_input_chunks_init()
+                try:
+                    itext = _M.mtmd_input_text(
+                        text=_prompt(marker).encode(),
+                        add_special=True, parse_special=True)
+                    arr = (_M.mtmd_bitmap_p_ctypes * 1)(bmp)
+                    rc = _M.mtmd_tokenize(
+                        mctx, chunks, ctypes.byref(itext),
+                        ctypes.cast(arr, ctypes.POINTER(ctypes.c_void_p)), 1)
+                    if rc != 0:
+                        raise RuntimeError(f"mtmd_tokenize failed ({rc})")
+                    _np = ctypes.c_int(0)
+                    rc = _M.mtmd_helper_eval_chunks(
+                        mctx, ctx, chunks, 0, 0, n_ctx, True,
+                        ctypes.byref(_np))
+                    if rc != 0:
+                        raise RuntimeError(f"mtmd eval failed ({rc})")
+                finally:
+                    _M.mtmd_input_chunks_free(chunks)
+                    _M.mtmd_bitmap_free(bmp)
+            else:
+                body = it.get('text') or ''
+                toks = llm.tokenize(_prompt(body).encode('utf-8', 'ignore'),
+                                    add_bos=True, special=True)
+                if len(toks) > n_ctx - 8:
+                    # over-long text: shrink the BODY, keep the prompt frame
+                    _bt = llm.tokenize(body.encode('utf-8', 'ignore'),
+                                       add_bos=False, special=False)
+                    _keep = max(16, (n_ctx - 8) - (len(toks) - len(_bt)))
+                    body = llm.detokenize(_bt[:_keep]).decode('utf-8', 'ignore')
+                    toks = llm.tokenize(_prompt(body).encode('utf-8', 'ignore'),
+                                        add_bos=True, special=True)[:n_ctx - 4]
+                batch = _L.llama_batch_init(len(toks), 0, 1)
+                try:
+                    batch.n_tokens = len(toks)
+                    for i, tk in enumerate(toks):
+                        batch.token[i] = tk
+                        batch.pos[i] = i
+                        batch.n_seq_id[i] = 1
+                        batch.seq_id[i][0] = 0
+                        batch.logits[i] = 1 if i == len(toks) - 1 else 0
+                    if _L.llama_decode(ctx, batch) != 0:
+                        raise RuntimeError("llama_decode failed")
+                finally:
+                    _L.llama_batch_free(batch)
+            out.append(_seq_embedding())
+    return _truncate_dims(out, dimensions)
+
+
 def _embed_texts(model_obj, texts: List[str], dimensions=None) -> List[List[float]]:
     backend, model = model_obj
     if backend == 'sentence_transformers':
@@ -443,6 +592,8 @@ def _embed_texts(model_obj, texts: List[str], dimensions=None) -> List[List[floa
         results = [row.cpu().tolist() for row in feats]
     elif backend == 'qwenvl':
         return _qwenvl_embed(model, [{'text': t} for t in texts], dimensions)
+    elif backend == 'llama-vl':
+        return _llama_vl_embed(model_obj, [{'text': t} for t in texts], dimensions)
     elif backend == 'llama':
         # llama.cpp embedding mode. With a pooling type baked into the GGUF the
         # result is one vector per input; without, per-token vectors — mean-pool
@@ -578,6 +729,9 @@ def _embed_images(model_obj, images: List[str], dimensions=None) -> List[List[fl
         results = [row.cpu().tolist() for row in feats]
     elif backend == 'qwenvl':
         return _qwenvl_embed(model, [{'image': im} for im in pil_images], dimensions)
+    elif backend == 'llama-vl':
+        return _llama_vl_embed(model_obj, [{'image': im} for im in pil_images],
+                               dimensions)
     elif backend == 'vision':
         # Image-only encoder (DINOv2/ViT…): CLS/pooled token of the vision
         # transformer is the image representation.

codai/frontproxy/app.py

@@ -1705,36 +1705,6 @@ class FrontProxy:
             body_bytes = await request.body()
             model = self._peek_model(body_bytes, request.headers.get("content-type", ""))
 
-        # Modality-aware embedding reroute: a GGUF embedder (llama.cpp) has no
-        # image tower, so an `image` request against it would 400. When the SAME
-        # model is also registered as a non-GGUF (HF) entry, serve the image
-        # request from that sibling instead — the client keeps one model name;
-        # text requests stay on the GGUF entry's (e.g. radeon) engine.
-        if body_bytes is not None and "/embeddings" in path and model:
-            _gpath = str(self._model_info(model).get("path") or "")
-            if _gpath.lower().endswith(".gguf"):
-                try:
-                    import json as _json
-                    import re as _re
-                    _b = _json.loads(body_bytes or b"{}")
-                except Exception:
-                    _b = None
-                if isinstance(_b, dict) and _b.get("image"):
-                    _stem = model.lower().split("/")[-1]
-                    if _stem.endswith(".gguf"):
-                        _stem = _stem[:-5]
-                    # strip a trailing quant tag (…-Q4_K_M, ….i1-IQ4_XS, …)
-                    _stem = _re.sub(r'[-.](i1-)?(iq|q)\d[\w-]*$', '', _stem)
-                    _sib = self._model_info(_stem)
-                    _spath = str(_sib.get("path") or "")
-                    if _spath and not _spath.lower().endswith(".gguf"):
-                        _b["model"] = _sib.get("model_id") or _spath
-                        body_bytes = _json.dumps(_b).encode()
-                        model = _b["model"]
-                        print(f"[front] embeddings: image request for a GGUF "
-                              f"(text-only) embedder rerouted to HF sibling "
-                              f"'{model}'", flush=True)
-
         engine = _router.pick_engine(
             self.registry, path, method, model,
             required_cap=self._required_cap(path, model),