dinov2.cpp GGUF backend on Vulkan + Tasks-page compute-device badges

dinov2: packaging/dinov2cpp/ builds dinov2.cpp with ggml's Vulkan
backend plus a coderai `embed` stdin/stdout server (HF-matching
preprocessing: short-side 256 bicubic, center-crop 224; CLS read via
ggml_backend_tensor_get so GPU backends work; DINOV2_FORCE_CPU escape
hatch). embeddings.py 'dinov2cpp' backend drives one persistent
subprocess per model (evict = terminate, reload = respawn); dinov2
GGUF entries route there. Verified on the RX 580: cos 0.9973 vs the
HF dinov2-large backend.

Tasks page: every task now shows the compute device it runs on and
whether the model is CPU-offloading. record_vram_delta records
per-model device+offload ("GPU RTX 3090 · CPU offload 12.4 GB");
api_tasks tags each task from that map (engine default as fallback);
tasks.html renders a green/amber/gray badge (GPU / offload / CPU).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014S8VtAvG499SsCbeESRK7V

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.43"
+__version__ = "0.1.44"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/admin/routes.py

@@ -2969,6 +2969,24 @@ def api_tasks(username: str = Depends(require_admin)):
     except Exception:
         pass
 
+    # Tag every task with the compute device it runs on ("GPU RTX 3090",
+    # "GPU Vulkan/AMD · CPU offload 12.4 GB", "CPU"). Per-model state comes from
+    # record_vram_delta (device + offload measured at load); tasks whose model
+    # hasn't loaded yet (or non-model tasks like downloads, which are CPU/disk)
+    # fall back to the engine's device / a sensible per-kind default.
+    try:
+        _default_dev = multi_model_manager.engine_device_desc()
+        for t in tasks:
+            if t.get("device"):
+                continue
+            if t.get("kind") == "download":
+                t["device"] = "CPU"
+                continue
+            t["device"] = (multi_model_manager.device_for_model(t.get("model"))
+                           or _default_dev)
+    except Exception:
+        pass
+
     # The queue-summary header must reflect ALL model activity, not just requests
     # that flow through queue_manager (text/pipelines/training). Image/video/audio
     # generations run their own paths and live only in the task registry, so derive

codai/admin/templates/tasks.html

@@ -192,7 +192,7 @@ function taskRow(t) {
   }
   return `<tr>
     <td><span class="badge badge-user">${esc(KIND_LABEL[t.kind] || t.kind)}</span></td>
-    <td><div class="td-name">${esc(title)}${t.engine?` <span class="badge badge-user" style="font-size:9px;padding:.05rem .3rem;vertical-align:middle" title="Running on engine">${esc(t.engine)}</span>`:''}</div><div class="dim small mono">${esc(t.model || '')}</div></td>
+    <td><div class="td-name">${esc(title)}${t.engine?` <span class="badge badge-user" style="font-size:9px;padding:.05rem .3rem;vertical-align:middle" title="Running on engine">${esc(t.engine)}</span>`:''}${t.device?` <span class="badge" style="font-size:9px;padding:.05rem .3rem;vertical-align:middle;background:${/offload/i.test(t.device)?'#7a5a1e':(/^GPU/i.test(t.device)?'#1e5a3a':'#444')};color:#fff" title="Compute device${/offload/i.test(t.device)?' — model partially offloaded to CPU RAM':''}">${esc(t.device)}</span>`:''}</div><div class="dim small mono">${esc(t.model || '')}</div></td>
     <td>${statusCell}</td>
     <td>${progressBar(t)}</td>
     <td class="dim small">${fmtTime(t.started_at)}</td>

codai/api/embeddings.py

@@ -65,7 +65,7 @@ class _EmbeddingModel:
         self.model = model
         # llama.cpp contexts are NOT thread-safe: two executor threads calling
         # embed() on one ctx segfault the engine. Serialize per model instance.
-        if backend in ('llama', 'llama-vl'):
+        if backend in ('llama', 'llama-vl', 'dinov2cpp'):
             import threading
             self.lock = threading.Lock()
         else:
@@ -77,7 +77,18 @@ class _EmbeddingModel:
 
     def cleanup(self):
         try:
-            if self.backend == 'llama-vl':
+            if self.backend == 'dinov2cpp':
+                # persistent embed server subprocess — terminating it frees
+                # its (V)RAM; eviction restarts it on the next request.
+                try:
+                    self.model.terminate()
+                    self.model.wait(timeout=10)
+                except Exception:
+                    try:
+                        self.model.kill()
+                    except Exception:
+                        pass
+            elif self.backend == 'llama-vl':
                 llm, mctx = self.model
                 try:
                     from llama_cpp import mtmd_cpp as _M
@@ -179,6 +190,11 @@ def _has_st_modules(model_name: str) -> bool:
         return False
 
 
+def _os_basename(p) -> str:
+    import os
+    return os.path.basename(str(p))
+
+
 def _load_embedding_model(model_name: str, device: str, model_config: dict = None):
     from codai.models.hf_loading import build_from_pretrained_kwargs
     trust = _trust_remote_code(model_config)
@@ -186,6 +202,49 @@ def _load_embedding_model(model_name: str, device: str, model_config: dict = Non
     # GGUF file → llama.cpp in embedding mode (works on whatever backend this
     # build targets: Vulkan on the radeon engine, CUDA on nvidia). Text-only —
     # llama.cpp's embedding path has no image tower wired here.
+    # DINOv2 GGUF → the dinov2.cpp `dinov2-embed` server (ggml Vulkan/CPU;
+    # llama.cpp cannot load a ViT-only arch). One persistent subprocess holds
+    # the model; requests stream image paths in and JSON embeddings out. Its
+    # CLS output matches the HF 'vision' backend at ~0.997 cosine.
+    if (str(model_name).lower().endswith('.gguf')
+            and 'dinov2' in _os_basename(model_name).lower()):
+        import os as _os
+        import subprocess
+        _bin = _os.environ.get('DINOV2_EMBED_BIN', '/opt/coderai/bin/dinov2-embed')
+        if not _os.path.isfile(_bin):
+            raise RuntimeError(
+                f"dinov2-embed binary not found at {_bin} — build it with "
+                "packaging/dinov2cpp/build.sh")
+        cfg = model_config or {}
+        raw = cfg.get('_raw_cfg') if isinstance(cfg.get('_raw_cfg'), dict) else {}
+        env = dict(_os.environ)
+        _ngl = cfg.get('n_gpu_layers', raw.get('n_gpu_layers', -1))
+        if _ngl == 0:
+            env['DINOV2_FORCE_CPU'] = '1'
+        proc = subprocess.Popen(
+            [_bin, '-m', model_name, '-t', '8'],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL, env=env, text=True, bufsize=1)
+        # wait for the ready line (model load), skipping loader chatter
+        import json as _json
+        import time as _time
+        _deadline = _time.time() + 300
+        while _time.time() < _deadline:
+            line = proc.stdout.readline()
+            if not line:
+                raise RuntimeError("dinov2-embed exited during load")
+            line = line.strip()
+            if line.startswith('{'):
+                try:
+                    if _json.loads(line).get('ready'):
+                        break
+                except Exception:
+                    pass
+        else:
+            proc.kill()
+            raise RuntimeError("dinov2-embed load timed out")
+        return _EmbeddingModel('dinov2cpp', proc)
+
     if str(model_name).lower().endswith('.gguf'):
         try:
             import os as _os
@@ -337,7 +396,7 @@ def _supports_images(model_obj) -> bool:
     """True if this loaded model can embed images (shared space for clip/ST
     multimodal; image-only space for the 'vision' backend)."""
     backend, model = model_obj
-    if backend in ('clip', 'vision', 'qwenvl', 'llama-vl'):
+    if backend in ('clip', 'vision', 'qwenvl', 'llama-vl', 'dinov2cpp'):
         return True
     if backend != 'sentence_transformers':
         return False
@@ -678,7 +737,7 @@ def _embed_texts(model_obj, texts: List[str], dimensions=None) -> List[List[floa
                     emb = [x / _tot for x in _acc]
                 norm = math.sqrt(sum(x * x for x in emb)) or 1.0
                 results.append([x / norm for x in emb])
-    elif backend == 'vision':
+    elif backend in ('vision', 'dinov2cpp'):
         raise ValueError(
             "this embedding model is image-only (no text tower) — send 'image' "
             "instead of 'input', or use a text/multimodal embedding model")
@@ -732,6 +791,47 @@ def _embed_images(model_obj, images: List[str], dimensions=None) -> List[List[fl
     elif backend == 'llama-vl':
         return _llama_vl_embed(model_obj, [{'image': im} for im in pil_images],
                                dimensions)
+    elif backend == 'dinov2cpp':
+        # dinov2-embed subprocess: temp-file the PIL images, send paths, read
+        # JSON lines back; normalize (the binary emits raw CLS values).
+        import contextlib
+        import json as _json
+        import math
+        import os
+        import tempfile
+        proc = model
+        results = []
+        _mlock = getattr(model_obj, 'lock', None)
+        with (_mlock if _mlock is not None else contextlib.nullcontext()):
+            if proc.poll() is not None:
+                raise RuntimeError("dinov2-embed process died — retry (it will reload)")
+            for im in pil_images:
+                with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as f:
+                    tmp = f.name
+                try:
+                    im.convert('RGB').save(tmp, 'PNG')
+                    proc.stdin.write(tmp + "\n")
+                    proc.stdin.flush()
+                    while True:
+                        line = proc.stdout.readline()
+                        if not line:
+                            raise RuntimeError("dinov2-embed died mid-request")
+                        line = line.strip()
+                        if not line.startswith('{'):
+                            continue
+                        d = _json.loads(line)
+                        if 'error' in d:
+                            raise ValueError(f"dinov2-embed: {d['error']}")
+                        v = d['embedding']
+                        n = math.sqrt(sum(x * x for x in v)) or 1.0
+                        results.append([x / n for x in v])
+                        break
+                finally:
+                    try:
+                        os.unlink(tmp)
+                    except OSError:
+                        pass
+        return _truncate_dims(results, dimensions)
     elif backend == 'vision':
         # Image-only encoder (DINOv2/ViT…): CLS/pooled token of the vision
         # transformer is the image representation.

codai/models/manager.py

@@ -838,6 +838,11 @@ class MultiModelManager:
         # load window and keeps sweep_orphan_vram from mislabeling an
         # in-progress load as a teardown leak.
         self._loading_reservations: Dict[str, float] = {}
+        # model_key -> human compute-device description ("GPU RTX 3090",
+        # "GPU Vulkan/AMD · CPU offload 12.4 GB", "CPU"). Populated by
+        # record_vram_delta after each load; the Tasks page reads it so every
+        # task shows WHERE it runs and whether the model is CPU-offloading.
+        self.model_devices: Dict[str, str] = {}
         # Set once a CUDA device-side assert / unrecoverable CUDA error is seen.
         # The CUDA context is corrupted process-wide after such an error, so all
         # further GPU work is futile until the server is restarted.  We surface
@@ -2346,6 +2351,54 @@ class MultiModelManager:
                 total += gb
         return total
 
+    _engine_device_cache: Optional[str] = None
+
+    @classmethod
+    def engine_device_desc(cls) -> str:
+        """Human name of this engine's compute device ("GPU RTX 3090",
+        "GPU Vulkan/AMD", "CPU"). torch is only consulted when ALREADY imported
+        — the GGUF engines must not pull in a ~1 GB torch just for a label."""
+        if cls._engine_device_cache is not None:
+            return cls._engine_device_cache
+        import os
+        import sys
+        desc = None
+        if 'torch' in sys.modules:
+            try:
+                _t = sys.modules['torch']
+                if _t.cuda.is_available():
+                    desc = f"GPU {_t.cuda.get_device_name(0)}"
+            except Exception:
+                pass
+        if desc is None:
+            be = (os.environ.get("CODERAI_ENGINE_BACKEND") or "").lower()
+            if 'vulkan' in be or 'radeon' in be or 'amd' in be:
+                desc = "GPU Vulkan/AMD"
+            elif be in ('nvidia', 'cuda'):
+                desc = "GPU CUDA"
+            else:
+                desc = "CPU"
+        cls._engine_device_cache = desc
+        return desc
+
+    def device_for_model(self, name) -> Optional[str]:
+        """Device description for a model in any id form (key, path, basename)."""
+        if not name:
+            return None
+        n = str(name)
+        d = self.model_devices.get(n)
+        if d:
+            return d
+        import os
+        base = os.path.basename(n)
+        base_noext = base[:-5] if base.lower().endswith('.gguf') else base
+        for k, v in self.model_devices.items():
+            kb = os.path.basename(str(k).split(':', 1)[-1])
+            kb_noext = kb[:-5] if kb.lower().endswith('.gguf') else kb
+            if kb == base or kb_noext == base_noext:
+                return v
+        return None
+
     def note_loading(self, model_key: str, gb: float = 0.0) -> None:
         """Reserve VRAM for a load that is about to start (see
         _loading_reservations). Pass the model's estimate, or 0 to look it up."""
@@ -2557,6 +2610,13 @@ class MultiModelManager:
         except (TypeError, ValueError):
             measured_layers = None
 
+        # Record the compute device + offload state for the Tasks page.
+        _dev = self.engine_device_desc()
+        if offloaded or (measured_ram and measured_ram > 0.3):
+            _dev += (f" · CPU offload {measured_ram:.1f} GB"
+                     if measured_ram else " · CPU offload")
+        self.model_devices[model_key] = _dev
+
         # measured_vram already INCLUDES the offloaded RAM portion (full footprint),
         # so don't add measured_ram again here — that would double-count.
         print(f"Measured footprint for '{model_key}': "

packaging/dinov2cpp/build.sh

+#!/bin/bash
+# Build dinov2.cpp with ggml's Vulkan backend + the coderai `embed` server.
+#
+# Produces: <workdir>/dinov2.cpp/build/bin/embed
+# Deps (debian/ubuntu): build-essential cmake pkg-config libvulkan-dev glslc
+#                       libopencv-dev
+#
+# Usage: build.sh [workdir]   (default: /tmp/dinov2cpp-build)
+set -euo pipefail
+
+WORKDIR="${1:-/tmp/dinov2cpp-build}"
+PIN=3d070782afc264b7d60aa5692c5b10cb79b9bd56
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+mkdir -p "$WORKDIR"
+cd "$WORKDIR"
+if [ ! -d dinov2.cpp ]; then
+    git clone https://github.com/lavaman131/dinov2.cpp.git
+fi
+cd dinov2.cpp
+git checkout "$PIN"
+# submodule URL is ssh (git@github.com:) — force anonymous https
+git -c url."https://github.com/".insteadOf="git@github.com:" \
+    submodule update --init --depth 1
+
+# ── coderai patches (idempotent) ─────────────────────────────────────────────
+# 1) Vulkan backend init in dino_model_load + header include.
+python3 - <<'EOF'
+import re
+
+src = open('dinov2.cpp').read()
+if 'GGML_USE_VULKAN' not in src:
+    src = src.replace(
+        '#ifdef GGML_USE_CUDA\n#include "ggml-cuda.h"\n#endif',
+        '#ifdef GGML_USE_VULKAN\n#include "ggml-vulkan.h"\n#endif\n\n'
+        '#ifdef GGML_USE_CUDA\n#include "ggml-cuda.h"\n#endif', 1)
+    vk_block = (
+        '#ifdef GGML_USE_VULKAN\n'
+        '    if (!getenv("DINOV2_FORCE_CPU")) {\n'
+        '        fprintf(stderr, "%s: using Vulkan backend\\n", __func__);\n'
+        '        model.backend = ggml_backend_vk_init(0);\n'
+        '        if (!model.backend) {\n'
+        '            fprintf(stderr, "%s: ggml_backend_vk_init() failed\\n", __func__);\n'
+        '        }\n'
+        '    }\n'
+        '#endif\n'
+        '#ifdef GGML_USE_CUDA\n')
+    src = src.replace('#ifdef GGML_USE_CUDA\n    fprintf(stderr, "%s: using CUDA backend',
+                      vk_block + '    fprintf(stderr, "%s: using CUDA backend', 1)
+    open('dinov2.cpp', 'w').write(src)
+    print('patched dinov2.cpp')
+
+cm = open('CMakeLists.txt').read()
+if 'add_executable(embed' not in cm:
+    cm = cm.replace(
+        'option(BUILD_QUANTIZE',
+        'add_executable(embed embed.cpp dinov2.cpp)\n'
+        'target_link_libraries(embed PRIVATE ${OpenCV_LIBS} PUBLIC ggml)\n'
+        'target_include_directories(embed PUBLIC .)\n'
+        'if (GGML_VULKAN)\n'
+        '    target_compile_definitions(embed PRIVATE GGML_USE_VULKAN)\n'
+        '    target_compile_definitions(inference PRIVATE GGML_USE_VULKAN)\n'
+        'endif ()\n\n'
+        'option(BUILD_QUANTIZE', 1)
+    open('CMakeLists.txt', 'w').write(cm)
+    print('patched CMakeLists.txt')
+EOF
+cp "$HERE/embed.cpp" .
+
+mkdir -p build && cd build
+cmake -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON \
+      -DBUILD_REALTIME=OFF -DBUILD_QUANTIZE=OFF ..
+make -j"$(nproc)" embed
+echo "BUILT: $(pwd)/bin/embed"

packaging/dinov2cpp/embed.cpp

+// coderai embedding server for dinov2.cpp
+//
+// Loads a DINOv2 GGUF once, then serves a stdin/stdout loop: each input line
+// is an image path; each output line is JSON {"embedding":[...]} (the CLS
+// token after the final layernorm — identical to HF Dinov2Model's
+// pooler-input, so vectors match the transformers 'vision' backend) or
+// {"error":"..."}. Preprocessing mirrors the HF AutoImageProcessor for
+// facebook/dinov2-*: shortest side -> 256 (bicubic), center-crop 224,
+// ImageNet mean/std normalization.
+//
+// GPU: built with GGML_VULKAN this runs the graph on the Vulkan device
+// (radeon); tensors are read back with ggml_backend_tensor_get (device-safe,
+// unlike the upstream tools' ggml_get_data_f32). DINOV2_FORCE_CPU=1 forces
+// the CPU backend.
+
+#include "dinov2.h"
+#include "ggml.h"
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+
+#include <opencv2/core.hpp>
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+static cv::Mat hf_preprocess(const cv::Mat &bgr) {
+    // shortest side -> 256, bicubic (matches HF resize)
+    const int short_side = std::min(bgr.cols, bgr.rows);
+    const double scale = 256.0 / short_side;
+    cv::Mat resized;
+    cv::resize(bgr, resized,
+               cv::Size(int(round(bgr.cols * scale)), int(round(bgr.rows * scale))),
+               0, 0, cv::INTER_CUBIC);
+    // center crop 224x224
+    const int x = (resized.cols - 224) / 2;
+    const int y = (resized.rows - 224) / 2;
+    cv::Mat crop = resized(cv::Rect(x, y, 224, 224)).clone();
+    // float [0,1] + ImageNet standardization (channels are BGR here; the
+    // mean/std constants are indexed reversed exactly like dino_preprocess)
+    cv::Mat image;
+    crop.convertTo(image, CV_32FC3, 1.0 / 255.0);
+    std::vector<cv::Mat> channels(3);
+    cv::split(image, channels);
+    for (int i = 0; i < 3; ++i) {
+        channels[i] = (channels[i] - IMAGENET_DEFAULT_MEAN[2 - i])
+                      / IMAGENET_DEFAULT_STD[2 - i];
+    }
+    cv::merge(channels, image);
+    return image;
+}
+
+static bool predict_cls(dino_model &model, const cv::Mat &img,
+                        const dino_params &params, ggml_gallocr_t allocr,
+                        std::vector<float> &out) {
+    struct ggml_init_params params0 = {
+        /*.mem_size   =*/ ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE +
+                          ggml_graph_overhead(),
+        /*.mem_buffer =*/ nullptr,
+        /*.no_alloc   =*/ true,
+    };
+    struct ggml_context *ctx_cgraph = ggml_init(params0);
+    struct ggml_cgraph *gf = build_graph(img.size(), ctx_cgraph, model, params);
+    ggml_gallocr_alloc_graph(allocr, gf);
+
+    // input image, planar RGB
+    struct ggml_tensor *input = ggml_graph_get_tensor(gf, "input");
+    std::vector<float> planar(img.total() * 3);
+    float *dst = planar.data();
+    std::vector<cv::Mat> bgr_channels(3);
+    cv::split(img, bgr_channels);
+    std::vector<cv::Mat> rgb_planar = {
+        cv::Mat(img.rows, img.cols, CV_32F, dst + 0 * img.total()),
+        cv::Mat(img.rows, img.cols, CV_32F, dst + 1 * img.total()),
+        cv::Mat(img.rows, img.cols, CV_32F, dst + 2 * img.total()),
+    };
+    bgr_channels[2].copyTo(rgb_planar[0]);
+    bgr_channels[1].copyTo(rgb_planar[1]);
+    bgr_channels[0].copyTo(rgb_planar[2]);
+    ggml_backend_tensor_set(input, planar.data(), 0, ggml_nbytes(input));
+
+    // interpolated position embeddings — copy the source tensor to HOST first
+    // (on a GPU backend ->data points at device memory; upstream reads it
+    // directly and only works on CPU)
+    const struct ggml_tensor *pos_embed =
+        ggml_get_tensor(model.ctx, "embeddings.position_embeddings");
+    std::vector<float> pos_host(ggml_nelements(pos_embed));
+    ggml_backend_tensor_get(const_cast<ggml_tensor *>(pos_embed),
+                            pos_host.data(), 0, ggml_nbytes(pos_embed));
+    const std::vector<float> pos_fixed =
+        interpolate_pos_embed(img.size(), pos_host.data(), model.hparams);
+    struct ggml_tensor *pos_embed_fixed =
+        ggml_graph_get_tensor(gf, "pos_embed_fixed");
+    ggml_backend_tensor_set(pos_embed_fixed, pos_fixed.data(), 0,
+                            ggml_nbytes(pos_embed_fixed));
+
+    if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) {
+        ggml_free(ctx_cgraph);
+        return false;
+    }
+
+    struct ggml_tensor *cls = ggml_graph_get_tensor(gf, "cls_token");
+    out.resize(model.hparams.hidden_size);
+    ggml_backend_tensor_get(cls, out.data(), 0, out.size() * sizeof(float));
+    ggml_free(ctx_cgraph);
+    return true;
+}
+
+int main(int argc, char **argv) {
+    ggml_time_init();
+    dino_params params;
+    params.classify = false;
+    for (int i = 1; i < argc; i++) {
+        const std::string arg = argv[i];
+        if (arg == "-m" && i + 1 < argc) params.model = argv[++i];
+        else if (arg == "-t" && i + 1 < argc) params.n_threads = std::stoi(argv[++i]);
+    }
+
+    dino_model model;
+    if (!dino_model_load(cv::Size(224, 224), params.model, model, params)) {
+        fprintf(stderr, "embed: failed to load model '%s'\n", params.model.c_str());
+        return 1;
+    }
+    ggml_gallocr_t allocr =
+        ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
+
+    fprintf(stderr, "embed: ready (hidden=%d)\n", model.hparams.hidden_size);
+    printf("{\"ready\":true,\"hidden\":%d}\n", model.hparams.hidden_size);
+    fflush(stdout);
+
+    std::string line;
+    while (std::getline(std::cin, line)) {
+        if (line.empty()) continue;
+        cv::Mat raw = cv::imread(line, cv::IMREAD_COLOR);
+        if (raw.empty()) {
+            printf("{\"error\":\"cannot read image\"}\n");
+            fflush(stdout);
+            continue;
+        }
+        cv::Mat img = hf_preprocess(raw);
+        std::vector<float> emb;
+        if (!predict_cls(model, img, params, allocr, emb)) {
+            printf("{\"error\":\"graph compute failed\"}\n");
+            fflush(stdout);
+            continue;
+        }
+        std::string out = "{\"embedding\":[";
+        char buf[32];
+        for (size_t i = 0; i < emb.size(); ++i) {
+            snprintf(buf, sizeof(buf), i ? ",%.6g" : "%.6g", emb[i]);
+            out += buf;
+        }
+        out += "]}";
+        puts(out.c_str());
+        fflush(stdout);
+    }
+
+    ggml_gallocr_free(allocr);
+    ggml_free(model.ctx);
+    ggml_backend_buffer_free(model.buffer);
+    ggml_backend_free(model.backend);
+    return 0;
+}