embeddings: detecting the head is not having it (v0.3.85)

0.3.84 made `_is_bge_m3` see `sparse_linear.pt` on a machine that had
never downloaded the repo, so the multi-vector backend was finally
selected on a pod. It then loaded the heads with the same cache-only
lookup, and `from_pretrained` does not fetch them: it pulls the config
and the weights the config NAMES, and bge-m3's sparse/colbert heads are
plain tensors sitting beside the model. So they were still absent, and
the request came back

    HTTP 400  "This model has no sparse head."

which reads as a property of the MODEL rather than of the disk.

`_fetch_repo_file` is the companion to `_resolve_repo_file` for files
that have to be READ rather than merely detected: cache first, then a
real hf_hub_download. Offline short-circuits, a local directory is the
whole answer, and a failure returns None -- the dense path, which the
caller already handles -- because a head that cannot be fetched must not
turn into a failed load.

Both halves of this were the same mistake made twice: a cache-only
helper answering a question about existence on a machine whose cache is
empty by design. A rented pod is that machine every single time.

Needs a coderai-embeddings rebuild: this code runs on the pod.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HXRMe9ZPUQCCCtBai9uqt3

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.84"
+__version__ = "0.3.85"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/embeddings.py

@@ -401,6 +401,40 @@ def _repo_has_file(model_name: str, filename: str) -> bool:
     return found
 
 
+def _fetch_repo_file(model_name: str, filename: str):
+    """Local path to ``filename``, DOWNLOADING it if it is not already here.
+
+    The companion to `_resolve_repo_file`, for files that must actually be
+    read rather than merely detected. Needed because a repo's extra files are
+    not part of what `from_pretrained` fetches: it pulls the config and the
+    weights the config names, and nothing else. bge-m3's `sparse_linear.pt`
+    and `colbert_linear.pt` are plain tensors sitting beside the model, so on
+    a machine that has never seen the repo they are simply absent after the
+    base model has loaded -- and a cache-only lookup then reports a model with
+    no sparse head, which is indistinguishable from a model that has none.
+
+    Returns None rather than raising: a head that cannot be fetched means the
+    dense path, which is what the caller already handles.
+    """
+    got = _resolve_repo_file(model_name, filename)
+    if got is not None:
+        return got
+    import os
+    if os.path.isdir(model_name):
+        return None                  # a local dir IS the whole answer
+    if str(os.environ.get("HF_HUB_OFFLINE") or "").strip().lower() in (
+            "1", "true", "yes", "on"):
+        return None
+    try:
+        from huggingface_hub import hf_hub_download
+        path = hf_hub_download(model_name, filename)
+        return path if path and os.path.isfile(path) else None
+    except Exception as exc:
+        print(f"[embeddings] could not fetch {filename} for {model_name!r}: "
+              f"{type(exc).__name__}: {exc}", flush=True)
+        return None
+
+
 def _is_bge_m3(model_name: str, trust: bool) -> bool:
     """True if the repo is a bge-m3-style multi-vector embedder.
 
@@ -440,7 +474,12 @@ def _load_bge_m3(model_name: str, device, model_config):
     model.eval()
 
     def _load_head(fname):
-        path = _resolve_repo_file(model_name, fname)
+        # FETCH, not resolve. `from_pretrained` above downloaded the config and
+        # the weights the config names -- these heads are neither, so on a
+        # machine that has never seen this repo they are still absent here.
+        # Resolving from cache therefore reported "no sparse head" on every
+        # freshly rented pod, which reads as a property of the MODEL.
+        path = _fetch_repo_file(model_name, fname)
         if not path:
             return None
         sd = torch.load(path, map_location='cpu', weights_only=True)

tests/test_embedding_repo_file_detection.py

@@ -140,3 +140,64 @@ def test_the_detector_uses_the_existence_check_not_the_path_one():
     blk = src.split("def _is_bge_m3")[1].split("\ndef ")[0]
     assert "_repo_has_file(" in blk
     assert "_resolve_repo_file(" not in blk
+
+
+# ------------------------------------------- and then the heads must ARRIVE #
+def test_the_head_is_downloaded_when_it_is_not_cached(monkeypatch, no_local):
+    """Detecting the head is not having it. `from_pretrained` fetches the
+    config and the weights the config NAMES — `sparse_linear.pt` is neither,
+    so after the base model loads it is still absent on a fresh machine, and a
+    cache-only lookup reports "This model has no sparse head." That reads as a
+    property of the model rather than of the disk."""
+    import huggingface_hub
+    calls = []
+
+    def _dl(repo, fname, *a, **k):
+        calls.append((repo, fname, k.get("local_files_only")))
+        return "/tmp/fetched.pt"
+    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _dl, raising=False)
+    monkeypatch.setattr(emb.os.path, "isfile", lambda p: p == "/tmp/fetched.pt")
+    got = emb._fetch_repo_file("BAAI/bge-m3", "sparse_linear.pt")
+    assert got == "/tmp/fetched.pt"
+    # Downloaded for real: local_files_only must NOT be set on this call.
+    assert calls and not calls[0][2]
+
+
+def test_a_cached_head_is_not_downloaded_again(monkeypatch):
+    monkeypatch.setattr(emb, "_resolve_repo_file", lambda *a, **k: "/cache/h.pt")
+    import huggingface_hub
+
+    def _boom(*a, **k):
+        raise AssertionError("downloaded a file that was already here")
+    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _boom, raising=False)
+    assert emb._fetch_repo_file("BAAI/bge-m3", "sparse_linear.pt") == "/cache/h.pt"
+
+
+def test_a_head_that_cannot_be_fetched_means_the_dense_path(monkeypatch, no_local):
+    """None, not an exception: the caller already handles a model with no
+    head, and a network blip must not turn into a failed load."""
+    import huggingface_hub
+
+    def _boom(*a, **k):
+        raise OSError("gated repo")
+    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _boom, raising=False)
+    assert emb._fetch_repo_file("BAAI/bge-m3", "sparse_linear.pt") is None
+
+
+def test_offline_does_not_fetch_heads(monkeypatch, no_local):
+    import huggingface_hub
+
+    def _boom(*a, **k):
+        raise AssertionError("reached the network while offline")
+    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _boom, raising=False)
+    monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+    assert emb._fetch_repo_file("BAAI/bge-m3", "sparse_linear.pt") is None
+
+
+def test_the_head_loader_fetches_rather_than_resolves():
+    """The two read almost identically at the call site and only one of them
+    works on a machine that has never seen the repo."""
+    src = (ROOT / "codai/api/embeddings.py").read_text()
+    blk = src.split("def _load_bge_m3")[1].split("\ndef ")[0]
+    assert "_fetch_repo_file(" in blk
+    assert "_resolve_repo_file(" not in blk