the bytes reached the pod, the entry did not (v0.3.93)

Measured on a live pod. The pool gate from v0.3.91 worked —

    [lora] no adapters yet, but this engine serves a pool — starting with LoRA
           enabled so one can be added without a reload
    [vllm] launching: … --enable-lora --max-lora…

— and the adapter was still dropped:

    pod seeded expecting : sha256:2e820c…     (the local adapter directory)
    re-registered as     : sha256:b8eac366…   (an uploaded blob)
    orchestrator pushed  : b8eac366… ✓ 159 MB

The bytes arrived. The pod's CATALOGUE did not. It was seeded at boot with the
first id, /v1/loras/register repointed the entry at the second, and nothing
told the pod — so its engine looked for an id nobody holds, dropped the
adapter, and served the base under its name. (The bundle hash is deterministic;
the two ids differ because the adapter genuinely changed, not because hashing
is unstable.)

Two keys missing a version, the same shape as every other bug in this area:

  * _ensure_adapters sent the BYTES only. It sends bytes and entry now.
  * teach_model keyed "already taught" on (pod, model), so a re-register could
    never reach a pod it had spoken to once. Keyed on the adapter's source.

/v1/loras listed only the TRAINING registry, so an adapter that arrived by
upload+register was invisible there — a successful registration looked exactly
like a failed one, which is how this was found. It lists both now, tagged
`trained` or `registered`, with the adapter id, base and pool.
Co-Authored-By: 's avatarClaude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HXRMe9ZPUQCCCtBai9uqt3
parent 7601bdf9
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
# Canonical product version for CoderAI — single source of truth. Both the API # Canonical product version for CoderAI — single source of truth. Both the API
# metadata and the admin web UI read from here. # metadata and the admin web UI read from here.
__version__ = "0.3.92" __version__ = "0.3.93"
__tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference." __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
# Configure the CUDA caching allocator BEFORE torch is imported anywhere. # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
......
...@@ -3058,7 +3058,38 @@ async def list_loras(_auth=Depends(_require_api_auth)): ...@@ -3058,7 +3058,38 @@ async def list_loras(_auth=Depends(_require_api_auth)):
meta = json.load(f) meta = json.load(f)
except Exception: except Exception:
pass pass
out.append({"name": name, "path": wf, **meta}) out.append({"name": name, "path": wf, "source": "trained", **meta})
# …and the adapters REGISTERED as servable names (/v1/loras/register).
# Those live in the model catalogue, not in the training registry, so this
# listed nothing after a successful registration — which reads exactly like
# the registration having failed.
seen = {e["name"] for e in out}
try:
from codai.admin.routes import config_manager
md = getattr(config_manager, "models_data", None) or {}
except Exception:
md = {}
for lst in md.values():
if not isinstance(lst, list):
continue
for entry in lst:
if not isinstance(entry, dict) or not entry.get("loras"):
continue
nm = (str(entry.get("id") or "").strip()
or str(entry.get("alias") or "").strip())
if not nm or nm in seen:
continue
seen.add(nm)
specs = entry["loras"] if isinstance(entry["loras"], list) else []
first = specs[0] if specs and isinstance(specs[0], dict) else {}
out.append({
"name": nm, "source": "registered",
"adapter": str(first.get("path") or first.get("source") or ""),
"weight": float(first.get("weight", 1.0) or 1.0),
"base": str(entry.get("path") or ""),
"pool": str((entry.get("runpod") or {}).get("pool") or "") or None,
"served": True,
})
return {"loras": out} return {"loras": out}
......
...@@ -1195,7 +1195,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool: ...@@ -1195,7 +1195,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool:
""" """
import requests import requests
if not model or (base, model) in _REGISTERED: if not model:
return False return False
try: try:
from codai.models.manager import _model_entry_for from codai.models.manager import _model_entry_for
...@@ -1204,6 +1204,17 @@ def teach_model(base: str, api_key: str, model: str) -> bool: ...@@ -1204,6 +1204,17 @@ def teach_model(base: str, api_key: str, model: str) -> bool:
entry = {} entry = {}
if not entry: if not entry:
return False return False
# Keyed on WHICH adapter, not just the name. A retrain keeps the name and
# changes the content id, and a remote still holding the old one answers
# from the old weights — or, when it holds neither, from the base.
try:
from codai.models.text_loras import configured_specs
stamp = tuple(sorted(str(s.get("source") or "")
for s in configured_specs(entry)))
except Exception:
stamp = ()
if (base, model, stamp) in _REGISTERED:
return False
from codai.api.runpod_worker import (parse_model_runpod, resolve_model_source, from codai.api.runpod_worker import (parse_model_runpod, resolve_model_source,
seed_model_env) seed_model_env)
...@@ -1212,7 +1223,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool: ...@@ -1212,7 +1223,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool:
kind, value = resolve_model_source(entry, mcfg) kind, value = resolve_model_source(entry, mcfg)
if kind == "upload": if kind == "upload":
ensure_model_uploaded(base, api_key, entry) ensure_model_uploaded(base, api_key, entry)
_REGISTERED.add((base, model)) _REGISTERED.add((base, model, stamp))
return True return True
if kind not in ("hf", "url") or not value: if kind not in ("hf", "url") or not value:
print(f"[remote-gateway] cannot teach {model!r} to {base}: no HuggingFace id " print(f"[remote-gateway] cannot teach {model!r} to {base}: no HuggingFace id "
...@@ -1242,7 +1253,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool: ...@@ -1242,7 +1253,7 @@ def teach_model(base: str, api_key: str, model: str) -> bool:
f"{r.status_code} {(r.text or '')[:160]}", flush=True) f"{r.status_code} {(r.text or '')[:160]}", flush=True)
return False return False
print(f"[remote-gateway] taught {base} about {model!r} ({value})", flush=True) print(f"[remote-gateway] taught {base} about {model!r} ({value})", flush=True)
_REGISTERED.add((base, model)) _REGISTERED.add((base, model, stamp))
return True return True
except Exception as exc: except Exception as exc:
print(f"[remote-gateway] could not teach {model!r} to {base}: {exc}", flush=True) print(f"[remote-gateway] could not teach {model!r} to {base}: {exc}", flush=True)
......
...@@ -250,10 +250,15 @@ class RunpodBackend(ModelBackend): ...@@ -250,10 +250,15 @@ class RunpodBackend(ModelBackend):
if key in self._ADAPTERS_SENT: if key in self._ADAPTERS_SENT:
return return
self._ADAPTERS_SENT.add(key) # once per version, even if it fails self._ADAPTERS_SENT.add(key) # once per version, even if it fails
from codai.api.remote_gateway import ensure_text_loras # Bytes AND entry. Pushing the bytes alone leaves the pod's
# catalogue naming the adapter it was SEEDED with, which after a
# re-register is a content id nobody holds — so the engine drops
# the adapter and serves the base under its name. Measured live:
# seeded sha256:2e820c…, pushed sha256:b8eac366…, and the pod went
# on looking for the first.
from codai.api.remote_gateway import teach_pod
pool = getattr(self, "_pool", None) pool = getattr(self, "_pool", None)
ensure_text_loras(root, getattr(pool, "api_key", "") if pool else "", teach_pod(root, getattr(pool, "api_key", "") if pool else "", name)
entry)
except Exception as exc: except Exception as exc:
print(f"[runpod] could not send {name!r}'s adapters to {root}: {exc}", print(f"[runpod] could not send {name!r}'s adapters to {root}: {exc}",
flush=True) flush=True)
......
...@@ -362,3 +362,28 @@ def test_the_adapter_store_is_never_forwarded(): ...@@ -362,3 +362,28 @@ def test_the_adapter_store_is_never_forwarded():
"/v1/loras/blob/abc", "/v1/loras"): "/v1/loras/blob/abc", "/v1/loras"):
assert resolve_target(path, "POST", "", b'{"name":"x","base":"y"}', assert resolve_target(path, "POST", "", b'{"name":"x","base":"y"}',
"application/json") is None, path "application/json") is None, path
@pytest.mark.anyio
async def test_a_registered_adapter_is_visible_in_the_listing(monkeypatch):
"""It listed only the TRAINING registry, so a blob-uploaded adapter was
invisible right after a successful register — which reads exactly like the
registration having failed."""
from codai.api import loras
_catalogue(monkeypatch, [dict(BASE_MODEL), {
"id": "tuned-alpha", "path": "vendor/Some-27B",
"loras": [{"name": "tuned-alpha", "path": "sha256:abc", "weight": 1.0}],
"runpod": {"pool": "shared"}}])
monkeypatch.setattr(loras, "_loras_dir", lambda: "/nonexistent")
got = (await loras.list_loras())["loras"]
mine = [e for e in got if e["name"] == "tuned-alpha"]
assert mine and mine[0]["adapter"] == "sha256:abc"
assert mine[0]["pool"] == "shared" and mine[0]["source"] == "registered"
@pytest.mark.anyio
async def test_the_base_model_is_not_listed_as_an_adapter(monkeypatch):
from codai.api import loras
_catalogue(monkeypatch, [dict(BASE_MODEL)])
monkeypatch.setattr(loras, "_loras_dir", lambda: "/nonexistent")
assert (await loras.list_loras())["loras"] == []
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment