a pool pod keeps the LoRA door open (v0.3.89)

v0.3.88 made the pod FIND the adapter. The live pod then said:

    [lora] 'diogene-alpha': adapter sha256:2e820c… is not in this machine's
           blob store — serving without it
    [vllm] launching: … --served-model-name cyankiwi/Qwen3.8-27B-AWQ-INT4 qwen38-awq

Found, and dropped. A seed names an adapter by CONTENT ID — computed here so
the pod can be told about it before it exists — and the bytes travel
separately. They were pushed on the first `acquire`, but the pool's WARM-UP
request loads the model seconds after boot, long before any acquire. So vLLM
started without the adapter, and `--enable-lora` cannot be turned on
afterwards.

Two changes, both gated on the model declaring a `pool`, because a pool is the
explicit statement that this engine serves several names over one copy of the
weights:

  * a pool engine launches with `--enable-lora` and the capacity flags even
    with no adapter resolvable yet. The flags that must be decided at LAUNCH
    are the capacity ones — the machinery, the slot size, how many adapters may
    be live in one batch — not the list. Opening the door costs a buffer
    allocation; keeping it shut costs a full reload of a 27B on a card billed
    by the hour, the first time an adapter arrives.
  * an adapter is then handed to the RUNNING engine
    (`/v1/load_lora_adapter`, vLLM >= 0.9 behind VLLM_ALLOW_RUNTIME_LORA_UPDATING,
    which the launch sets whenever it enabled LoRA) instead of restarting it.

The restart stays as the fallback and is still reached on every path that
cannot use the API — an engine started without `--enable-lora`, an older vLLM,
an adapter whose rank exceeds the slot size. A pod rented an hour ago may be
running an older image than the one we would launch today, so "it should be
supported" is not something this can assume.

What this buys: one rented card, one copy of the base model, and QLoRAs added
or swapped per request by name with no reload.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HXRMe9ZPUQCCCtBai9uqt3

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.88"
+__version__ = "0.3.89"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_worker.py

@@ -1255,6 +1255,14 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
     # look like they fell over at 32 concurrent pages when they were simply out
     # of window. Admission here is the pod's parallel-request and queue limits.
     env["CODERAI_SERVER_RATE_LIMIT_ENABLED"] = "0"
+    # This pod serves a shared POOL: several names over one copy of the weights,
+    # which for text models means a base and its adapters. vLLM cannot turn
+    # `--enable-lora` on after it has started, so an engine launched without it
+    # has to RELOAD the base model the first time an adapter arrives — minutes,
+    # on a card billed by the hour. Telling the pod now costs a buffer
+    # allocation and lets an adapter be added to the running engine instead.
+    if str(getattr(mcfg, "pool", "") or "").strip():
+        env["CODERAI_VLLM_ENABLE_LORA"] = "1"
     seeds = []
     if entry:
         kind, value = resolve_model_source(entry, mcfg)

codai/api/vllm_worker.py

@@ -316,15 +316,50 @@ def _served_names(model_config: dict, served_name: str, fallback: str) -> list:
     return out or [fallback]
 
 
+#: This engine serves a shared POOL — several names over one copy of the
+#: weights — so the LoRA machinery is allocated whether or not an adapter is
+#: known yet. Set by the orchestrator when the model's runpod block names a
+#: pool (see runpod_worker._plan); a model entry's own `runpod.pool` works too,
+#: for an install driving vLLM directly.
+_ENABLE_LORA_ENV = "CODERAI_VLLM_ENABLE_LORA"
+
+
+def _serves_a_pool(model_config: dict) -> bool:
+    raw = (os.environ.get(_ENABLE_LORA_ENV) or "").strip().lower()
+    if raw in ("1", "true", "on", "yes"):
+        return True
+    block = (model_config or {}).get("runpod")
+    return bool(isinstance(block, dict) and str(block.get("pool") or "").strip())
+
+
+def _lora_capacity_args(n_mods: int) -> list:
+    """The flags that must be decided at LAUNCH, separate from the adapter list.
+
+    vLLM takes `--lora-modules` as a convenience; what it cannot change later is
+    the CAPACITY. `--enable-lora` allocates the machinery, `--max-lora-rank`
+    sizes the slots, `--max-loras` decides how many distinct adapters may be
+    live in one batch. Get those right and the list itself can grow at runtime.
+    """
+    # vLLM rejects an adapter whose rank exceeds --max-lora-rank (default 16),
+    # and the message points at the flag rather than the adapter, so raise the
+    # ceiling to what vLLM supports rather than have a trained LoRA refused.
+    #
+    # --max-loras default is 1, so with several registered, requests naming
+    # different ones cannot batch together and serialise instead — the cost
+    # lands on latency, not on an error, which is why it goes unnoticed. The
+    # floor of 4 is for a pool that has no adapters YET: the engine must be able
+    # to batch across the ones it is about to be given.
+    return ["--max-lora-rank", "64",
+            "--max-loras", str(max(4, min(n_mods, 8)) if n_mods else 4)]
+
+
 def _lora_args(model_config: dict) -> list:
     """`--enable-lora --lora-modules …` for the adapters this engine must serve."""
     try:
         from codai.models.text_loras import describe, is_portable, local_path
         specs = _sibling_lora_specs(model_config or {})
     except Exception:
-        return []
-    if not specs:
-        return []
+        specs = []
     mods, usable = [], []
     for spec in specs:
         src = str(spec.get("source") or "")
@@ -346,17 +381,19 @@ def _lora_args(model_config: dict) -> list:
         mods.append(f"{spec['name']}={resolved}")
         usable.append(spec)
     if not mods:
-        return []
+        if not _serves_a_pool(model_config):
+            return []
+        # A pool pod with no adapter yet. `--enable-lora` cannot be turned on
+        # afterwards, so an engine launched without it has to RELOAD the base
+        # model — minutes, on a card being paid for by the hour — the first
+        # time an adapter arrives. Opening the door costs a buffer allocation
+        # and nothing else, and only a model that declares a pool pays it.
+        print("[lora] no adapters yet, but this engine serves a pool — "
+              "starting with LoRA enabled so one can be added without a "
+              "reload", flush=True)
+        return ["--enable-lora"] + _lora_capacity_args(0)
     args = ["--enable-lora", "--lora-modules"] + mods
-    # vLLM rejects an adapter whose rank exceeds --max-lora-rank (default 16),
-    # and the message points at the flag rather than the adapter, so raise the
-    # ceiling to what vLLM supports rather than have a trained LoRA refused.
-    args += ["--max-lora-rank", "64"]
-    # How many DISTINCT adapters may be active in one batch. The default is 1,
-    # so with several registered, requests naming different ones cannot batch
-    # together and serialise instead — the cost lands on latency, not on an
-    # error, which is why it goes unnoticed.
-    args += ["--max-loras", str(max(1, min(len(mods), 8)))]
+    args += _lora_capacity_args(len(mods))
     print(f"[lora] vLLM will serve adapters: {describe(usable)}", flush=True)
     return args
 
@@ -371,6 +408,64 @@ def _model_config_for(model_name: str) -> dict:
         return {}
 
 
+def _load_lora_runtime(url: str, model_config: dict, served_name: str) -> bool:
+    """Register an adapter into a RUNNING engine. True when it will now serve.
+
+    vLLM takes `--lora-modules` at launch, so for a long time the only way to
+    add one was to restart — a full reload of the base model, minutes, on a card
+    billed by the hour, and every request in flight lost. `/v1/load_lora_adapter`
+    does it in place (vLLM >= 0.9, behind VLLM_ALLOW_RUNTIME_LORA_UPDATING,
+    which the launch sets whenever it enabled LoRA).
+
+    Returns False for every reason it might not work — an engine started without
+    `--enable-lora`, an older vLLM, the flag unset, an adapter whose rank exceeds
+    the slot size — and the caller falls back to the restart. A pod rented an
+    hour ago may be running an older image than the one we would launch today,
+    so "it should be supported" is not something this can assume.
+    """
+    import json as _json
+    import urllib.error
+    import urllib.request
+
+    want = (served_name or "").strip()
+    if not want:
+        return False
+    spec = next((s for s in _sibling_lora_specs(model_config or {})
+                 if str(s.get("name") or "") == want), None)
+    if spec is None:
+        return False
+    try:
+        from codai.models.text_loras import local_path
+        src = str(spec.get("source") or "")
+        path = local_path(src) if src.startswith(("sha256:", "name:")) else src
+    except Exception:
+        path = ""
+    if not path:
+        return False
+    body = _json.dumps({"lora_name": want, "lora_path": path}).encode()
+    req = urllib.request.Request(url.rstrip("/") + "/v1/load_lora_adapter",
+                                 data=body,
+                                 headers={"Content-Type": "application/json"})
+    try:
+        urllib.request.urlopen(req, timeout=120)
+    except urllib.error.HTTPError as exc:
+        detail = ""
+        try:
+            detail = exc.read().decode("utf-8", "replace")[:200]
+        except Exception:
+            pass
+        print(f"[lora] {url} refused {want!r} at runtime ({exc.code} {detail}) "
+              f"— falling back to a restart", flush=True)
+        return False
+    except Exception as exc:
+        print(f"[lora] could not reach {url} to add {want!r} ({exc}) — falling "
+              f"back to a restart", flush=True)
+        return False
+    print(f"[lora] added {want!r} to the running engine on {url} — no reload",
+          flush=True)
+    return True
+
+
 def _serves_name(url: str, name: str):
     """True / False / None — the last meaning "could not tell".
 
@@ -498,6 +593,12 @@ def _ensure_service_once(cfg, model_path: Optional[str] = None,
             # timeout would be far worse than the doubt.
             if not served_name or _serves_name(svc["url"], served_name) is not False:
                 return svc["url"]
+            # An ADAPTER can usually just be handed to the running engine. Try
+            # that before throwing away a loaded base model.
+            if _load_lora_runtime(svc["url"],
+                                  _model_config_for(resolved or model_path or ""),
+                                  served_name):
+                return svc["url"]
             print(f"[vllm] the engine on {svc['url']} does not serve "
                   f"{served_name!r} — it started before this model was known "
                   f"here; restarting it so one engine serves the base and every "
@@ -536,6 +637,13 @@ def _ensure_service_once(cfg, model_path: Optional[str] = None,
         # via extra_env. (Set before extra_env so an explicit override wins.)
         env.setdefault("VLLM_USE_FLASHINFER", "0")
         env.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
+        # An adapter can then be registered into the RUNNING engine
+        # (/v1/load_lora_adapter) instead of costing a reload of the base
+        # model. Only where the launch actually enabled LoRA: the flag is
+        # meaningless otherwise, and setting it everywhere would be a claim we
+        # cannot honour.
+        if "--enable-lora" in cmd:
+            env.setdefault("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "1")
         # Pin this vLLM instance to specific NVIDIA GPU(s) if configured (CUDA-only).
         gpu = (getattr(cfg, "gpu", "") or "").strip()
         if gpu:

tests/test_shared_pool_adapter_identity.py

@@ -373,3 +373,98 @@ def test_a_model_with_no_local_adapters_sends_nothing(monkeypatch):
     b._pool = type("P", (), {"api_key": "k"})()
     rb.RunpodBackend._ADAPTERS_SENT.clear()
     b._ensure_adapters("https://pod")
+
+
+# --------------------------------------------------------------------------- #
+# a pool keeps the LoRA door open, and an adapter walks through it
+# --------------------------------------------------------------------------- #
+def test_a_pool_engine_enables_lora_before_it_has_any_adapter(monkeypatch):
+    """--enable-lora cannot be turned on later, so an engine without it has to
+    RELOAD the base model the first time an adapter arrives."""
+    import codai.admin.routes as routes
+    monkeypatch.setattr(routes, "config_manager", _CM({"text_models": [SEEDED_BASE]}),
+                        raising=False)
+    monkeypatch.setenv("CODERAI_VLLM_ENABLE_LORA", "1")
+    args = vw._lora_args(SEEDED_BASE)
+    assert "--enable-lora" in args
+    assert "--lora-modules" not in args
+    assert "--max-lora-rank" in args and "--max-loras" in args
+
+
+def test_a_model_that_serves_no_pool_pays_nothing(monkeypatch):
+    import codai.admin.routes as routes
+    monkeypatch.delenv("CODERAI_VLLM_ENABLE_LORA", raising=False)
+    monkeypatch.setattr(routes, "config_manager", _CM({"text_models": [SEEDED_BASE]}),
+                        raising=False)
+    assert vw._lora_args(SEEDED_BASE) == []
+
+
+def test_the_models_own_pool_also_opens_the_door(monkeypatch):
+    """An install driving vLLM directly has no pod env to read."""
+    import codai.admin.routes as routes
+    monkeypatch.delenv("CODERAI_VLLM_ENABLE_LORA", raising=False)
+    entry = dict(BASE)
+    monkeypatch.setattr(routes, "config_manager", _CM({"text_models": [entry]}),
+                        raising=False)
+    assert "--enable-lora" in vw._lora_args(entry)
+
+
+def test_the_pod_is_told_it_serves_a_pool():
+    import json
+    mcfg = rw.parse_model_runpod({"pool": "shared", "engine": "coderai-vllm",
+                                  "hf_repo": BASE_REPO})
+    plan = rw._plan("coderai", "img", "", mcfg, "key", dict(BASE), "base-awq")
+    assert plan["env"]["CODERAI_VLLM_ENABLE_LORA"] == "1"
+
+
+def test_a_model_with_no_pool_is_not():
+    mcfg = rw.parse_model_runpod({"engine": "coderai-vllm", "hf_repo": BASE_REPO})
+    plan = rw._plan("coderai", "img", "", mcfg, "key", dict(BASE), "base-awq")
+    assert "CODERAI_VLLM_ENABLE_LORA" not in plan["env"]
+
+
+def test_a_new_adapter_is_handed_to_the_running_engine_not_a_restart(monkeypatch):
+    """The whole point: one A40, the base loaded once, adapters added live."""
+    import codai.admin.routes as routes
+    monkeypatch.setattr(routes, "config_manager",
+                        _CM({"text_models": [SEEDED_BASE, SEEDED_ADAPTER]}),
+                        raising=False)
+    monkeypatch.setattr("codai.models.text_loras.local_path",
+                        lambda src: "/blobs/tuned-alpha")
+    posted = {}
+
+    def _urlopen(req, timeout=0):
+        posted["url"] = req.full_url
+        posted["body"] = req.data
+        return type("R", (), {"read": lambda self: b"{}"})()
+
+    monkeypatch.setattr("urllib.request.urlopen", _urlopen)
+    assert vw._load_lora_runtime("http://e", SEEDED_BASE, "tuned-alpha") is True
+    assert posted["url"].endswith("/v1/load_lora_adapter")
+    assert b'"lora_name": "tuned-alpha"' in posted["body"]
+
+
+def test_an_engine_that_refuses_at_runtime_still_gets_restarted(monkeypatch):
+    """A pod rented an hour ago may run an older image than we would launch."""
+    import urllib.error
+    import codai.admin.routes as routes
+    monkeypatch.setattr(routes, "config_manager",
+                        _CM({"text_models": [SEEDED_BASE, SEEDED_ADAPTER]}),
+                        raising=False)
+    monkeypatch.setattr("codai.models.text_loras.local_path", lambda src: "/blobs/a")
+
+    def _boom(req, timeout=0):
+        raise urllib.error.HTTPError(req.full_url, 404, "Not Found", {}, None)
+
+    monkeypatch.setattr("urllib.request.urlopen", _boom)
+    assert vw._load_lora_runtime("http://e", SEEDED_BASE, "tuned-alpha") is False
+
+
+def test_an_unknown_name_is_not_sent_anywhere(monkeypatch):
+    import codai.admin.routes as routes
+    monkeypatch.setattr(routes, "config_manager",
+                        _CM({"text_models": [SEEDED_BASE, SEEDED_ADAPTER]}),
+                        raising=False)
+    monkeypatch.setattr("urllib.request.urlopen",
+                        lambda *a, **k: pytest.fail("nothing to register"))
+    assert vw._load_lora_runtime("http://e", SEEDED_BASE, "not-an-adapter") is False