runpod: a preferred GPU, not the only GPU (v0.3.1)

`gpu_type` was a filter: "NVIDIA A40" meant an A40 or nothing, so when RunPod
had no A40 free the request failed rather than taking a card that would have
served it. Now `gpu_types` is an ordered PREFERENCE and `allow_other_gpus` lets
the search fall through to anything else that still satisfies min_vram_gb and
max_hourly_usd.

A preferred card ranks ahead of every fallback whatever selection_criteria
says — otherwise "cheaper" would put the fallback first and naming a card would
mean nothing. Each ranked option carries `preferred`, and provisioning a card
nobody asked for prints "[FALLBACK — no preferred card available]": correct
behaviour, but never silent.

Off by default. A pinned card is sometimes pinned for a reason, and an install
that quietly started renting something else would be a surprise on the invoice.

Name matching is exact on the id or display name, case-insensitive, tolerating
a missing "NVIDIA " prefix, and deliberately NOT a substring: "A40" would
otherwise match "NVIDIA RTX A4000" — 16 GB for a job that asked for 48. GPU
names split on commas only, because "NVIDIA RTX A6000" has spaces in it.

Also: the admin handler rebuilds the runpod block from a whitelist, so all six
keys added today (global_volume_id, data_centers, gpu_types, allow_other_gpus,
allow_client_scale, client_scale_ttl_s) were being dropped the next time anyone
saved a model from the GUI — taking the warm schedule and the volume with them.
They are in the whitelist now, with the two ordered lists kept in order.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/admin/routes.py

@@ -3105,7 +3105,9 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                   "network_volume_id", "volume_mount_path", "volume_path",
                   "venv_name", "slim_image",
                   # region + warm-pod schedule
-                  "data_center", "schedule_start", "schedule_end", "schedule_tz"):
+                  "data_center", "schedule_start", "schedule_end", "schedule_tz",
+                  # a GLOBAL (region-independent) volume
+                  "global_volume_id"):
             v = src.get(k)
             if isinstance(v, str) and v.strip():
                 rpo[k] = v.strip()
@@ -3119,6 +3121,20 @@ async def api_model_configure(request: Request, username: str = Depends(require_
             ct = None
         if ct:
             rpo["cloud_types"] = ct
+        # Ordered lists: regions to try, and preferred GPUs. Order IS meaning
+        # here (it is the fallback order), so neither is sorted or upper-cased
+        # beyond what the region ids need. GPU names split on commas only —
+        # "NVIDIA RTX A6000" has spaces in it.
+        dcs = src.get("data_centers")
+        if isinstance(dcs, str):
+            dcs = [d.strip() for d in dcs.replace(",", " ").split() if d.strip()]
+        if isinstance(dcs, list) and dcs:
+            rpo["data_centers"] = [str(d).strip().upper() for d in dcs if str(d).strip()]
+        gts = src.get("gpu_types")
+        if isinstance(gts, str):
+            gts = [g.strip() for g in gts.split(",") if g.strip()]
+        if isinstance(gts, list) and gts:
+            rpo["gpu_types"] = [str(g).strip() for g in gts if str(g).strip()]
         # schedule_days: list of 0=Mon…6=Sun, or a comma string of those/names
         sd = src.get("schedule_days")
         if isinstance(sd, str):
@@ -3127,7 +3143,10 @@ async def api_model_configure(request: Request, username: str = Depends(require_
             rpo["schedule_days"] = [str(d).strip().lower() for d in sd if str(d).strip()]
         # bools
         for k in ("allow_spot", "allow_open_pod", "sticky_sessions", "keep_warm",
-                  "model_url_is_tar", "venv_on_volume", "schedule_enabled"):
+                  "model_url_is_tar", "venv_on_volume", "schedule_enabled",
+                  # rent a different card when no preferred one is free, and let
+                  # a client lease warm pods
+                  "allow_other_gpus", "allow_client_scale"):
             if k in src and src.get(k) is not None and src.get(k) != "":
                 sv = src.get(k)
                 rpo[k] = (sv.lower() in ("1", "true", "on", "yes")
@@ -3139,7 +3158,7 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                   # how hard the POD's own server may be driven
                   "pod_max_parallel_requests", "pod_queue_max_size",
                   "boot_timeout_s", "load_timeout_s", "min_workers", "max_workers",
-                  "gpu_count"):
+                  "gpu_count", "client_scale_ttl_s"):
             v = src.get(k)
             if v not in (None, ""):
                 try:

codai/api/runpod_worker.py

@@ -42,6 +42,18 @@ class RunpodModelConfig:
     cloud_types: list = field(default_factory=lambda: ["SECURE"])   # allowed pools
     selection_criteria: str = "cheaper"      # cheaper | faster
     gpu_type: str = ""                       # explicit RunPod gpuTypeId (optional)
+    # Ordered PREFERENCE instead of a single card: tried best-first, and with
+    # allow_other_gpus the search falls through to anything else that fits.
+    # `gpu_type` above is folded in as a single-entry preference.
+    gpu_types: list = field(default_factory=list)
+    # When every preferred card is unavailable, may we rent a different one that
+    # still satisfies min_vram_gb and max_hourly_usd? OFF by default: a pinned
+    # card is sometimes pinned for a reason (an architecture a kernel needs, or
+    # a price), and silently renting something else would be a surprise. ON is
+    # what you want for "serve the request, A40 or not" — and then the VRAM
+    # floor and the $/hr ceiling are the real constraints, so check that
+    # something actually fits under them.
+    allow_other_gpus: bool = False
     min_vram_gb: float = 0.0
     max_hourly_usd: float = 0.0              # per-model $/hr GPU ceiling (0 = none)
     # GPUs per pod. RunPod rents multi-GPU pods, and a 154 GB model on one 80 GB
@@ -1366,6 +1378,43 @@ def _as_int(v, default=0):
 _DAY_NAMES = ("mon", "tue", "wed", "thu", "fri", "sat", "sun")
 
 
+def _parse_name_list(value) -> list:
+    """An ordered list of GPU names from a list or a comma string.
+
+    Comma only, not whitespace: a GPU display name has spaces in it ("NVIDIA
+    RTX A6000"), so splitting on those would turn one card into three.
+    """
+    if value in (None, ""):
+        return []
+    items = value if isinstance(value, (list, tuple)) else str(value).split(",")
+    out = []
+    for raw in items:
+        tok = str(raw).strip()
+        if tok and tok not in out:
+            out.append(tok)
+    return out
+
+
+def _gpu_matches(gpu: dict, want: str) -> bool:
+    """Whether a catalogue entry is the card `want` names.
+
+    Exact on the id or the display name, case-insensitively, and tolerant of a
+    missing "NVIDIA " prefix so "A40" finds "NVIDIA A40". NOT a substring match:
+    "A40" would otherwise also match "NVIDIA RTX A4000", which is a 16 GB card
+    where someone asked for a 48 GB one.
+    """
+    w = (want or "").strip().lower()
+    if not w:
+        return False
+    for field_ in (gpu.get("id"), gpu.get("display_name")):
+        name = str(field_ or "").strip().lower()
+        if not name:
+            continue
+        if name == w or name.removeprefix("nvidia ") == w.removeprefix("nvidia "):
+            return True
+    return False
+
+
 def _parse_dc_list(value) -> list:
     """A region allow-list from a list or a comma/space string, order kept.
 
@@ -1530,6 +1579,8 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
     cfg.network_volume_id = (b.get("network_volume_id") or "").strip()
     cfg.global_volume_id = (b.get("global_volume_id") or "").strip()
     cfg.data_centers = _parse_dc_list(b.get("data_centers"))
+    cfg.gpu_types = _parse_name_list(b.get("gpu_types"))
+    cfg.allow_other_gpus = _as_bool(b.get("allow_other_gpus"), cfg.allow_other_gpus)
     if cfg.network_volume_id:
         # RunPod offers network volumes on Secure Cloud only; leaving COMMUNITY
         # in the list just produces candidates the attach would reject.
@@ -1691,11 +1742,24 @@ def _rank_gpus(client, mcfg: "RunpodModelConfig", account_cfg) -> list:
     # holds N times the VRAM, so both the ceiling and the VRAM floor are checked
     # against the pod, not the card — or a 4-GPU pod would pass a $1/hr ceiling
     # at $0.90 a card and bill $3.60.
-    explicit = (mcfg.gpu_type or getattr(account_cfg, "default_gpu_type", "") or "").strip()
-
-    cands = []   # (price, is_spot, cloud_type, gpu, on_demand_price)
+    # Preference, not a filter. The old single `gpu_type` was a hard filter, so
+    # "A40" meant "A40 or nothing" — and when RunPod had no A40 the request
+    # failed instead of taking a card that fit just as well.
+    prefer = ([p for p in (getattr(mcfg, "gpu_types", None) or []) if str(p).strip()]
+              or ([mcfg.gpu_type] if mcfg.gpu_type.strip() else [])
+              or ([getattr(account_cfg, "default_gpu_type", "")]
+                  if str(getattr(account_cfg, "default_gpu_type", "") or "").strip()
+                  else []))
+    allow_other = bool(getattr(mcfg, "allow_other_gpus", False))
+
+    cands = []   # (pref_rank, price, is_spot, cloud_type, gpu, on_demand_price)
     for g in catalog:
-        if explicit and g["id"] != explicit and g["display_name"] != explicit:
+        rank = len(prefer)
+        for idx_p, want in enumerate(prefer):
+            if _gpu_matches(g, want):
+                rank = idx_p
+                break
+        if prefer and rank == len(prefer) and not allow_other:
             continue
         mem = (g.get("memory_gb") or 0) * n
         if mcfg.min_vram_gb and mem < mcfg.min_vram_gb:
@@ -1712,26 +1776,31 @@ def _rank_gpus(client, mcfg: "RunpodModelConfig", account_cfg) -> list:
                 price = price * n
                 if ceiling > 0 and price > ceiling:
                     continue
-                cands.append((price, is_spot, ct, g, (on_demand or price) * n))
+                cands.append((rank, price, is_spot, ct, g, (on_demand or price) * n))
     if not cands:
         raise RunpodError(
             f"RunPod: no GPU in pools {allowed} with ≥{mcfg.min_vram_gb}GB under "
             f"${ceiling}/hr (spot={'on' if mcfg.allow_spot else 'off'}"
-            + (f", gpu={explicit}" if explicit else "") + ").")
+            + (f", preferred={prefer}" if prefer else "")
+            + (", allow_other_gpus=off — set it to let another card that fits "
+               "serve the request" if prefer and not allow_other else "") + ").")
 
     if (mcfg.selection_criteria or "cheaper").lower() == "faster":
         # Reliability + throughput: SECURE before COMMUNITY, on-demand before spot,
         # more VRAM (throughput proxy) first, then cheapest.
         ct_rank = {"SECURE": 0, "COMMUNITY": 1}
-        cands.sort(key=lambda c: (ct_rank.get(c[2], 9), c[1] is True,
-                                  -(c[3].get("memory_gb") or 0), c[0]))
+        cands.sort(key=lambda c: (c[0], ct_rank.get(c[3], 9), c[2] is True,
+                                  -(c[4].get("memory_gb") or 0), c[1]))
     else:  # cheaper
-        cands.sort(key=lambda c: (c[0], c[1] is False))
+        cands.sort(key=lambda c: (c[0], c[1], c[2] is False))
+    # A preferred card always sorts ahead of a fallback, whatever the criteria:
+    # the point of naming one is to get it when it is there.
     return [{"gpu_type_id": g["id"],
              "display_name": (f"{n}x {g['display_name']}" if n > 1 else g["display_name"]),
              "memory_gb": (g.get("memory_gb") or 0) * n, "cloud_type": ct, "price": price,
-             "is_spot": is_spot, "bid": on_demand if is_spot else 0.0, "gpu_count": n}
-            for (price, is_spot, ct, g, on_demand) in cands]
+             "is_spot": is_spot, "bid": on_demand if is_spot else 0.0, "gpu_count": n,
+             "preferred": rank < len(prefer)}
+            for (rank, price, is_spot, ct, g, on_demand) in cands]
 
 
 def _select_gpu(client, mcfg: "RunpodModelConfig", account_cfg) -> dict:
@@ -2158,9 +2227,13 @@ class RunpodPodPool:
                     + str(self.model_key)[:20].replace("/", "_").replace(" ", "_")
                     + "-" + uuid.uuid4().hex[:6])
             _dc = sel.get("_dc") or self._data_center()
+            # Say so when this is not the card that was asked for. Renting a
+            # different GPU than the configuration names is correct here — it is
+            # what allow_other_gpus is for — but it must never be silent.
+            _alt = "" if sel.get("preferred", True) else " [FALLBACK — no preferred card available]"
             print(f"[runpod] provisioning pod for {self.model_key!r}: {sel['display_name']} "
                   f"({sel['cloud_type']}{'/spot' if sel['is_spot'] else ''}) ${sel['price']}/hr"
-                  + (f" in {_dc}" if _dc else " (region: RunPod's choice)"),
+                  + (f" in {_dc}" if _dc else " (region: RunPod's choice)") + _alt,
                   flush=True)
             try:
                 _disk_gb, _disk_note = disk_for(

docs/orchestrator-runpod-guide.md

@@ -608,6 +608,8 @@ Four numbers decide the shape of a pool:
 | `client_scale_ttl_s` | how long such a lease lasts unless renewed (default 1800). A lease that never expired would be a bill that never stopped. |
 | `data_centers` | several acceptable regions, tried in order on a capacity miss — how you say "anywhere in the EU". A single `data_center` pins one region; blank lets RunPod pick any region **on earth**. |
 | `global_volume_id` | a RunPod **global** volume (region-independent). Unlike `network_volume_id` it does NOT pin the pod's region, which is the only reason to use one. |
+| `gpu_types` | ordered GPU **preference**, best first (comma-separated; names have spaces, so commas only). |
+| `allow_other_gpus` | when no preferred card is free, rent another that still fits `min_vram_gb` and `max_hourly_usd`. Off by default. |
 
 **Then the two that are easy to miss**, and the reason a pod can look slow while
 its GPU is idle:
@@ -697,6 +699,44 @@ Caveats worth knowing before you rely on it:
 
 ---
 
+### When the card you asked for is not available
+
+`gpu_type` is a **filter**, not a preference: `"NVIDIA A40"` has always meant
+"an A40 or nothing", so when RunPod has no A40 free the request fails rather
+than taking a card that would have served it just as well.
+
+To let it fall through:
+
+```json
+"gpu_types": "NVIDIA A40,NVIDIA RTX A6000",
+"allow_other_gpus": true,
+"min_vram_gb": 40,
+"max_hourly_usd": 1.20
+```
+
+* `gpu_types` is an ordered **preference**. A preferred card always ranks ahead
+  of a fallback, whatever `selection_criteria` says — the point of naming one is
+  to get it when it is there.
+* `allow_other_gpus` is what permits anything else. It is **off by default**: a
+  pinned card is sometimes pinned for a reason, and quietly renting a different
+  one would be a surprise on the invoice.
+* **`min_vram_gb` and `max_hourly_usd` are the real constraints** once the
+  fallback is on, and this is the part that catches people. With
+  `max_hourly_usd: 0.60`, an A40 at $0.59 fits and nothing else does — so the
+  fallback finds nothing and you are back where you started. Raise the ceiling
+  to the most you are willing to pay for the *alternative*, and keep
+  `min_vram_gb` honest so a cheap 16 GB card cannot win a job that needs 48.
+* A fallback is never silent. The provisioning line says
+  `[FALLBACK — no preferred card available]`, and each ranked option carries a
+  `preferred` flag.
+
+Name matching is exact on the id or display name, case-insensitive, and
+tolerates a missing `NVIDIA ` prefix — so `A40` finds `NVIDIA A40`. It is
+deliberately **not** a substring match: `A40` would otherwise also match
+`NVIDIA RTX A4000`, a 16 GB card, for a job that asked for 48.
+
+---
+
 ### Let a client ask for pods
 
 The autoscaler reacts to load: it adds a pod once the least-loaded one already

tests/test_runpod_placement.py

+"""Where a pod goes, and on what card: volumes, regions and GPU fallback.
+
+A global volume must not pin the pod to a region — that is the whole point.
+
+RunPod's Global Volumes (beta, Sept 2026) are region-independent storage backed
+by object storage. A NETWORK volume lives in one data centre and a pod elsewhere
+cannot attach it, which is why coderai forces the pod into the volume's region.
+Applying that same pinning to a global volume would throw away the only reason
+to use one: store the weights once, rent a GPU in whichever region has one.
+
+So the two are resolved separately, and these tests hold them apart.
+"""
+import pathlib
+
+from codai.api import runpod_worker as rw
+
+ROOT = pathlib.Path(rw.__file__).resolve().parents[2]
+
+
+class _Acct:
+    def __init__(self, net="", glob="", dc="", mount="/workspace"):
+        self.network_volume_id = net
+        self.global_volume_id = glob
+        self.data_center = dc
+        self.volume_mount_path = mount
+
+
+def test_the_account_setting_applies_to_every_model():
+    mcfg = rw.parse_model_runpod({})
+    assert rw.global_volume_for(mcfg, _Acct(glob="gv-123")) == "gv-123"
+
+
+def test_a_model_can_override_the_account_volume():
+    mcfg = rw.parse_model_runpod({"global_volume_id": "gv-model"})
+    assert rw.global_volume_for(mcfg, _Acct(glob="gv-account")) == "gv-model"
+
+
+def test_no_volume_configured_is_empty_not_an_error():
+    assert rw.global_volume_for(rw.parse_model_runpod({}), _Acct()) == ""
+
+
+def test_a_global_volume_does_not_pin_the_data_centre():
+    """The network volume's region wins over everything; a global volume must
+    leave the choice alone so the scaler can rank cards across all regions."""
+    pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
+    pool.account = _Acct(glob="gv-123")
+    pool.mcfg = rw.parse_model_runpod({"global_volume_id": "gv-123"})
+    pool._volume_dc = None
+    assert pool._data_center() == ""      # free to go anywhere
+    pool.mcfg.data_center = "EU-RO-1"
+    assert pool._data_center() == "EU-RO-1"   # only because the model asked
+
+
+def test_only_the_network_volume_triggers_the_region_lookup():
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    blk = src.split("vol_id, vol_mount = volume_for")[1].split("self._tls = False")[0]
+    # The region lookup sits under the network-volume branch, and the global
+    # branch that follows must not contain it.
+    net, glob = blk.split("global_vol = global_volume_for")
+    assert "_data_center_for_volume" in net
+    assert "_data_center_for_volume" not in glob
+
+
+def test_the_two_volumes_are_not_attached_together_by_accident():
+    """With both attached RunPod moves the global one to /workspace-global, so
+    pointing the pod env at /workspace would aim it at the wrong volume."""
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    blk = src.split("global_vol = global_volume_for")[1][:600]
+    assert "if global_vol and not vol_id:" in blk
+    assert "/workspace-global" in src
+
+
+def test_the_client_sends_the_field_on_both_api_shapes():
+    """create_pod has a GraphQL body and a REST body; a volume wired into only
+    one of them attaches on some code paths and silently not on others."""
+    src = (ROOT / "codai/api/runpod_client.py").read_text()
+    assert src.count("GLOBAL_VOLUME_FIELD] = global_volume_id") == 2
+    assert "global_volume_id: str = \"\"" in src
+
+
+def test_the_field_name_is_overridable_because_it_is_undocumented():
+    """RunPod documents global volumes as console-only and the v1 validator
+    accepts unknown body keys silently, so a wrong name cannot be detected by
+    probing — it yields a pod with no volume that re-downloads its weights."""
+    from codai.api import runpod_client
+    assert runpod_client.GLOBAL_VOLUME_FIELD
+    src = (ROOT / "codai/api/runpod_client.py").read_text()
+    assert "CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD" in src
+    assert "VERIFY" in src        # says out loud that it is unconfirmed
+
+
+def test_the_config_documents_that_it_cannot_be_created_by_api():
+    cfgsrc = (ROOT / "codai/config.py").read_text()
+    blk = cfgsrc.split("global_volume_id")[0][-1400:]
+    assert "console" in blk.lower()
+    assert "STANDARD" in blk      # the enum the API actually allows
+
+
+# ------------------------------------------- "anywhere in the EU", said safely
+
+class _AcctDC(_Acct):
+    def __init__(self, dcs=None, dc="", **kw):
+        super().__init__(**kw)
+        self.data_centers = dcs or []
+        self.data_center = dc
+
+
+def _pool(mcfg_block=None, acct=None):
+    pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
+    pool.account = acct or _AcctDC()
+    pool.mcfg = rw.parse_model_runpod(mcfg_block or {})
+    pool._volume_dc = None
+    return pool
+
+
+def test_a_region_list_is_parsed_from_a_string_or_a_list():
+    for value in ("EU-RO-1,EU-FR-1", "EU-RO-1 EU-FR-1", ["eu-ro-1", "EU-FR-1"]):
+        assert rw._parse_dc_list(value) == ["EU-RO-1", "EU-FR-1"], value
+    assert rw._parse_dc_list("") == []
+    assert rw._parse_dc_list(None) == []
+
+
+def test_the_order_is_the_fallback_order_and_is_preserved():
+    """Not sorted: the first entry is the preferred region and the scaler only
+    moves on when it cannot get a card there."""
+    assert rw._parse_dc_list("EU-RO-1,EU-FR-1,EUR-NO-1")[0] == "EU-RO-1"
+    assert rw._parse_dc_list("EU-FR-1,EU-RO-1")[0] == "EU-FR-1"
+
+
+def test_duplicates_collapse():
+    assert rw._parse_dc_list("EU-RO-1, eu-ro-1 ,EU-FR-1") == ["EU-RO-1", "EU-FR-1"]
+
+
+def test_the_model_list_beats_the_account_list():
+    pool = _pool({"data_centers": "EU-FR-1"}, _AcctDC(dcs=["EU-RO-1"]))
+    assert pool._data_centers() == ["EU-FR-1"]
+
+
+def test_a_single_data_center_still_works_exactly_as_before():
+    pool = _pool({"data_center": "EU-SE-1"})
+    assert pool._data_centers() == ["EU-SE-1"]
+    assert pool._data_center() == "EU-SE-1"
+
+
+def test_nothing_configured_still_means_runpod_chooses():
+    pool = _pool()
+    assert pool._data_centers() == [""]
+    assert pool._data_center() == ""
+
+
+def test_a_network_volume_overrides_every_region_setting():
+    """It exists in ONE region and no pod elsewhere can attach it, so a
+    multi-region list must not send a pod where the attach would fail."""
+    pool = _pool({"data_centers": "EU-FR-1,EU-RO-1"}, _AcctDC(dcs=["EUR-NO-1"]))
+    pool._volume_dc = "EU-NL-1"
+    assert pool._data_centers() == ["EU-NL-1"]
+
+
+def test_candidates_are_expanded_across_regions_for_the_capacity_fallback():
+    """The existing retry walks `ranked` on a capacity miss; pairing each card
+    with each region makes it walk regions too, with no new control flow."""
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    blk = src.split("ranked = _rank_gpus(client, self.mcfg, self.account)")[1][:700]
+    assert 'dict(sel, _dc=dc) for sel in ranked for dc in dcs' in blk
+    assert "if len(dcs) > 1:" in blk
+
+
+def test_the_pod_is_created_in_and_records_the_region_it_got():
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    assert src.count('sel.get("_dc") or self._data_center()') >= 3   # create, log, handle
+
+
+# ----------------------------- no A40 free must not mean "no pod for you"
+
+class _Client:
+    """A catalogue stand-in. Prices are per card, as RunPod's is."""
+
+    CATALOG = [
+        {"id": "NVIDIA A40", "display_name": "NVIDIA A40", "memory_gb": 48,
+         "secure_price": 0.59, "community_price": 0.44, "spot_price": 0.22},
+        {"id": "NVIDIA RTX A6000", "display_name": "NVIDIA RTX A6000", "memory_gb": 48,
+         "secure_price": 0.76, "community_price": 0.49, "spot_price": 0.25},
+        {"id": "NVIDIA L40S", "display_name": "NVIDIA L40S", "memory_gb": 48,
+         "secure_price": 1.03, "community_price": 0.86, "spot_price": 0.40},
+        {"id": "NVIDIA RTX A4000", "display_name": "NVIDIA RTX A4000", "memory_gb": 16,
+         "secure_price": 0.17, "community_price": 0.12, "spot_price": 0.05},
+    ]
+
+    def list_gpu_types(self):
+        return [dict(g) for g in self.CATALOG]
+
+
+def _rank(block, acct_default=""):
+    class _A:
+        default_gpu_type = acct_default
+        data_center = ""
+    return rw._rank_gpus(_Client(), rw.parse_model_runpod(block), _A())
+
+
+def test_a_single_gpu_type_is_still_exclusive_by_default():
+    """Existing installs must not start renting other cards because of this."""
+    out = _rank({"gpu_type": "NVIDIA A40", "max_hourly_usd": 2.0})
+    assert {o["display_name"] for o in out} == {"NVIDIA A40"}
+
+
+def test_allow_other_gpus_lets_another_card_serve_the_request():
+    out = _rank({"gpu_type": "NVIDIA A40", "allow_other_gpus": True,
+                 "min_vram_gb": 40, "max_hourly_usd": 2.0})
+    names = [o["display_name"] for o in out]
+    assert names[0] == "NVIDIA A40", "the preferred card still comes first"
+    assert "NVIDIA RTX A6000" in names and "NVIDIA L40S" in names
+
+
+def test_the_vram_floor_still_excludes_a_card_that_is_too_small():
+    """The 16 GB A4000 is cheapest, so without the floor it would win."""
+    out = _rank({"gpu_type": "NVIDIA A40", "allow_other_gpus": True,
+                 "min_vram_gb": 40, "max_hourly_usd": 2.0})
+    assert "NVIDIA RTX A4000" not in [o["display_name"] for o in out]
+
+
+def test_the_price_ceiling_still_governs_the_fallback():
+    """"Up to max price" means exactly that: a ceiling too low for any
+    alternative leaves the preferred card alone rather than overspending."""
+    out = _rank({"gpu_type": "NVIDIA A40", "allow_other_gpus": True,
+                 "min_vram_gb": 40, "max_hourly_usd": 0.6})
+    assert [o["display_name"] for o in out] == ["NVIDIA A40"]
+
+
+def test_a_preference_order_is_honoured():
+    out = _rank({"gpu_types": "NVIDIA L40S,NVIDIA A40", "allow_other_gpus": True,
+                 "min_vram_gb": 40, "max_hourly_usd": 2.0})
+    names = [o["display_name"] for o in out]
+    assert names[0] == "NVIDIA L40S" and names[1] == "NVIDIA A40"
+    assert names.index("NVIDIA RTX A6000") > 1, "fallbacks come after every preference"
+
+
+def test_a_preferred_card_beats_a_cheaper_fallback():
+    """Otherwise 'cheaper' would rank the fallback first and the preference
+    would mean nothing."""
+    out = _rank({"gpu_types": "NVIDIA L40S", "allow_other_gpus": True,
+                 "min_vram_gb": 40, "max_hourly_usd": 2.0,
+                 "selection_criteria": "cheaper"})
+    assert out[0]["display_name"] == "NVIDIA L40S"
+    assert out[0]["price"] > out[1]["price"], "and it really was the dearer one"
+
+
+def test_each_option_says_whether_it_was_preferred():
+    out = _rank({"gpu_type": "NVIDIA A40", "allow_other_gpus": True,
+                 "min_vram_gb": 40, "max_hourly_usd": 2.0})
+    assert out[0]["preferred"] is True
+    assert any(o["preferred"] is False for o in out)
+
+
+def test_renting_an_unasked_card_is_logged_loudly():
+    src = (ROOT / "codai/api/runpod_worker.py").read_text()
+    assert "FALLBACK — no preferred card available" in src
+
+
+def test_a40_does_not_match_the_a4000():
+    """Substring matching would hand a 16 GB card to a request for a 48 GB one."""
+    a40 = {"id": "NVIDIA A40", "display_name": "NVIDIA A40"}
+    a4000 = {"id": "NVIDIA RTX A4000", "display_name": "NVIDIA RTX A4000"}
+    assert rw._gpu_matches(a40, "A40") is True
+    assert rw._gpu_matches(a40, "NVIDIA A40") is True
+    assert rw._gpu_matches(a40, "nvidia a40") is True
+    assert rw._gpu_matches(a4000, "A40") is False
+    assert rw._gpu_matches(a40, "") is False
+
+
+def test_gpu_names_split_on_commas_only():
+    """"NVIDIA RTX A6000" has spaces in it; splitting on those makes three
+    cards out of one."""
+    assert rw._parse_name_list("NVIDIA RTX A6000, NVIDIA A40") == \
+        ["NVIDIA RTX A6000", "NVIDIA A40"]
+
+
+def test_the_error_says_how_to_allow_a_fallback():
+    import pytest
+    from codai.api.runpod_client import RunpodError
+    with pytest.raises(RunpodError) as e:
+        _rank({"gpu_type": "NVIDIA A40", "min_vram_gb": 999})
+    assert "allow_other_gpus" in str(e.value)
+
+
+# -------------------- the GUI must not erase what it does not know about
+
+def test_every_new_runpod_key_survives_a_save_from_the_admin_ui():
+    """The admin handler REBUILDS the runpod block from a whitelist, so a key
+    missing from it is silently dropped the next time anyone saves the model —
+    taking the warm schedule, the volume or the GPU fallback with it."""
+    src = (ROOT / "codai/admin/routes.py").read_text()
+    block = src.split("# Per-model runpod block")[1].split("# Per-model `host` block")[0]
+    for key in ("global_volume_id", "data_centers", "gpu_types",
+                "allow_other_gpus", "allow_client_scale", "client_scale_ttl_s"):
+        assert f'"{key}"' in block, key
+
+
+def test_the_two_ordered_lists_keep_their_order_through_the_ui():
+    """Order is the fallback order; sorting it would change behaviour."""
+    src = (ROOT / "codai/admin/routes.py").read_text()
+    block = src.split("# Per-model runpod block")[1].split("# Per-model `host` block")[0]
+    seg = block.split('rpo["data_centers"]')[0][-500:] + block.split('rpo["data_centers"]')[1][:400]
+    assert "sorted(" not in seg