runpod: global volumes and a region allow-list (v0.3.0)

Weights in one place, pods wherever a card is free.

`global_volume_id` attaches a RunPod GLOBAL volume (beta since Sept 2026):
region-independent storage, so unlike `network_volume_id` it does NOT pin the
pod's data centre — which is the only reason to use one. Resolved by its own
function precisely so the two cannot be confused: the region lookup stays under
the network-volume branch, and a test holds them apart.

`data_centers` is the other half. A single `data_center` pins one region and a
blank one lets RunPod pick any region on earth, so there was no way to say
"anywhere in the EU" — which is what data that may not leave the EU requires.
The list is tried in order, and each (card, region) pair becomes its own
candidate so the capacity fallback that already walks GPU options walks regions
too, with no new control flow. The pod handle records the region it actually
got, not the pool's first choice.

Two honest limits, both documented in the code and the guide. A global volume
must be created in the RunPod CONSOLE: POST /v2/network-volumes allows only
STANDARD|HIGH_PERFORMANCE and demands a dataCenter. And attaching is
console-documented only — no field exists in the Pod API reference, and the v1
REST validator accepts unknown body keys silently, so a wrong name cannot be
caught by probing; it yields a pod with no volume that re-downloads its
weights. We send globalNetworkVolumeId, overridable by
CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD, and it must be verified on a real pod.
Co-Authored-By: 's avatarClaude Opus 5 <noreply@anthropic.com>
parent 7b87efd7
......@@ -16,7 +16,7 @@
# Canonical product version for CoderAI — single source of truth. Both the API
# metadata and the admin web UI read from here.
__version__ = "0.2.99"
__version__ = "0.3.0"
__tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
# Configure the CUDA caching allocator BEFORE torch is imported anywhere.
......
......@@ -17,10 +17,23 @@ This module is pure transport: no scaling/policy logic (that lives in
NEVER logged — only masked in error strings.
"""
import os
import time
from typing import Optional
#: Request field that attaches a GLOBAL volume (RunPod's region-independent
#: storage, beta since Sept 2026). RunPod documents global volumes as console
#: only — no field appears in the Pod API reference — and the v1 REST validator
#: accepts unknown body keys silently, so this cannot be confirmed by probing:
#: a wrong name yields a pod that boots with no volume and re-downloads its
#: weights, which is expensive and invisible. Hence the override, so the name
#: can be corrected without a release once RunPod publishes it. VERIFY on the
#: first pod that the weights are actually at the mount before relying on it.
GLOBAL_VOLUME_FIELD = os.environ.get("CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD",
"globalNetworkVolumeId")
class RunpodError(RuntimeError):
"""A RunPod API call failed (network, auth, or GraphQL error)."""
......@@ -209,6 +222,7 @@ class RunpodClient:
volume_mount_path: str = "/workspace", env: Optional[dict] = None,
docker_args: str = "", is_spot: bool = False,
bid_per_gpu: float = 0.0, data_center_id: str = "",
global_volume_id: str = "",
registry_auth_id: str = "",
entrypoint: Optional[list] = None,
start_cmd: Optional[list] = None,
......@@ -261,6 +275,8 @@ class RunpodClient:
# cold model download per pod and paying it once.
if network_volume_id:
common["networkVolumeId"] = network_volume_id
if global_volume_id:
common[GLOBAL_VOLUME_FIELD] = global_volume_id
if is_spot:
common["bidPerGpu"] = float(bid_per_gpu)
mutation = "podRentInterruptable"
......@@ -309,6 +325,8 @@ class RunpodClient:
body["containerRegistryAuthId"] = registry_auth_id
if network_volume_id:
body["networkVolumeId"] = network_volume_id
if global_volume_id:
body[GLOBAL_VOLUME_FIELD] = global_volume_id
if is_spot:
body["interruptible"] = True
body["bidPerGpu"] = float(bid_per_gpu)
......
......@@ -70,6 +70,9 @@ class RunpodModelConfig:
# Attach a RunPod network volume to this pool's pods. Blank falls back to the
# account-wide one. Weights, uploads and adapters on it outlive the pod.
network_volume_id: str = ""
# Per-model override for the account's GLOBAL volume (see RunpodConfig).
# Region-independent, so it never pins the pod's data centre.
global_volume_id: str = ""
volume_mount_path: str = "" # blank = the account default
volume_path: str = "" # weights already on the volume (file or dir)
# Reach the pod at its public ip:port instead of through RunPod's HTTP proxy.
......@@ -158,6 +161,12 @@ class RunpodModelConfig:
# over both: a volume lives in one region and a pod elsewhere cannot attach
# it. Pin a region for latency or for where the data is allowed to be.
data_center: str = ""
# Several acceptable regions, tried in order on a capacity miss. This is how
# "anywhere in the EU" is said: a single data_center pins one region and a
# blank one lets RunPod pick ANY region on earth, which is the wrong answer
# for data that may not leave the EU. Ignored when a NETWORK volume is
# attached — that volume exists in one region and nothing else can reach it.
data_centers: list = field(default_factory=list)
# --- how hard the pod's own server may be driven ---
# A rented pod carries no config file, so these ride as env. The defaults
# (2 in flight, 6 queued) are sized for a shared workstation engine and
......@@ -764,6 +773,18 @@ def volume_for(mcfg: "RunpodModelConfig", account) -> tuple:
return vol, mount.rstrip("/") or "/workspace"
def global_volume_for(mcfg: "RunpodModelConfig", account) -> str:
"""The GLOBAL volume id for this pool, or ''.
Deliberately separate from :func:`volume_for`: a global volume is
region-independent, so unlike a network volume it must NOT pin the pod's
data centre. Keeping them apart is what lets the scaler rent whichever
region has a card while the weights stay in one place.
"""
return ((getattr(mcfg, "global_volume_id", "") or "").strip()
or (getattr(account, "global_volume_id", "") or "").strip())
#: The dependency-free image that runs coderai from a venv on a volume.
SLIM_POD_IMAGE = os.environ.get("CODERAI_SLIM_POD_IMAGE",
f"{CAPABILITY_IMAGE_REPO}-slim:{CAPABILITY_IMAGE_TAG}")
......@@ -1345,6 +1366,24 @@ def _as_int(v, default=0):
_DAY_NAMES = ("mon", "tue", "wed", "thu", "fri", "sat", "sun")
def _parse_dc_list(value) -> list:
"""A region allow-list from a list or a comma/space string, order kept.
Order is the fallback order, so it is preserved rather than sorted: put the
region you prefer first and the scaler only moves on when it cannot get a
card there.
"""
if value in (None, ""):
return []
items = value if isinstance(value, (list, tuple)) else str(value).replace(",", " ").split()
out = []
for raw in items:
tok = str(raw).strip().upper()
if tok and tok not in out:
out.append(tok)
return out
def _parse_days(value) -> list:
"""Normalise a weekday selection to a sorted list of 0=Mon … 6=Sun.
......@@ -1489,6 +1528,8 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
cfg.docker_args = (b.get("docker_args") or "").strip()
cfg.registry_auth_id = (b.get("registry_auth_id") or "").strip()
cfg.network_volume_id = (b.get("network_volume_id") or "").strip()
cfg.global_volume_id = (b.get("global_volume_id") or "").strip()
cfg.data_centers = _parse_dc_list(b.get("data_centers"))
if cfg.network_volume_id:
# RunPod offers network volumes on Secure Cloud only; leaving COMMUNITY
# in the list just produces candidates the attach would reject.
......@@ -2080,6 +2121,14 @@ class RunpodPodPool:
env.update(volume_env(vol_mount))
if getattr(self, "_volume_dc", None) is None:
self._volume_dc = self._data_center_for_volume(vol_id)
# A GLOBAL volume instead: same caches, no data-centre pinning. RunPod
# mounts it at /workspace by itself when it is the only volume, which is
# the mount the pod env already points at; with a network volume also
# attached it moves to /workspace-global, so attaching both and
# expecting /workspace to be the global one would be wrong.
global_vol = global_volume_for(self.mcfg, self.account)
if global_vol and not vol_id:
env.update(volume_env(vol_mount or GLOBAL_VOLUME_MOUNT))
# Direct TCP puts nothing between here and the pod, so the pod brings
# its own TLS: a certificate from this install's CA, sent like the
# token. Only a coderai image knows what to do with it — vLLM's and
......@@ -2108,8 +2157,10 @@ class RunpodPodPool:
name = (f"coderai-{_deploy_tag(self.account)}-"
+ str(self.model_key)[:20].replace("/", "_").replace(" ", "_")
+ "-" + uuid.uuid4().hex[:6])
_dc = sel.get("_dc") or self._data_center()
print(f"[runpod] provisioning pod for {self.model_key!r}: {sel['display_name']} "
f"({sel['cloud_type']}{'/spot' if sel['is_spot'] else ''}) ${sel['price']}/hr",
f"({sel['cloud_type']}{'/spot' if sel['is_spot'] else ''}) ${sel['price']}/hr"
+ (f" in {_dc}" if _dc else " (region: RunPod's choice)"),
flush=True)
try:
_disk_gb, _disk_note = disk_for(
......@@ -2122,8 +2173,9 @@ class RunpodPodPool:
cloud_type=sel["cloud_type"], container_disk_gb=_disk_gb,
volume_gb=self.mcfg.volume_gb, env=env, docker_args=args,
is_spot=sel["is_spot"], bid_per_gpu=sel["bid"],
data_center_id=self._data_center(),
data_center_id=(sel.get("_dc") or self._data_center()),
network_volume_id=vol_id,
global_volume_id=global_vol,
volume_mount_path=(vol_mount or "/workspace"),
registry_auth_id=(self.mcfg.registry_auth_id
or getattr(self.account, "registry_auth_id", "") or ""),
......@@ -2235,6 +2287,27 @@ class RunpodPodPool:
hourly_usd=float(shared_rate or 0.0))
return True
def _data_centers(self) -> list:
"""Regions to try, in order. Always at least one entry.
A network volume overrides everything: it lives in one region and a pod
elsewhere cannot attach it. Otherwise the model's list, the model's
single setting, the account's list, the account's single setting, and
finally [""] — let RunPod choose, which is what blank has always meant.
"""
pinned = getattr(self, "_volume_dc", "") or ""
if pinned:
return [pinned]
for src in (getattr(self.mcfg, "data_centers", None),
[self.mcfg.data_center] if self.mcfg.data_center else None,
getattr(self.account, "data_centers", None),
[getattr(self.account, "data_center", "")]
if getattr(self.account, "data_center", "") else None):
vals = [str(v).strip() for v in (src or []) if str(v).strip()]
if vals:
return vals
return [""]
def _data_center(self) -> str:
"""Which RunPod region this pool's pods go to.
......@@ -2248,9 +2321,7 @@ class RunpodPodPool:
NameError, and the pool never recorded the pod it had just rented — so
provisioning retried and the untracked A40 billed on.
"""
return (getattr(self, "_volume_dc", "")
or self.mcfg.data_center
or getattr(self.account, "data_center", "") or "")
return self._data_centers()[0]
def _provision_one(self):
"""Create + boot one pod; append it healthy. Blocking (minutes).
......@@ -2259,6 +2330,14 @@ class RunpodPodPool:
from codai.api.runpod_client import RunpodClient, RunpodError, pod_console_url
client = RunpodClient(self.account)
ranked = _rank_gpus(client, self.mcfg, self.account)
# Each (card, region) pair is its own candidate, so the existing
# capacity fallback walks regions too: no A40 in the first EU region
# falls through to the next instead of failing the request.
dcs = self._data_centers()
if len(dcs) > 1:
ranked = [dict(sel, _dc=dc) for sel in ranked for dc in dcs]
print(f"[runpod] {len(dcs)} regions allowed for {self.model_key!r}: "
f"{', '.join(dcs)}", flush=True)
port = self.mcfg.port or 8000
idx, attempts, last_exc = 0, 0, None
......@@ -2338,7 +2417,7 @@ class RunpodPodPool:
started_at=time.time(), gpu=sel["display_name"],
is_spot=sel["is_spot"], healthy=True, last_used=time.time(),
console_url=console,
data_center=self._data_center(),
data_center=(sel.get("_dc") or self._data_center()),
cloud_type=str(sel.get("cloud_type") or ""),
gpu_count=int(sel.get("gpu_count") or 1))
with self._cv:
......@@ -3239,6 +3318,11 @@ def _known_pod_ids() -> set:
return ids
#: Where RunPod mounts a global volume when it is the only one attached. With a
#: network volume beside it, RunPod moves the global one to /workspace-global.
GLOBAL_VOLUME_MOUNT = "/workspace"
#: Never reap a pod younger than this, whatever the registry says. Longer than
#: any boot budget, so a pod still pulling its image cannot be mistaken for a
#: leftover — the backstop for the race a registry cannot close (a process that
......
......@@ -759,6 +759,12 @@ class RunpodConfig:
serverless_base: str = "https://api.runpod.ai/v2" # serverless invoke base
default_gpu_type: str = "" # fallback RunPod gpuTypeId when a model gives none
data_center: str = "" # optional data-center id filter (blank = any)
# Account-wide region allow-list, tried in order on a capacity miss. The way
# to say "anywhere in the EU": `data_center` pins ONE region, and leaving it
# blank lets RunPod pick any region on earth — not what you want for data
# that may not leave the EU. A per-model `data_centers` overrides this, and a
# network volume overrides both (it exists in one region only).
data_centers: list = field(default_factory=list)
# RunPod "Container Registry Credentials" id, for pods whose image lives in a
# private registry (your own coderai image, typically). Per-model/capability
# `registry_auth_id` overrides this. Blank = public images only.
......@@ -771,6 +777,18 @@ class RunpodConfig:
# volume's own data center.
network_volume_id: str = ""
volume_mount_path: str = "/workspace" # where pods mount it
# A RunPod GLOBAL volume (beta, Sept 2026): region-INDEPENDENT storage, so
# unlike network_volume_id above it does NOT pin pods to one data centre —
# which is the whole reason to use it. Store the weights once and rent a GPU
# in whichever region has one. Backed by object storage: fine for weights,
# adapters and tokenizers, wrong for checkpoints (no file locking, no atomic
# rename, eventual consistency, last-write-wins). RunPod mounts it at
# /workspace on its own, or /workspace-global when a network volume is
# attached too — so do not set both unless you mean that.
# Must be created in the RunPod console: the public API still refuses the
# type (POST /v2/network-volumes allows only STANDARD|HIGH_PERFORMANCE and
# demands a dataCenter).
global_volume_id: str = ""
# Stable tag baked into every pod name (coderai-<deployment_id>-<model>-<rand>) so the
# reaper can identify OUR pods across restarts and never touch another deployment's.
deployment_id: str = "default"
......
......@@ -606,6 +606,8 @@ Four numbers decide the shape of a pool:
| `idle_timeout_s` | destroy a pod this long after its last request. |
| `allow_client_scale` | let a client lease a warm floor up to `max_pods` through `POST /v1/runpod/scale`. Off by default. |
| `client_scale_ttl_s` | how long such a lease lasts unless renewed (default 1800). A lease that never expired would be a bill that never stopped. |
| `data_centers` | several acceptable regions, tried in order on a capacity miss — how you say "anywhere in the EU". A single `data_center` pins one region; blank lets RunPod pick any region **on earth**. |
| `global_volume_id` | a RunPod **global** volume (region-independent). Unlike `network_volume_id` it does NOT pin the pod's region, which is the only reason to use one. |
**Then the two that are easy to miss**, and the reason a pod can look slow while
its GPU is idle:
......@@ -648,6 +650,53 @@ start runs overnight and belongs to the day it started on, so `22:00`–`06:00`
`fri` covers Friday night into Saturday morning. Leave `schedule_days` empty for
every day.
### Weights in one place, pods in any region
A **network volume** lives in exactly one data centre and a pod elsewhere
cannot attach it, so `network_volume_id` also decides where every pod runs —
coderai pins it for you and says so (`volume … is in X — pinning pods there`).
That is the right trade when you want fast starts in one region.
A **global volume** (RunPod beta, Sept 2026) is region-independent storage
backed by object storage. Store the weights once and rent a card wherever one
is free:
```json
"global_volume_id": "gv-abc123",
"data_centers": "EU-RO-1,EU-FR-1,EUR-NO-1,EUR-IS-3"
```
`data_centers` is the other half, and it matters more than it looks. A single
`data_center` pins one region, and leaving it blank lets RunPod pick **any
region on earth** — rarely what you want for data that may not leave the EU.
A list is tried in order, so the first entry is the preferred region and the
scaler only moves on when it cannot get a card there. Each (card, region) pair
becomes its own candidate, so the capacity fallback that already walks GPU
options walks regions too.
Caveats worth knowing before you rely on it:
* **It has to be created in the RunPod console** — Storage → **+ New volume** →
**Global volume**. The public API still refuses: `POST /v2/network-volumes`
allows only `STANDARD` and `HIGH_PERFORMANCE` and demands a `dataCenter`.
* **Attaching is console-documented only.** No field appears in the Pod API
reference, and the v1 REST validator accepts unknown body keys silently — so
a wrong name cannot be caught by probing, it just yields a pod with no volume
that re-downloads its weights. coderai sends `globalNetworkVolumeId`,
overridable with `CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD`. **Verify on the first
pod** that the weights are really at the mount.
* **Read-heavy only.** No file locking, no atomic rename, eventual consistency,
last-write-wins on concurrent writes. Fine for weights, adapters, tokenizers
and configs; wrong for checkpoints — use a network volume to train.
* RunPod mounts it at `/workspace` when it is the only volume, and moves it to
`/workspace-global` if a network volume is attached too. coderai therefore
points the pod env at the global volume only when no network volume is set.
* GPU pods and GPU serverless only; CPU is not supported. And if the account
balance reaches $0 the volume is flagged and permanently deleted after 15
days.
---
### Let a client ask for pods
The autoscaler reacts to load: it adds a pod once the least-loaded one already
......
......@@ -67,7 +67,10 @@ def test_each_pod_reports_where_it_is_and_what_it_serves():
# The handle gets its region from the pool's own resolver. It used to read
# a local `dc` that did not exist in this function, so the pod was created
# and then the handle raised NameError — the A40 billed untracked.
assert "data_center=self._data_center()" in src
# The handle records the region the pod ACTUALLY got, which with a region
# allow-list is the candidate's own region rather than the pool's first
# choice — otherwise the fleet page would report every pod in region #1.
assert 'data_center=(sel.get("_dc") or self._data_center())' in src
def test_status_and_spend_are_the_same_document_behind_the_same_gate():
......
"""A global volume must not pin the pod to a region — that is the whole point.
RunPod's Global Volumes (beta, Sept 2026) are region-independent storage backed
by object storage. A NETWORK volume lives in one data centre and a pod elsewhere
cannot attach it, which is why coderai forces the pod into the volume's region.
Applying that same pinning to a global volume would throw away the only reason
to use one: store the weights once, rent a GPU in whichever region has one.
So the two are resolved separately, and these tests hold them apart.
"""
import pathlib
from codai.api import runpod_worker as rw
ROOT = pathlib.Path(rw.__file__).resolve().parents[2]
class _Acct:
def __init__(self, net="", glob="", dc="", mount="/workspace"):
self.network_volume_id = net
self.global_volume_id = glob
self.data_center = dc
self.volume_mount_path = mount
def test_the_account_setting_applies_to_every_model():
mcfg = rw.parse_model_runpod({})
assert rw.global_volume_for(mcfg, _Acct(glob="gv-123")) == "gv-123"
def test_a_model_can_override_the_account_volume():
mcfg = rw.parse_model_runpod({"global_volume_id": "gv-model"})
assert rw.global_volume_for(mcfg, _Acct(glob="gv-account")) == "gv-model"
def test_no_volume_configured_is_empty_not_an_error():
assert rw.global_volume_for(rw.parse_model_runpod({}), _Acct()) == ""
def test_a_global_volume_does_not_pin_the_data_centre():
"""The network volume's region wins over everything; a global volume must
leave the choice alone so the scaler can rank cards across all regions."""
pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
pool.account = _Acct(glob="gv-123")
pool.mcfg = rw.parse_model_runpod({"global_volume_id": "gv-123"})
pool._volume_dc = None
assert pool._data_center() == "" # free to go anywhere
pool.mcfg.data_center = "EU-RO-1"
assert pool._data_center() == "EU-RO-1" # only because the model asked
def test_only_the_network_volume_triggers_the_region_lookup():
src = (ROOT / "codai/api/runpod_worker.py").read_text()
blk = src.split("vol_id, vol_mount = volume_for")[1].split("self._tls = False")[0]
# The region lookup sits under the network-volume branch, and the global
# branch that follows must not contain it.
net, glob = blk.split("global_vol = global_volume_for")
assert "_data_center_for_volume" in net
assert "_data_center_for_volume" not in glob
def test_the_two_volumes_are_not_attached_together_by_accident():
"""With both attached RunPod moves the global one to /workspace-global, so
pointing the pod env at /workspace would aim it at the wrong volume."""
src = (ROOT / "codai/api/runpod_worker.py").read_text()
blk = src.split("global_vol = global_volume_for")[1][:600]
assert "if global_vol and not vol_id:" in blk
assert "/workspace-global" in src
def test_the_client_sends_the_field_on_both_api_shapes():
"""create_pod has a GraphQL body and a REST body; a volume wired into only
one of them attaches on some code paths and silently not on others."""
src = (ROOT / "codai/api/runpod_client.py").read_text()
assert src.count("GLOBAL_VOLUME_FIELD] = global_volume_id") == 2
assert "global_volume_id: str = \"\"" in src
def test_the_field_name_is_overridable_because_it_is_undocumented():
"""RunPod documents global volumes as console-only and the v1 validator
accepts unknown body keys silently, so a wrong name cannot be detected by
probing — it yields a pod with no volume that re-downloads its weights."""
from codai.api import runpod_client
assert runpod_client.GLOBAL_VOLUME_FIELD
src = (ROOT / "codai/api/runpod_client.py").read_text()
assert "CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD" in src
assert "VERIFY" in src # says out loud that it is unconfirmed
def test_the_config_documents_that_it_cannot_be_created_by_api():
cfgsrc = (ROOT / "codai/config.py").read_text()
blk = cfgsrc.split("global_volume_id")[0][-1400:]
assert "console" in blk.lower()
assert "STANDARD" in blk # the enum the API actually allows
# ------------------------------------------- "anywhere in the EU", said safely
class _AcctDC(_Acct):
def __init__(self, dcs=None, dc="", **kw):
super().__init__(**kw)
self.data_centers = dcs or []
self.data_center = dc
def _pool(mcfg_block=None, acct=None):
pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
pool.account = acct or _AcctDC()
pool.mcfg = rw.parse_model_runpod(mcfg_block or {})
pool._volume_dc = None
return pool
def test_a_region_list_is_parsed_from_a_string_or_a_list():
for value in ("EU-RO-1,EU-FR-1", "EU-RO-1 EU-FR-1", ["eu-ro-1", "EU-FR-1"]):
assert rw._parse_dc_list(value) == ["EU-RO-1", "EU-FR-1"], value
assert rw._parse_dc_list("") == []
assert rw._parse_dc_list(None) == []
def test_the_order_is_the_fallback_order_and_is_preserved():
"""Not sorted: the first entry is the preferred region and the scaler only
moves on when it cannot get a card there."""
assert rw._parse_dc_list("EU-RO-1,EU-FR-1,EUR-NO-1")[0] == "EU-RO-1"
assert rw._parse_dc_list("EU-FR-1,EU-RO-1")[0] == "EU-FR-1"
def test_duplicates_collapse():
assert rw._parse_dc_list("EU-RO-1, eu-ro-1 ,EU-FR-1") == ["EU-RO-1", "EU-FR-1"]
def test_the_model_list_beats_the_account_list():
pool = _pool({"data_centers": "EU-FR-1"}, _AcctDC(dcs=["EU-RO-1"]))
assert pool._data_centers() == ["EU-FR-1"]
def test_a_single_data_center_still_works_exactly_as_before():
pool = _pool({"data_center": "EU-SE-1"})
assert pool._data_centers() == ["EU-SE-1"]
assert pool._data_center() == "EU-SE-1"
def test_nothing_configured_still_means_runpod_chooses():
pool = _pool()
assert pool._data_centers() == [""]
assert pool._data_center() == ""
def test_a_network_volume_overrides_every_region_setting():
"""It exists in ONE region and no pod elsewhere can attach it, so a
multi-region list must not send a pod where the attach would fail."""
pool = _pool({"data_centers": "EU-FR-1,EU-RO-1"}, _AcctDC(dcs=["EUR-NO-1"]))
pool._volume_dc = "EU-NL-1"
assert pool._data_centers() == ["EU-NL-1"]
def test_candidates_are_expanded_across_regions_for_the_capacity_fallback():
"""The existing retry walks `ranked` on a capacity miss; pairing each card
with each region makes it walk regions too, with no new control flow."""
src = (ROOT / "codai/api/runpod_worker.py").read_text()
blk = src.split("ranked = _rank_gpus(client, self.mcfg, self.account)")[1][:700]
assert 'dict(sel, _dc=dc) for sel in ranked for dc in dcs' in blk
assert "if len(dcs) > 1:" in blk
def test_the_pod_is_created_in_and_records_the_region_it_got():
src = (ROOT / "codai/api/runpod_worker.py").read_text()
assert src.count('sel.get("_dc") or self._data_center()') >= 3 # create, log, handle
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment