runpod: a configured global volume must not fail every pod (v0.3.2)

Probed every API surface coderai can use for attaching a global volume, each
with a body valid enough that the extra-key check actually runs:

  POST /v1/pods            key … not in input schema: 'globalNetworkVolumeId'
  podFindAndDeployOnDemand field not defined by the input type
  POST /v2/pods            additional properties 'globalVolumeId' not allowed
  templates (mounts)       every candidate key refused

There is no public API for it: global volumes are console-only, as the docs
say. Which makes `global_volume_id` a loaded gun — sending the field fails the
create, so configuring a global volume would have stopped the install renting
pods at all.

Both create paths now recognise that specific rejection, drop the field and
retry. The pod boots without the volume and downloads its weights: slower and
it costs bandwidth, and far better than serving nothing. Logged once per
process, naming the consequence and the override to set when RunPod publishes
the field name. An unrelated failure is NOT treated this way — a capacity error
retried with the volume stripped would bring a pod up silently wrong.

Earlier probes that reported the field as "accepted" were wrong: with an
otherwise-invalid body the validators stop before the extra-key check. Worth
remembering next time I probe a schema.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_client.py

@@ -38,6 +38,42 @@ class RunpodError(RuntimeError):
     """A RunPod API call failed (network, auth, or GraphQL error)."""
 
 
+#: Said once per process, not once per pod: the scaler would otherwise repeat it
+#: every boot.
+_GLOBAL_VOLUME_WARNED = False
+
+
+def _global_field_refused(text: str) -> bool:
+    """Whether a create failed *because* of the global-volume field.
+
+    RunPod words it two ways — "not in input schema" (v1) and "additional
+    properties … not allowed" (v2) — and both name the field.
+    """
+    t = (text or "")
+    return GLOBAL_VOLUME_FIELD in t and (
+        "not in input schema" in t or "not allowed" in t or "Extra input keys" in t)
+
+
+def _global_field_refused_gql(text: str) -> bool:
+    """GraphQL's wording for an input field its schema does not define."""
+    t = (text or "")
+    return GLOBAL_VOLUME_FIELD in t and (
+        "is not defined by type" in t or "invalid value" in t or "Unknown" in t)
+
+
+def _warn_global_volume_unsupported(volume_id: str) -> None:
+    global _GLOBAL_VOLUME_WARNED
+    if _GLOBAL_VOLUME_WARNED:
+        return
+    _GLOBAL_VOLUME_WARNED = True
+    print(f"[runpod] global volume {volume_id!r} is configured but RunPod's API "
+          f"rejects {GLOBAL_VOLUME_FIELD!r}: global volumes are console-only for "
+          "now (checked against v1 REST, GraphQL, v2 pods and templates). Pods "
+          "will boot WITHOUT it and download their weights each time. Set "
+          "CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD once RunPod publishes the field "
+          "name.", flush=True)
+
+
 def _mask(key: str) -> str:
     if not key:
         return "<none>"
@@ -288,7 +324,18 @@ class RunpodClient:
         mutation Deploy($input: {var_type}!) {{
           {mutation}(input: $input) {{ id imageName machineId }}
         }}"""
-        data = self._gql(q, {"input": common}, timeout=90.0)
+        try:
+            data = self._gql(q, {"input": common}, timeout=90.0)
+        except RunpodError as exc:
+            # Same guard as the REST path: GraphQL calls the global-volume field
+            # an undefined input field, and a configured global volume must not
+            # fail every create while RunPod keeps it console-only.
+            if not (global_volume_id and GLOBAL_VOLUME_FIELD in common
+                    and _global_field_refused_gql(str(exc))):
+                raise
+            _warn_global_volume_unsupported(global_volume_id)
+            common.pop(GLOBAL_VOLUME_FIELD, None)
+            data = self._gql(q, {"input": common}, timeout=90.0)
         node = data.get(mutation) or {}
         pod_id = node.get("id")
         if not pod_id:
@@ -330,10 +377,20 @@ class RunpodClient:
         if is_spot:
             body["interruptible"] = True
             body["bidPerGpu"] = float(bid_per_gpu)
-        r = requests.post(f"{rest}/pods", json=body,
-                          headers={"Authorization": f"Bearer {self._api_key}",
-                                   "Content-Type": "application/json"},
-                          timeout=90)
+        hdrs = {"Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json"}
+        r = requests.post(f"{rest}/pods", json=body, headers=hdrs, timeout=90)
+        # RunPod has no public API for attaching a GLOBAL volume yet: v1 REST,
+        # GraphQL, the v2 pods API and templates all reject the field (probed
+        # 2026-10-09). Sending it therefore fails the whole create — so a
+        # configured global volume must not take every pod down with it. Drop
+        # the field and rent the pod without it: the weights get downloaded
+        # instead of mounted, which is slower and costs bandwidth, and is far
+        # better than serving nothing. Said once per process, loudly.
+        if r.status_code >= 400 and global_volume_id and _global_field_refused(r.text):
+            _warn_global_volume_unsupported(global_volume_id)
+            body.pop(GLOBAL_VOLUME_FIELD, None)
+            r = requests.post(f"{rest}/pods", json=body, headers=hdrs, timeout=90)
         if r.status_code >= 400:
             raise RunpodError(f"RunPod REST create failed ({r.status_code}): "
                               f"{(r.text or '')[:400]}")

docs/orchestrator-runpod-guide.md

@@ -681,12 +681,23 @@ Caveats worth knowing before you rely on it:
 * **It has to be created in the RunPod console** — Storage → **+ New volume** →
   **Global volume**. The public API still refuses: `POST /v2/network-volumes`
   allows only `STANDARD` and `HIGH_PERFORMANCE` and demands a `dataCenter`.
-* **Attaching is console-documented only.** No field appears in the Pod API
-  reference, and the v1 REST validator accepts unknown body keys silently — so
-  a wrong name cannot be caught by probing, it just yields a pod with no volume
-  that re-downloads its weights. coderai sends `globalNetworkVolumeId`,
-  overridable with `CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD`. **Verify on the first
-  pod** that the weights are really at the mount.
+* **There is no API to attach one — at all, yet.** Probed 2026-10-09 against
+  every surface coderai can use, each with a body valid enough that the
+  extra-key check actually runs:
+
+  | surface | answer |
+  |---|---|
+  | `POST /v1/pods` (REST) | `key provided in request body which is not in input schema` |
+  | `podFindAndDeployOnDemand` (GraphQL) | field not defined by the input type |
+  | `POST /v2/pods` | `additional properties … not allowed` |
+  | templates (`mounts`) | every candidate key refused |
+
+  So `global_volume_id` is wired and inert: coderai sends
+  `globalNetworkVolumeId`, and when the API refuses it the create is **retried
+  without the field** rather than failing — the pod boots and downloads its
+  weights, and the reason is logged once per process. Set
+  `CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD` the day RunPod publishes the real name
+  and it starts attaching with no code change.
 * **Read-heavy only.** No file locking, no atomic rename, eventual consistency,
   last-write-wins on concurrent writes. Fine for weights, adapters, tokenizers
   and configs; wrong for checkpoints — use a network volume to train.

tests/test_runpod_placement.py

@@ -301,3 +301,58 @@ def test_the_two_ordered_lists_keep_their_order_through_the_ui():
     block = src.split("# Per-model runpod block")[1].split("# Per-model `host` block")[0]
     seg = block.split('rpo["data_centers"]')[0][-500:] + block.split('rpo["data_centers"]')[1][:400]
     assert "sorted(" not in seg
+
+
+# ------- a configured global volume must not take every pod down with it
+
+def test_the_rejection_is_recognised_in_both_api_wordings():
+    from codai.api import runpod_client as rc
+    f = rc.GLOBAL_VOLUME_FIELD
+    assert rc._global_field_refused(
+        f'{{"error":"Extra input keys provided in request body",'
+        f'"problems":["key provided in request body which is not in input '
+        f'schema: \'{f}\'"]}}') is True
+    assert rc._global_field_refused(
+        f'{{"errors":["$: additional properties \'{f}\' not allowed"]}}') is True
+    assert rc._global_field_refused_gql(
+        f'RunPod GraphQL error: Field "{f}" is not defined by type '
+        f'"PodFindAndDeployOnDemandInput"') is True
+
+
+def test_an_unrelated_failure_is_not_mistaken_for_it():
+    """Otherwise a real error would be retried with the volume stripped and the
+    pod would come up silently wrong."""
+    from codai.api import runpod_client as rc
+    assert rc._global_field_refused('{"error":"no capacity for NVIDIA A40"}') is False
+    assert rc._global_field_refused('{"error":"additional properties \'foo\' '
+                                    'not allowed"}') is False
+    assert rc._global_field_refused_gql("RunPod GraphQL error: unauthorized") is False
+    assert rc._global_field_refused("") is False
+
+
+def test_both_create_paths_retry_without_the_field():
+    """REST and GraphQL are two code paths; a guard on one leaves the other
+    failing every pod."""
+    src = (ROOT / "codai/api/runpod_client.py").read_text()
+    assert src.count("_warn_global_volume_unsupported(global_volume_id)") == 2
+    assert src.count("pop(GLOBAL_VOLUME_FIELD, None)") == 2
+
+
+def test_the_warning_is_said_once_not_once_per_pod():
+    from codai.api import runpod_client as rc
+    saved = rc._GLOBAL_VOLUME_WARNED
+    rc._GLOBAL_VOLUME_WARNED = False
+    try:
+        rc._warn_global_volume_unsupported("gv-1")
+        assert rc._GLOBAL_VOLUME_WARNED is True
+        rc._warn_global_volume_unsupported("gv-1")    # must not print again
+    finally:
+        rc._GLOBAL_VOLUME_WARNED = saved
+
+
+def test_the_warning_says_what_to_do_about_it():
+    src = (ROOT / "codai/api/runpod_client.py").read_text()
+    blk = src.split("def _warn_global_volume_unsupported")[1].split("class ")[0]
+    assert "console-only" in blk
+    assert "CODERAI_RUNPOD_GLOBAL_VOLUME_FIELD" in blk
+    assert "download their weights" in blk    # names the real consequence