hold RunPod GraphQL to a 1,500/h budget, then route to REST v2 (v0.4.58)

RunPod's live GraphQL limits for our key are 180/min, 3,600/h and
43,200/day. The day limit is the one that ran out, and the ban lasts until
midnight UTC. Averaged over a day it allows 1,800/h, so the hourly cap is
not the binding one.

This process now counts its own GraphQL calls over a rolling hour. Past
CODERAI_RUNPOD_GQL_HOURLY_BUDGET (default 1500; 0 = off), _gql_banned()
reports true, so everything REST v2 can do goes to v2 before RunPod ever
answers 429. The budget only reroutes. It never refuses a GraphQL call on
its own, so spot rentals, which v2 cannot do, still work. The first time
the budget is reached, and every 10 min while it stays reached, it says so.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KnfkxBpcZ1v6ypRFFyhrVM

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.4.57"
+__version__ = "0.4.58"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/runpod_client.py

@@ -18,6 +18,7 @@ NEVER logged — only masked in error strings.
 """
 
 import os
+import threading
 import time
 from typing import Optional
 
@@ -367,9 +368,57 @@ def _v2_pod_row(p: dict, pod_id: str = "", with_ports: bool = False) -> dict:
     return row
 
 
+#: GraphQL calls this process may make per rolling hour before it routes
+#: everything REST v2 can do to REST v2. RunPod's own GraphQL limits (live
+#: headers, 2026-10-20) are 3,600/h but 43,200/DAY -- an average of 1,800/h --
+#: and the daily one is the one that ran out: a ban until midnight UTC. 1,500/h
+#: stays under the daily average with room for other clients of the same key.
+#: CODERAI_RUNPOD_GQL_HOURLY_BUDGET overrides; 0 disables the budget.
+GQL_HOURLY_BUDGET = int(os.environ.get("CODERAI_RUNPOD_GQL_HOURLY_BUDGET", "1500") or 0)
+_GQL_RECENT: "collections.deque" = None
+_GQL_RECENT_LOCK = threading.Lock()
+_GQL_OVER_SAID = [0.0]
+
+
+def _gql_record_call() -> None:
+    global _GQL_RECENT
+    with _GQL_RECENT_LOCK:
+        if _GQL_RECENT is None:
+            import collections
+            _GQL_RECENT = collections.deque()
+        _GQL_RECENT.append(time.time())
+
+
+def _gql_calls_last_hour() -> int:
+    with _GQL_RECENT_LOCK:
+        if not _GQL_RECENT:
+            return 0
+        cut = time.time() - 3600
+        while _GQL_RECENT and _GQL_RECENT[0] < cut:
+            _GQL_RECENT.popleft()
+        return len(_GQL_RECENT)
+
+
+def _gql_over_budget() -> bool:
+    if GQL_HOURLY_BUDGET <= 0:
+        return False
+    n = _gql_calls_last_hour()
+    if n < GQL_HOURLY_BUDGET:
+        return False
+    now = time.time()
+    if now - _GQL_OVER_SAID[0] > 600:
+        _GQL_OVER_SAID[0] = now
+        print(f"[runpod] GraphQL budget reached ({n} calls in the last hour, budget "
+              f"{GQL_HOURLY_BUDGET}/h) — routing everything REST v2 can do to REST "
+              f"v2 until it drops; only spot rentals still need GraphQL", flush=True)
+    return True
+
+
 def _gql_banned() -> bool:
-    """True while RunPod's GraphQL rate limit is in force (see _note_rate_limit)."""
-    return _GQL_BACKOFF_UNTIL[0] > time.time()
+    """True while GraphQL should not be used where REST v2 can do the job:
+    RunPod's rate limit is in force (see _note_rate_limit), or this process has
+    spent its own hourly GraphQL budget."""
+    return _GQL_BACKOFF_UNTIL[0] > time.time() or _gql_over_budget()
 
 
 class RunpodClient:
@@ -391,6 +440,7 @@ class RunpodClient:
         # inside that window only re-confirms the ban, so callers get the answer
         # here, without a request.
         _count_gql(query)
+        _gql_record_call()
         until = _GQL_BACKOFF_UNTIL[0]
         if until > time.time():
             raise RunpodTransient(

tests/conftest.py

@@ -163,5 +163,7 @@ def _no_runpod_rate_limit_carryover():
         yield
         return
     _rc._GQL_BACKOFF_UNTIL[0] = 0.0
+    _rc._GQL_RECENT = None
     yield
     _rc._GQL_BACKOFF_UNTIL[0] = 0.0
+    _rc._GQL_RECENT = None

tests/test_runpod_rest_fallback.py

@@ -177,3 +177,53 @@ def test_graphql_calls_are_counted_by_operation():
     rc._count_gql("mutation { podTerminate(input: {podId: \"x\"}) }")
     got = rc.gql_call_counts()
     assert got.get("pod") == 1 and got.get("podTerminate") == 1
+
+
+# ------------------------------------------- staying under the daily quota #
+
+def test_over_the_hourly_budget_routes_to_v2(monkeypatch):
+    """RunPod's GraphQL day quota (43,200) averages 1,800/h; spending it ends
+    in a ban until midnight UTC. Past our own budget, v2 does the work."""
+    import collections
+    monkeypatch.setattr(rc, "GQL_HOURLY_BUDGET", 5)
+    monkeypatch.setattr(rc, "_GQL_RECENT", collections.deque([time.time()] * 5))
+    assert rc._gql_banned() is True
+    c, calls = _client(monkeypatch, lambda m, p, b: {"id": "new1"})
+    assert c.create_pod(name="n", image="i", gpu_type_id="NVIDIA A40", port=8000,
+                        direct_tcp=True) == "new1"
+    assert calls[0][:2] == ("POST", "/pods")
+
+
+def test_old_calls_fall_out_of_the_window(monkeypatch):
+    import collections
+    monkeypatch.setattr(rc, "GQL_HOURLY_BUDGET", 5)
+    monkeypatch.setattr(rc, "_GQL_RECENT", collections.deque([time.time() - 4000] * 50))
+    assert rc._gql_banned() is False
+    assert rc._gql_calls_last_hour() == 0
+
+
+def test_the_budget_does_not_block_what_only_graphql_can_do(monkeypatch):
+    """Spot rentals have no v2 equivalent: the budget reroutes, it never
+    refuses a GraphQL call on its own -- only RunPod's 429 does that."""
+    import collections, requests
+    monkeypatch.setattr(rc, "GQL_HOURLY_BUDGET", 1)
+    monkeypatch.setattr(rc, "_GQL_RECENT", collections.deque([time.time()] * 3))
+
+    class _R:
+        status_code = 200
+        text = ""
+
+        def json(self):
+            return {"data": {"podRentInterruptable": {"id": "spot1"}}}
+    monkeypatch.setattr(requests, "post", lambda *a, **k: _R())
+    c = rc.RunpodClient.__new__(rc.RunpodClient)
+    c._api_key, c._gql_base, c._cfg = "k" * 20, "https://api.runpod.io/graphql", None
+    assert c._gql("mutation { podRentInterruptable(input: {}) { id } }") == {
+        "podRentInterruptable": {"id": "spot1"}}
+
+
+def test_zero_disables_the_budget(monkeypatch):
+    import collections
+    monkeypatch.setattr(rc, "GQL_HOURLY_BUDGET", 0)
+    monkeypatch.setattr(rc, "_GQL_RECENT", collections.deque([time.time()] * 99999))
+    assert rc._gql_over_budget() is False