ratelimit: 60 req/min was measured as a GPU cliff, and throttled rented pods (v0.3.38)

`/v1/chat/completions` was capped at 60 requests per 60 s per client IP as a
module constant — no config, no env, no way to move it.

On a RENTED POD that cap is the orchestrator's own: a pod has exactly one client,
the install paying for it by the hour, so every request shares one bucket. An
olmOCR pod measured 3.45 pages/s (207/min) and was being refused above 60.

It also produced three wrong measurements that were reported as hardware facts.
Concurrency ladders ran every level inside one window, so refusals began exactly
where the cumulative count crossed 60:

    surya   16+24=40, then 20 of 32 pass, 12 refused   (13 measured)
    surya   +48 = 72 over,  all 48 refused             (all 48 measured)
    chandra 8+16+24=48, then 12 pass, 20 refused       (21 measured)
    olmOCR  same ladder, same arithmetic               (21 measured, twice)

"Both peak at 16 and both fail outright by 32 — a cliff" was the rate limiter.
Paced across windows, olmOCR runs N=32 with ZERO errors at its best throughput;
it is flat at ~3.4 pages/s from N=8 to N=32 while p50 goes 2.3s -> 7.9s, which is
saturation at 8, not a cliff at 32.

So: the limits take env overrides (CODERAI_SERVER_RATE_LIMITS="prefix=count/window",
CODERAI_SERVER_RATE_LIMIT_ENABLED=0), and a pod is launched with the limiter OFF —
its admission is pod_max_parallel_requests + pod_queue_max_size, which describe
the GPU rather than guessing at an abusive client that cannot exist there.

docs/ocr.md keeps the old table with the refusals marked as refusals, because the
numbers themselves are still the only ones measured and the reading of them was
the error.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.37"
+__version__ = "0.3.38"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/ratelimit.py

@@ -177,6 +177,41 @@ _QUEUED_PREFIXES = ("/v1/",)
 RATE_LIMITING_ENABLED = True
 
 
+def _env_limits() -> Dict[str, Tuple[int, int]]:
+    """Per-prefix overrides from the environment, as ``prefix=count/window``.
+
+    This exists because the limit was a module constant: 60 requests per minute
+    on /v1/chat/completions, with no config and no way to move it. On a RENTED
+    POD that is the orchestrator's own ceiling — one client IP, so one bucket —
+    and it silently discarded most of a GPU that was being paid for by the hour.
+    It was also measured as if it were hardware: three OCR engines appeared to
+    "fall over" at 32 concurrent pages, and all three were simply out of window.
+
+        CODERAI_SERVER_RATE_LIMITS="/v1/chat/completions=600/60,/v1/audio/=300/60"
+        CODERAI_SERVER_RATE_LIMIT_ENABLED=0     # off entirely
+    """
+    raw = (os.environ.get("CODERAI_SERVER_RATE_LIMITS") or "").strip()
+    out: Dict[str, Tuple[int, int]] = {}
+    for item in raw.split(","):
+        item = item.strip()
+        if not item or "=" not in item:
+            continue
+        prefix, _, spec = item.partition("=")
+        count, _, window = spec.partition("/")
+        try:
+            out[prefix.strip()] = (int(count), int(window or 60))
+        except ValueError:
+            continue
+    return out
+
+
+def _limits_enabled() -> bool:
+    flag = (os.environ.get("CODERAI_SERVER_RATE_LIMIT_ENABLED") or "").strip().lower()
+    if flag in ("0", "false", "no", "off"):
+        return False
+    return RATE_LIMITING_ENABLED
+
+
 class _Bucket:
     """Fixed-window counter."""
     __slots__ = ("count", "window_start")
@@ -192,7 +227,16 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
 
     def __init__(self, app, limits: Dict[str, Tuple[int, int]] = None):
         super().__init__(app)
-        self._limits = limits or _DEFAULT_LIMITS
+        self._limits = dict(limits or _DEFAULT_LIMITS)
+        self._limits.update(_env_limits())
+        self._enabled = _limits_enabled()
+        if not self._enabled:
+            print("[ratelimit] disabled by CODERAI_SERVER_RATE_LIMIT_ENABLED",
+                  flush=True)
+        else:
+            moved = {k: v for k, v in _env_limits().items()}
+            if moved:
+                print(f"[ratelimit] overridden: {moved}", flush=True)
         # (client_ip, prefix) → _Bucket
         self._buckets: Dict[Tuple[str, str], _Bucket] = defaultdict(lambda: _Bucket(time.monotonic()))
         self._lock = threading.Lock()
@@ -207,7 +251,7 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
     _EXEMPT_PATHS = _PROGRESS_PATHS
 
     async def dispatch(self, request: Request, call_next):
-        if not RATE_LIMITING_ENABLED:
+        if not self._enabled:
             return await call_next(request)
 
         path = request.url.path

codai/api/runpod_worker.py

@@ -1084,6 +1084,13 @@ def _plan(engine, image, args, mcfg, api_key, entry, served,
         env["CODERAI_SERVER_MAX_PARALLEL_REQUESTS"] = str(int(mcfg.pod_max_parallel_requests))
     if getattr(mcfg, "pod_queue_max_size", 0):
         env["CODERAI_SERVER_QUEUE_MAX_SIZE"] = str(int(mcfg.pod_queue_max_size))
+    # A pod has ONE client — the install that rented it — so per-IP rate limiting
+    # throttles its owner and nobody else. The default 60 requests/min on
+    # /v1/chat/completions is below what a single A40 serves, so it threw away
+    # GPU time that is being paid for by the hour, and made three OCR engines
+    # look like they fell over at 32 concurrent pages when they were simply out
+    # of window. Admission here is the pod's parallel-request and queue limits.
+    env["CODERAI_SERVER_RATE_LIMIT_ENABLED"] = "0"
     seeds = []
     if entry:
         kind, value = resolve_model_source(entry, mcfg)

docs/ocr.md

@@ -417,12 +417,28 @@ adapts to wherever the checkpoint is actually served and tells you to set
 | 8 | 3.79 | 2.0s | 1.31 | 6.1s |
 | **16** | **7.11** | **2.2s** | **1.88** | **8.4s** |
 | 24 | 5.89 | 4.1s | 2.09 | 10.6s |
-| 32 | 5.44 + 13 failed | 3.5s | 1.44 + 21 failed | 7.5s |
-| 48 | **all failed** | — | — | — |
-
-**Both peak at 16 and both fail outright by 32** — that is a cliff, not a graceful
-degradation, so `runpod.scale_up_inflight_per_pod` is set to 16 per model: the point
-where a second pod starts earning its cost, with margin before the edge.
+| 32 | 5.44 + 13 *rejected* | 3.5s | 1.44 + 21 *rejected* | 7.5s |
+| 48 | **all 48 rejected** | — | — | — |
+
+> **The failures in the last two rows were not the GPU.** They were HTTP 429 from
+> coderai's own rate limiter — `/v1/chat/completions` was capped at 60 requests per
+> 60 seconds per client IP, as a module constant with no config. These ladders ran
+> every level inside one window, so the rejections begin exactly where the
+> cumulative count crosses 60: surya 16+24=40 then 20 of 32 pass and 12 are
+> refused (13 measured); chandra and olmOCR 8+16+24=48 then 12 pass and 20 are
+> refused (21 measured, twice). Re-running one level alone returns 32/32 OK.
+>
+> So the reading once given here — "both peak at 16 and both fail outright by 32,
+> a cliff" — was wrong: the engines never fell over, and the true knee is **above**
+> what these numbers show. A pod now ships with the limiter off
+> (`CODERAI_SERVER_RATE_LIMIT_ENABLED=0`), since a pod has exactly one client — the
+> install that rented it — and its admission is governed by
+> `pod_max_parallel_requests` and `pod_queue_max_size` instead. **Any ladder like
+> this must either keep each level under the limit or pace itself across windows.**
+
+`runpod.scale_up_inflight_per_pod` is 16 for all three engines. That remains a
+defensible spill point — latency is already climbing at 16 on these cards — but it
+is a *chosen* setting, not the measured edge of the hardware.
 
 Three settings bound this and they are easy to confuse:
 

tests/test_text_lora_to_pod.py

@@ -349,3 +349,36 @@ def test_a_disabled_model_is_not_seeded_onto_a_shared_pod(monkeypatch):
 
     monkeypatch.setattr("codai.admin.routes.config_manager", _CM, raising=False)
     assert rw.shared_pool_seed_entries("digesta-qwen38") == []
+
+
+# ---------------------------------------------------------------- the limiter
+
+def test_a_pod_does_not_rate_limit_the_install_that_rented_it():
+    """60 requests/min per IP is below one A40's output, and a pod has exactly
+    one client: its owner. Three OCR engines were measured "falling over" at 32
+    concurrent pages when all three had simply run out of window."""
+    from codai.api import runpod_worker as rw
+    mcfg = rw.parse_model_runpod({"pod_max_parallel_requests": 16,
+                                  "pod_queue_max_size": 64})
+    plan = rw._plan("vllm", "img", "", mcfg, "key", None, "")
+    env = plan.get("env") or {}
+    assert env.get("CODERAI_SERVER_RATE_LIMIT_ENABLED") == "0"
+    # The real admission controls must still be passed.
+    assert env.get("CODERAI_SERVER_MAX_PARALLEL_REQUESTS") == "16"
+    assert env.get("CODERAI_SERVER_QUEUE_MAX_SIZE") == "64"
+
+
+def test_the_rate_limit_can_be_moved_without_editing_the_source(monkeypatch):
+    from codai.api import ratelimit
+    monkeypatch.setenv("CODERAI_SERVER_RATE_LIMITS",
+                       "/v1/chat/completions=600/60,/v1/audio/=300/30")
+    got = ratelimit._env_limits()
+    assert got["/v1/chat/completions"] == (600, 60)
+    assert got["/v1/audio/"] == (300, 30)
+
+
+def test_the_limiter_can_be_turned_off_by_env(monkeypatch):
+    from codai.api import ratelimit
+    assert ratelimit._limits_enabled() is True
+    monkeypatch.setenv("CODERAI_SERVER_RATE_LIMIT_ENABLED", "0")
+    assert ratelimit._limits_enabled() is False