config: per-engine AMD GPU power-state lock (Polaris compute-hang mitigation)

New server.dpm_force_performance_level_overrides (engine name → level,
e.g. {"radeon":"high"}). At engine startup the engine writes the level
to every amdgpu card's power_dpm_force_performance_level — locking a
Polaris/GCN card to fixed top clocks avoids the DPM power-state
transitions that hang these cards under sustained Vulkan compute
(current default is 'auto', the hang-prone mode). Card is matched by PCI
vendor id (0x1002), robust to DRM card renumbering across resets.
Best-effort: an unprivileged container logs the exact host command on
PermissionError.

Pairs with two config-only stability levers applied for the radeon:
n_ctx reduction (qwen3 1024→512, gme 1536→1024) to keep all three
embedders in real VRAM with headroom (no GTT-over-PCIe spill during
compute — another Polaris hang trigger), and max_parallel_requests
override {"radeon":1} to serialize Vulkan submissions.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014S8VtAvG499SsCbeESRK7V

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.52"
+__version__ = "0.1.53"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/config.py

@@ -40,6 +40,13 @@ class ServerConfig:
     # override lets a bigger card run more concurrently than a smaller one. Blank =
     # use the default above.
     max_parallel_requests_overrides: dict = field(default_factory=dict)
+    # Per-engine AMD GPU power/clock lock (engine name → level, e.g.
+    # {"radeon": "high"}). Applied at engine startup: writes the level to each
+    # amdgpu card's power_dpm_force_performance_level. Stabilises Polaris/GCN
+    # cards that hang under sustained Vulkan compute due to DPM clock switching.
+    # Levels: auto | low | high | manual | profile_standard | profile_peak.
+    # Best-effort — an unprivileged container logs the manual host command.
+    dpm_force_performance_level_overrides: dict = field(default_factory=dict)
     # ─── Frontend/engine split ───────────────────────────────────────────────
     # coderai boots a thin, always-responsive *front* reverse proxy on the public
     # host/port and supervises one or more *engine* subprocesses (which do all
@@ -659,6 +666,7 @@ class ConfigManager:
                 "queue_max_size": self.config.server.queue_max_size,
                 "max_parallel_requests": self.config.server.max_parallel_requests,
                 "max_parallel_requests_overrides": self.config.server.max_parallel_requests_overrides,
+                "dpm_force_performance_level_overrides": self.config.server.dpm_force_performance_level_overrides,
                 "internal_port_base": self.config.server.internal_port_base,
                 "engines": self.config.server.engines,
                 "engine_gpus": self.config.server.engine_gpus,

codai/frontproxy/engine_supervisor.py

@@ -372,6 +372,13 @@ class EngineSupervisor:
             env["CODERAI_MAX_PARALLEL"] = str(int(par))
         if inst is not None:
             env["CODERAI_MAX_MODEL_INSTANCES"] = str(int(inst))
+        # Per-engine AMD GPU power-state lock (e.g. {"radeon": "high"}) — the
+        # engine applies it to its amdgpu card(s) at startup to avoid Polaris
+        # DPM-transition hangs under sustained Vulkan compute.
+        _dpm = (getattr(srv, "dpm_force_performance_level_overrides", None)
+                or {}).get(engine.name)
+        if _dpm:
+            env["CODERAI_DPM_FORCE"] = str(_dpm)
         # Force this engine's backend (the engine reads this in --engine-only mode
         # and overrides config.backend.type) so a Vulkan/Radeon engine doesn't
         # auto-pick CUDA, and vice-versa.

codai/frontproxy/gpu_detect.py

@@ -401,6 +401,52 @@ def engine_gpu_stats() -> list:
             if c.get("vendor") in sels or (c.get("uuid") and c["uuid"] in sels)]
 
 
+_VALID_DPM_LEVELS = ("auto", "low", "high", "manual",
+                     "profile_standard", "profile_min_sclk", "profile_min_mclk",
+                     "profile_peak")
+
+
+def apply_amd_dpm_force_level(level: str) -> None:
+    """Write ``level`` to every amdgpu card's power_dpm_force_performance_level.
+
+    Locking a Polaris/GCN card to 'high' (fixed top clocks) avoids the DPM
+    power-state transitions that hang these cards under sustained Vulkan
+    compute. Best-effort: an unprivileged container can't write root-owned
+    sysfs, so on PermissionError we log the exact host command to run instead."""
+    import glob
+    level = (level or "").strip()
+    if level not in _VALID_DPM_LEVELS:
+        print(f"[dpm] ignoring invalid level '{level}' "
+              f"(valid: {', '.join(_VALID_DPM_LEVELS)})", flush=True)
+        return
+    wrote, denied = [], []
+    for card in sorted(glob.glob("/sys/class/drm/card*")):
+        dev = os.path.join(card, "device")
+        try:
+            with open(os.path.join(dev, "vendor")) as f:
+                if f.read().strip().lower() != "0x1002":   # AMD PCI vendor id
+                    continue
+        except OSError:
+            continue
+        node = os.path.join(dev, "power_dpm_force_performance_level")
+        if not os.path.exists(node):
+            continue
+        try:
+            with open(node, "w") as f:
+                f.write(level + "\n")
+            wrote.append(os.path.basename(card))
+        except PermissionError:
+            denied.append(node)
+        except OSError as e:
+            print(f"[dpm] failed to set {node}: {e}", flush=True)
+    if wrote:
+        print(f"[dpm] locked AMD power level '{level}' on {', '.join(wrote)}",
+              flush=True)
+    for node in denied:
+        print(f"[dpm] cannot set power level (unprivileged container). Run once "
+              f"on the HOST: echo {level} | sudo tee {node}", flush=True)
+
+
 def card_key(vendor: str, uuid: str = "", pci: str = "") -> str:
     """Stable identifier for one physical GPU, computable identically by the front
     (which sees every card) and by an engine (which sees only its own, in llama.cpp

codai/main.py

@@ -635,6 +635,18 @@ def main():
         if _forced_backend:
             config.backend.type = _forced_backend
             print(f"[engine] backend forced to '{_forced_backend}' by the front")
+        # Optional: pin this engine's AMD GPU(s) to a fixed power/clock state.
+        # Polaris (and other GCN) cards frequently HANG under sustained Vulkan
+        # compute when the driver switches DPM power states mid-kernel; locking
+        # to 'high' avoids those transitions. Configured per-engine via
+        # server.dpm_force_performance_level_overrides (env CODERAI_DPM_FORCE).
+        _dpm = os.environ.get("CODERAI_DPM_FORCE")
+        if _dpm:
+            try:
+                from codai.frontproxy.gpu_detect import apply_amd_dpm_force_level
+                apply_amd_dpm_force_level(_dpm.strip())
+            except Exception as _de:
+                print(f"[dpm] apply failed: {_de}", flush=True)
         # The front owns the AISBF broker (always-responsive, one registration for
         # the whole node, routes to engines). So no engine runs its own broker
         # client — that would double-register and stall when the engine loads.