colibri: pre-compile like ds4 (don't compile in the runtime container)

The engine is a pre-built binary, bundled in the image and built on a host with the
CUDA toolkit — never compiled per-request in the CUDA-*runtime* container (which has
no nvcc). Build it with the host CUDA 13.x toolkit like ds4: the binary links
libcudart.so.13, resolved in-container from /opt/coderai/local-libs (the CUDA-13
runtime coderai already ships for PyTorch), so it adds no portability constraint.

- worker: _detect_build_target now needs a REAL nvcc (a runtime container has
  /usr/local/cuda but no compiler); ensure_built fails early with a clear
  "pre-compile the binary" message instead of cloning and dying deep in `make` with
  the confusing "nvcc not found … backend_cuda.o Error". _make_args defaults CUDA to
  a PORTABLE arch (sm_80..120 + PTX) and points at the resolved nvcc/CUDA_HOME.
- build.sh --colibri: require a real nvcc and default CUDA_ARCH=portable (like ds4's
  cuda-generic), so the bundled binary isn't locked to the build host's GPU.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LoSpEthysqmseCc6Geizty

build.sh

@@ -806,9 +806,13 @@ build_colibri() {
         git clone --depth 1 https://github.com/JustVugg/colibri "$COLIBRI_DIR" || {
             echo -e "${YELLOW}Warning: could not clone colibri; skipping.${NC}"; return 0; }
     fi
+    # Default to a PORTABLE CUDA arch (SASS for Ampere..Blackwell + PTX fallback), like
+    # ds4's cuda-generic, so the bundled binary isn't locked to the build host's GPU.
+    # Requires a real nvcc (a CUDA *runtime* has /usr/local/cuda but no compiler); the
+    # binary links libcudart.so.13, resolved in the container from /opt/coderai/local-libs.
     local MAKE_ARGS="colibri"
-    if command -v nvcc &> /dev/null || [ -d "/usr/local/cuda" ]; then
-        MAKE_ARGS="colibri CUDA=1 CUDA_ARCH=${COLI_CUDA_ARCH:-native}"
+    if command -v nvcc &> /dev/null; then
+        MAKE_ARGS="colibri CUDA=1 CUDA_ARCH=${COLI_CUDA_ARCH:-portable}"
     elif command -v hipcc &> /dev/null || [ -d "/opt/rocm" ]; then
         MAKE_ARGS="colibri HIP=1 HIP_ARCH=${COLI_HIP_ARCH:-native}"
     elif [ "$(uname -s)" = "Darwin" ]; then

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.71"
+__version__ = "0.1.72"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/api/colibri_worker.py

@@ -78,13 +78,30 @@ def _engine_bin(install_dir: Path) -> Path:
     return cdir / "colibri"
 
 
+def _find_nvcc() -> Optional[str]:
+    """Path to a real ``nvcc`` binary, or None. A CUDA *runtime* container has
+    ``/usr/local/cuda`` (the libraries) but NO compiler — checking the directory is
+    not enough, we must find the nvcc binary itself."""
+    import glob
+    n = shutil.which("nvcc")
+    if n:
+        return n
+    for c in sorted(glob.glob("/usr/local/cuda*/bin/nvcc"), reverse=True):
+        if os.path.isfile(c):
+            return c
+    return None
+
+
 def _detect_build_target() -> str:
-    """Pick a build flavour from the host: CUDA when the toolkit is present."""
+    """Pick the *intended* build flavour. A CUDA runtime/GPU means the intent is a
+    CUDA build even when the toolkit (nvcc) is missing — :func:`ensure_built` then
+    surfaces a clear 'pre-compile the binary' error rather than silently building a
+    CPU-only engine on a GPU box."""
     if platform.system() == "Darwin":
         return "metal"
-    if shutil.which("nvcc") or os.path.isdir("/usr/local/cuda"):
+    if _find_nvcc() or os.path.isdir("/usr/local/cuda") or shutil.which("nvidia-smi"):
         return "cuda"
-    if shutil.which("hipcc") or os.path.isdir("/opt/rocm"):
+    if shutil.which("hipcc") or os.path.isfile("/opt/rocm/bin/hipcc"):
         return "hip"
     return "cpu"
 
@@ -95,9 +112,15 @@ def _make_args(cfg) -> list:
     if target in ("", "auto"):
         target = _detect_build_target()
     if target == "cuda":
-        # native SASS is fine — coderai builds on the same GPU host it runs on. For a
-        # portable image build pass CUDA_ARCH=portable via extra make env if needed.
-        return ["colibri", "CUDA=1", f"CUDA_ARCH={os.environ.get('COLI_CUDA_ARCH', 'native')}"]
+        # Default to a PORTABLE arch (SASS for Ampere..Blackwell + a PTX fallback) so
+        # a pre-compiled/bundled binary runs on any modern card; override with
+        # COLI_CUDA_ARCH (e.g. sm_86 for a single 3090 build). Point the Makefile at
+        # a real nvcc/CUDA_HOME so a non-PATH toolkit still builds.
+        args = ["colibri", "CUDA=1", f"CUDA_ARCH={os.environ.get('COLI_CUDA_ARCH', 'portable')}"]
+        nvcc = _find_nvcc()
+        if nvcc:
+            args += [f"NVCC={nvcc}", f"CUDA_HOME={os.path.dirname(os.path.dirname(nvcc))}"]
+        return args
     if target == "hip":
         return ["colibri", "HIP=1", f"HIP_ARCH={os.environ.get('COLI_HIP_ARCH', 'native')}"]
     if target == "metal":
@@ -135,6 +158,23 @@ def ensure_built(cfg) -> Path:
             f"manually (git clone {cfg.repo_url}; cd c; make colibri [CUDA=1]) or enable "
             "auto_build.")
 
+    # A CUDA build needs the toolkit (nvcc), which a CUDA *runtime* container does
+    # NOT ship. Rather than clone + fail deep in `make` (the confusing
+    # "nvcc not found … backend_cuda.o Error" the user hit), fail early with the fix:
+    # pre-compile the binary. The engine is meant to be pre-built (bundled in the
+    # image or built once on a box with the toolkit), never compiled per-request.
+    _target = (getattr(cfg, "build_target", "auto") or "auto").strip().lower()
+    if _target in ("", "auto"):
+        _target = _detect_build_target()
+    if _target == "cuda" and _find_nvcc() is None:
+        raise RuntimeError(
+            f"colibri needs a CUDA build but no CUDA toolkit (nvcc) is available here "
+            f"— this looks like a CUDA *runtime* environment. Pre-compile the engine "
+            f"and place it at {binary}: build once on a box/image with the matching "
+            f"CUDA toolkit (e.g. `build.sh --colibri`, or a cuda:*-devel container) "
+            f"targeting the same CUDA runtime as this container. Runtime auto-build "
+            f"cannot compile CUDA without nvcc.")
+
     tail = collections.deque(maxlen=40)
     install_dir.parent.mkdir(parents=True, exist_ok=True)
     if not (install_dir / ".git").exists() and not (install_dir / "c" / "Makefile").exists():