hf: resolve relative offload_dir to the configured dir; forward HF_TOKEN

Loading an HF model (e.g. Qwen3.5-9B, 4-bit) failed with 'Permission denied:
./offload'. Cause: the model's per-model offload_dir was the relative './offload'
(a stale auto-saved default), which _cfg_or_global lets win over the global
config; './offload' resolves to the CWD = the READ-ONLY /opt/coderai/app tree in
the image. The config WAS respected — a relative offload path is just meaningless
where the CWD isn't writable.

* hf_loading.resolve_offload_dir(): an absolute offload_dir is respected as-is; a
  relative/empty one INHERITS the configured GLOBAL offload directory
  (global_args.offload_dir) when absolute, then CODERAI_OFFLOAD_DIR, then the user
  cache — never the CWD. Applied in the manager (both load sites, always passed),
  hf_loading, and defensively in the cuda backend.
* main.py + entrypoint: a container-writable CODERAI_OFFLOAD_DIR (=/cache/offload,
  created by the entrypoint) is used when the GLOBAL config is still the bare
  './offload' default; explicit config wins.
* run_oci.sh: forward HF_TOKEN / HUGGING_FACE_HUB_TOKEN from the host env so the
  engines authenticate to the HF Hub (the 'unauthenticated requests' warning) for
  higher rate limits + gated models. HF_HOME/cache dir was already honoured
  (main.py from config.models.hf_cache_dir).

Bump version to 0.1.8.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RdMufYvtTbtGDWsiZVoXce

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.1.7"
+__version__ = "0.1.8"
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.
 # expandable_segments lets the allocator return freed pages to the driver even

codai/backends/cuda.py

@@ -751,6 +751,14 @@ class NvidiaBackend(ModelBackend):
         self._kv_prefix_ok = None
 
         offload_dir = kwargs.get('offload_dir')
+        # Defensive: ensure an absolute, writable offload dir even if a caller passed
+        # a relative './offload' (resolves to the read-only app tree in the image) —
+        # a relative value inherits the configured global offload dir.
+        try:
+            from codai.models.hf_loading import resolve_offload_dir as _resolve_off
+            offload_dir = _resolve_off(offload_dir) if offload_dir else offload_dir
+        except Exception:
+            pass
         load_in_4bit = kwargs.get('load_in_4bit', False)
         load_in_8bit = kwargs.get('load_in_8bit', False)
         manual_ram_gb = kwargs.get('manual_ram_gb')

codai/main.py

@@ -1058,7 +1058,17 @@ def main():
     global_args.https = config.server.https
     global_args.privkey = config.server.https_key_path
     global_args.pubkey = config.server.https_cert_path
-    global_args.offload_dir = config.offload.directory
+    # Disk-offload directory. Honour the configured value (per-model offload_dir and
+    # this global both respected downstream), but when it's still the built-in
+    # relative default './offload' — which resolves to the CWD, the READ-ONLY
+    # /opt/coderai/app tree in the OCI image (EACCES on makedirs) — fall back to a
+    # container-provided writable default (CODERAI_OFFLOAD_DIR, set by the entrypoint
+    # to /cache/offload). Explicit config still wins; bare-metal keeps './offload'.
+    _off_dir = config.offload.directory
+    if (not _off_dir or str(_off_dir).strip() in ("", "./offload")) \
+            and os.environ.get("CODERAI_OFFLOAD_DIR"):
+        _off_dir = os.environ["CODERAI_OFFLOAD_DIR"]
+    global_args.offload_dir = _off_dir
     global_args.ram = config.offload.manual_ram_gb
     global_args.offload_strategy = config.offload.strategy
     global_args.no_ram = config.offload.no_ram

codai/models/hf_loading.py

@@ -25,6 +25,40 @@ import os
 from typing import Any, Dict, Optional
 
 
+def resolve_offload_dir(offload_dir) -> str:
+    """Return an ABSOLUTE, writable disk-offload directory.
+
+    A RELATIVE value (e.g. the legacy './offload' — whether the built-in default or
+    a stale per-model override saved into models.json) resolves against the CWD,
+    which in the OCI image is the READ-ONLY /opt/coderai/app tree → makedirs raises
+    EACCES. So:
+      * an absolute configured path is respected as-is;
+      * a relative/empty one INHERITS the configured GLOBAL offload dir
+        (global_args.offload_dir / config.offload.directory) when that's absolute,
+        then CODERAI_OFFLOAD_DIR (the container's writable default), then the user
+        cache — never the CWD.
+    This keeps the configuration authoritative while making a relative value mean
+    'use the configured offload location', not 'write next to the (read-only) app'.
+    """
+    if offload_dir:
+        p = os.path.expanduser(str(offload_dir))
+        if os.path.isabs(p):
+            return p
+    try:
+        from codai.api.state import get_global_args
+        g = getattr(get_global_args(), "offload_dir", None)
+        if g:
+            gp = os.path.expanduser(str(g))
+            if os.path.isabs(gp):
+                return gp
+    except Exception:
+        pass
+    env = os.environ.get("CODERAI_OFFLOAD_DIR")
+    if env:
+        return os.path.expanduser(env)
+    return os.path.join(os.path.expanduser("~"), ".cache", "coderai", "offload")
+
+
 def _norm(cfg: Optional[Dict[str, Any]]) -> Dict[str, Any]:
     """Return the per-model config dict, unwrapping a forwarded `_raw_cfg`."""
     if not cfg:
@@ -372,10 +406,9 @@ def build_from_pretrained_kwargs(
         kwargs['device_map'] = 'auto'
         kwargs['max_memory'] = {0: gpu_budget, 'cpu': cpu_budget}
 
-        # Disk overflow when offloading is allowed.
-        offload_dir = c.get('offload_dir') or os.path.join(
-            os.path.expanduser('~'), '.cache', 'coderai', 'offload')
-        offload_dir = os.path.expanduser(offload_dir)
+        # Disk overflow when offloading is allowed. Resolve to an absolute writable
+        # dir (a relative per-model './offload' inherits the configured global dir).
+        offload_dir = resolve_offload_dir(c.get('offload_dir'))
         os.makedirs(offload_dir, exist_ok=True)
         kwargs['offload_folder'] = offload_dir
         kwargs['offload_buffers'] = True

codai/models/manager.py

@@ -1085,9 +1085,14 @@ class MultiModelManager:
                 n_gpu_layers = _cfg_or_global('n_gpu_layers', 'n_gpu_layers')
                 if n_gpu_layers is not None:
                     kwargs['n_gpu_layers'] = n_gpu_layers
-                offload_dir = _cfg_or_global('offload_dir', 'offload_dir')
-                if offload_dir:
-                    kwargs['offload_dir'] = offload_dir
+                # Resolve to an absolute, writable offload dir: a relative per-model
+                # './offload' (or the bare default) inherits the configured GLOBAL
+                # offload directory instead of the read-only CWD (app tree in the OCI
+                # image). Always pass it so backends never fall back to accelerate's
+                # './offload' default.
+                from codai.models.hf_loading import resolve_offload_dir as _resolve_off
+                kwargs['offload_dir'] = _resolve_off(
+                    _cfg_or_global('offload_dir', 'offload_dir'))
                 manual_ram = _cfg_or_global('manual_ram_gb', 'ram')
                 if manual_ram is not None:
                     kwargs['manual_ram_gb'] = manual_ram
@@ -1235,9 +1240,14 @@ class MultiModelManager:
                 n_gpu_layers = _cfg_or_global('n_gpu_layers', 'n_gpu_layers')
                 if n_gpu_layers is not None:
                     kwargs['n_gpu_layers'] = n_gpu_layers
-                offload_dir = _cfg_or_global('offload_dir', 'offload_dir')
-                if offload_dir:
-                    kwargs['offload_dir'] = offload_dir
+                # Resolve to an absolute, writable offload dir: a relative per-model
+                # './offload' (or the bare default) inherits the configured GLOBAL
+                # offload directory instead of the read-only CWD (app tree in the OCI
+                # image). Always pass it so backends never fall back to accelerate's
+                # './offload' default.
+                from codai.models.hf_loading import resolve_offload_dir as _resolve_off
+                kwargs['offload_dir'] = _resolve_off(
+                    _cfg_or_global('offload_dir', 'offload_dir'))
                 manual_ram = _cfg_or_global('manual_ram_gb', 'ram')
                 if manual_ram is not None:
                     kwargs['manual_ram_gb'] = manual_ram

packaging/linux/launcher/coderai-entrypoint

@@ -15,6 +15,11 @@ set -eu
 # server's built-in janitor age-prunes it; see CODERAI_TMP below.
 : "${CODERAI_TMP:=$CODERAI_CACHE_DIR/coderai-tmp}"
 export TMPDIR="$CODERAI_TMP" TMP="$CODERAI_TMP" TEMP="$CODERAI_TMP"
+# Writable disk-offload dir on the cache volume. Used as the default ONLY when the
+# config still has the relative './offload' (which would land in the read-only
+# /opt/coderai/app tree). An explicit offload.directory in config wins.
+: "${CODERAI_OFFLOAD_DIR:=$CODERAI_CACHE_DIR/offload}"
+export CODERAI_OFFLOAD_DIR
 # Don't write .pyc into the read-only /opt/coderai tree (esp. when run as --user).
 export PYTHONDONTWRITEBYTECODE=1
 # Demo tool web-UI autostart toggles, consumed by supervisord.conf's
@@ -44,6 +49,7 @@ mkdir -p \
   "$CODERAI_CACHE_DIR/videogen_output" \
   "$CODERAI_CACHE_DIR/video_editor" \
   "$CODERAI_CACHE_DIR/video_editor/sessions" \
+  "$CODERAI_OFFLOAD_DIR" \
   "$CODERAI_TMP" \
   /tmp/nginx-client-body /tmp/nginx-proxy /tmp/nginx-fastcgi \
   /tmp/nginx-uwsgi /tmp/nginx-scgi

packaging/linux/run_oci.sh

@@ -337,6 +337,12 @@ else
   PUBLISH="$PORT:8776"
 fi
 args=(run --rm --name "$NAME" --ipc=host -p "$PUBLISH" -e CODERAI_HOST=0.0.0.0 -e CODERAI_PORT=8776)
+# Forward a HuggingFace token from the host env so the engines authenticate to the
+# HF Hub (higher rate limits + gated models) instead of sending unauthenticated
+# requests. huggingface_hub auto-reads these; no-op when unset.
+for _hv in HF_TOKEN HUGGING_FACE_HUB_TOKEN; do
+  if [[ -n "${!_hv:-}" ]]; then args+=(-e "$_hv=${!_hv}"); fi
+done
 # Pass-through coderai server flags (appended by the in-image launcher's argv).
 if [[ -n "$CODERAI_EXTRA_ARGS" ]]; then
   args+=(-e "CODERAI_EXTRA_ARGS=$CODERAI_EXTRA_ARGS")