choose the dtype default from a device that has been detected

NvidiaBackend.__init__ leaves self.device None. The loader picked the precision
default as `'fp16' if self.device == 'cuda' else 'fp32'` at line 897 and only
detected the device at line 1029, so the FIRST load in a fresh engine process
defaulted to fp32 on a CUDA card -- while a second load in the same process
inherited 'cuda' from the first and quietly got fp16. The same model loaded at
two precisions depending on load order.

fp32 costs twice the VRAM of the dtype asked for, which on a 24 GB card is the
whole headroom argument for quantizing in the first place: Qwen3.5-9B is
carded load_in_4bit with an estimate of 7.795 GB and measured 14.403 GB, a
1.85x gap that is the fp32 tell. FlashAttention also refuses fp32 outright, so
transformers warned twice at load, cast the inputs back to fp32 at inference,
and the model answered every request with "FlashAttention only support fp16 and
bf16 data type" -- and before that error was surfaced at all, with an empty
completion. It had been unusable since 2026-09-30.

Also resolve flash_attn by PRESENCE rather than truthiness: `config.get(k) or
global` cannot express "off", so a card saying `flash_attention: false` fell
through to the global default. Not the cause here (the global is false too),
but the same class of bug one layer up.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014T3rZnLhydX9QqUcfBbqLu

codai/backends/cuda.py

@@ -887,6 +887,21 @@ class NvidiaBackend(ModelBackend):
             offload_dir = _resolve_off(offload_dir) if offload_dir else offload_dir
         except Exception:
             pass
+        # The device has to be known BEFORE the dtype default is chosen from it.
+        # It used to be detected 130 lines further down, while __init__ leaves it
+        # None, so on the first load in a fresh engine process `self.device ==
+        # 'cuda'` was False and the default came out fp32: every HF text model
+        # with no explicit `precision` loaded in fp32 on a CUDA card. That is
+        # twice the VRAM on a 24 GB card, and FA2 refuses fp32 outright --
+        # measured 2026-10-28 on Qwen3.5-9B (load_in_4bit, precision unset),
+        # which transformers loaded as torch.float32 and which then answered
+        # every request with "FlashAttention only support fp16 and bf16 data
+        # type", and before that error was surfaced, with an empty completion.
+        #
+        # A SECOND load in the same process saw self.device still set to 'cuda'
+        # from the previous one and quietly got fp16, so the same model loaded
+        # at two different precisions depending on load order.
+        self.device = self._detect_device()
         # Resolve the model's configured `precision` → torch dtype (parity with
         # hf_loading, which non-text loaders use). Default fp16 on CUDA / fp32 on CPU
         # when unset, preserving this loader's historical behaviour.
@@ -1026,8 +1041,8 @@ class NvidiaBackend(ModelBackend):
                 self.use_flash_attn = False
                 print(f"  Flash Attention 2 disabled: {reason} → using SDPA instead.")
 
-        self.device = self._detect_device()
-        
+        # Already detected above, before the dtype default needed it; re-probing
+        # here would only repeat the log line.
         self.tokenizer = AutoTokenizer.from_pretrained(
             model_name,
             trust_remote_code=True,

codai/models/manager.py

@@ -1929,16 +1929,28 @@ class MultiModelManager:
                 manual_ram = _cfg_or_global('manual_ram_gb', 'ram')
                 if manual_ram is not None:
                     kwargs['manual_ram_gb'] = manual_ram
-                # flash_attn comes from the per-model config (source of truth).
-                # build_kwargs_from_config populates it from the model's
-                # 'flash_attention' setting; CLI/global is NOT consulted here.
-                # Honour either per-model key (flash_attn / flash_attention) and the
-                # global default (global_args.flash_attn ← offload.flash_attention),
-                # matching hf_loading — so the standard config key isn't silently
-                # ignored on the nvidia text path.
-                kwargs['flash_attn'] = bool(
-                    config.get('flash_attn') or config.get('flash_attention')
-                    or getattr(_ga, 'flash_attn', False))
+                # flash_attn: the per-model config is the source of truth, and
+                # the global (global_args.flash_attn ← offload.flash_attention)
+                # is a default for models that say nothing.
+                #
+                # Resolved by PRESENCE, not truthiness. `config.get(k) or global`
+                # cannot express "off": False and "unset" are the same value to
+                # `or`, so a card that explicitly said `flash_attention: false`
+                # fell through to the global and FA2 stayed on. Measured
+                # 2026-10-28 on Qwen3.5-9B (flash_attention: false,
+                # load_in_4bit: true): every request died with "FlashAttention
+                # only support fp16 and bf16 data type", and before that error
+                # was surfaced at all, with an empty completion -- the model had
+                # been unusable since 2026-09-30 with its own off switch
+                # already set.
+                _fa = None
+                for _fa_key in ('flash_attn', 'flash_attention'):
+                    if _fa_key in config and config[_fa_key] is not None:
+                        _fa = bool(config[_fa_key])
+                        break
+                if _fa is None:
+                    _fa = bool(getattr(_ga, 'flash_attn', False))
+                kwargs['flash_attn'] = _fa
                 # Configured precision → torch dtype (parity with hf_loading); the
                 # nvidia backend defaults fp16/fp32 when unset.
                 kwargs['precision'] = config.get('precision')
@@ -2105,16 +2117,28 @@ class MultiModelManager:
                 manual_ram = _cfg_or_global('manual_ram_gb', 'ram')
                 if manual_ram is not None:
                     kwargs['manual_ram_gb'] = manual_ram
-                # flash_attn comes from the per-model config (source of truth).
-                # build_kwargs_from_config populates it from the model's
-                # 'flash_attention' setting; CLI/global is NOT consulted here.
-                # Honour either per-model key (flash_attn / flash_attention) and the
-                # global default (global_args.flash_attn ← offload.flash_attention),
-                # matching hf_loading — so the standard config key isn't silently
-                # ignored on the nvidia text path.
-                kwargs['flash_attn'] = bool(
-                    config.get('flash_attn') or config.get('flash_attention')
-                    or getattr(_ga, 'flash_attn', False))
+                # flash_attn: the per-model config is the source of truth, and
+                # the global (global_args.flash_attn ← offload.flash_attention)
+                # is a default for models that say nothing.
+                #
+                # Resolved by PRESENCE, not truthiness. `config.get(k) or global`
+                # cannot express "off": False and "unset" are the same value to
+                # `or`, so a card that explicitly said `flash_attention: false`
+                # fell through to the global and FA2 stayed on. Measured
+                # 2026-10-28 on Qwen3.5-9B (flash_attention: false,
+                # load_in_4bit: true): every request died with "FlashAttention
+                # only support fp16 and bf16 data type", and before that error
+                # was surfaced at all, with an empty completion -- the model had
+                # been unusable since 2026-09-30 with its own off switch
+                # already set.
+                _fa = None
+                for _fa_key in ('flash_attn', 'flash_attention'):
+                    if _fa_key in config and config[_fa_key] is not None:
+                        _fa = bool(config[_fa_key])
+                        break
+                if _fa is None:
+                    _fa = bool(getattr(_ga, 'flash_attn', False))
+                kwargs['flash_attn'] = _fa
                 # Configured precision → torch dtype (parity with hf_loading); the
                 # nvidia backend defaults fp16/fp32 when unset.
                 kwargs['precision'] = config.get('precision')

tests/test_cuda_dtype_default_before_device.py

+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""The dtype default must be chosen from a device that has been detected.
+
+`NvidiaBackend.__init__` leaves `self.device = None`. The loader picks the
+precision default as `'fp16' if self.device == 'cuda' else 'fp32'`, so while
+the detection sat further down the function, the FIRST load in a fresh engine
+process defaulted to fp32 -- twice the VRAM on a 24 GB card, and FlashAttention
+refuses fp32, so the model answered every request with "FlashAttention only
+support fp16 and bf16 data type" (and, before that was surfaced, an empty
+completion). A second load in the same process inherited `self.device` from the
+first and quietly got fp16, so the same card loaded at two precisions
+depending on load order.
+"""
+
+import ast
+import io
+import os
+
+SRC = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                   "codai", "backends", "cuda.py")
+
+
+def _load_fn():
+    tree = ast.parse(io.open(SRC, encoding="utf-8").read())
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef) and node.name == "NvidiaBackend":
+            for fn in node.body:
+                if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    src = ast.dump(fn)
+                    if "_detect_device" in src and "precision" in src:
+                        return fn
+    raise AssertionError("no NvidiaBackend method both detects the device "
+                         "and resolves a precision default")
+
+
+def _detect_lines(fn):
+    """Lines assigning self.device from self._detect_device()."""
+    out = []
+    for node in ast.walk(fn):
+        if not isinstance(node, ast.Assign):
+            continue
+        tgts = [t for t in node.targets
+                if isinstance(t, ast.Attribute) and t.attr == "device"]
+        if not tgts:
+            continue
+        if any(isinstance(c, ast.Attribute) and c.attr == "_detect_device"
+               for c in ast.walk(node.value)):
+            out.append(node.lineno)
+    return sorted(out)
+
+
+def _device_compare_lines(fn):
+    """Lines comparing self.device to the literal 'cuda'."""
+    out = []
+    for node in ast.walk(fn):
+        if not isinstance(node, ast.Compare):
+            continue
+        left = node.left
+        if not (isinstance(left, ast.Attribute) and left.attr == "device"):
+            continue
+        for comp in node.comparators:
+            if isinstance(comp, ast.Constant) and comp.value == "cuda":
+                out.append(node.lineno)
+    return sorted(out)
+
+
+def test_the_device_is_detected_before_the_dtype_default_reads_it():
+    fn = _load_fn()
+    detects = _detect_lines(fn)
+    compares = _device_compare_lines(fn)
+    assert detects, "the loader never detects the device"
+    assert compares, (
+        "no `self.device == 'cuda'` comparison found; if the dtype default "
+        "stopped depending on the device, retarget this test")
+    assert min(detects) < min(compares), (
+        f"self.device is compared to 'cuda' at line {min(compares)} but only "
+        f"assigned from _detect_device() at line {min(detects)}: on the first "
+        f"load of a fresh backend self.device is still None, so the dtype "
+        f"default silently becomes fp32 on a CUDA card")
+
+
+def test_the_device_is_detected_once():
+    """A second detection would re-log and invite the old ordering back."""
+    fn = _load_fn()
+    detects = _detect_lines(fn)
+    assert len(detects) == 1, (
+        f"self.device is assigned from _detect_device() at lines {detects}; "
+        f"keep a single detection ahead of every use")
+
+
+def test_an_unset_precision_on_cuda_resolves_to_a_flash_attention_dtype():
+    """fp32 is not a dtype FlashAttention accepts, so it cannot be the default."""
+    import torch
+    from codai.models.hf_loading import resolve_dtype
+    got = resolve_dtype({"precision": None}, default="fp16")
+    assert got in (torch.float16, torch.bfloat16), (
+        f"an unset precision on CUDA resolved to {got}; FlashAttention only "
+        f"supports fp16/bf16, and fp32 also doubles the VRAM a card must hold")