h3: run the newer diffusers as a 75 MB overlay, and train through it

H3 and Krea 2 need diffusers >= 0.40 while the server runs 0.38. Upgrading in
place is not free: measured on this box, SDXL comes out bit-identical but Z-Image
drifts (mean |diff| 0.0146, max 0.32, against a same-version control that is
bit-identical) — so the upgrade would silently change image output for a model
that is working fine.

Since diffusers is imported once per process, the split only has to be
per-process. The H3 venv is now created with --system-site-packages and given
nothing but diffusers>=0.40 and huggingface_hub>=1.23; torch, transformers and the
rest are inherited from the server's venv. That is 75 MB instead of the several GB
a standalone venv costs, and keeps one torch on the machine.
CODERAI_H3_STANDALONE_VENV=1 still builds a fully independent venv from
requirements-h3.txt, for the day the parent's torch is too old.

Training follows the same boundary: tools/lora_train_worker.py runs one job in the
overlay — references written out as PNGs, request flattened to JSON, progress
returned as JSON lines and mirrored onto the usual /v1/loras/progress so polling is
unchanged. _train_via_overlay() drives it.

Routing is by capability, not hardcoded: _diffusers_has(arch) tries to load the
classes, so wan and ltx2 train in-process (0.38 has them) while h3 and krea go
out-of-process — and if the main venv is ever upgraded they move back in-process
with no config change.

Verified: the overlay resolves diffusers 0.40 with torch/transformers inherited and
leaves the parent untouched; a probe job ran under it, reached _train_flow_dit and
reported back through the job file. The trainers themselves are still unrun against
real weights.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/api/h3_worker.py

@@ -176,6 +176,20 @@ def _diffusers_ok(py: Path) -> bool:
 
 
 def _bootstrap_venv(config: dict = None) -> Path:
+    """Build (or reuse) the venv H3 runs in, as an OVERLAY on this one.
+
+    H3 needs diffusers >= 0.40; the server runs 0.38 and upgrading it in place
+    changes output for models that are fine as they are (Z-Image drifts
+    measurably). Since diffusers is imported once per process, the split only has
+    to be per-process — so this venv is created with --system-site-packages and
+    given nothing but diffusers and huggingface_hub. torch, transformers and the
+    rest are inherited from the parent, which makes it ~75 MB instead of the
+    several GB a standalone venv costs, and keeps exactly one torch on the box.
+
+    Set CODERAI_H3_STANDALONE_VENV=1 to build a fully independent venv instead
+    (from requirements-h3.txt) — needed only if the parent's torch is ever too old
+    for H3.
+    """
     global _bootstrapped
     venv = resolve_venv_dir(config)
     py = _venv_python(venv)
@@ -184,19 +198,33 @@ def _bootstrap_venv(config: dict = None) -> Path:
     if py.exists() and _diffusers_ok(py):
         _bootstrapped = True
         return py
+    standalone = os.environ.get("CODERAI_H3_STANDALONE_VENV", "") in ("1", "true", "yes")
     if not py.exists():
-        print(f"[h3] creating isolated venv at {venv} …", flush=True)
+        kind = "standalone" if standalone else "overlay"
+        print(f"[h3] creating {kind} venv at {venv} …", flush=True)
         venv.parent.mkdir(parents=True, exist_ok=True)
-        subprocess.run([sys.executable, "-m", "venv", str(venv)], check=True)
+        cmd = [sys.executable, "-m", "venv"]
+        if not standalone:
+            cmd.append("--system-site-packages")
+        subprocess.run(cmd + [str(venv)], check=True)
         py = _venv_python(venv)
     if not _diffusers_ok(py):
+        # NOTE: always `python -m pip`, never the venv's pip script — a venv that
+        # was copied carries a stale shebang and would install into the ORIGINAL
+        # interpreter's site-packages.
+        if standalone:
             if not _REQUIREMENTS.exists():
                 raise RuntimeError(f"H3 requirements file missing: {_REQUIREMENTS}")
-        print("[h3] installing diffusers>=0.40 + torch into the isolated venv "
+            print("[h3] installing diffusers>=0.40 + torch into a standalone venv "
                   "(first run downloads several GB, this takes a while) …", flush=True)
             subprocess.run([str(py), "-m", "pip", "install", "-U", "pip"], check=True)
             subprocess.run([str(py), "-m", "pip", "install", "-r", str(_REQUIREMENTS)],
                            check=True)
+        else:
+            print("[h3] adding diffusers>=0.40 to the overlay (torch and the rest "
+                  "are inherited from the server's venv) …", flush=True)
+            subprocess.run([str(py), "-m", "pip", "install", "-q",
+                            "diffusers>=0.40.0", "huggingface_hub>=1.23"], check=True)
         if not _diffusers_ok(py):
             raise RuntimeError(
                 "H3 venv built but 'from diffusers import MiniMaxH3ModularPipeline' "
@@ -205,6 +233,14 @@ def _bootstrap_venv(config: dict = None) -> Path:
     return py
 
 
+def overlay_python(config: dict = None) -> str:
+    """Path to the interpreter that has diffusers >= 0.40.
+
+    Shared with the LoRA trainers: H3 and Krea 2 adapters can't be trained in the
+    server process while it is on 0.38, so they run out-of-process against this."""
+    return str(_bootstrap_venv(config))
+
+
 # ── service lifecycle ─────────────────────────────────────────────────────────
 
 def _free_port() -> int:

codai/api/loras.py

@@ -1019,6 +1019,13 @@ def _train_lora_sync(req: LoraTrainRequest) -> dict:
         _arch = _detect_arch(video_path, _target) or "wan"
         if _arch != "wan":
             print(f"  [lora] training a {_arch} adapter for {video_path}")
+            # H3 and Krea 2 need diffusers >= 0.40 and this process has 0.38 — and
+            # diffusers is imported once per process, so they train in the overlay
+            # venv instead. LTX-2 needs nothing newer and stays in-process.
+            if _arch in ("h3", "krea") and not _diffusers_has(_arch):
+                return _train_via_overlay(_arch, req, video_path, images,
+                                          instance_prompt, steps, rank, resolution,
+                                          lr, seed, device)
             return _train_flow_dit(_arch, req, video_path, images, instance_prompt,
                                    steps, rank, resolution, lr, seed, device)
         return _train_wan(req, video_path, images, instance_prompt,
@@ -1729,6 +1736,100 @@ def _train_dit(req, base_path, images, instance_prompt,
     return {"name": name, "path": path}
 
 
+def _diffusers_has(arch: str) -> bool:
+    """Can THIS process train that architecture, or does it need the overlay?"""
+    try:
+        from codai.api.lora_archs import load_classes
+        load_classes(arch)
+        return True
+    except Exception:
+        return False
+
+
+def _train_via_overlay(arch, req, base_path, images, instance_prompt,
+                       steps, rank, resolution, lr, seed, device):
+    """Train in the overlay venv (diffusers >= 0.40) as a subprocess.
+
+    The reference images are written out as PNGs and the request is flattened to
+    JSON — nothing is shared but the filesystem, because the child runs a different
+    diffusers than we do. Progress is tailed back from the job file so the usual
+    /v1/loras/progress polling keeps working.
+    """
+    import json as _json
+    import subprocess
+    import tempfile
+    import threading
+
+    from codai.api import h3_worker
+    try:
+        py = h3_worker.overlay_python({})
+    except Exception as exc:
+        raise HTTPException(status_code=501, detail=(
+            f"{arch} LoRA training needs diffusers >= 0.40, which this process does "
+            f"not have, and the overlay venv could not be built: {exc}"))
+
+    work = tempfile.mkdtemp(prefix=f"lora_{arch}_")
+    img_paths = []
+    for i, im in enumerate(images):
+        pth = os.path.join(work, f"ref_{i:03d}.png")
+        im.save(pth)
+        img_paths.append(pth)
+    job_path = os.path.join(work, "job.json")
+    _json.dump({
+        "arch": arch, "base_path": base_path, "images": img_paths,
+        "instance_prompt": instance_prompt, "steps": steps, "rank": rank,
+        "resolution": resolution, "lr": lr, "seed": seed, "device": device,
+        "request": {k: getattr(req, k) for k in
+                    ("name", "num_frames", "quantize_4bit", "session", "target")
+                    if hasattr(req, k)},
+    }, open(job_path, "w"))
+
+    script = os.path.join(os.path.dirname(os.path.dirname(
+        os.path.dirname(os.path.abspath(__file__)))), "tools", "lora_train_worker.py")
+    print(f"  [lora][{arch}] training out-of-process in the overlay venv", flush=True)
+    proc = subprocess.Popen([py, script, "--job", job_path],
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                            text=True, bufsize=1)
+
+    def _pump():
+        for line in proc.stdout:
+            line = line.rstrip()
+            if line:
+                print(f"  [lora][{arch}] {line}", flush=True)
+    threading.Thread(target=_pump, daemon=True).start()
+
+    # Mirror the child's progress onto ours so the normal polling endpoint works.
+    prog_path = job_path + ".progress"
+    seen = 0
+    while proc.poll() is None:
+        time.sleep(2.0)
+        try:
+            _check_train_cancel()
+        except Exception:
+            proc.terminate()
+            raise
+        try:
+            lines = open(prog_path).read().splitlines() if os.path.exists(prog_path) else []
+            for ln in lines[seen:]:
+                try:
+                    _set_progress(**{k: v for k, v in _json.loads(ln).items() if k != "at"})
+                except Exception:
+                    pass
+            seen = len(lines)
+        except Exception:
+            pass
+
+    res_path = job_path + ".result"
+    if not os.path.exists(res_path):
+        raise HTTPException(status_code=500, detail=(
+            f"{arch} LoRA trainer exited (code {proc.returncode}) without a result"))
+    out = _json.load(open(res_path))
+    if not out.get("ok"):
+        raise HTTPException(status_code=500,
+                            detail=f"{arch} LoRA training failed: {out.get('error')}")
+    return out["result"]
+
+
 def _train_flow_dit(arch: str, req, base_path, images, instance_prompt,
                     steps, rank, resolution, lr, seed, device):
     """LoRA for a flow-matching DiT — LTX-2, MiniMax-H3, Krea 2.

requirements-h3.txt

-# Isolated venv for MiniMax-H3 video+audio generation.
+# Extra packages for the MiniMax-H3 venv.
 #
-# H3 exists ONLY as Modular Diffusers blocks (MiniMaxH3Blocks /
-# MiniMaxH3ModularPipeline), which landed in diffusers 0.40.0. The main coderai
-# venv is on diffusers 0.38.x, and moving the whole stack to 0.40 would shift the
-# floor under every Wan / LTX / Z-Image / SDXL pipeline at once — so H3 gets its
-# own venv, the same way pyannote, NeMo-Canary and vLLM do.
-# codai/api/h3_worker.py builds it (on demand under ~/.coderai/h3_venv or the
-# /cache mount) and runs tools/h3_service.py inside it.
-# Do NOT merge these into requirements.txt.
+# This venv is an OVERLAY: codai/api/h3_worker.py creates it with
+# --system-site-packages and installs only what follows, so torch, transformers and
+# everything else are inherited from the server's venv. That keeps it ~75 MB
+# instead of the several GB a standalone copy of torch costs, and keeps exactly one
+# torch on the machine.
+#
+# Why an overlay at all: H3 exists only as Modular Diffusers blocks
+# (MiniMaxH3Blocks / MiniMaxH3ModularPipeline) added in diffusers 0.40, while the
+# server runs 0.38 — and upgrading in place changes output for models that are fine
+# as they are (measured: Z-Image drifts, mean |diff| 0.0146, while SDXL is
+# bit-identical). diffusers is imported once per process, so the split only has to
+# be per-process.
+#
+# This file is used ONLY when CODERAI_H3_STANDALONE_VENV=1 forces a fully
+# independent venv — needed if the parent's torch is ever too old for H3.
 
-# The transformer is 33B and the text encoder is Qwen3-VL, so this venv wants a
-# recent torch. Left unpinned deliberately: unlike pyannote there is no known-bad
-# version to dodge, and the H3 blocks track diffusers main.
 --extra-index-url https://download.pytorch.org/whl/cu130
 torch
 torchvision
 torchaudio
 
-# The floor for MiniMax-H3 support.
 diffusers>=0.40.0
-# Qwen3VLForConditionalGeneration (H3's text encoder) needs transformers 5.x.
 transformers>=5.0
 accelerate>=1.0
 safetensors>=0.4
-huggingface_hub>=0.34
+huggingface_hub>=1.23
 
-# Media: export_to_video writes the frames, soundfile writes the jointly-generated
-# soundtrack, and ffmpeg (system binary, already on PATH in the image) muxes them.
 imageio>=2.34
 imageio-ffmpeg>=0.5
 soundfile>=0.12
 Pillow>=10.0
 numpy
-
-# peft carries the LoRA adapters MiniMaxH3LoraLoaderMixin loads.
 peft>=0.14

tools/lora_train_worker.py

+#!/usr/bin/env python3
+# CoderAI - OpenAI-compatible API server
+# Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+"""Run one LoRA training job in a venv that has a newer diffusers.
+
+MiniMax-H3 and Krea 2 need diffusers >= 0.40 while the server runs 0.38, and
+diffusers is imported once per process — so those two architectures train here,
+in the overlay venv (codai/api/h3_worker.overlay_python), instead of in the engine.
+
+Invoked by codai/api/loras.py, never by hand:
+
+    <overlay python> tools/lora_train_worker.py --job /path/to/job.json
+
+The job file carries the request fields plus the resolved image paths; progress is
+appended to <job>.progress as JSON lines so the parent can surface it, and the
+final result is written to <job>.result.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+import traceback
+
+# The server's own modules — importable because the overlay inherits the parent's
+# site-packages; only diffusers/huggingface_hub differ.
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Out-of-process LoRA trainer")
+    ap.add_argument("--job", required=True, help="Path to the job JSON")
+    args = ap.parse_args(argv)
+
+    job = json.load(open(args.job))
+    progress_path = args.job + ".progress"
+    result_path = args.job + ".result"
+
+    def emit(**kw):
+        kw["at"] = time.time()
+        with open(progress_path, "a") as fh:
+            fh.write(json.dumps(kw) + "\n")
+
+    try:
+        import diffusers
+        emit(status="starting", message=f"diffusers {diffusers.__version__} "
+                                        f"(overlay), arch={job['arch']}")
+        from PIL import Image
+        from codai.api import loras
+
+        # Progress from inside the trainer goes to the job file, so the parent can
+        # poll it without sharing memory.
+        def _set_progress(**kw):
+            emit(**kw)
+        loras._set_progress = _set_progress
+        loras._check_train_cancel = lambda: None   # cancellation is the parent's job
+
+        images = [Image.open(p).convert("RGB") for p in job["images"]]
+        if not images:
+            raise RuntimeError("no reference images were passed to the trainer")
+
+        class _Req:
+            pass
+        req = _Req()
+        for k, v in job["request"].items():
+            setattr(req, k, v)
+
+        result = loras._train_flow_dit(
+            job["arch"], req, job["base_path"], images, job["instance_prompt"],
+            int(job["steps"]), int(job["rank"]), int(job["resolution"]),
+            float(job["lr"]), int(job["seed"]), job["device"])
+        json.dump({"ok": True, "result": result}, open(result_path, "w"))
+        emit(status="done", message="training finished")
+        return 0
+    except Exception as exc:
+        json.dump({"ok": False, "error": f"{type(exc).__name__}: {exc}",
+                   "traceback": traceback.format_exc()[-2000:]}, open(result_path, "w"))
+        emit(status="error", message=str(exc)[:300])
+        print(traceback.format_exc(), file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())