longcat: say why the venv build failed (v0.2.55)

  training failed: Command '[…py310…, '-m', 'venv', '/AI/guffcache/longcat_venv-train']'
  returned non-zero exit status 1.

An exit status is not a diagnosis. The actual reason was in the log only because
the child's stderr happened to land there:

  Fatal Python error: init_fs_encoding: failed to get the Python codec of the
  filesystem encoding
  ModuleNotFoundError: No module named 'encodings'

Every step of the venv build went through subprocess.run(check=True), which
reports a number and discards what the process said. They now go through a runner
that raises with the child's output, truncated at both ends if it is long. The
next failure here names itself instead of needing a log archaeologist.

The environment is also stripped for anything that spawns the 3.10 interpreter —
PYTHONHOME, PYTHONPATH, VIRTUAL_ENV, PYTHONEXECUTABLE and friends — because that
is exactly the class of variable that produces the error above, and the failure is
fatal and silent in the parent. HF cache paths, CUDA_VISIBLE_DEVICES and
everything else still reach the child. This covers the venv build, the pip
installs, the service launch and the train/quantise workers; the service used
dict(os.environ) directly before.

Worth being straight about: I could NOT reproduce the failure. The same command
from a throwaway container on the same image, same user, same mount, exits 0 and
writes a complete venv, and the engine process has no PYTHONHOME, PYTHONPATH or
VIRTUAL_ENV in its environment. So this is not a confirmed fix for that specific
failure — it is the removal of the reason I cannot tell you what went wrong, plus
hardening against the one cause the error text points at. The retry will either
work or finally say why.

12 tests.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.54"
+__version__ = "0.2.55"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/longcat_worker.py

@@ -215,6 +215,41 @@ def _venv_ok(py: Path) -> bool:
         return False
 
 
+def _py310_env() -> dict:
+    """Environment for spawning the standalone Python 3.10.
+
+    Anything that points a Python at ANOTHER interpreter's files makes the child
+    die at startup with "No module named 'encodings'" — PYTHONHOME is the usual
+    one, but PYTHONPATH, VIRTUAL_ENV and friends all leak the 3.13 runtime into a
+    3.10 process. coderai does not set them today; it is stripped anyway because
+    the failure mode is fatal, silent in the parent, and costs nothing to prevent.
+    """
+    env = os.environ.copy()
+    for var in ("PYTHONHOME", "PYTHONPATH", "PYTHONSTARTUP", "PYTHONEXECUTABLE",
+                "VIRTUAL_ENV", "PYTHONNOUSERSITE", "PYTHONUSERBASE",
+                "__PYVENV_LAUNCHER__"):
+        env.pop(var, None)
+    return env
+
+
+def _run_py310(cmd: list, what: str) -> None:
+    """Run a step of the venv build, and RAISE WITH WHAT THE CHILD SAID.
+
+    subprocess.run(check=True) reports an exit status and nothing else, so the
+    first real failure here was a bare "returned non-zero exit status 1" while the
+    reason — a fatal interpreter error — only reached the log by accident.
+    """
+    proc = subprocess.run(cmd, env=_py310_env(), capture_output=True, text=True)
+    if proc.returncode == 0:
+        return
+    detail = ((proc.stderr or "") + ("\n" + proc.stdout if proc.stdout else "")).strip()
+    if len(detail) > 2000:
+        detail = detail[:1000] + "\n  …\n" + detail[-900:]
+    raise RuntimeError(
+        f"{what} failed (exit {proc.returncode}): {' '.join(str(c) for c in cmd)}\n"
+        f"{detail or '(the command produced no output)'}")
+
+
 def _find_python310(config: dict = None) -> Optional[str]:
     """A Python 3.10 interpreter to build the venv with.
 
@@ -261,14 +296,15 @@ def ensure_built(config: dict = None) -> Path:
     if not py.exists():
         print(f"[longcat] creating the venv at {venv} with {py310} …", flush=True)
         venv.parent.mkdir(parents=True, exist_ok=True)
-        subprocess.run([py310, "-m", "venv", str(venv)], check=True)
+        _run_py310([py310, "-m", "venv", str(venv)], "creating the LongCat venv")
         py = _venv_python(venv)
     print("[longcat] installing torch 2.6+cu124 and the rest (several GB, this takes "
           "a while) …", flush=True)
     # Always `python -m pip`, never the venv's pip script: a copied or re-pointed venv
     # carries a stale shebang and would install into the ORIGINAL interpreter.
-    subprocess.run([str(py), "-m", "pip", "install", "-U", "pip"], check=True)
-    subprocess.run([str(py), "-m", "pip", "install", "-r", str(_REQUIREMENTS)], check=True)
+    _run_py310([str(py), "-m", "pip", "install", "-U", "pip"], "upgrading pip")
+    _run_py310([str(py), "-m", "pip", "install", "-r", str(_REQUIREMENTS)],
+               "installing requirements-longcat.txt")
     if not _venv_ok(py):
         raise RuntimeError(
             f"the venv at {venv} was built but still does not report Python 3.10 with "
@@ -360,7 +396,10 @@ def ensure_service(model_path: str, config: dict = None,
             else "127.0.0.1"
         url = f"http://{'127.0.0.1' if host in ('0.0.0.0', '') else host}:{port}"
 
-        env = dict(os.environ)
+        # Same sanitising as the venv build: this launches the 3.10 interpreter,
+        # and a leaked PYTHONHOME/PYTHONPATH kills it before it prints anything
+        # useful.
+        env = _py310_env()
         try:
             from codai.models.cache import get_hf_hub_cache_dir
             hub = get_hf_hub_cache_dir()
@@ -556,11 +595,11 @@ def ensure_train_built(config: dict = None) -> Path:
     if not py.exists():
         print(f"[longcat] creating the training venv at {venv} …", flush=True)
         venv.parent.mkdir(parents=True, exist_ok=True)
-        subprocess.run([py310, "-m", "venv", str(venv)], check=True)
+        _run_py310([py310, "-m", "venv", str(venv)], "creating the LongCat training venv")
         py = _venv_python(venv)
-    subprocess.run([str(py), "-m", "pip", "install", "-U", "pip"], check=True)
-    subprocess.run([str(py), "-m", "pip", "install", "-r", str(_TRAIN_REQUIREMENTS)],
-                   check=True)
+    _run_py310([str(py), "-m", "pip", "install", "-U", "pip"], "upgrading pip")
+    _run_py310([str(py), "-m", "pip", "install", "-r", str(_TRAIN_REQUIREMENTS)],
+               "installing requirements-longcat-train.txt")
     if not _train_venv_ok(py):
         raise RuntimeError(f"the training venv at {venv} was built but `import "
                            f"simpletuner` still fails")
@@ -599,7 +638,8 @@ def train_lora(job: dict, workdir: str, on_progress=None) -> dict:
     cmd = [str(py), str(_TRAIN_SCRIPT), "--job", str(job_path)]
     print(f"[longcat] training: {' '.join(cmd)}", flush=True)
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                            text=True, bufsize=1, cwd=str(_REPO_ROOT))
+                            text=True, bufsize=1, cwd=str(_REPO_ROOT),
+                            env=_py310_env())
     threading.Thread(target=_pump_logs, args=(proc, collections.deque(maxlen=20)),
                      daemon=True).start()
 
@@ -650,7 +690,8 @@ def quantize_int8(checkpoint_dir: str, workdir: str, on_progress=None,
     cmd = [str(py), str(_QUANTIZE_SCRIPT), "--job", str(job_path)]
     print(f"[longcat] quantising: {' '.join(cmd)}", flush=True)
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                            text=True, bufsize=1, cwd=str(_REPO_ROOT))
+                            text=True, bufsize=1, cwd=str(_REPO_ROOT),
+                            env=_py310_env())
     threading.Thread(target=_pump_logs, args=(proc, collections.deque(maxlen=20)),
                      daemon=True).start()
 

tests/test_longcat_venv_env.py

+"""Spawning LongCat's Python 3.10 from coderai's 3.13.
+
+Pointing a Python at another interpreter's files kills it during startup:
+
+    Fatal Python error: init_fs_encoding: failed to get the Python codec of the
+    filesystem encoding
+    ModuleNotFoundError: No module named 'encodings'
+
+which is what the first venv build produced. The parent reported only
+"returned non-zero exit status 1" — the reason reached the log by accident, not
+by design. Both halves of that are fixed here: the environment is stripped of
+anything interpreter-specific, and a failed step raises with what the child said.
+"""
+import pathlib
+
+import pytest
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+lw = pytest.importorskip("codai.api.longcat_worker")
+
+
+@pytest.mark.parametrize("var", [
+    "PYTHONHOME", "PYTHONPATH", "PYTHONSTARTUP", "PYTHONEXECUTABLE",
+    "VIRTUAL_ENV", "PYTHONNOUSERSITE", "PYTHONUSERBASE", "__PYVENV_LAUNCHER__",
+])
+def test_interpreter_variables_are_stripped(monkeypatch, var):
+    monkeypatch.setenv(var, "/opt/coderai/python")
+    assert var not in lw._py310_env()
+
+
+def test_everything_else_is_kept(monkeypatch):
+    """Only the interpreter-specific ones go — HF cache paths, CUDA_VISIBLE_DEVICES
+    and the rest still have to reach the child."""
+    monkeypatch.setenv("HF_HOME", "/AI/hfcache/huggingface")
+    monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
+    env = lw._py310_env()
+    assert env["HF_HOME"] == "/AI/hfcache/huggingface"
+    assert env["CUDA_VISIBLE_DEVICES"] == "0"
+
+
+def test_a_failing_step_raises_with_the_childs_output():
+    """The whole point: an exit code alone is not a diagnosis."""
+    import sys
+    with pytest.raises(RuntimeError) as err:
+        lw._run_py310([sys.executable, "-c",
+                       "import sys; sys.stderr.write('THE REAL REASON\\n'); sys.exit(3)"],
+                      "a failing step")
+    msg = str(err.value)
+    assert "THE REAL REASON" in msg
+    assert "exit 3" in msg
+    assert "a failing step" in msg
+
+
+def test_a_successful_step_is_silent():
+    import sys
+    lw._run_py310([sys.executable, "-c", "pass"], "a fine step")
+
+
+def test_every_py310_spawn_uses_the_clean_environment():
+    src = (ROOT / "codai" / "api" / "longcat_worker.py").read_text(encoding="utf-8")
+    # no raw inheritance left on any subprocess that runs the 3.10 side
+    assert "env = dict(os.environ)" not in src
+    assert src.count("_py310_env()") >= 4
+    # and no step left reporting only an exit status. Matched as a CALL argument
+    # (", check=True)"), so the prose in _run_py310's docstring explaining the old
+    # behaviour does not trip it.
+    assert ", check=True)" not in src