longcat: a slow venv check is not a broken venv (v0.2.61)

Three clip generations failed while OCR and other work was running, with:

  RuntimeError: the venv at /AI/guffcache/longcat_venv was built but still does
  not report Python 3.10 with torch 2.6 / transformers 4.41

The venv reports exactly that. It holds torch 2.6.0+cu124, transformers 4.41.0,
numpy 1.26.4, on Python 3.10.18 — read from its own dist-info, and the probe
passes when run by hand.

The check imports torch from a 6.2 GB venv. Timed on an IDLE box: 106 seconds,
against a 120 second limit. Under the load that necessarily accompanies a
generation request it expired, TimeoutExpired was caught by a bare `except
Exception: return False`, and the caller turned "I could not finish asking" into
"the answer is no" — then rebuilt, re-probed, and failed again.

Both probes are now tri-state: True verified, False a real mismatch, None could
not be determined. An unverified venv is USED; only a verified-bad one is rebuilt
or rejected. The limit is 900s, and a genuine failure now prints what the probe
actually said instead of asserting versions nobody checked.

Nothing was wrong with the OCR/paddle side, and there were no VRAM, eviction,
swap or queue failures in the window — the 1626 apparent "503"s are those digits
occurring inside OCR'd Italian legal documents in the request debug dumps, and
the 1792 "ERROR"s are mostly the same plus vLLM warnings. The only real
exceptions in two hours were this one and the LongCat training chain.

32 tests.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.60"
+__version__ = "0.2.61"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/api/longcat_worker.py

@@ -208,12 +208,33 @@ def _venv_ok(py: Path) -> bool:
     probe = ("import sys,torch,transformers;"
              "assert sys.version_info[:2]==(3,10);"
              "assert torch.__version__.startswith('2.6');"
-             "assert transformers.__version__.startswith('4.41')")
+             "assert transformers.__version__.startswith('4.41');"
+             "print(sys.version_info[0],sys.version_info[1],"
+             "torch.__version__,transformers.__version__)")
     try:
-        return subprocess.run([str(py), "-c", probe],
-                              capture_output=True, timeout=120).returncode == 0
-    except Exception:
+        # Importing torch 2.6 from a 6 GB venv takes ~106s on an IDLE machine and
+        # longer while the GPU box is busy, which is exactly when a generation is
+        # requested. The old 120s limit therefore expired under load, the timeout
+        # was caught as a failure, and a perfectly good venv was declared broken.
+        done = subprocess.run([str(py), "-c", probe], capture_output=True,
+                              text=True, timeout=900, env=_clean_py_env())
+    except subprocess.TimeoutExpired:
+        # NOT the same as a failed check: we could not finish asking. Saying the
+        # venv is wrong here is a lie, and it is the lie that sent three
+        # generations into a rebuild loop.
+        print(f"[longcat] the venv check at {py} did not finish in 900s — the box "
+              f"is probably loaded; treating it as unverified, not broken",
+              flush=True)
+        return None
+    except Exception as exc:                                   # noqa: BLE001
+        print(f"[longcat] could not run the venv check: {exc}", flush=True)
         return False
+    if done.returncode == 0:
+        return True
+    detail = (done.stderr or done.stdout or "").strip().splitlines()
+    print(f"[longcat] the venv at {py.parent.parent} is not usable: "
+          f"{detail[-1] if detail else 'the probe failed with no output'}", flush=True)
+    return False
 
 
 def _clean_py_env() -> dict:
@@ -271,7 +292,11 @@ def ensure_built(config: dict = None) -> Path:
     """The interpreter the service runs under. Builds the venv only if allowed to."""
     venv = resolve_venv_dir(config)
     py = _venv_python(venv)
-    if py.exists() and _venv_ok(py):
+    # `is not False` on purpose: the check is tri-state. True is verified, False
+    # is a real mismatch, None means the probe could not finish (a loaded box —
+    # importing torch 2.6 takes ~106s idle). An unverified venv is used; only a
+    # verified-bad one is rebuilt.
+    if py.exists() and _venv_ok(py) is not False:
         return py
 
     sec = _cfg_section()
@@ -306,10 +331,11 @@ def ensure_built(config: dict = None) -> Path:
     _run_py310([str(py), "-m", "pip", "install", "-U", "pip"], "upgrading pip")
     _run_py310([str(py), "-m", "pip", "install", "-r", str(_REQUIREMENTS)],
                "installing requirements-longcat.txt")
-    if not _venv_ok(py):
+    if _venv_ok(py) is False:
         raise RuntimeError(
             f"the venv at {venv} was built but still does not report Python 3.10 with "
-            f"torch 2.6 / transformers 4.41")
+            f"torch 2.6 / transformers 4.41 — see the [longcat] line above for what "
+            f"it did report")
     return py
 
 
@@ -562,12 +588,27 @@ def resolve_train_venv(config: dict = None) -> Path:
     return inference.parent / (inference.name + "-train")
 
 
-def _train_venv_ok(py: Path) -> bool:
+def _train_venv_ok(py: Path):
+    """True / False / None, the same tri-state as _venv_ok and for the same reason:
+    importing simpletuner pulls its own torch, which is slow on a busy box, and a
+    timeout is not evidence that the venv is wrong."""
     try:
-        return subprocess.run([str(py), "-c", "import simpletuner"],
-                              capture_output=True, timeout=180).returncode == 0
-    except Exception:
+        done = subprocess.run([str(py), "-c", "import simpletuner"],
+                              capture_output=True, text=True, timeout=900,
+                              env=_clean_py_env())
+    except subprocess.TimeoutExpired:
+        print(f"[longcat] the training venv check at {py} did not finish in 900s — "
+              f"treating it as unverified, not broken", flush=True)
+        return None
+    except Exception as exc:                                   # noqa: BLE001
+        print(f"[longcat] could not run the training venv check: {exc}", flush=True)
         return False
+    if done.returncode == 0:
+        return True
+    detail = (done.stderr or "").strip().splitlines()
+    print(f"[longcat] `import simpletuner` fails: "
+          f"{detail[-1] if detail else 'no output'}", flush=True)
+    return False
 
 
 def _venv_python_version(py: Path) -> tuple:
@@ -621,7 +662,7 @@ def ensure_train_built(config: dict = None) -> Path:
     """The interpreter SimpleTuner runs under. Builds it only if allowed to."""
     venv = resolve_train_venv(config)
     py = _venv_python(venv)
-    if py.exists() and _train_venv_ok(py):
+    if py.exists() and _train_venv_ok(py) is not False:
         return py
     sec = _cfg_section()
     auto = bool(getattr(sec, "train_auto_build", False)) if sec is not None else False
@@ -663,9 +704,9 @@ def ensure_train_built(config: dict = None) -> Path:
     _run_py310([str(py), "-m", "pip", "install", "-U", "pip"], "upgrading pip")
     _run_py310([str(py), "-m", "pip", "install", "-r", str(_TRAIN_REQUIREMENTS)],
                "installing requirements-longcat-train.txt")
-    if not _train_venv_ok(py):
+    if _train_venv_ok(py) is False:
         raise RuntimeError(f"the training venv at {venv} was built but `import "
-                           f"simpletuner` still fails")
+                           f"simpletuner` still fails — see the [longcat] line above")
     return py
 
 

tests/test_longcat_venv_env.py

@@ -160,3 +160,59 @@ def test_the_builder_rebuilds_an_unusable_training_venv():
     body = body[:body.index("\ndef ")]
     assert "_venv_python_version(py)" in body
     assert "rmtree" in body, "an unusable venv must be replaced, not reused"
+
+
+# ── a slow check is not a broken venv ───────────────────────────────────────
+# Measured: importing torch 2.6 from the 6.2 GB inference venv takes ~106s on an
+# IDLE box, against a 120s limit. Under the load that accompanies an actual
+# generation request it expired, the TimeoutExpired was caught as a failure, and
+# three generations were told
+#   "the venv ... was built but still does not report Python 3.10 with torch 2.6"
+# about a venv holding exactly torch 2.6.0+cu124 and transformers 4.41.0.
+
+import subprocess as _sp
+
+
+def _probe(monkeypatch, outcome):
+    def fake(*a, **kw):
+        if outcome == "timeout":
+            raise _sp.TimeoutExpired(cmd="probe", timeout=900)
+        if outcome == "boom":
+            raise OSError("exec failed")
+        rc, err = (0, "") if outcome == "ok" else (1, "AssertionError\n")
+        return _sp.CompletedProcess(args=["probe"], returncode=rc, stdout="", stderr=err)
+    monkeypatch.setattr(lw.subprocess, "run", fake)
+
+
+@pytest.mark.parametrize("probe_fn", ["_venv_ok", "_train_venv_ok"])
+def test_a_timeout_is_unverified_not_broken(monkeypatch, probe_fn, tmp_path):
+    _probe(monkeypatch, "timeout")
+    assert getattr(lw, probe_fn)(tmp_path / "bin" / "python") is None
+
+
+@pytest.mark.parametrize("probe_fn", ["_venv_ok", "_train_venv_ok"])
+def test_a_real_mismatch_is_still_false(monkeypatch, probe_fn, tmp_path):
+    _probe(monkeypatch, "bad")
+    assert getattr(lw, probe_fn)(tmp_path / "bin" / "python") is False
+
+
+@pytest.mark.parametrize("probe_fn", ["_venv_ok", "_train_venv_ok"])
+def test_a_passing_probe_is_true(monkeypatch, probe_fn, tmp_path):
+    _probe(monkeypatch, "ok")
+    assert getattr(lw, probe_fn)(tmp_path / "bin" / "python") is True
+
+
+def test_the_timeout_is_long_enough_for_a_cold_torch_import():
+    """106s measured idle; 120 was the old limit."""
+    src = (ROOT / "codai" / "api" / "longcat_worker.py").read_text(encoding="utf-8")
+    assert "timeout=120" not in src, "the limit that caused this is back"
+    assert src.count("timeout=900") >= 2
+
+
+def test_unverified_venvs_are_used_rather_than_rebuilt():
+    """The tri-state only helps if the call sites respect it."""
+    src = (ROOT / "codai" / "api" / "longcat_worker.py").read_text(encoding="utf-8")
+    assert "_venv_ok(py) is not False" in src
+    assert "_train_venv_ok(py) is not False" in src
+    assert "if not _venv_ok(py)" not in src
+    assert "if not _train_venv_ok(py)" not in src