vllm: size the share down too, from the cache a successful boot reports (v0.2.28)

Escalation alone only ratchets upward, so an opening bid that is too generous was
permanent: 0.60 of a 24 GB card for an engine that needs 0.25 is 8 GB no other model
could have. A successful boot states how much of the budget was left for the cache,
and `footprint = budget - cache` is the only direct measurement of an engine's real
cost there is — so it is now read back and used.

- _pump_logs captures "Available KV cache memory: N GiB" as it STREAMS. By the time the
  server answers its health check that line has scrolled out of the 80-line tail behind
  vLLM's route listing, so it cannot be recovered afterwards.
- On a boot with clear slack (> 1 GiB beyond the ~2 GiB target), a smaller share is
  computed from the measured footprint and recorded for the next start.
- The learned store is now a record per key, {"ok": booted, "failed": too small}, and
  the downward half may never propose a share at or below "failed" + a step. That floor
  is what stops it oscillating between a budget that works and one that does not.
- Over-provisioning now costs one boot instead of the life of the process, and the flat
  float format the previous version wrote still loads.

6 more tests (shrinking, the tight-fit no-op, the anti-oscillation floor, a missing
cache figure, the old format, and the footprint round-trip): 16 in the file, 255 across
the OCR/gate/cluster/remote-execution/placement files together.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0182DhR1eFNPQyGmDrpHbedY

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.2.27"
+__version__ = "0.2.28"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

docs/ocr.md

@@ -136,9 +136,15 @@ refuses to start when that much is not free. Three rules follow, all enforced by
   the larger figure, retries, and records what booted**, keyed by model + card + context +
   batch limits (`~/.coderai/vllm_gmu_learned.json`). A different model, a bigger card or
   different limits rediscover their own value; the recorded one is reused on later starts,
-  and a configured share lower than what is known to work is ignored. Nothing is added
-  back for other residents — vLLM charges only its own allocations against `total × gmu` —
-  so the evict-first pass is what guarantees the room.
+  and a configured share lower than what is known to work is ignored. It also sizes
+  **down**: a successful boot reports the cache it got, and `footprint = budget - cache`
+  is the only direct measurement of an engine's real cost there is, so a share with
+  clear slack is recorded smaller for the next start (keeping ~2 GiB of cache, and
+  never at or below a share already proven too small — that floor is what stops it
+  oscillating). Over-provisioning therefore costs one boot, not the life of the
+  process. Nothing is added back for other residents — vLLM charges only its own
+  allocations against `total × gmu` — so the evict-first pass is what guarantees the
+  room.
 
 - **The batch limits shrink the footprint a little; the encoder is what dominates.**
   `ocr.vlm_max_num_batched_tokens` (default 4096), `ocr.vlm_max_num_seqs` (16) and

tests/test_vllm_share_self_tuning.py

@@ -43,8 +43,11 @@ def _cfg(**kw):
     return type("C", (), base)()
 
 
-def _launcher(monkeypatch, succeed_at: float = None, error=KV_ERR):
-    """Stub the one-shot launcher: record every share tried, fail until `succeed_at`."""
+def _launcher(monkeypatch, succeed_at: float = None, error=KV_ERR, kv_gib=None):
+    """Stub the one-shot launcher: record every share tried, fail until `succeed_at`.
+
+    `kv_gib` is what the booted engine reports as its remaining cache — the figure the
+    downward half reads back."""
     tried = []
 
     def _once(cfg, model_path=None, served_name=None, ready_timeout=3600.0,
@@ -56,6 +59,8 @@ def _launcher(monkeypatch, succeed_at: float = None, error=KV_ERR):
         raise RuntimeError(error)
 
     monkeypatch.setattr(vllm_worker, "_ensure_service_once", _once)
+    monkeypatch.setattr(vllm_worker, "_service_meta",
+                        lambda *a, **k: ({"kv_gib": kv_gib} if kv_gib is not None else {}))
     return tried
 
 
@@ -161,3 +166,70 @@ def test_an_unwritable_store_costs_rediscovery_not_a_failure(monkeypatch):
     url = vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
                                      gpu_memory_utilization=0.60)
     assert url == "http://127.0.0.1:1/" and len(tried) >= 2
+
+
+# ---------------------------------------------------------------- shrinking
+def test_a_share_with_cache_to_spare_comes_down_next_start(monkeypatch):
+    """Escalation only ratchets up; without this an over-generous opening bid would have
+    OCR sitting on memory another model could have used."""
+    _launcher(monkeypatch, succeed_at=0.0, kv_gib=8.0)   # 0.60*24 - 8.0 => 6.4 footprint
+    vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
+                               gpu_memory_utilization=0.60)
+    key = vllm_worker._learned_gmu_key(_cfg(), "m", 18432, None)
+    learned = vllm_worker._load_learned(key)["ok"]
+    assert learned < 0.60
+    # Still enough for the 2 GB target plus the margin, not a starved budget.
+    assert learned * 24.0 >= 6.4 + vllm_worker._TARGET_KV_GB
+
+    tried = _launcher(monkeypatch, succeed_at=0.0, kv_gib=2.2)
+    vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
+                               gpu_memory_utilization=0.60)
+    assert tried == [pytest.approx(learned, abs=1e-3)]   # started at the smaller share
+
+
+def test_a_tight_fit_is_left_alone(monkeypatch):
+    """Only clear slack is worth a move — shaving a nearly-exact budget risks a dead one."""
+    _launcher(monkeypatch, succeed_at=0.0, kv_gib=vllm_worker._TARGET_KV_GB + 0.2)
+    vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
+                               gpu_memory_utilization=0.60)
+    key = vllm_worker._learned_gmu_key(_cfg(), "m", 18432, None)
+    assert vllm_worker._load_learned(key)["ok"] == pytest.approx(0.60, abs=1e-3)
+
+
+def test_shrinking_never_proposes_a_share_already_proven_dead(monkeypatch):
+    """The anti-oscillation rule: a budget that failed is a floor for the downward half."""
+    # Fails at 0.60, boots at 0.72, and reports a lot of spare cache.
+    _launcher(monkeypatch, succeed_at=0.72, kv_gib=9.0)
+    vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
+                               gpu_memory_utilization=0.60)
+    key = vllm_worker._learned_gmu_key(_cfg(), "m", 18432, None)
+    rec = vllm_worker._load_learned(key)
+    assert rec["failed"] == pytest.approx(0.60, abs=1e-3)
+    assert rec["ok"] > 0.60      # never back down to or below the share that failed
+
+
+def test_no_reported_cache_means_no_change(monkeypatch):
+    _launcher(monkeypatch, succeed_at=0.0, kv_gib=None)
+    vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
+                               gpu_memory_utilization=0.60)
+    key = vllm_worker._learned_gmu_key(_cfg(), "m", 18432, None)
+    assert vllm_worker._load_learned(key)["ok"] == pytest.approx(0.60, abs=1e-3)
+
+
+def test_the_flat_format_this_used_to_write_still_loads(monkeypatch, tmp_path):
+    import json
+    key = vllm_worker._learned_gmu_key(_cfg(), "m", 18432, None)
+    (tmp_path / "vllm_gmu_learned.json").write_text(json.dumps({key: 0.72}))
+    assert vllm_worker._load_learned(key) == {"ok": 0.72}
+    tried = _launcher(monkeypatch, succeed_at=0.72, kv_gib=2.1)
+    vllm_worker.ensure_service(_cfg(), model_path="m", served_name="m",
+                               gpu_memory_utilization=0.60)
+    assert tried == [pytest.approx(0.72, abs=1e-3)]
+
+
+def test_the_measured_footprint_round_trips(monkeypatch):
+    """footprint = budget - cache, which is the whole basis for sizing down."""
+    cand = vllm_worker._shrink_candidate(0.60, 8.0, 2.0, {})
+    assert cand is not None
+    # 0.60*24 = 14.4 budget, 8.0 cache => 6.4 footprint; +2.0 target => 8.4/24 = 0.35
+    assert cand == pytest.approx(0.35 + vllm_worker._SHRINK_MARGIN, abs=0.01)