runpod: a fleet that grew for a burst has to give the pods back (v0.3.11)

Only half the scaler existed. The pool adds a pod once every pod it has carries
scale_up_inflight_per_pod requests, and pods went away again only through the
idle reaper — which fires on a pod with NO work and no request for
idle_timeout_s.

That misses the expensive case. A pool that grew for a burst and is now serving
a trickle keeps every pod: each is touched often enough that last_used never
ages out, each is a fraction busy, and the bill stays at the peak for as long as
the trickle lasts. With max_pods raised to 64 that is the difference between a
few dollars an hour and a few hundred.

So the pool also asks how many pods the CURRENT concurrency justifies and
retires the excess. The rules that matter are not "fewer pods":

  * work is never dropped to save money. The pod retired is the least loaded,
    and if it still has requests it is marked draining — no new work, and
    terminated only once it has finished what it had;
  * one pod per pass, after the surplus has persisted for scale_down_after_s,
    and the window restarts after each retirement. A dip of a few seconds must
    not cost a pod wanted again seconds later: a cold one takes minutes;
  * a quiet pool is NOT this routine's business. With nothing in flight,
    teardown stays entirely with idle_timeout_s — a number the operator chose,
    weighing the bill against a cold start — so this never silently shortens it.

A draining pod is also excluded from the growth decision, which reads the LEAST
loaded pod: an emptying drainer would otherwise answer "no need to grow" with
the one pod that cannot take the work.

Two existing tests had to change. A _Pod stub stood in for PodHandle without
the new field, and a source-text assertion used a 600-character window that a
new field plus its comment pushed data_center out of — now asserted against the
whole class, which is what it was always about.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PxYHWbCAjjjD7A1rhFKtaW

codai/__init__.py

@@ -16,7 +16,7 @@
 
 # Canonical product version for CoderAI — single source of truth. Both the API
 # metadata and the admin web UI read from here.
-__version__ = "0.3.10"
+__version__ = "0.3.11"
 __tagline__ = "Complete orchestration, distribution and escalation of remotizable advanced inference."
 
 # Configure the CUDA caching allocator BEFORE torch is imported anywhere.

codai/admin/routes.py

@@ -3154,7 +3154,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
                           if isinstance(sv, str) else bool(sv))
         # ints
         for k in ("container_disk_gb", "volume_gb", "port", "ctx", "min_pods",
-                  "max_pods", "scale_up_inflight_per_pod", "max_inflight_per_pod",
+                  "max_pods", "scale_up_inflight_per_pod", "scale_down_after_s",
+                  "max_inflight_per_pod",
                   "idle_timeout_s",
                   # how hard the POD's own server may be driven
                   "pod_max_parallel_requests", "pod_queue_max_size",

codai/admin/templates/models.html

@@ -1104,6 +1104,10 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
               <input type="number" id="cfg-rp-scale-up" class="form-input" min="1" placeholder="4">
               <span class="form-hint" style="font-size:11px">Add a pod once the least-loaded one carries this many requests.</span>
             </div>
+            <div class="form-row" style="margin:0"><label class="form-label">Scale down after (s)</label>
+              <input type="number" id="cfg-rp-scale-down" class="form-input" min="0" placeholder="180">
+              <span class="form-hint" style="font-size:11px">Give a pod back once the load no longer needs it and has not for this long. One per pass, least-loaded first, draining rather than dropping work. <b>0</b> = keep every pod until it goes fully idle (<code>idle_timeout_s</code>), which on a trickle of traffic means paying for the peak.</span>
+            </div>
             <div class="form-row" style="margin:0">
               <label class="form-label">Max in-flight per pod</label>
               <input type="number" id="cfg-rp-max-inflight" class="form-input" min="0" placeholder="0 = unlimited">
@@ -4927,6 +4931,7 @@ function _populateRunpod(s){
   set('cfg-rp-health-path', rp.health_path || '');
   set('cfg-rp-docker-args', rp.docker_args || '');
   set('cfg-rp-scale-up', rp.scale_up_inflight_per_pod != null ? rp.scale_up_inflight_per_pod : '');
+  set('cfg-rp-scale-down', rp.scale_down_after_s != null ? rp.scale_down_after_s : '');
   set('cfg-rp-max-inflight', rp.max_inflight_per_pod != null ? rp.max_inflight_per_pod : '');
   chk('cfg-rp-sticky', rp.sticky_sessions !== false);
   _updateRunpodVisibility();
@@ -5107,6 +5112,7 @@ function _collectRunpod(){
     health_path:        (val('cfg-rp-health-path') || '').trim(),
     docker_args:        (val('cfg-rp-docker-args') || '').trim(),
     scale_up_inflight_per_pod: intOrU('cfg-rp-scale-up'),
+    scale_down_after_s:        intOrU('cfg-rp-scale-down'),
     max_inflight_per_pod:      intOrU('cfg-rp-max-inflight'),
     sticky_sessions:    document.getElementById('cfg-rp-sticky').checked,
     schedule_enabled:   document.getElementById('cfg-rp-schedule-enabled').checked,

codai/api/runpod_worker.py

@@ -158,6 +158,16 @@ class RunpodModelConfig:
     # pool only ever grows when it has NO healthy pod, so fifty concurrent
     # requests would all pile onto pod #1 while max_pods sat unused.
     scale_up_inflight_per_pod: int = 4
+    # Scaling DOWN again when the pressure that justified the pods is gone.
+    # Idle teardown alone is not enough: it only fires on a pod with NO work and
+    # no request for idle_timeout_s, so a fleet that grew to absorb a burst and
+    # is now trickling keeps every pod alive — each one touched just often enough
+    # to look used, all of them a fraction busy, and the bill is the peak for as
+    # long as the trickle lasts. So the pool also asks how many pods the CURRENT
+    # concurrency needs and retires the excess, after holding the observation
+    # this long (a burst that dips for a moment must not cost a pod that is
+    # needed again seconds later, and a cold pod takes minutes to replace).
+    scale_down_after_s: int = 180
     # Hard ceiling on concurrent requests per pod (0 = no ceiling, let the remote
     # server's own batching absorb them). Above it, a request waits for a slot
     # instead of being piled on — use it for engines that degrade badly under
@@ -1639,6 +1649,9 @@ def parse_model_runpod(block: Optional[dict]) -> RunpodModelConfig:
                                              cfg.client_scale_ttl_s) or 1800)
     cfg.scale_up_inflight_per_pod = max(1, _as_int(b.get("scale_up_inflight_per_pod"),
                                                    cfg.scale_up_inflight_per_pod))
+    # 0 disables shedding entirely (pods then go only when idle_timeout_s says so).
+    cfg.scale_down_after_s = max(0, _as_int(b.get("scale_down_after_s"),
+                                            cfg.scale_down_after_s))
     cfg.max_inflight_per_pod = max(0, _as_int(b.get("max_inflight_per_pod"),
                                               cfg.max_inflight_per_pod))
     cfg.sticky_sessions = _as_bool(b.get("sticky_sessions"), cfg.sticky_sessions)
@@ -1837,6 +1850,10 @@ class PodHandle:
     healthy: bool = True
     inflight: int = 0
     last_used: float = 0.0
+    #: Chosen for retirement and no longer given new work. It stays healthy and
+    #: keeps serving what it already has: killing a pod with requests on it to
+    #: save money would be paying for the work twice.
+    draining: bool = False
     console_url: str = ""
     #: Where this pod actually is and what it is. Reported to monitoring: the
     #: region matters for latency and for where the data is allowed to live, and
@@ -2094,6 +2111,10 @@ class RunpodPodPool:
         self._client_floor_logged = -1
         self._provisioning = False
         self._closed = False
+        #: When the fleet first looked larger than the load needs. Scale-down
+        #: waits out scale_down_after_s from here, so a momentary dip in a burst
+        #: does not cost a pod that is wanted again seconds later.
+        self._surplus_since = 0.0
         # One log line per window transition, not one every scaler tick.
         self._sched_closed_logged = False
 
@@ -2569,7 +2590,8 @@ class RunpodPodPool:
         """Which pod serves this request: the one that already holds the
         conversation's cache, else the least loaded."""
         cap = int(getattr(self.mcfg, "max_inflight_per_pod", 0) or 0)
-        free = [p for p in healthy if cap <= 0 or p.inflight < cap]
+        free = [p for p in healthy
+                if not p.draining and (cap <= 0 or p.inflight < cap)]
         if not free:
             return None
         if affinity and getattr(self.mcfg, "sticky_sessions", True):
@@ -2595,10 +2617,15 @@ class RunpodPodPool:
         """
         if len(self.pods) >= self.mcfg.max_pods or self._provisioning:
             return False
-        if not healthy:
+        # A draining pod is on its way out and takes no new work, so counting it
+        # as capacity would answer "no need to grow" with the one pod that cannot
+        # help — and counting its emptying in-flight as the minimum would say the
+        # opposite. It is simply not part of the fleet any more.
+        serving = [p for p in healthy if not p.draining]
+        if not serving:
             return True
         threshold = max(1, int(self.mcfg.scale_up_inflight_per_pod or 1))
-        return min(p.inflight for p in healthy) >= threshold
+        return min(p.inflight for p in serving) >= threshold
 
     def _grow_in_background(self):
         """Provision another pod without making the triggering request wait for it.
@@ -2792,9 +2819,88 @@ class RunpodPodPool:
             self._client_floor_logged = leased
         return max(scheduled, min(leased, max(1, self.mcfg.max_pods)))
 
+    def _pods_needed(self, serving: list) -> int:
+        """How many pods the CURRENT concurrency justifies.
+
+        The same arithmetic as scaling up, read backwards: one pod per
+        ``scale_up_inflight_per_pod`` requests in flight. Never below the warm
+        floor, never above the ceiling, and never zero while anything is in
+        flight.
+        """
+        threshold = max(1, int(self.mcfg.scale_up_inflight_per_pod or 1))
+        total = sum(p.inflight for p in serving)
+        need = -(-total // threshold)          # ceil, in integers
+        return max(self.effective_min_pods(), min(int(self.mcfg.max_pods), need))
+
+    def _scale_down(self, now: float) -> None:
+        """Retire one surplus pod per pass, once the surplus has persisted.
+
+        One per pass, not all of them at once: each retirement changes the
+        arithmetic, and a fleet that shed everything on a single quiet moment
+        would then cold-start for minutes when the next burst arrived. The
+        surplus has to hold for ``scale_down_after_s`` first, for the same
+        reason.
+
+        The pod picked is the least loaded; if it still has work it is marked
+        draining and left to finish, so no request is ever thrown away to save a
+        few cents.
+        """
+        if self._closed or int(getattr(self.mcfg, "scale_down_after_s", 0) or 0) <= 0:
+            return
+        with self._cv:
+            serving = [p for p in self.pods if p.healthy and not p.draining]
+            draining = [p for p in self.pods if p.healthy and p.draining]
+            need = self._pods_needed(serving)
+            surplus = len(serving) - need
+            total = sum(p.inflight for p in serving)
+
+            # A pod that was draining and has now emptied is done: terminate it
+            # here rather than waiting for the idle reaper's timeout, which is
+            # what it was taken out of service to avoid paying.
+            for p in draining:
+                if p.inflight == 0:
+                    p.healthy = False
+                    self.pods.remove(p)
+                    threading.Thread(target=self._terminate, args=(p, "scaled down"),
+                                     daemon=True).start()
+
+            if surplus <= 0 or total == 0:
+                # Nothing in flight at all is NOT this routine's business. The
+                # idle reaper already walks a quiet pool down to the warm floor,
+                # on idle_timeout_s — a number the operator chose, weighing the
+                # bill against a cold start. Shedding here too would quietly
+                # replace 900s with scale_down_after_s and make that setting a
+                # lie. This exists for the other case: pods still serving, but
+                # more of them than the load needs.
+                self._surplus_since = 0.0
+                return
+            if not self._surplus_since:
+                self._surplus_since = now
+                return
+            if (now - self._surplus_since) < float(self.mcfg.scale_down_after_s):
+                return
+            # Hold the observation window again before the next one, so a pass
+            # does not walk the fleet down in one sweep.
+            self._surplus_since = now
+            victim = min(serving, key=lambda x: (x.inflight, -x.last_used))
+            if victim.inflight == 0:
+                victim.healthy = False
+                self.pods.remove(victim)
+                threading.Thread(target=self._terminate, args=(victim, "scaled down"),
+                                 daemon=True).start()
+                why = "terminated"
+            else:
+                victim.draining = True
+                why = f"draining ({victim.inflight} in flight)"
+        print(f"[runpod] {self.model_key!r}: {len(serving)} pod(s) for {total} "
+              f"request(s) in flight, {need} needed — {why} {victim.pod_id}",
+              flush=True)
+
     def maintain(self):
-        """One scaler pass: reap idle/dead pods to min_pods, keep min warm."""
+        """One scaler pass: shed pods the load no longer needs, reap idle/dead
+        ones to min_pods, keep min warm."""
         now = time.time()
+        self._scale_down(now)
         with self._cv:
             pods = list(self.pods)
         # Idle teardown (down to min_pods), only pods with no in-flight work.

docs/modelli-digesta-e-container.md

+# I modelli di Digesta e delle altre postazioni — cosa deve servire coderai
+
+> Documento per l'agente di **coderai**. Elenca tutti i modelli (LLM, embedding,
+> reranker, OCR) usati da Digesta e dai container fratelli, distinguendo:
+> **(A)** i modelli *nostri* che coderai deve ospitare, **(B)** i modelli esterni
+> (cloud / RunPod) che restano dove sono, **(C)** i modelli locali in-container.
+> Aggiornato al 09/10/2026. Stringhe e default verificati nel codice (file:riga).
+
+## Topologia dell'inferenza (per orientarsi)
+
+- **coderai (zeiss, 192.168.42.3:8000, 3090 condivisa)** = la *nostra* inferenza
+  dietro la *nostra* API OpenAI-compatibile. È qui che devono stare i modelli
+  della sezione **A**.
+- **coderai su prod (ru002848:8777)** = orchestratore **senza GPU** (CPU);
+  oggi `models.json` è vuoto. Non serve modelli di chat/embedding.
+- **Pod RunPod A40 on-demand** = oggi ospitano OCR-VLM, l'estrattore casi
+  (`qwen38-awq`/`diogene-alpha`) e il trio di estrazione. Accesi a richiesta.
+- **diogene-alpha** = il nostro modello condiviso (base + LoRA), servito da un pod
+  vLLM; il controllo accensione/spegnimento è **alpha-ctl** su
+  `127.0.0.1:8628` (NON è l'endpoint di inferenza, è il controllore).
+- **Cloud** = OpenAI / Anthropic / Mistral (sezione B), con chiavi nostre.
+
+---
+
+## A. Modelli che coderai deve servire (inferenza nostra)
+
+Nomi serviti (`--served-model-name`) che i client già si aspettano. Dove un
+modello oggi gira su un pod RunPod, l'indirizzo d'elezione è coderai.
+
+| nome servito | modello base / file | quant. | tipo | chi lo usa |
+|---|---|---|---|---|
+| `qwen38` | Qwen3.8-27B | AWQ | chat/RAG | Digesta compositore `coderai-qwen38`; verti/omnia `coderai-qwen38` |
+| `qwen38-awq` | `cyankiwi/Qwen3.8-27B-AWQ-INT4` | AWQ int4 | chat/estrazione (JSON) | Digesta: classificazione L2 (`modello`), estrattore casi, opzione residenze; `casi_pod.py` lo serve con `--served-model-name qwen38-awq` |
+| `diogene-alpha` | `cyankiwi/Qwen3.8-27B-AWQ-INT4` + adattatore QLoRA **`sib4`** | AWQ int4 + LoRA | chat/estrazione (il nostro «reader» addestrato) | verti/omnia (`diogene-alpha`, default del confronto admin); Digesta estrattore casi modo `alpha` |
+| `lisa-32k` | **Gemma-4-26B-A4B-it** QAT q4_0 «uncensored/heretic», file `/AI/guffcache/gemma-4-26B-A4B-it-qat-q4_0-uncensored-heretic-Q4_0.gguf` + mmproj BF16 (multimodale, image_to_text); `parser: gemma`, contesto 32768 (alias `lisa` = stesso file a 178k) | GGUF Q4_0 | chat/RAG (vision) | Digesta compositore `coderai-lisa`; verti/omnia `coderai-lisa` |
+| `Gemma-4-14B-A4B-Heretic.i1-IQ4_XS.gguf` | Gemma-4-14B-A4B «Heretic» | GGUF IQ4_XS | chat/RAG | Digesta compositore `coderai-gemma14-heretic`; verti/omnia idem |
+| `bge-m3` | `BAAI/bge-m3` | fp16 | **embedding** (dense 1024 + sparse) | Digesta: indicizzazione+query del corpus (vedi nota embeddings) |
+| `allenai/olmOCR-2-7B-1025-FP8` | olmOCR-2-7B | FP8 | OCR vision (ripiego) | Digesta: OCR di ripiego per le pagine lette male da Surya |
+| `surya` (servizio OCR) | Surya | — | OCR primario | Digesta: prima passata OCR (il codice la chiama «surya su coderai») |
+
+**Reranker** (oggi locali nel container Digesta; **passano a coderai-prod in CPU
+locale**, non su pod GPU; reranking **off** di default):
+
+| nome | modello | tipo | dove |
+|---|---|---|---|
+| profondo | `BAAI/bge-reranker-v2-m3` | cross-encoder | coderai-prod, CPU |
+| veloce | `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` | cross-encoder | coderai-prod, CPU |
+
+**Trio di estrazione (campagna alpha)** — il modello è passato dal chiamante a
+`estrazione.estrai(modello=…)`, non è un default di config. I tre usati:
+
+| modello | quant. | dove gira oggi |
+|---|---|---|
+| `gemma-4-31B` | AWQ 8-bit | pod A40 |
+| `llama-3.3-70B` | AWQ | pod A40 |
+| `Qwen3.8-27B` | AWQ int4 | pod A40 (= `qwen38-awq`) |
+
+> Nota embeddings (richiesta esplicita): gli **embedding vanno su coderai**. Il
+> modello reale è `BAAI/bge-m3`; il default nel codice di Digesta
+> (`EMBED_MODEL = "text-embedding-3-small"`, `app/config.py:315`) è solo un
+> segnaposto OpenAI. In pratica Digesta deve puntare `DIGESTA_LLM_BASE_URL` al
+> `/v1` di coderai (oggi vale `https://api.openai.com/v1`) e usare `bge-m3`.
+> Esiste anche un bge-m3 *locale* in-container (`FlagEmbedding.BGEM3FlagModel`,
+> `app/embed_locale.py`) usato solo come ripiego per la query quando la scheda
+> condivisa sfratta il modello remoto.
+
+---
+
+## B. Modelli cloud
+
+**Unico cloud rimasto: Anthropic `claude-opus-5`.**
+
+| provider | modello | dove / default | uso |
+|---|---|---|---|
+| Anthropic | `claude-opus-5` | `ANTHROPIC_CHAT_MODEL` `config.py:352`; `OCR_CLOUD_MODEL_ANTHROPIC` `config.py:104` | compositore `anthropic` (default chat di verti/omnia/sibilla); OCR cloud di ultima istanza; teacher dell'addestramento di `diogene-alpha` |
+
+**Dismessi il 09/10/2026 (nessun impiego):**
+- `gpt-5.6-luna` (OpenAI, `OPENAI_CHAT_MODEL`) — compositore `openai` /
+  verti-omnia `openai` ora morti.
+- `mistral-ocr-latest` (Mistral, `OCR_CLOUD_MODEL_MISTRAL`) — OCR cloud di
+  ultima istanza resta solo Anthropic (surya → olmOCR → opus).
+- `text-embedding-3-small` (OpenAI, `EMBED_MODEL` segnaposto) — sostituito da
+  `bge-m3`; mettere `DIGESTA_EMBED_MODEL=bge-m3`.
+
+Nota: `llama-3.3-70b` NON è più un endpoint RunPod *serverless* — è servito da
+coderai-prod (sezione A), che unifica il pod del trio e l'ex-serverless.
+
+---
+
+## C. Modelli locali in-container (NON coderai)
+
+- **Digesta — classificazione L2 CPU**: `tfidf1`, scikit-learn LogisticRegression
+  su TF-IDF, file `DATA_DIR/classificazione/cpu-tfidf1.joblib`
+  (`app/classificazione_l2.py`). Nessuna GPU.
+- **Digesta — bge-m3 ripiego**: query embedding locale su CPU (vedi nota).
+
+> I due **reranker** (`BAAI/bge-reranker-v2-m3`, `cross-encoder/mmarco-mMiniLMv2-
+> L12-H384-v1`), prima qui in-container, **passano a coderai-prod servita in CPU
+> locale** (coderai è una macchina CPU): sono nella sezione A.
+- **nexus**: modello **proprio** di triage = regressione logistica a pesi
+  (`app/coda.py`, pesi in DB `pesi_coda`); niente LLM/embedding, nessun endpoint.
+
+---
+
+## D. Chi usa cosa, per container
+
+- **Digesta** (il nucleo): tutto quanto sopra — OCR (surya + olmOCR + cloud di
+  ripiego), embeddings (`bge-m3`), reranker (bge), classificazione L2 (`tfidf1`
+  + `qwen38-awq`), estrazione campi (trio), estrattore casi
+  (`qwen38-awq`/`diogene-alpha`+`sib4`), compositore RAG (menu: `qwen38`,
+  `lisa-32k`, `Gemma-4-14B…gguf`, `openai`, `anthropic`, `serverless`), reader e
+  guardia-linguaggio (modello da env/admin, nessun default letterale).
+- **verti / omnia / groupama** (stesso codice): modello **scelto dall'admin**,
+  salvato nella tabella SQLite `impostazioni`. Catalogo: `anthropic`
+  (`claude-opus-5`), `openai` (`gpt-5.6-luna`), `coderai-qwen38` (`qwen38`),
+  `coderai-lisa` (`lisa-32k`), `coderai-gemma14-heretic`
+  (`Gemma-4-14B-A4B-Heretic.i1-IQ4_XS.gguf`), `runpod` (`llama-3.3-70b`),
+  `diogene-alpha`. Default chat = `anthropic`; confronto admin = `diogene-alpha`.
+  L'URL dei profili `coderai-*` è il campo `coderai_url` (l'admin lo punta al
+  `/v1` di coderai).
+- **sibilla-face**: **nessun modello proprio** — delega al nucleo
+  (`/api/cluster/ask`, `/api/cluster/compositori`); conserva solo la scelta di
+  default del menu (`"anthropic"`). I modelli e le chiavi stanno in Digesta.
+- **vitruvio-face**: **nessun passo LLM** ad oggi; usa solo Aristotele
+  (pseudonimizzazione, cripto) e le API di Digesta. In futuro lo stesso menu di
+  Sibilla.
+- **veridoc-face**: **nessun modello** — delega tutta l'analisi al nucleo.
+- **nexus**: solo il suo logreg locale (sezione C).
+
+### Container senza alcun modello
+`cautio` (fatturazione SdI), `caelum` (meteo Open-Meteo, «modello» = modello
+numerico meteo, non ML), `netlex`, `topos`, `ae-console`, `fasti`,
+`giustiziamap`, `ianua`, `aristotele` (cripto), `mcp-aggregator` (solo gateway;
+espone il controllore alpha-ctl su 8628, non un `/v1` di chat).
+
+---
+
+## In breve, per coderai
+
+Ospitati/orchestrati da coderai-prod (RunPod A40 on-demand): **`qwen38` /
+`qwen38-awq`**, **`diogene-alpha`** (base + LoRA `sib4`), **`lisa` / `lisa-32k`**,
+**`Gemma-4-14B-A4B-Heretic.i1-IQ4_XS.gguf`**, **`bge-m3`** (embedding),
+**`allenai/olmOCR-2-7B-1025-FP8`**, il servizio **`surya`**, e il trio
+(`gemma-4-31B`, **`llama-3.3-70b`** — che unifica pod-trio + ex-serverless).
+Piano in `piano-inferenza-su-coderai-prod.md`.
+
+I due **reranker** stanno anch'essi su coderai-prod ma in **CPU locale** (non pod
+GPU). L'unico **cloud** è **`claude-opus-5`** (chat + teacher). Dismessi il 09/10:
+`gpt-5.6-luna`, `mistral-ocr-latest`, `text-embedding-3-small` (sostituito da
+`bge-m3`).

docs/piano-inferenza-su-coderai-prod.md

+# Istruzioni per l'agente coderai — tutta l'inferenza sotto la coderai di produzione
+
+> **Decisione (09/10/2026).** Tutti i modelli che oggi girano **su RunPod** e
+> **sulla coderai di zeiss** vanno spostati a girare **orchestrati dalla coderai
+> di produzione** (ru002848). La coderai di prod diventa l'**unico piano di
+> controllo** e l'**unico endpoint OpenAI-compatibile** (`/v1`): non ha GPU,
+> quindi **orchestra pod RunPod A40 on-demand** (come fanno oggi i launcher di
+> Digesta `app/casi_pod.py` e `app/ocr_pod.py`, che vanno assorbiti qui).
+>
+> **Uniche eccezioni che NON si spostano:**
+> 1. i **reranker locali** di Digesta (restano in-container, vedi sotto);
+> 2. il modello **cloud Opus** (`claude-opus-5`) usato nelle chat (resta su Anthropic).
+>
+> I modelli cloud di terzi (OpenAI `gpt-5.6-luna`, Mistral `mistral-ocr-latest`,
+> OpenAI `text-embedding-3-small`) **non fanno parte di questo spostamento**: non
+> sono «su RunPod o coderai», non sono auto-ospitabili; cosa farne si decide a
+> parte (vedi §4).
+
+Chi avvia materialmente l'orchestrazione dei pod lo dice l'utente/gli altri
+agenti; **qui c'è solo la configurazione richiesta alla coderai di prod**.
+
+---
+
+## 1. Obiettivo
+
+- **Un solo `/v1`**: `ru002848` coderai (porta interna 8776, pubblicata
+  `127.0.0.1:8777`, davanti il proxy SSO di Digesta su `digesta.tech/coderai/`;
+  l'API `/v1` resta coi suoi bearer). Tutti i container chiamano **questo**.
+- **coderai = control plane**: accende/spegne pod **RunPod A40** a richiesta e
+  instrada le richieste; nessuna GPU locale a ru002848. Riusare la logica di
+  `casi_pod.py`/`ocr_pod.py` (served-model-name, regioni, `RUNPOD_API_KEY`,
+  ledger) dentro coderai.
+
+### Dove stanno i pesi
+**Tutti i file dei modelli vivono su uno storage RunPod** (network volume
+persistente): i GGUF (`lisa`/gemma), le basi HF/AWQ (Qwen, Llama, olmOCR), gli
+adattatori **QLoRA** (`sib4` e futuri), i modelli Surya. I pod A40 che coderai
+accende **montano quello storage** e caricano da lì — niente download da zeiss
+né ri-upload a ogni avvio. coderai deve sapere il path di ciascun modello/
+adattatore su quel volume (non più percorsi locali di zeiss tipo
+`/AI/guffcache/…`).
+
+## 2. Modelli che la coderai di prod deve servire (orchestrando RunPod)
+
+| served-model-name | motore | pesi / fonte | quant | GPU | ruolo | consumatori |
+|---|---|---|---|---|---|---|
+| `qwen38-awq` (+ alias `qwen38`) | vLLM | `cyankiwi/Qwen3.8-27B-AWQ-INT4` (HF) | AWQ int4 | A40 | classificazione L2 (`modello`), estrazione casi, chat | Digesta (`classificazione_l2`, `estrazione_casi`, compositore `coderai-qwen38`), verti/omnia `coderai-qwen38` |
+| `diogene-alpha` | vLLM + LoRA | base `cyankiwi/Qwen3.8-27B-AWQ-INT4` **+ adattatore QLoRA `sib4`** (sullo storage RunPod) | AWQ int4 + LoRA | A40 (può condividere la base con `qwen38-awq` via multi-LoRA) | il nostro **reader addestrato** per l'estrazione casi | Digesta (`estrazione_casi` modo `alpha`), verti/omnia `diogene-alpha`. **Assorbe anche il ruolo di alpha-ctl :8628** (accensione/stato del pod) |
+| `lisa` e `lisa-32k` | llama.cpp (GGUF), `parser: gemma`, **multimodale** (mmproj) | `gemma-4-26B-A4B-it-qat-q4_0-uncensored-heretic-Q4_0.gguf` + `…-mmproj-BF16.gguf` (sullo storage RunPod) | GGUF Q4_0 | A40 | chat/RAG con visione; `lisa`=178k ctx, `lisa-32k`=32k ctx | Digesta compositore `coderai-lisa`, verti/omnia `coderai-lisa` |
+| `Gemma-4-14B-A4B-Heretic.i1-IQ4_XS.gguf` | llama.cpp (GGUF) | file omonimo | GGUF IQ4_XS | A40 | chat/RAG più piccolo/veloce | Digesta compositore `coderai-gemma14-heretic`, verti/omnia idem |
+| `allenai/olmOCR-2-7B-1025-FP8` | vLLM | HF omonimo | FP8 | A40 | OCR vision di ripiego | Digesta OCR (ripiego). **Assorbe `ocr_pod.py`** |
+| `surya` (servizio OCR) | server Surya (`/v1/ocr`) | modelli Surya | — | A40 | OCR primario | Digesta OCR (prima passata) |
+| `gemma-4-31B` | vLLM | Gemma-4-31B | AWQ 8-bit | A40 | leg del **trio** di estrazione campi / generazione del secondo gold-set | Digesta `estrazione` (trio) |
+| `llama-3.3-70b` | vLLM | Llama-3.3-70B | AWQ | A40 | leg del **trio** + opzione chat | Digesta `estrazione` (trio) e compositore `serverless`; verti/omnia `runpod`. **Unifica il pod-trio E il serverless RunPod in un solo modello orchestrato** |
+| `bge-m3` | backend embedding (vLLM / FlagEmbedding) | `BAAI/bge-m3` | fp16 | GPU piccola o CPU | **embedding** (dense 1024 + sparse) del corpus e delle query | Digesta (indicizzazione + query) |
+| `BAAI/bge-reranker-v2-m3` | HF seq-classification | `BAAI/bge-reranker-v2-m3` (storage RunPod o pull HF) | — | **CPU, locale su ru002848** (non RunPod) | reranker profondo | Digesta (reranking, off di default) |
+| `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` | HF seq-classification | id omonimo | — | **CPU, locale su ru002848** | reranker veloce | Digesta (reranking, off di default) |
+
+> **Reranker su coderai in CPU.** Oggi girano locali su CPU dentro il container
+> Digesta; siccome coderai-prod è anch'essa una macchina CPU, si servono **lì**
+> in locale (CPU), senza accendere un pod RunPod. Così anche i reranker passano
+> per l'unico `/v1`.
+
+Note:
+- **Il trio NON si ritira** (richiesta esplicita): `gemma-4-31B`, `llama-3.3-70b`
+  e `qwen38-awq` restano disponibili, ma orchestrati da coderai. Il reader
+  addestrato `diogene-alpha` resta il singolo modello di punta per i campi già
+  coperti; il trio serve per i nuovi campi (secondo gold-set).
+- **`qwen38` / `Qwen3.8-27B` del trio** = lo stesso checkpoint `qwen38-awq`: un
+  solo modello servito, non tre istanze diverse.
+- **`llama-3.3-70b`**: oggi esiste in due posti (pod A40 del trio + endpoint
+  RunPod *serverless*); vanno unificati in **un** modello servito da coderai.
+
+## 3. NON spostare (unica eccezione)
+
+- **`claude-opus-5`** (Anthropic), resta cloud: è il modello di chat di
+  default e il teacher dell'addestramento di `diogene-alpha`.
+
+(I reranker, prima eccezione, **passano anch'essi a coderai** ma in CPU locale su
+ru002848 — vedi §2.)
+
+## 4. Cloud di terzi — DISMESSI (09/10/2026)
+
+**Non si usano più, nessun impiego.** L'unico modello cloud che resta è
+`claude-opus-5` (§3).
+- `gpt-5.6-luna` (OpenAI) — **via**: il compositore `openai` di Digesta e il
+  profilo `openai` di verti/omnia diventano morti.
+- `mistral-ocr-latest` (Mistral) — **via**: la catena OCR cloud di ultima
+  istanza resta solo su Anthropic (`claude-opus-5`): surya → olmOCR → opus.
+- `text-embedding-3-small` (OpenAI) — **via**: sostituito da `bge-m3`;
+  `DIGESTA_EMBED_MODEL` va messo a `bge-m3` (niente più segnaposto OpenAI).
+
+Al cutover di Digesta: togliere/lasciare non valorizzati `OPENAI_*` e
+`MISTRAL_API_KEY`, l'opzione `OCR_CLOUD=mistral`, la voce compositore `openai`;
+in verti/omnia il profilo `openai` sparisce dal catalogo utile.
+
+## 4b. Ritiro di alpha-ctl (dopo il cutover)
+
+**alpha-ctl** (il controllore della scheda `diogene-alpha` su `127.0.0.1:8628`:
+accensione/stato/spegnimento del pod) **va ritirato** una volta che coderai
+orchestra il pod `diogene-alpha`: quel ruolo (power/stato) lo fa coderai. È un
+passo **sequenziato** — si spegne alpha-ctl solo quando `diogene-alpha` è servito
+e verificato su coderai-prod. Pulizie collegate:
+- **verti/omnia**: l'impostazione `diogene_alpha_controllore` (default
+  `http://127.0.0.1:8628`) diventa inutile → rimuoverla; resta solo
+  `diogene_alpha_url` puntato al `/v1` di coderai.
+- **mcp-aggregator**: togliere il backend che espone alpha-ctl
+  (`registra_rest.py` upstream `http://127.0.0.1:8628`, doc `GATEWAY_REST.md`).
+- spegnere il servizio alpha-ctl su ru002848 e il relativo proxy
+  `127.0.0.1:8628/v1`.
+
+## 5. Ripuntamento dei consumatori (lo guida l'utente/gli altri agenti)
+
+Quando i modelli sono su coderai-prod, ripuntare:
+- **Digesta** → tutti al `/v1` di coderai-prod: `DIGESTA_LLM_BASE_URL`
+  (embedding `bge-m3` + base chat), `DIGESTA_OCR_BASE_URL` (surya),
+  `DIGESTA_OCR_VLM_BASE_URL` (olmOCR), `DIGESTA_CASI_ALPHA_URL` /
+  `DIGESTA_CASI_ESTRATTORE_*`, `SERVERLESS_BASE_URL`+`SERVERLESS_MODEL`
+  (`llama-3.3-70b`), `coderai_url` dei compositori; impostare
+  `DIGESTA_EMBED_MODEL=bge-m3`.
+- **verti / omnia / groupama** → impostazioni admin: `coderai_url`,
+  `diogene_alpha_url`, `runpod_url` tutti al `/v1` di coderai-prod.
+- **sibilla / vitruvio / veridoc**: nessuna modifica (delegano a Digesta).

docs/remote-execution.md

@@ -742,6 +742,36 @@ and the pool rents a second pod only once the first is carrying
 `scale_up_inflight_per_pod` requests. The same `pool` field works on a burst
 target, so two models' overflow shares one card.
 
+### Giving the pods back
+
+Growth has a matching contraction. The pool asks how many pods the **current**
+concurrency justifies — the same arithmetic as scaling up, read backwards, one
+pod per `scale_up_inflight_per_pod` requests in flight — and retires the excess
+once the surplus has held for `scale_down_after_s` (default 180; `0` switches
+shedding off).
+
+`idle_timeout_s` alone does not cover this. It fires on a pod with **no** work
+and no request for its whole timeout, so a fleet that grew for a burst and is now
+serving a trickle keeps every pod: each is touched often enough to look used,
+each is a fraction busy, and the bill stays at the peak until the trickle stops.
+With a high `max_pods` that is the difference between a few dollars an hour and a
+few hundred.
+
+Three rules make it safe to leave on:
+
+* **work is never dropped to save money** — the pod retired is the least loaded,
+  and if it still has requests it is marked draining, taking no new work and
+  terminating only once it has finished what it had;
+* **one pod per pass**, after the surplus has persisted, and the window restarts
+  after each retirement: a dip of a few seconds must not cost a pod that is
+  wanted again seconds later, because a cold one takes minutes to replace;
+* **a quiet pool is not this routine's business.** With nothing in flight,
+  teardown is left entirely to `idle_timeout_s` — a number you chose, weighing
+  the bill against a cold start — so shedding never silently shortens it.
+
+The warm floor (`min_pods`, or a client's `/v1/runpod/scale` lease) is a floor
+here too and is never scaled through.
+
 **This needs a multi-model pod image.** A vLLM pod is launched `--model X` and a
 llama.cpp pod `-hf one.gguf`: they serve exactly one model, so sharing is refused
 with an explicit error rather than sending requests to a pod that would answer

tests/test_burst_placement.py

@@ -124,12 +124,15 @@ def test_model_is_renamed_for_the_remote():
 # pod pool: real concurrency scaling, sticky cache, and the bearer token
 # --------------------------------------------------------------------------- #
 class _Pod:
-    def __init__(self, pid, inflight=0):
+    def __init__(self, pid, inflight=0, draining=False):
         self.pod_id = pid
         self.url = f"http://{pid}"
         self.healthy = True
         self.inflight = inflight
         self.last_used = 0.0
+        # A pod chosen for retirement takes no new work and is not counted as
+        # capacity (see test_runpod_scale_down).
+        self.draining = draining
 
 
 def _pool(**block):

tests/test_runpod_fleet_status.py

@@ -63,7 +63,8 @@ def test_each_pod_reports_where_it_is_and_what_it_serves():
                   '"gpu_count"', '"model"'):
         assert field in blk, field
     # and the placement has to be carried on the pod to be reportable
-    assert "data_center: str = \"\"" in src.split("class PodHandle")[1][:600]
+    _handle = src.split("class PodHandle")[1].split("\nclass ")[0].split("\n@")[0]
+    assert "data_center: str = \"\"" in _handle
     # The handle gets its region from the pool's own resolver. It used to read
     # a local `dc` that did not exist in this function, so the pod was created
     # and then the handle raised NameError — the A40 billed untracked.

tests/test_runpod_scale_down.py

+"""A fleet that grew for a burst has to give the pods back.
+
+Scaling up was the only half that existed. The pool adds a pod once every pod it
+has carries `scale_up_inflight_per_pod` requests, and pods went away again only
+through the idle reaper — which fires on a pod with NO work and no request for
+`idle_timeout_s`.
+
+That leaves the expensive case uncovered. A pool that grew to absorb a burst and
+is now serving a trickle keeps every pod: each one is touched often enough that
+`last_used` never ages out, each is a fraction busy, and the bill stays at the
+peak for as long as the trickle lasts. With `max_pods: 64` on an A40 that is the
+difference between a few dollars an hour and a few hundred.
+
+So the pool also asks how many pods the CURRENT concurrency justifies — the
+scale-up arithmetic read backwards — and retires the excess. The rules that
+matter are not "fewer pods": they are that work is never dropped to save money,
+that a momentary dip does not cost a pod needed seconds later, and that the warm
+floor is still a floor.
+"""
+
+import pathlib
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from codai.api import runpod_worker as rw                         # noqa: E402
+
+
+def _pool(max_pods=8, min_pods=0, threshold=4, after=180, pods=()):
+    """A PodPool with its scaling state and nothing else."""
+    pool = rw.RunpodPodPool.__new__(rw.RunpodPodPool)
+    import threading
+    pool._cv = threading.Condition()
+    pool.model_key = "datalab-to/surya-ocr-2"
+    pool._closed = False
+    pool._provisioning = False
+    pool._surplus_since = 0.0
+    pool.pods = list(pods)
+    cfg = rw.RunpodModelConfig()
+    cfg.max_pods = max_pods
+    cfg.min_pods = min_pods
+    cfg.scale_up_inflight_per_pod = threshold
+    cfg.scale_down_after_s = after
+    cfg.schedule_enabled = False
+    pool.mcfg = cfg
+    pool.effective_min_pods = lambda: min_pods
+    killed = []
+    pool._terminate = lambda pod, why: killed.append((pod.pod_id, why))
+    pool._killed = killed
+    return pool
+
+
+def _pod(pid, inflight=0, last_used=0.0, draining=False):
+    return rw.PodHandle(pod_id=pid, url=f"http://{pid}", hourly_usd=0.6,
+                        started_at=0.0, inflight=inflight, last_used=last_used,
+                        draining=draining)
+
+
+def _settle(pool, now=1000.0):
+    """Two passes: the first notices the surplus, the second acts on it once the
+    window has passed. That delay is the point, so the tests go through it."""
+    pool._scale_down(now)
+    pool._scale_down(now + pool.mcfg.scale_down_after_s + 1)
+
+
+# ------------------------------------------------------------ the arithmetic #
+def test_how_many_pods_the_load_needs():
+    """One pod per scale_up_inflight_per_pod requests — up-scaling, backwards."""
+    pool = _pool(threshold=4)
+    assert pool._pods_needed([_pod("a", 4), _pod("b", 4)]) == 2
+    assert pool._pods_needed([_pod("a", 1), _pod("b", 0)]) == 1
+    assert pool._pods_needed([_pod("a", 5), _pod("b", 0)]) == 2   # ceil, not floor
+
+
+def test_no_traffic_needs_no_pods_but_the_floor_still_holds():
+    assert _pool(min_pods=0).\
+        _pods_needed([_pod("a", 0)]) == 0
+    assert _pool(min_pods=2)._pods_needed([_pod("a", 0)]) == 2
+
+
+def test_it_never_asks_for_more_than_the_ceiling():
+    """The ceiling is the budget."""
+    assert _pool(max_pods=3, threshold=1)._pods_needed([_pod("a", 99)]) == 3
+
+
+# ------------------------------------------------------------- shedding pods #
+def test_a_fleet_larger_than_the_load_sheds_one():
+    pool = _pool(pods=[_pod("busy", 4), _pod("idle", 0), _pod("spare", 0)])
+    _settle(pool)
+    assert len(pool._killed) == 1
+    assert pool._killed[0][1] == "scaled down"
+    assert "busy" in [p.pod_id for p in pool.pods]
+
+
+def test_only_one_pod_goes_per_pass():
+    """Each retirement changes the arithmetic, and a fleet that shed everything
+    on one quiet moment would cold-start for minutes on the next burst."""
+    pool = _pool(pods=[_pod("a", 1), _pod("b", 0), _pod("c", 0), _pod("d", 0)])
+    _settle(pool)
+    assert len(pool._killed) == 1
+    assert len(pool.pods) == 3
+
+
+def test_the_least_loaded_pod_is_the_one_that_goes():
+    pool = _pool(threshold=4, pods=[_pod("heavy", 3), _pod("light", 1),
+                                    _pod("empty", 0)])
+    _settle(pool)
+    assert pool._killed == [("empty", "scaled down")]
+
+
+# --------------------------------------------- never drop work to save money #
+def test_a_pod_with_work_is_drained_not_killed():
+    """Killing a pod mid-request would make the caller pay for that work twice —
+    once on the pod that died and once on the retry."""
+    pool = _pool(threshold=10, pods=[_pod("a", 1), _pod("b", 2)])
+    _settle(pool)
+    assert pool._killed == []
+    assert sum(1 for p in pool.pods if p.draining) == 1
+    assert [p.pod_id for p in pool.pods if p.draining] == ["a"]
+
+
+def test_a_drained_pod_is_terminated_once_it_empties():
+    """And straight away, rather than waiting out idle_timeout_s — which is the
+    cost it was taken out of service to stop paying."""
+    draining = _pod("old", 2, draining=True)
+    pool = _pool(threshold=10, pods=[_pod("a", 1), draining])
+    draining.inflight = 0
+    pool._scale_down(1000.0)
+    assert ("old", "scaled down") in pool._killed
+    assert "old" not in [p.pod_id for p in pool.pods]
+
+
+def test_a_draining_pod_is_given_no_new_work():
+    pool = _pool(pods=[_pod("drain", 0, draining=True), _pod("live", 3)])
+    chosen = pool._pick([p for p in pool.pods if p.healthy], "")
+    assert chosen.pod_id == "live"
+
+
+def test_a_draining_pod_is_not_mistaken_for_spare_capacity():
+    """_should_grow reads the LEAST loaded pod. An empty drainer would answer
+    "no need to grow" with the one pod that cannot take the work."""
+    pool = _pool(max_pods=4, threshold=4,
+                 pods=[_pod("drain", 0, draining=True), _pod("live", 4)])
+    assert pool._should_grow([p for p in pool.pods if p.healthy]) is True
+
+
+def test_everything_draining_still_counts_as_needing_a_pod():
+    pool = _pool(max_pods=4, pods=[_pod("drain", 1, draining=True)])
+    assert pool._should_grow([p for p in pool.pods if p.healthy]) is True
+
+
+# ----------------------------------------------------- not on a momentary dip #
+def test_a_surplus_must_persist_before_it_costs_a_pod():
+    """A burst that dips for a second must not cost a pod that is wanted again a
+    second later: a cold one takes minutes to replace."""
+    pool = _pool(after=180, pods=[_pod("a", 1), _pod("b", 0)])
+    pool._scale_down(1000.0)                     # first sighting
+    assert pool._killed == []
+    pool._scale_down(1000.0 + 179)               # still inside the window
+    assert pool._killed == []
+    pool._scale_down(1000.0 + 181)
+    assert len(pool._killed) == 1
+
+
+def test_pressure_returning_cancels_the_countdown():
+    pool = _pool(threshold=4, after=180, pods=[_pod("a", 0), _pod("b", 0)])
+    pool._scale_down(1000.0)
+    pool.pods[0].inflight = 4                    # the burst is back
+    pool.pods[1].inflight = 4
+    pool._scale_down(1100.0)
+    assert pool._surplus_since == 0.0
+    pool._scale_down(2000.0)                     # and the clock starts over
+    assert pool._killed == []
+
+
+def test_the_window_restarts_after_each_retirement():
+    """Otherwise one long quiet spell would walk the whole fleet down in a single
+    pass, which is the same mistake as shedding everything at once."""
+    pool = _pool(pods=[_pod("a", 1), _pod("b", 0), _pod("c", 0)])
+    _settle(pool)
+    assert len(pool._killed) == 1
+    pool._scale_down(1000.0 + pool.mcfg.scale_down_after_s + 2)
+    assert len(pool._killed) == 1                # not yet: the window restarted
+
+
+# ------------------------------------------------------------- the floor holds #
+def test_the_warm_floor_is_never_scaled_through():
+    """min_pods means "keep this many ready", and an idle pool is exactly when
+    that promise matters."""
+    pool = _pool(min_pods=2, pods=[_pod("a", 0), _pod("b", 0)])
+    _settle(pool)
+    assert pool._killed == []
+    assert len(pool.pods) == 2
+
+
+def test_zero_disables_shedding_entirely():
+    """An install that would rather hold its pods can say so, and then only the
+    idle reaper takes them."""
+    pool = _pool(after=0, pods=[_pod("a", 1), _pod("b", 0)])
+    _settle(pool)
+    assert pool._killed == []
+
+
+def test_a_closed_pool_sheds_nothing():
+    pool = _pool(pods=[_pod("a", 1), _pod("b", 0)])
+    pool._closed = True
+    _settle(pool)
+    assert pool._killed == []
+
+
+def test_a_pool_with_nothing_in_flight_is_left_to_the_idle_reaper():
+    """The important boundary. The idle reaper walks a quiet pool down to the
+    warm floor on idle_timeout_s — a number the operator chose, weighing the bill
+    against a cold start. Shedding here as well would quietly replace 900s with
+    scale_down_after_s and make that setting a lie."""
+    pool = _pool(min_pods=0, pods=[_pod("a", 0), _pod("b", 0), _pod("c", 0)])
+    _settle(pool)
+    assert pool._killed == []
+    assert len(pool.pods) == 3
+
+
+def test_one_request_is_enough_for_shedding_to_apply():
+    """The pool is serving, just not with this many pods — which is exactly the
+    case the idle reaper cannot see, because every pod looks recently used."""
+    pool = _pool(threshold=4, pods=[_pod("a", 1), _pod("b", 0)])
+    _settle(pool)
+    assert len(pool._killed) == 1