Skip to content

C
coderai
Commit 1ad76dfb authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

front: configurable compaction model + live progress to the client 

Auto-compaction can now summarize with a DIFFERENT model than the one
serving the request, with a global default (config.json `compaction`) and
a per-model override (models.json `auto_compact_model`). Empty = the
request's own model, as before.

- config: new CompactionConfig (enabled/pct/strategy/model) + round-trip
- text.py: resolve effective settings (per-model over global), resolve the
  summarizer LAZILY (only when actually over threshold, so a separate model
  isn't loaded on every request); map-reduce the dropped history into chunks
  sized to the CHOSEN summarizer's own context, reducing iteratively until
  it fits; stream status + live per-chunk progress to the client as content
  deltas (queue-bridged from the summarizer's callback)
- admin: global compaction card (settings) + per-model summarizer dropdown
  (models, shown only for the summarize strategy)

Raw two-pass path is skipped (prompt is built from system + last user turn).
Co-Authored-By: 's avatarClaude Opus 4.8 <noreply@anthropic.com>
parent 2b2bdc72
Show whitespace changes
Inline Side-by-side
Showing with 377 additions and 58 deletions
codai/admin/routes.py
View file @ 1ad76dfb
......@@ -2682,7 +2682,8 @@ async def api_model_configure(request: Request, username: str = Depends(require_
"balanced_gpu_percent", "acceleration",
"cache_type_k", "cache_type_v", "turboquant", "engine", "engine_fallback",
"quant_backend", "kv_cache_budget_mb", "kv_cache_slots", "mmproj",
"auto_compact", "auto_compact_pct", "auto_compact_strategy"):
"auto_compact", "auto_compact_pct", "auto_compact_strategy",
"auto_compact_model"):
if key in data:
entry[key] = data[key]
......@@ -3432,6 +3433,12 @@ async def api_get_settings(username: str = Depends(require_admin)):
"extra_env": c.ds4.extra_env,
"auto_build": c.ds4.auto_build,
},
"compaction": {
"enabled": c.compaction.enabled,
"pct": c.compaction.pct,
"strategy": c.compaction.strategy,
"model": c.compaction.model,
},
"broker": {
"enabled": c.broker.enabled,
"base_url": c.broker.base_url,
......@@ -3720,6 +3727,22 @@ async def api_save_settings(request: Request, username: str = Depends(require_ad
if "auto_build" in d:
c.ds4.auto_build = bool(d["auto_build"])
if "compaction" in data:
cp = data["compaction"] or {}
if "enabled" in cp:
c.compaction.enabled = bool(cp["enabled"])
if "pct" in cp:
try:
c.compaction.pct = max(50, min(99, int(cp["pct"])))
except (TypeError, ValueError):
pass
if "strategy" in cp:
_st = (cp.get("strategy") or "drop_oldest").strip()
if _st in ("drop_oldest", "keep_head_tail", "summarize"):
c.compaction.strategy = _st
if "model" in cp:
c.compaction.model = (cp.get("model") or "").strip()
if "broker" in data:
bro = data["broker"]
c.broker.enabled = bool(bro.get("enabled", c.broker.enabled))
......
codai/admin/templates/models.html
View file @ 1ad76dfb
......@@ -707,12 +707,19 @@ window.__DEFAULT_WHISPER_SERVER_PATH__ = {{ default_whisper_server_path|tojson }
</div>
<div class="form-row" style="margin:0">
<label class="form-label">Strategy</label>
<select id="cfg-autocompact-strategy" class="form-input">
<select id="cfg-autocompact-strategy" class="form-input" onchange="_toggleAutoCompact()">
<option value="drop_oldest">Drop oldest — keep system + most recent turns</option>
<option value="keep_head_tail">Keep head + tail — also keep the first turn, drop the middle</option>
<option value="summarize">Summarize — replace the dropped middle with an LLM summary</option>
</select>
</div>
<div class="form-row" id="cfg-autocompact-model-row" style="margin:0;grid-column:1/-1;display:none">
<label class="form-label">Summarizer model <span class="muted">(summarize only)</span></label>
<select id="cfg-autocompact-model" class="form-input">
<option value="">Same as request model (default)</option>
</select>
<span class="form-hint" style="font-size:11px">A separate (e.g. smaller/faster) model can write the summary while the main model answers. The dropped history is chunked to fit the chosen model's own context. Empty = the request's own model summarizes. Leave empty to inherit the global default.</span>
</div>
</div>
<div style="display:flex;gap:1.5rem;flex-wrap:wrap;margin-top:.75rem">
......@@ -2812,6 +2819,36 @@ async function freeDiskConfirm(idx){
function _toggleAutoCompact(){
const on = document.getElementById('cfg-autocompact').checked;
document.getElementById('cfg-autocompact-opts').style.display = on ? 'grid' : 'none';
const strat = document.getElementById('cfg-autocompact-strategy').value;
const mrow = document.getElementById('cfg-autocompact-model-row');
if(mrow) mrow.style.display = (on && strat === 'summarize') ? 'block' : 'none';
}
// Populate the summarizer-model dropdown from the configured text/LLM models,
// preserving the current selection.
function _populateCompactModelSelect(selected){
const sel = document.getElementById('cfg-autocompact-model');
if(!sel) return;
const cur = selected != null ? selected : sel.value;
const seen = new Set();
let html = '<option value="">Same as request model (default)</option>';
(_localModels||[]).forEach(m=>{
const caps = m.capabilities||[];
const type = m.defaultType||'text_models';
const isText = type==='text_models' || type==='gguf_models' || type==='vision_models'
|| caps.includes('chat') || caps.includes('text') || caps.includes('tools');
if(!isText) return;
const id = m.path || m.label;
if(!id || seen.has(id)) return;
seen.add(id);
html += `<option value="${esc(id)}">${esc(m.label||id)}</option>`;
});
// Keep a previously-saved value even if its model isn't in the local list.
if(cur && !seen.has(cur)){
html += `<option value="${esc(cur)}">${esc(cur)}</option>`;
}
sel.innerHTML = html;
sel.value = cur || '';
}
/* ── type checkbox helpers ─────────────────────────────── */
......@@ -3197,6 +3234,7 @@ function openCfgModal(idx, cfgIdx){
document.getElementById('cfg-autocompact').checked = !!s.auto_compact;
document.getElementById('cfg-autocompact-pct').value = s.auto_compact_pct != null ? s.auto_compact_pct : 85;
document.getElementById('cfg-autocompact-strategy').value = s.auto_compact_strategy || 'drop_oldest';
_populateCompactModelSelect(s.auto_compact_model || '');
_toggleAutoCompact();
document.getElementById('cfg-4bit').checked = !!s.load_in_4bit;
document.getElementById('cfg-8bit').checked = !!s.load_in_8bit;
......@@ -3586,6 +3624,7 @@ async function saveModelConfig(){
auto_compact: document.getElementById('cfg-autocompact').checked,
auto_compact_pct: parseInt(document.getElementById('cfg-autocompact-pct').value) || 85,
auto_compact_strategy: document.getElementById('cfg-autocompact-strategy').value || 'drop_oldest',
auto_compact_model: document.getElementById('cfg-autocompact-model').value || '',
max_gpu_percent: isNaN(maxGpu) ? null : maxGpu,
manual_ram_gb: isNaN(ramGb) ? null : ramGb,
load_in_4bit: document.getElementById('cfg-4bit').checked,
......
codai/admin/templates/settings.html
View file @ 1ad76dfb
......@@ -493,6 +493,40 @@
</div>
</div>
</div>
<!-- Auto-compact context -->
<div class="card">
<div class="card-title">Auto-compact context</div>
<p class="form-hint" style="margin-bottom:.6rem">Global <b>defaults</b> for shrinking an over-long chat history before generation so requests don't error out on context overflow. Per-model settings on the <b>Models</b> page override these. OFF by default.</p>
<div class="form-row">
<label style="display:flex;align-items:center;gap:.5rem;cursor:pointer">
<input type="checkbox" id="s-compact-enabled" onchange="toggleCompactFields()">
<span style="font-size:13px;font-weight:500">Enable auto-compaction by default</span>
</label>
</div>
<div id="compact-fields" style="display:none">
<div style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem">
<div class="form-row">
<label class="form-label">Trigger at <span class="muted">(% of context)</span></label>
<input type="number" id="s-compact-pct" class="form-input" min="50" max="99" step="1" placeholder="85">
<span class="form-hint">Compacts to ~65% of the context window when the prompt reaches this %.</span>
</div>
<div class="form-row">
<label class="form-label">Strategy</label>
<select id="s-compact-strategy" class="form-input">
<option value="drop_oldest">Drop oldest — keep system + most recent turns</option>
<option value="keep_head_tail">Keep head + tail — also keep the first turn, drop the middle</option>
<option value="summarize">Summarize — replace the dropped middle with an LLM summary</option>
</select>
</div>
</div>
<div class="form-row">
<label class="form-label">Summarizer model <span class="muted">(summarize strategy)</span></label>
<input type="text" id="s-compact-model" class="form-input" placeholder="(empty = the model serving the request)">
<span class="form-hint">Model id/alias that writes the summary. Empty = the request's own model summarizes itself. Point it at a smaller/faster model to summarize old turns while the main model answers — the dropped history is chunked to fit the chosen model's own context window. The client receives a status message and live progress while compaction runs.</span>
</div>
</div>
</div>
{% endblock %}
{% block scripts %}
......@@ -561,6 +595,10 @@ function toggleDs4Fields(){
document.getElementById('ds4-fields').style.display =
document.getElementById('s-ds4-enabled').checked ? 'block' : 'none';
}
function toggleCompactFields(){
document.getElementById('compact-fields').style.display =
document.getElementById('s-compact-enabled').checked ? 'block' : 'none';
}
function toggleHttps(){
document.getElementById('https-fields').style.display =
document.getElementById('s-https').checked ? 'block' : 'none';
......@@ -675,6 +713,13 @@ async function loadSettings(){
document.getElementById('s-broker-reconnect-max').value = broker.reconnect_max_delay_seconds ?? 60;
document.getElementById('s-broker-ws-ping').value = broker.websocket_ping_interval ?? 20;
toggleBrokerFields();
// Auto-compact context
const compact = d.compaction || {};
document.getElementById('s-compact-enabled').checked = !!compact.enabled;
document.getElementById('s-compact-pct').value = compact.pct ?? 85;
document.getElementById('s-compact-strategy').value = compact.strategy || 'drop_oldest';
document.getElementById('s-compact-model').value = compact.model ?? '';
toggleCompactFields();
// Thermal protection
const therm = d.thermal || {};
document.getElementById('s-therm-gpu-enabled').checked = therm.gpu_enabled !== false;
......@@ -788,6 +833,12 @@ async function saveSettings(){
expert_cache_reserve_gb: parseInt(document.getElementById('s-ds4-expert-cache-reserve').value) || 0,
extra_env: document.getElementById('s-ds4-extra-env').value.trim(),
},
compaction:{
enabled: document.getElementById('s-compact-enabled').checked,
pct: parseInt(document.getElementById('s-compact-pct').value) || 85,
strategy: document.getElementById('s-compact-strategy').value || 'drop_oldest',
model: document.getElementById('s-compact-model').value.trim(),
},
broker:{
enabled: document.getElementById('s-broker-enabled').checked,
base_url: document.getElementById('s-broker-base-url').value.trim(),
......
codai/api/text.py
View file @ 1ad76dfb
......@@ -407,12 +407,29 @@ async def _summarize_one(manager, text: str, max_tokens: int = 400):
return (out or "").strip()
def _summary_chunk_chars(compact_n_ctx: int) -> int:
"""Per-chunk char budget for the summarizer so each summarization prompt fits
the SUMMARIZING model's own context. Leaves headroom for the summary system
prompt (~120 tok) and the generated summary (~500 tok); ~4 chars/token with a
0.75 safety factor."""
usable = max(int(compact_n_ctx or 0) - 700, 512)
return max(2000, int(usable * 4 * 0.75))
async def _summarize_for_compact(manager, messages, keep_recent: int = 2,
chunk_chars: int = 8000):
"""Best-effort map-reduce summary of the older turns using the loaded model:
CHUNK the history, summarize each chunk, then summarize the combined chunk
summaries. Returns a summary string or None (caller falls back to a count
note). Chunking keeps the summarization prompt itself from overflowing."""
compact_n_ctx: int = 8192, progress=None):
"""Best-effort map-reduce summary of the older turns using ``manager`` (which
may be a DIFFERENT model than the one serving the request): CHUNK the history
to fit ``compact_n_ctx``, summarize each chunk, then iteratively reduce the
combined chunk summaries until they fit one chunk. ``progress`` is an optional
async callable(str) used to stream status to the client. Returns a summary
string or None (caller falls back to a count note)."""
async def _emit(msg):
if progress:
try:
await progress(msg)
except Exception:
pass
try:
body = [m for m in messages if m.get("role") != "system"]
older = body[:-keep_recent] if len(body) > keep_recent else body
......@@ -425,10 +442,16 @@ async def _summarize_for_compact(manager, messages, keep_recent: int = 2,
c = " ".join(it.get("text", "") for it in c if isinstance(it, dict))
lines.append(f"{m.get('role', '?')}: {str(c)}")
text = "\n".join(lines)
chunk_chars = _summary_chunk_chars(compact_n_ctx)
chunks = [text[i:i + chunk_chars] for i in range(0, len(text), chunk_chars)] or [text]
# Map: summarize each chunk (cap the number of chunks so this stays bounded).
# Map → Reduce, looping the reduce until the combined summaries fit one chunk.
level = 0
while True:
total = len(chunks)
await _emit(f"summarizing {total} chunk(s) of earlier context…")
summaries = []
for ch in chunks[:12]:
for i, ch in enumerate(chunks):
await _emit(f"summarizing chunk {i + 1}/{total}…")
s = await _summarize_one(manager, ch)
if s:
summaries.append(s)
......@@ -436,15 +459,150 @@ async def _summarize_for_compact(manager, messages, keep_recent: int = 2,
return None
if len(summaries) == 1:
return summaries[0]
# Reduce: summarize the combined chunk summaries.
combined = "\n".join(summaries)
final = await _summarize_one(manager, combined[:chunk_chars * 2], max_tokens=500)
if len(combined) <= chunk_chars or level >= 3:
await _emit("combining chunk summaries…")
final = await _summarize_one(manager, combined[:chunk_chars], max_tokens=500)
return final or combined
# Still too big — reduce another level.
chunks = [combined[i:i + chunk_chars] for i in range(0, len(combined), chunk_chars)]
level += 1
except Exception as e:
print(f"[auto-compact] summary generation failed: {e}", flush=True)
return None
def _resolve_compaction(request, current_manager):
"""Resolve effective auto-compaction settings for a request by merging the
per-model config over the global ``compaction`` defaults. Returns a plan dict
or None when compaction is disabled. The over-threshold decision is made later
against the live token estimate (see ``_auto_compact_events``)."""
try:
from codai.models.manager import multi_model_manager as _mmm
_cc = _mmm._config_for_model(getattr(request, "model", None) or "") or {}
except Exception:
_mmm = None
_cc = {}
_g = None
try:
from codai.admin.routes import config_manager as _cm
if _cm is not None and getattr(_cm, "config", None) is not None:
_g = _cm.config.compaction
except Exception:
_g = None
def _gv(attr, default):
return getattr(_g, attr, default) if _g is not None else default
enabled = _cc.get("auto_compact", _gv("enabled", False))
if not enabled:
return None
pct = _cc.get("auto_compact_pct", _gv("pct", 85)) or 85
strategy = (_cc.get("auto_compact_strategy") or _gv("strategy", "drop_oldest") or "drop_oldest").strip()
compact_model = (_cc.get("auto_compact_model") or _gv("model", "") or "").strip()
try:
n_ctx = current_manager.get_context_size() if current_manager else 0
except Exception:
n_ctx = 0
# NOTE: the summarizer model (``compact_model``) is resolved LAZILY in
# _auto_compact_events, only when the prompt is actually over threshold — so a
# configured separate model isn't loaded on every (under-threshold) request.
return {
"pct": float(pct), "strategy": strategy, "n_ctx": n_ctx,
"compact_model": compact_model, "current_manager": current_manager,
}
def _resolve_compact_manager(plan):
"""Lazily pick the manager that performs summarization for ``plan`` and its
context size. Returns (manager, name, compact_n_ctx). Falls back to the
request's own model when no separate model is configured or it can't load."""
current_manager = plan.get("current_manager")
compact_manager = current_manager
try:
compact_name = getattr(current_manager, "model_name", None) or "the model"
except Exception:
compact_name = "the model"
compact_model = plan.get("compact_model")
if compact_model:
try:
from codai.models.manager import multi_model_manager as _mmm
_cand = _mmm.get_model_for_request(compact_model)
if _cand is not None and getattr(_cand, "backend", None) is not None:
compact_manager = _cand
compact_name = compact_model
except Exception:
pass
try:
compact_n_ctx = compact_manager.get_context_size() if compact_manager else 0
except Exception:
compact_n_ctx = 0
return compact_manager, compact_name, (compact_n_ctx or plan.get("n_ctx") or 4096)
async def _auto_compact_events(plan, messages):
"""Drive auto-compaction for ``plan`` (from ``_resolve_compaction``), yielding
('status', text) progress events and finally one ('done', messages, info,
error) event. ``error`` is a string when the request still overflows after
compaction (caller decides whether to raise or stream it), else None. When the
prompt is under threshold, yields only the terminal ('done', messages, None,
None)."""
n_ctx = plan["n_ctx"]
pct = plan["pct"]
strategy = plan["strategy"]
est = _estimate_tokens(messages)
if not n_ctx or est < n_ctx * pct / 100.0:
yield ("done", messages, None, None)
return
summary = None
if strategy == "summarize":
# Resolve the summarizer model now (may load a separate, smaller model).
compact_manager, compact_name, compact_n_ctx = _resolve_compact_manager(plan)
via = f" via {compact_name}" if compact_name and compact_name != "the model" else ""
yield ("status", f"🗜 Compacting context (~{est} tokens ≥ {int(pct)}% of {n_ctx})"
f" using '{strategy}'{via}…\n")
# Bridge the summarizer's progress callback to this generator through a
# queue so status lines stream to the client LIVE while it summarizes
# (summarization can take minutes on a large model).
_q: asyncio.Queue = asyncio.Queue()
_DONE = object()
async def _cb(msg):
await _q.put(f" • {msg}\n")
async def _run():
try:
return await _summarize_for_compact(
compact_manager, messages,
compact_n_ctx=compact_n_ctx, progress=_cb)
finally:
await _q.put(_DONE)
_task = asyncio.create_task(_run())
while True:
_ev = await _q.get()
if _ev is _DONE:
break
yield ("status", _ev)
summary = await _task
else:
yield ("status", f"🗜 Compacting context (~{est} tokens ≥ {int(pct)}% of {n_ctx})"
f" using '{strategy}'…\n")
new_messages, info = _compact_messages(messages, n_ctx, pct, strategy, summary)
if info:
yield ("status", f"✅ Context compacted: dropped {info['dropped']} message(s), "
f"~{info['before_tokens']}→{info['after_tokens']} tokens.\n")
err = None
if _estimate_tokens(new_messages) > n_ctx:
err = ("The request is too large for this model's context window "
f"(~{_estimate_tokens(new_messages)} tokens vs n_ctx={n_ctx}) "
"even after auto-compaction. Shorten the latest message or "
"increase the model's context size (n_ctx).")
yield ("done", new_messages, info, err)
@router.post("/v1/chat/completions", summary="Chat completions")
async def chat_completions(request: ChatCompletionRequest, http_request: Request = None):
"""Chat completions endpoint with streaming and tool support."""
......@@ -992,42 +1150,27 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
elif not isinstance(m["content"], str) and not isinstance(m["content"], list):
messages_dict[i]["content"] = str(m["content"])
# Auto-compact (per-model, OFF by default): when the prompt would exceed
# `auto_compact_pct`% of the model's context window, shrink it to ~65% using
# the configured strategy (drop_oldest | keep_head_tail | summarize) instead of
# erroring out on overflow.
try:
from codai.models.manager import multi_model_manager as _mmm
_cc = _mmm._config_for_model(getattr(request, "model", None) or "") or {}
except Exception:
_cc = {}
if _cc.get("auto_compact"):
try:
_nctx = current_manager.get_context_size() if current_manager else 0
except Exception:
_nctx = 0
_pct = _cc.get("auto_compact_pct", 85)
_strategy = (_cc.get("auto_compact_strategy") or "drop_oldest").strip()
if _nctx and _estimate_tokens(messages_dict) >= _nctx * float(_pct or 85) / 100.0:
_summary = None
if _strategy == "summarize":
_summary = await _summarize_for_compact(current_manager, messages_dict)
messages_dict, _info = _compact_messages(
messages_dict, _nctx, _pct, _strategy, _summary)
# Auto-compact (per-model or global, OFF by default): when the prompt nears
# the model's context window, shrink it using the configured strategy
# (drop_oldest | keep_head_tail | summarize). Resolve the effective settings
# now; the streaming path applies it inside stream_chat_response so it can
# stream progress to the client, while the non-streaming path applies it
# inline just below. The raw two-pass path builds its prompt from only the
# system + last user turn, so compaction there is a no-op and is skipped.
_compact_plan = _resolve_compaction(request, current_manager)
if _compact_plan and not request.stream:
async for _ev in _auto_compact_events(_compact_plan, messages_dict):
if _ev[0] == "status":
print(f"[auto-compact] {_ev[1].strip()}", flush=True)
else:
_, messages_dict, _info, _cerr = _ev
if _info:
print(f"[auto-compact] {getattr(request, 'model', '?')}: "
f"~{_info['before_tokens']}→{_info['after_tokens']} tokens "
f"(n_ctx={_nctx}, dropped {_info['dropped']} msgs via "
f"{_info['strategy']})", flush=True)
# If compaction couldn't get it under the window (e.g. a single huge
# final message), signal a clear "request too big for context" error
# instead of letting generation fail mid-stream.
if _estimate_tokens(messages_dict) > _nctx:
raise HTTPException(status_code=400, detail=(
"The request is too large for this model's context window "
f"(~{_estimate_tokens(messages_dict)} tokens vs n_ctx={_nctx}) "
"even after auto-compaction. Shorten the latest message or "
"increase the model's context size (n_ctx)."))
f"(dropped {_info['dropped']} msgs via {_info['strategy']})",
flush=True)
if _cerr:
raise HTTPException(status_code=400, detail=_cerr)
# Convert tools to dict format if present
......@@ -1630,6 +1773,7 @@ async def chat_completions(request: ChatCompletionRequest, http_request: Request
request.response_format,
_prefix_key,
enable_thinking=reasoning_enabled,
compact_plan=_compact_plan,
):
yield chunk
finally:
......@@ -1765,6 +1909,7 @@ async def stream_chat_response(
response_format: Optional[Dict] = None,
prefix_key: str = "",
enable_thinking: bool = False,
compact_plan: Optional[Dict] = None,
) -> AsyncGenerator[str, None]:
"""Stream chat completion response with queue notifications."""
completion_id = f"chatcmpl-{uuid.uuid4().hex}"
......@@ -1774,6 +1919,41 @@ async def stream_chat_response(
generated_text = ""
# Auto-compact an over-long history before generation, streaming progress to
# the client as status content deltas (the same mechanism as the "Waiting for
# model reply…" notices — visible text, not part of the saved completion).
if compact_plan:
try:
async for _ev in _auto_compact_events(compact_plan, messages):
if _ev[0] == "status":
_sc = {
"id": completion_id, "object": "chat.completion.chunk",
"created": created, "model": model_name,
"choices": [{"index": 0, "delta": {"content": _ev[1]},
"finish_reason": None}],
"x_compaction": {"status": "compacting"},
}
yield f"data: {json.dumps(_sc)}\n\n"
else:
_, messages, _cinfo, _cerr = _ev
if _cinfo:
print(f"[auto-compact] {model_name}: "
f"~{_cinfo['before_tokens']}→{_cinfo['after_tokens']} tokens "
f"(dropped {_cinfo['dropped']} msgs via {_cinfo['strategy']})",
flush=True)
if _cerr:
_ec = {
"id": completion_id, "object": "chat.completion.chunk",
"created": created, "model": model_name,
"choices": [{"index": 0, "delta": {"content": "\n⚠ " + _cerr},
"finish_reason": "stop"}],
}
yield f"data: {json.dumps(_ec)}\n\n"
yield "data: [DONE]\n\n"
return
except Exception as _ce:
print(f"[auto-compact] streaming compaction failed: {_ce}", flush=True)
# Check if model is loaded - if not, notify waiting clients
# The model manager exists but backend may not be loaded yet in on-demand mode
model_loaded = False
......
codai/config.py
View file @ 1ad76dfb
......@@ -214,6 +214,24 @@ class EnhanceConfig:
allow_rife_ncnn: bool = False # allow the external rife-ncnn-vulkan binary instead of a torch model
@dataclass
class CompactionConfig:
"""Global defaults for auto-compaction of an over-long chat history.
Per-model settings in a models.json entry (``auto_compact``,
``auto_compact_pct``, ``auto_compact_strategy``, ``auto_compact_model``)
OVERRIDE the values here; when a model leaves one unset, the global default
below applies. ``model`` selects which model performs the summarization for
the ``summarize`` strategy — empty means use the same model that serves the
request. Pointing it at a smaller/faster model lets that model summarize the
old turns while the big model answers; the dropped history is chunked to fit
the chosen summarizer's own context window before it is summarized."""
enabled: bool = False
pct: int = 85 # compact when the prompt reaches this % of n_ctx
strategy: str = "drop_oldest" # drop_oldest | keep_head_tail | summarize
model: str = "" # model id/alias that summarizes; "" = same as request
@dataclass
class Ds4Config:
"""DeepSeek V4 via ds4 (antirez/DwarfStar) external-worker configuration.
......@@ -273,6 +291,7 @@ class Config:
jobs: JobsConfig = field(default_factory=JobsConfig)
enhance: EnhanceConfig = field(default_factory=EnhanceConfig)
ds4: Ds4Config = field(default_factory=Ds4Config)
compaction: CompactionConfig = field(default_factory=CompactionConfig)
broker: BrokerConfig = field(default_factory=BrokerConfig)
system_prompt: Optional[str] = None
tools_closer_prompt: bool = False
......@@ -457,6 +476,7 @@ class ConfigManager:
jobs=_dc(JobsConfig, config_data.get("jobs", {})),
enhance=_dc(EnhanceConfig, config_data.get("enhance", {})),
ds4=_dc(Ds4Config, config_data.get("ds4", {})),
compaction=_dc(CompactionConfig, config_data.get("compaction", {})),
broker=_dc(BrokerConfig, config_data.get("broker", {})),
system_prompt=config_data.get("system_prompt"),
tools_closer_prompt=config_data.get("tools_closer_prompt", False),
......@@ -625,6 +645,12 @@ class ConfigManager:
"extra_env": self.config.ds4.extra_env,
"auto_build": self.config.ds4.auto_build,
},
"compaction": {
"enabled": self.config.compaction.enabled,
"pct": self.config.compaction.pct,
"strategy": self.config.compaction.strategy,
"model": self.config.compaction.model,
},
"broker": {
"enabled": self.config.broker.enabled,
"base_url": self.config.broker.base_url,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment