fix(studio): isolate persisted capability metadata

parent e1f4e104
......@@ -409,7 +409,9 @@ async def _auto_detect_provider_models(provider_key: str, provider: dict) -> lis
'name': model_id,
'rate_limit': 0,
'max_request_tokens': int(context_size) if context_size else 100000,
'context_size': int(context_size) if context_size else 100000
'context_size': int(context_size) if context_size else 100000,
'capabilities': model_data.get('capabilities'),
'architecture': model_data.get('architecture'),
})
detected_models = [stamp_inferred_capabilities(model, provider_type) for model in detected_models]
......
......@@ -98,12 +98,11 @@ def stamp_inferred_capabilities(model: Dict[str, Any], provider_type: str) -> Di
capability_result = infer_model_capabilities(
model_name=stamped.get("name") or stamped.get("id") or "",
provider_type=provider_type,
explicit_capabilities=stamped.get("capabilities") or stamped.get("studio_capabilities"),
explicit_capabilities=stamped.get("studio_capabilities") or stamped.get("capabilities"),
architecture=stamped.get("architecture"),
provider_metadata=stamped,
)
stamped["capabilities"] = capability_result.capabilities
stamped["studio_capabilities"] = capability_result.capabilities
stamped["studio_capability_source"] = capability_result.source
stamped["studio_capability_unknown"] = capability_result.unknown
......@@ -220,6 +219,8 @@ def build_catalog_entry(
availability_reason: Optional[str],
metadata: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
metadata = metadata or {}
effective_capabilities = metadata.get("studio_capabilities") or capabilities
return {
"id": f"provider/{source_id}/{target_id}",
"kind": kind,
......@@ -229,10 +230,10 @@ def build_catalog_entry(
"target_id": target_id,
"label": label,
"description": description,
"capabilities": normalize_capabilities(capabilities),
"capabilities": normalize_capabilities(effective_capabilities),
"availability_state": availability_state,
"availability_reason": availability_reason,
"metadata": metadata or {},
"metadata": metadata,
}
......@@ -324,13 +325,26 @@ def _build_provider_entries(scope: str, owner_id: Optional[int], providers: Dict
if not target_id:
continue
capability_result = infer_model_capabilities(
model_name=target_id,
provider_type=provider_type,
explicit_capabilities=model.get("capabilities"),
architecture=model.get("architecture"),
provider_metadata=model,
)
persisted_studio_capabilities = model.get("studio_capabilities")
persisted_source = model.get("studio_capability_source")
persisted_unknown = model.get("studio_capability_unknown")
persisted_notes = model.get("studio_capability_notes")
if persisted_studio_capabilities is not None:
capability_result = StudioCapabilityResult(
capabilities=normalize_capabilities(persisted_studio_capabilities),
source=persisted_source or "persisted",
unknown=bool(persisted_unknown),
notes=list(persisted_notes or []),
)
else:
capability_result = infer_model_capabilities(
model_name=target_id,
provider_type=provider_type,
explicit_capabilities=None,
architecture=model.get("architecture"),
provider_metadata=model,
)
metadata = {
"provider_type": provider_type,
}
......@@ -338,6 +352,14 @@ def _build_provider_entries(scope: str, owner_id: Optional[int], providers: Dict
metadata["context_length"] = model.get("context_length")
if model.get("architecture") is not None:
metadata["architecture"] = model.get("architecture")
if model.get("studio_capabilities") is not None:
metadata["studio_capabilities"] = normalize_capabilities(model.get("studio_capabilities"))
if model.get("studio_capability_source") is not None:
metadata["studio_capability_source"] = model.get("studio_capability_source")
if model.get("studio_capability_unknown") is not None:
metadata["studio_capability_unknown"] = model.get("studio_capability_unknown")
if model.get("studio_capability_notes"):
metadata["studio_capability_notes"] = model.get("studio_capability_notes")
if capability_result.source:
metadata["capability_source"] = capability_result.source
if capability_result.notes:
......
......@@ -394,6 +394,135 @@ def test_dashboard_providers_save_persists_inferred_studio_metadata_for_bulk_sav
assert model["studio_capabilities"] == ["audio_input", "transcription"]
assert model["studio_capability_source"] in {"provider_metadata", "heuristic"}
assert model["studio_capability_unknown"] is False
assert "capabilities" not in model
def test_build_studio_catalog_prefers_persisted_studio_capabilities_over_legacy_capabilities():
class ConfigStub:
providers = {
"openai": {
"type": "openai",
"models": [
{
"name": "whisper-large-v3",
"capabilities": ["chat"],
"studio_capabilities": ["audio_input", "transcription"],
"studio_capability_source": "heuristic",
"studio_capability_unknown": False,
}
],
}
}
rotations = {}
autoselect = {}
catalog = build_studio_catalog(scope="admin", owner_id=None, config=ConfigStub())
provider_entry = next(entry for entry in catalog["entries"] if entry["kind"] == "provider_model")
assert provider_entry["capabilities"] == ["audio_input", "transcription"]
assert provider_entry["metadata"]["studio_capabilities"] == ["audio_input", "transcription"]
@pytest.mark.asyncio
async def test_auto_detect_provider_models_persists_studio_metadata_without_overwriting_legacy_capabilities(monkeypatch):
class ResponseStub:
status_code = 200
def json(self):
return {
"data": [
{
"id": "whisper-large-v3",
"capabilities": ["legacy-audio"],
"architecture": {"input_modalities": ["audio"], "output_modalities": ["text"]},
}
]
}
class ClientStub:
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
async def get(self, url, headers=None):
return ResponseStub()
monkeypatch.setattr(dashboard_providers.httpx, "AsyncClient", lambda *args, **kwargs: ClientStub())
models = await dashboard_providers._auto_detect_provider_models(
"openai",
{"type": "openai", "endpoint": "https://example.test", "api_key": "secret"},
)
assert models[0]["capabilities"] == ["legacy-audio"]
assert models[0]["studio_capabilities"] == ["legacy-audio"]
assert models[0]["studio_capability_source"] == "explicit"
def test_search_provider_models_refresh_uses_autodetect_flow_without_exposing_studio_metadata(monkeypatch):
client = TestClient(app)
_login_as_admin(client)
class SourceOpen:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self):
return json.dumps(
{
"providers": {
"openai": {
"type": "openai",
"endpoint": "https://example.test",
"api_key": "secret",
"models": [],
}
}
}
)
async def fake_fetch_provider_models(provider_id, user_id=None):
return await dashboard_providers._auto_detect_provider_models(
provider_id,
{"type": "openai", "endpoint": "https://example.test", "api_key": "secret"},
)
class ResponseStub:
status_code = 200
def json(self):
return {
"data": [
{
"id": "whisper-large-v3",
"architecture": {"input_modalities": ["audio"], "output_modalities": ["text"]},
}
]
}
class ClientStub:
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
async def get(self, url, headers=None):
return ResponseStub()
monkeypatch.setattr(dashboard_providers, "open", lambda *args, **kwargs: SourceOpen(), raising=False)
monkeypatch.setattr(dashboard_providers, "fetch_provider_models", fake_fetch_provider_models)
monkeypatch.setattr(dashboard_providers.httpx, "AsyncClient", lambda *args, **kwargs: ClientStub())
response = client.get("/dashboard/providers/openai/search-models?refresh=true")
assert response.status_code == 200
assert response.json() == {"models": ["whisper-large-v3"], "fetched_live": True}
class DummyOpen:
......
......@@ -112,7 +112,21 @@ def test_stamp_inferred_capabilities_persists_inferred_metadata_for_model_dict()
assert stamped["studio_capabilities"] == ["audio_input", "transcription"]
assert stamped["studio_capability_source"] in {"provider_metadata", "heuristic"}
assert stamped["studio_capability_unknown"] is False
assert stamped["capabilities"] == ["audio_input", "transcription"]
assert "capabilities" not in stamped
def test_stamp_inferred_capabilities_preserves_existing_legacy_capabilities():
stamped = stamp_inferred_capabilities(
{
"name": "whisper-large-v3",
"capabilities": ["legacy-audio"],
"architecture": {"input_modalities": ["audio"], "output_modalities": ["text"]},
},
provider_type="openai",
)
assert stamped["capabilities"] == ["legacy-audio"]
assert stamped["studio_capabilities"] == ["legacy-audio"]
def test_build_catalog_entry_uses_persisted_studio_capability_metadata_when_available():
......@@ -140,3 +154,25 @@ def test_build_catalog_entry_uses_persisted_studio_capability_metadata_when_avai
assert entry["metadata"]["studio_capabilities"] == ["audio_input", "transcription"]
assert entry["metadata"]["studio_capability_source"] == "heuristic"
assert entry["metadata"]["studio_capability_unknown"] is False
def test_build_catalog_entry_prefers_persisted_studio_capabilities_over_legacy_capabilities():
entry = build_catalog_entry(
scope="admin",
owner_id=None,
kind="provider_model",
source_id="openai",
target_id="whisper-large-v3",
label="Whisper Large V3",
description=None,
capabilities=["chat"],
availability_state="ready",
availability_reason=None,
metadata={
"studio_capabilities": ["audio_input", "transcription"],
"studio_capability_source": "heuristic",
"studio_capability_unknown": False,
},
)
assert entry["capabilities"] == ["audio_input", "transcription"]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment