Merge branch 'feat/per-rotation-provider-disable'

aisbf/handlers.py

@@ -39,7 +39,6 @@ from fastapi import HTTPException, Request
 from fastapi.responses import JSONResponse, StreamingResponse, Response
 from .models import ChatCompletionRequest, ChatCompletionResponse
 from .providers import get_provider_handler, RateLimitError
-from .providers.base import is_provider_disabled_cheap
 from .config import config
 from .studio import infer_model_capabilities
 from .studio_adapters import effective_studio_adapter, infer_studio_adapter_profile, adapt_studio_payload_with_profile
@@ -106,14 +105,15 @@ def _raise_if_provider_unavailable(handler, provider_id: str) -> None:
     """Reject a direct request with 503 only when the provider is genuinely
     unavailable, saying why.
 
-    is_rate_limited() collapses three very different states — the dashboard
-    disable toggle, the failure cooldown and the usage-limit cooldown — into one
-    boolean. Those are NOT equivalent for a direct call:
+    is_rate_limited() collapses three very different states — the legacy global
+    manual-disable flag, the failure cooldown and the usage-limit cooldown — into
+    one boolean. Those are NOT equivalent for a direct call:
 
-    * The dashboard "disable" toggle only removes a provider from *rotation* and
-      *autoselect* (those scans call is_provider_disabled_cheap, which honours the
-      manual flag). A direct request that names the provider/model explicitly must
-      still go through — manually disabling a provider must never take it offline
+    * Disabling a provider is now expressed per rotation entry (``enabled: false``
+      in that rotation's config), honoured by the rotation scan. The legacy global
+      manual-disable flag no longer gates anything: not direct calls, not rotation.
+      A direct request that names the provider/model explicitly must always go
+      through — disabling a provider for a rotation must never take it offline
       across the whole API.
     * A failure cooldown or a usage-limit cooldown is a real availability problem:
       calling upstream would just fail again, so a direct call is still rejected,
@@ -144,6 +144,21 @@ def _raise_if_provider_unavailable(handler, provider_id: str) -> None:
     raise HTTPException(status_code=503, detail=f"Provider '{provider_id}' is {reason}")
 
 
+def _provider_in_availability_cooldown(handler) -> bool:
+    """True only when the provider is in a genuine auto-disable cooldown — a
+    failure-threshold cooldown or a usage-limit cooldown.
+
+    Deliberately excludes the manual dashboard flag (is_manually_disabled): that
+    flag is now rotation-*membership* only, expressed per rotation entry via
+    ``enabled: false``. It must never knock a provider out of a rotation it is
+    still a member of, nor out of other rotations, nor out of direct calls.
+    """
+    now = time.time()
+    failure_until = handler.error_tracking.get('disabled_until') if getattr(handler, 'error_tracking', None) else None
+    usage_until = getattr(handler, '_usage_disabled_until', None)
+    return bool((failure_until and failure_until > now) or (usage_until and usage_until > now))
+
+
 # Registry for proxied content — shared across all RequestHandler instances.
 # Values: {"type": "broker", "provider_id": str, "path": str}
 #      or {"type": "http",   "url": str}
@@ -3330,25 +3345,28 @@ class RotationHandler:
                 skipped_providers.append(provider_id)
                 continue
             
-            # Cheap pre-check: skip providers that are already disabled — manually
-            # (dashboard toggle) or by an auto-disable cooldown — BEFORE building the
-            # handler and validating credentials, which for some provider types
-            # performs a network round-trip. This keeps disabled providers from being
-            # contacted at all and makes a dashboard enable/disable take effect on the
-            # very next request.
-            if is_provider_disabled_cheap(provider_id, self.user_id):
-                logger.warning(f"  [SKIPPED] Provider {provider_id} is disabled (manual toggle or cooldown)")
+            # Per-rotation membership: a provider entry disabled *in this rotation*
+            # (enabled: false) is skipped. This is the ONLY way to disable a provider
+            # for a rotation — it does not touch the provider in other rotations or in
+            # direct API calls. Cheap check, so we never build the handler (which for
+            # some provider types performs a credential-validation network round-trip)
+            # for an entry the operator turned off here.
+            if provider.get('enabled') is False:
+                logger.info(f"  [SKIPPED] Provider {provider_id} is disabled in this rotation (enabled=false)")
                 skipped_providers.append(provider_id)
                 continue
 
             # Get API key: first from provider config, then from rotation config
             api_key = self._get_api_key(provider_id, provider.get('api_key'))
 
-            # Check if provider is rate limited/deactivated (authoritative check)
+            # Authoritative availability check: skip only on a genuine auto-disable
+            # cooldown (failure threshold / usage limit). The manual dashboard flag is
+            # intentionally NOT consulted here — disabling for a rotation is expressed
+            # per entry above, so a manually-disabled provider is still usable in any
+            # rotation that lists it as enabled.
             provider_handler = get_provider_handler(provider_id, api_key, user_id=self.user_id)
-            if provider_handler.is_rate_limited():
-                logger.warning(f"  [SKIPPED] Provider {provider_id} is rate limited/deactivated")
-                logger.warning(f"  Reason: Provider has exceeded failure threshold or is in cooldown period")
+            if _provider_in_availability_cooldown(provider_handler):
+                logger.warning(f"  [SKIPPED] Provider {provider_id} is in an auto-disable cooldown (failure threshold or usage limit)")
                 skipped_providers.append(provider_id)
                 continue
             

templates/dashboard/rotations.html

@@ -455,7 +455,7 @@ function renderRotationProviders(rotationKey) {
 
         const body = `
             <div style="display:flex; justify-content:flex-end; align-items:center; gap:6px; margin-bottom:10px;">
-                ${(_pid && !META_PROVIDERS.includes(_pid)) ? `<button type="button" class="btn btn-secondary provider-toggle-btn" data-provider="${escHtmlAttr(_pid)}" onclick="toggleProviderDisabled(this)" style="padding:5px 10px;font-size:12px;">${window.i18n.t('rotations.disable') || 'Disable'}</button>` : ''}
+                ${(_pid && !META_PROVIDERS.includes(_pid)) ? `<button type="button" class="btn btn-secondary provider-toggle-btn" data-rotation="${escHtmlAttr(rotationKey)}" data-pindex="${providerIndex}" data-disabled="${provider.enabled === false ? '1' : '0'}" onclick="toggleProviderDisabled(this)" style="padding:5px 10px;font-size:12px;background:${provider.enabled === false ? '#198754' : ''};">${provider.enabled === false ? (window.i18n.t('rotations.enable') || 'Enable') : (window.i18n.t('rotations.disable') || 'Disable')}</button>` : ''}
                 <button type="button" class="btn btn-secondary" onclick="removeRotationProvider('${safeKey}', ${providerIndex})" style="background: #dc3545; padding: 5px 10px; font-size: 12px;">${window.i18n.t('rotations.remove')}</button>
             </div>
 
@@ -485,26 +485,18 @@ function renderRotationProviders(rotationKey) {
     applyProviderToggleStates();
 }
 
-// --- Manual provider disable/enable -------------------------------------
-let providerManualStatus = {};
-
-async function loadProviderManualStatus() {
-    try {
-        const r = await fetch(`${BASE_PATH}/dashboard/api/providers/manual-status`);
-        if (!r.ok) return;
-        const data = await r.json();
-        if (data && data.success) {
-            providerManualStatus = data.statuses || {};
-            applyProviderToggleStates();
-        }
-    } catch (e) { /* non-fatal */ }
-}
+// --- Per-rotation provider disable/enable -------------------------------
+// Disabling is membership in THIS rotation only: it flips `enabled` on the
+// provider entry inside this rotation's config and persists it. It never touches
+// the provider in other rotations or in direct API calls.
 
 function applyProviderToggleStates() {
     document.querySelectorAll('.provider-toggle-btn').forEach(btn => {
-        const pid = btn.getAttribute('data-provider');
-        const st = providerManualStatus[pid] || {};
-        _setToggleBtn(btn, !!st.manual_disabled);
+        const rotationKey = btn.getAttribute('data-rotation');
+        const pindex = parseInt(btn.getAttribute('data-pindex'), 10);
+        const rot = rotationsConfig.rotations[rotationKey];
+        const provider = rot && rot.providers && rot.providers[pindex];
+        _setToggleBtn(btn, !!(provider && provider.enabled === false));
     });
 }
 
@@ -519,31 +511,51 @@ function _setToggleBtn(btn, disabled) {
     }
 }
 
+// POST the current rotation config without navigating away (used by the instant
+// per-entry disable toggle). Returns true on success.
+async function persistRotationsSilent() {
+    try {
+        const orderedRotations = {};
+        rotationMasterOrder.forEach(k => { if (rotationsConfig.rotations[k]) orderedRotations[k] = rotationsConfig.rotations[k]; });
+        const response = await fetch(rotationsData.saveUrl, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            body: 'config=' + encodeURIComponent(JSON.stringify({ ...rotationsConfig, rotations: orderedRotations }, null, 2))
+        });
+        return response.ok;
+    } catch (e) {
+        return false;
+    }
+}
+
 async function toggleProviderDisabled(btn) {
-    const pid = btn.getAttribute('data-provider');
-    if (!pid) return;
-    const currentlyDisabled = btn.dataset.disabled === '1';
-    const action = currentlyDisabled ? 'manual-enable' : 'manual-disable';
+    const rotationKey = btn.getAttribute('data-rotation');
+    const pindex = parseInt(btn.getAttribute('data-pindex'), 10);
+    const rot = rotationsConfig.rotations[rotationKey];
+    if (!rot || !rot.providers || !rot.providers[pindex]) return;
+    const provider = rot.providers[pindex];
+    // Currently enabled (enabled !== false) → this click disables it, and vice versa.
+    const willDisable = provider.enabled !== false;
     btn.disabled = true;
+    // Optimistically flip, persist, revert on failure.
+    provider.enabled = willDisable ? false : true;
+    _setToggleBtn(btn, willDisable);
     try {
-        const r = await fetch(`${BASE_PATH}/dashboard/api/provider/${encodeURIComponent(pid)}/${action}`, { method: 'POST' });
-        const data = await r.json();
-        if (data && data.success) {
-            providerManualStatus[pid] = { ...(providerManualStatus[pid] || {}), manual_disabled: !!data.manual_disabled };
-            // Update every toggle button for this provider (it may appear in several rotations).
-            document.querySelectorAll(`.provider-toggle-btn[data-provider="${CSS.escape(pid)}"]`).forEach(b => _setToggleBtn(b, !!data.manual_disabled));
-        } else {
-            alert((data && data.error) || 'Action failed');
+        const ok = await persistRotationsSilent();
+        if (!ok) {
+            provider.enabled = willDisable ? true : false;
+            _setToggleBtn(btn, !willDisable);
+            alert('Failed to save rotation change');
         }
     } catch (e) {
+        provider.enabled = willDisable ? true : false;
+        _setToggleBtn(btn, !willDisable);
         alert('Request failed: ' + e);
     } finally {
         btn.disabled = false;
     }
 }
 
-document.addEventListener('DOMContentLoaded', () => setTimeout(loadProviderManualStatus, 600));
-
 function renderRotationModels(rotationKey, providerIndex) {
     const container = document.getElementById(`models-${rotationKey}-${providerIndex}`);
     const provider = rotationsConfig.rotations[rotationKey].providers[providerIndex];

templates/dashboard/user_rotations.html

@@ -471,7 +471,7 @@ function renderRotationProviders(rotationKey) {
                     ${_usageBadge}
                 </div>
                 <div style="display:flex;align-items:center;gap:6px;">
-                    ${(_pid && !META_PROVIDERS.includes(_pid)) ? `<button type="button" class="btn btn-secondary provider-toggle-btn" data-provider="${escHtmlAttr(_pid)}" onclick="toggleProviderDisabled(this)" style="padding:5px 10px;font-size:12px;">${window.i18n.t('rotations.disable') || 'Disable'}</button>` : ''}
+                    ${(_pid && !META_PROVIDERS.includes(_pid)) ? `<button type="button" class="btn btn-secondary provider-toggle-btn" data-rotation="${escHtmlAttr(rotationKey)}" data-pindex="${providerIndex}" data-disabled="${provider.enabled === false ? '1' : '0'}" onclick="toggleProviderDisabled(this)" style="padding:5px 10px;font-size:12px;background:${provider.enabled === false ? '#198754' : ''};">${provider.enabled === false ? (window.i18n.t('rotations.enable') || 'Enable') : (window.i18n.t('rotations.disable') || 'Disable')}</button>` : ''}
                     <button type="button" class="btn btn-secondary" onclick="removeRotationProvider('${safeKey}', ${providerIndex})" style="background: #dc3545; padding: 5px 10px; font-size: 12px;">${window.i18n.t('rotations.remove')}</button>
                 </div>
             </div>
@@ -499,26 +499,18 @@ function renderRotationProviders(rotationKey) {
     applyProviderToggleStates();
 }
 
-// --- Manual provider disable/enable -------------------------------------
-let providerManualStatus = {};
-
-async function loadProviderManualStatus() {
-    try {
-        const r = await fetch(`${BASE_PATH}/dashboard/api/providers/manual-status`);
-        if (!r.ok) return;
-        const data = await r.json();
-        if (data && data.success) {
-            providerManualStatus = data.statuses || {};
-            applyProviderToggleStates();
-        }
-    } catch (e) { /* non-fatal */ }
-}
+// --- Per-rotation provider disable/enable -------------------------------
+// Disabling is membership in THIS rotation only: it flips `enabled` on the
+// provider entry inside this rotation's config and persists just that rotation.
+// It never touches the provider in other rotations or in direct API calls.
 
 function applyProviderToggleStates() {
     document.querySelectorAll('.provider-toggle-btn').forEach(btn => {
-        const pid = btn.getAttribute('data-provider');
-        const st = providerManualStatus[pid] || {};
-        _setToggleBtn(btn, !!st.manual_disabled);
+        const rotationKey = btn.getAttribute('data-rotation');
+        const pindex = parseInt(btn.getAttribute('data-pindex'), 10);
+        const rot = rotationsConfig.rotations[rotationKey];
+        const provider = rot && rot.providers && rot.providers[pindex];
+        _setToggleBtn(btn, !!(provider && provider.enabled === false));
     });
 }
 
@@ -534,29 +526,36 @@ function _setToggleBtn(btn, disabled) {
 }
 
 async function toggleProviderDisabled(btn) {
-    const pid = btn.getAttribute('data-provider');
-    if (!pid) return;
-    const currentlyDisabled = btn.dataset.disabled === '1';
-    const action = currentlyDisabled ? 'manual-enable' : 'manual-disable';
+    const rotationKey = btn.getAttribute('data-rotation');
+    const pindex = parseInt(btn.getAttribute('data-pindex'), 10);
+    const rot = rotationsConfig.rotations[rotationKey];
+    if (!rot || !rot.providers || !rot.providers[pindex]) return;
+    const provider = rot.providers[pindex];
+    // Currently enabled (enabled !== false) → this click disables it, and vice versa.
+    const willDisable = provider.enabled !== false;
     btn.disabled = true;
+    // Optimistically flip, persist just this rotation, revert on failure.
+    provider.enabled = willDisable ? false : true;
+    _setToggleBtn(btn, willDisable);
     try {
-        const r = await fetch(`${BASE_PATH}/dashboard/api/provider/${encodeURIComponent(pid)}/${action}`, { method: 'POST' });
-        const data = await r.json();
-        if (data && data.success) {
-            providerManualStatus[pid] = { ...(providerManualStatus[pid] || {}), manual_disabled: !!data.manual_disabled };
-            document.querySelectorAll(`.provider-toggle-btn[data-provider="${CSS.escape(pid)}"]`).forEach(b => _setToggleBtn(b, !!data.manual_disabled));
-        } else {
-            alert((data && data.error) || 'Action failed');
+        const result = await apiCall('POST', `${BASE_PATH}/dashboard/api/rotation`, {
+            rotation_id: rotationKey,
+            config: rot,
+        });
+        if (!result || !result.success) {
+            provider.enabled = willDisable ? true : false;
+            _setToggleBtn(btn, !willDisable);
+            alert((result && result.error) || 'Failed to save rotation change');
         }
     } catch (e) {
+        provider.enabled = willDisable ? true : false;
+        _setToggleBtn(btn, !willDisable);
         alert('Request failed: ' + e);
     } finally {
         btn.disabled = false;
     }
 }
 
-document.addEventListener('DOMContentLoaded', () => setTimeout(loadProviderManualStatus, 600));
-
 function renderRotationModels(rotationKey, providerIndex) {
     const container = document.getElementById(`models-${rotationKey}-${providerIndex}`);
     const provider = rotationsConfig.rotations[rotationKey].providers[providerIndex];