Manual disable/enable provider button (persistent across reboot) + cooldown recovery

Adds a per-provider Disable/Enable button to the rotations page for both admin
and users, exposing a manual disable that — unlike the failure cooldown — never
auto-expires and persists across reboots.

- providers/base.py: manual_disable()/manual_enable()/is_manually_disabled();
  manual disable takes precedence in is_rate_limited(); manual_enable() also
  clears any cooldown and resets the failure budget. Source of truth is the
  database (mirrored to cache as a fast path) so state survives restarts and
  cache flushes. Also: when a failure cooldown elapses, the provider is
  reactivated with a fresh failure budget instead of a hair-trigger.
- database.py: dedicated provider_manual_disabled table (+migration) and
  get/set/clear helpers, user-scoped so a user's toggle never affects others.
- dashboard: manual-disable / manual-enable / bulk manual-status endpoints; the
  rotations pages render and toggle the button state.

Bump to 0.99.73.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

aisbf/__init__.py

@@ -55,7 +55,7 @@ from .auth.qwen import QwenOAuth2
 from .handlers import RequestHandler, RotationHandler, AutoselectHandler
 from .utils import count_messages_tokens, split_messages_into_chunks, get_max_request_tokens_for_model
 
-__version__ = "0.99.71"
+__version__ = "0.99.73"
 __all__ = [
     # Config
     "config",

aisbf/database.py

@@ -3849,6 +3849,48 @@ class DatabaseManager:
                 cursor.execute(f'DELETE FROM provider_disabled_state WHERE user_id = {placeholder} AND provider_id = {placeholder}', (user_id, provider_id))
             conn.commit()
 
+    def is_provider_manually_disabled(self, user_id, provider_id: str) -> bool:
+        """Return True if a provider was manually disabled (persists across reboot).
+
+        Kept in a dedicated table so it never auto-expires and never collides with
+        the usage/cooldown disabled state."""
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            placeholder = '?' if self.db_type == 'sqlite' else '%s'
+            if user_id is None:
+                cursor.execute(f'SELECT 1 FROM provider_manual_disabled WHERE user_id IS NULL AND provider_id = {placeholder}', (provider_id,))
+            else:
+                cursor.execute(f'SELECT 1 FROM provider_manual_disabled WHERE user_id = {placeholder} AND provider_id = {placeholder}', (user_id, provider_id))
+            return cursor.fetchone() is not None
+
+    def set_provider_manually_disabled(self, user_id, provider_id: str):
+        """Persist a manual provider disable (idempotent)."""
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            placeholder = '?' if self.db_type == 'sqlite' else '%s'
+            if user_id is None:
+                cursor.execute(f'DELETE FROM provider_manual_disabled WHERE user_id IS NULL AND provider_id = {placeholder}', (provider_id,))
+                cursor.execute(f'INSERT INTO provider_manual_disabled (user_id, provider_id) VALUES (NULL, {placeholder})', (provider_id,))
+            elif self.db_type == 'sqlite':
+                cursor.execute(f'INSERT OR REPLACE INTO provider_manual_disabled (user_id, provider_id) VALUES ({placeholder}, {placeholder})', (user_id, provider_id))
+            else:
+                cursor.execute(f'''
+                    INSERT INTO provider_manual_disabled (user_id, provider_id) VALUES ({placeholder}, {placeholder})
+                    ON DUPLICATE KEY UPDATE updated_at=CURRENT_TIMESTAMP
+                ''', (user_id, provider_id))
+            conn.commit()
+
+    def clear_provider_manually_disabled(self, user_id, provider_id: str):
+        """Remove a manual provider disable."""
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            placeholder = '?' if self.db_type == 'sqlite' else '%s'
+            if user_id is None:
+                cursor.execute(f'DELETE FROM provider_manual_disabled WHERE user_id IS NULL AND provider_id = {placeholder}', (provider_id,))
+            else:
+                cursor.execute(f'DELETE FROM provider_manual_disabled WHERE user_id = {placeholder} AND provider_id = {placeholder}', (user_id, provider_id))
+            conn.commit()
+
     # Sort order methods
 
     def get_sort_order(self, user_id, entity_type: str) -> Optional[List[str]]:
@@ -6797,6 +6839,40 @@ def DatabaseManager__run_config_migrations(self, cursor, auto_increment, timesta
     except Exception as e:
         logger.warning(f"Migration check for provider_disabled_state table: {e}")
 
+    # Migration: Create provider_manual_disabled table if missing
+    try:
+        if self.db_type == 'sqlite':
+            cursor.execute("PRAGMA table_info(provider_manual_disabled)")
+            if not cursor.fetchall():
+                cursor.execute(f'''
+                    CREATE TABLE provider_manual_disabled (
+                        id INTEGER PRIMARY KEY {auto_increment},
+                        user_id INTEGER,
+                        provider_id VARCHAR(255) NOT NULL,
+                        updated_at TIMESTAMP DEFAULT {timestamp_default},
+                        UNIQUE(user_id, provider_id)
+                    )
+                ''')
+                logger.info("✅ Migration: Created provider_manual_disabled table")
+        else:
+            cursor.execute("""
+                SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES
+                WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'provider_manual_disabled'
+            """)
+            if not cursor.fetchone():
+                cursor.execute(f'''
+                    CREATE TABLE provider_manual_disabled (
+                        id INTEGER PRIMARY KEY {auto_increment},
+                        user_id INTEGER,
+                        provider_id VARCHAR(255) NOT NULL,
+                        updated_at TIMESTAMP DEFAULT {timestamp_default},
+                        UNIQUE(user_id, provider_id)
+                    )
+                ''')
+                logger.info("✅ Migration: Created provider_manual_disabled table")
+    except Exception as e:
+        logger.warning(f"Migration check for provider_manual_disabled table: {e}")
+
     # Migration: Create runpod_provider_state table if missing
     try:
         if self.db_type == 'sqlite':

aisbf/providers/base.py

@@ -855,6 +855,9 @@ class BaseProviderHandler:
         self.adaptive_limiter = get_adaptive_rate_limiter(provider_id, adaptive_config, user_id)
         # Load rate-limit disabled state from cache (persists across restarts)
         self._load_disabled_until_from_cache()
+        # Load manual (admin/user-triggered) disable flag from cache
+        self._manual_disabled: bool = False
+        self._load_manual_disabled_from_cache()
         # Load usage-based disabled state from DB (persists across restarts)
         self._usage_disabled_until: Optional[float] = None
         try:
@@ -1284,10 +1287,102 @@ class BaseProviderHandler:
         except Exception:
             pass
 
+    def _manual_disabled_cache_key(self) -> str:
+        # User-scoped so a user's manual toggle never affects the global provider.
+        scope = self.user_id if self.user_id is not None else 'global'
+        return f"aisbf:provider_manual_disabled:{scope}:{self.provider_id}"
+
+    def _load_manual_disabled_from_cache(self):
+        """Restore the manual-disable flag on startup.
+
+        The cache is a fast path; the database is the source of truth so the state
+        survives reboots and cache flushes. On a cache miss we read the DB and warm
+        the cache (storing the boolean either way)."""
+        try:
+            from ..cache import get_cache_manager
+            cached = get_cache_manager().get(self._manual_disabled_cache_key())
+        except Exception:
+            cached = None
+        if cached is not None:
+            self._manual_disabled = bool(cached)
+            return
+        # Cache miss → authoritative database lookup
+        value = False
+        try:
+            db = DatabaseRegistry.get_config_database()
+            if db:
+                value = bool(db.is_provider_manually_disabled(self.user_id, self.provider_id))
+        except Exception:
+            value = False
+        self._manual_disabled = value
+        try:
+            from ..cache import get_cache_manager
+            get_cache_manager().set(self._manual_disabled_cache_key(), value, ttl=10 * 365 * 24 * 3600)
+        except Exception:
+            pass
+
+    def manual_disable(self):
+        """Manually disable this provider until it is manually re-enabled.
+
+        Unlike the failure cooldown this never auto-expires. It is persisted in the
+        database (source of truth, survives reboot) and mirrored to the cache."""
+        import logging
+        self._manual_disabled = True
+        try:
+            db = DatabaseRegistry.get_config_database()
+            if db:
+                db.set_provider_manually_disabled(self.user_id, self.provider_id)
+        except Exception as e:
+            logging.getLogger(__name__).warning(f"Failed to persist manual disable for {self.provider_id}: {e}")
+        try:
+            from ..cache import get_cache_manager
+            get_cache_manager().set(self._manual_disabled_cache_key(), True, ttl=10 * 365 * 24 * 3600)
+        except Exception:
+            pass
+        logging.getLogger(__name__).warning(f"Provider {self.provider_id} manually disabled (scope: {self.user_id if self.user_id is not None else 'global'})")
+
+    def manual_enable(self):
+        """Clear a manual disable AND any failure cooldown, restoring a fresh
+        failure budget so the provider is immediately available again. Persisted
+        to the database and mirrored to the cache."""
+        import logging
+        self._manual_disabled = False
+        try:
+            db = DatabaseRegistry.get_config_database()
+            if db:
+                db.clear_provider_manually_disabled(self.user_id, self.provider_id)
+        except Exception as e:
+            logging.getLogger(__name__).warning(f"Failed to persist manual enable for {self.provider_id}: {e}")
+        try:
+            from ..cache import get_cache_manager
+            get_cache_manager().set(self._manual_disabled_cache_key(), False, ttl=10 * 365 * 24 * 3600)
+        except Exception:
+            pass
+        # Also clear any auto-disable cooldown and reset the failure counter.
+        self._clear_disabled_until()
+        self.error_tracking['failures'] = 0
+        logging.getLogger(__name__).info(f"Provider {self.provider_id} manually re-enabled (scope: {self.user_id if self.user_id is not None else 'global'})")
+
+    def is_manually_disabled(self) -> bool:
+        return bool(getattr(self, '_manual_disabled', False))
+
     def is_rate_limited(self) -> bool:
-        disabled_until = self.error_tracking.get('disabled_until')
-        if disabled_until and disabled_until > time.time():
+        # Manual disable takes precedence and never auto-expires.
+        if getattr(self, '_manual_disabled', False):
             return True
+        disabled_until = self.error_tracking.get('disabled_until')
+        if disabled_until:
+            if disabled_until > time.time():
+                return True
+            # Cooldown has elapsed: this is a temporary measure, so reactivate the
+            # provider with a fresh failure budget instead of leaving it on a
+            # hair-trigger (failures still at 3) with a stale disabled_until.
+            import logging
+            logging.getLogger(__name__).info(
+                f"Provider {self.provider_id} cooldown elapsed — reactivating with a fresh failure budget"
+            )
+            self._clear_disabled_until()
+            self.error_tracking['failures'] = 0
         # Check usage-based disable (loaded from DB on init, persists across restarts)
         if self._usage_disabled_until and self._usage_disabled_until > time.time():
             return True

aisbf/routes/dashboard/providers.py

@@ -2127,6 +2127,73 @@ async def api_provider_delete(request: Request, provider_id: str):
         return JSONResponse({"success": False, "error": str(e)}, status_code=500)
 
 
+@router.get("/dashboard/api/providers/manual-status")
+async def api_providers_manual_status(request: Request):
+    """Return manual-disable / rate-limited status for every provider in scope,
+    so the rotations page can render the correct disable/enable button state."""
+    auth_check = require_dashboard_auth(request)
+    if auth_check:
+        return JSONResponse({"success": False, "error": "Not authenticated"}, status_code=401)
+
+    current_user_id = request.session.get('user_id')
+    from aisbf.providers import get_provider_handler
+    from aisbf.config import config as global_config
+
+    if current_user_id is None:
+        provider_ids = list((global_config.providers or {}).keys())
+    else:
+        db = DatabaseRegistry.get_config_database()
+        provider_ids = [p['provider_id'] for p in db.get_user_providers(current_user_id)]
+
+    statuses = {}
+    for pid in provider_ids:
+        try:
+            handler = get_provider_handler(pid, user_id=current_user_id)
+            manual = handler.is_manually_disabled()
+            statuses[pid] = {
+                "manual_disabled": manual,
+                # rate_limited includes the manual flag; expose the cooldown-only state too
+                "rate_limited": handler.is_rate_limited(),
+            }
+        except Exception:
+            continue
+    return JSONResponse({"success": True, "statuses": statuses})
+
+
+@router.post("/dashboard/api/provider/{provider_id:path}/manual-disable")
+async def api_provider_manual_disable(request: Request, provider_id: str):
+    """Manually disable a provider (admin = global scope, user = their scope)."""
+    auth_check = require_dashboard_auth(request)
+    if auth_check:
+        return JSONResponse({"success": False, "error": "Not authenticated"}, status_code=401)
+    current_user_id = request.session.get('user_id')
+    from aisbf.providers import get_provider_handler
+    try:
+        handler = get_provider_handler(provider_id, user_id=current_user_id)
+        handler.manual_disable()
+        return JSONResponse({"success": True, "manual_disabled": True})
+    except Exception as e:
+        logger.warning(f"manual_disable error for {provider_id}: {e}")
+        return JSONResponse({"success": False, "error": str(e)}, status_code=500)
+
+
+@router.post("/dashboard/api/provider/{provider_id:path}/manual-enable")
+async def api_provider_manual_enable(request: Request, provider_id: str):
+    """Manually re-enable a provider, clearing manual disable and any cooldown."""
+    auth_check = require_dashboard_auth(request)
+    if auth_check:
+        return JSONResponse({"success": False, "error": "Not authenticated"}, status_code=401)
+    current_user_id = request.session.get('user_id')
+    from aisbf.providers import get_provider_handler
+    try:
+        handler = get_provider_handler(provider_id, user_id=current_user_id)
+        handler.manual_enable()
+        return JSONResponse({"success": True, "manual_disabled": False})
+    except Exception as e:
+        logger.warning(f"manual_enable error for {provider_id}: {e}")
+        return JSONResponse({"success": False, "error": str(e)}, status_code=500)
+
+
 @router.get("/dashboard/api/provider/{provider_id:path}/usage")
 async def api_provider_usage(request: Request, provider_id: str):
     """Return cached usage data for a provider, refreshing from source if stale (>5 min)."""

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aisbf"
-version = "0.99.71"
+version = "0.99.73"
 description = "AISBF - AI Service Broker Framework || AI Should Be Free - A modular proxy server for managing multiple AI provider integrations"
 readme = "README.md"
 license = "GPL-3.0-or-later"

setup.py

@@ -106,7 +106,7 @@ class InstallCommand(_install):
 
 setup(
     name="aisbf",
-    version="0.99.71",
+    version="0.99.73",
     author="AISBF Contributors",
     author_email="stefy@nexlab.net",
     description="AISBF - AI Service Broker Framework || AI Should Be Free - A modular proxy server for managing multiple AI provider integrations",

static/i18n/en.json

@@ -340,7 +340,9 @@
     "error_saving": "Error saving configuration",
     "forward_to": "Forward to",
     "target_name": "Target rotation / autoselect",
-    "target_placeholder": "rotation or autoselect name…"
+    "target_placeholder": "rotation or autoselect name…",
+    "disable": "Disable",
+    "enable": "Enable"
   },
   "autoselect": {
     "model_name": "Model Name",
@@ -1241,4 +1243,4 @@
     "lbl_pending": "Pending Payments",
     "lbl_failed": "Failed Payments"
   }
-}
+}
\ No newline at end of file

templates/dashboard/rotations.html

@@ -447,7 +447,10 @@ function renderRotationProviders(rotationKey) {
                     <strong>${window.i18n.t('rotations.provider_label')} ${providerIndex + 1}</strong>
                     ${_usageBadge}
                 </div>
-                <button type="button" class="btn btn-secondary" onclick="removeRotationProvider('${safeKey}', ${providerIndex})" style="background: #dc3545; padding: 5px 10px; font-size: 12px;">${window.i18n.t('rotations.remove')}</button>
+                <div style="display:flex;align-items:center;gap:6px;">
+                    ${(_pid && !META_PROVIDERS.includes(_pid)) ? `<button type="button" class="btn btn-secondary provider-toggle-btn" data-provider="${escHtmlAttr(_pid)}" onclick="toggleProviderDisabled(this)" style="padding:5px 10px;font-size:12px;">${window.i18n.t('rotations.disable') || 'Disable'}</button>` : ''}
+                    <button type="button" class="btn btn-secondary" onclick="removeRotationProvider('${safeKey}', ${providerIndex})" style="background: #dc3545; padding: 5px 10px; font-size: 12px;">${window.i18n.t('rotations.remove')}</button>
+                </div>
             </div>
 
             <div class="form-group">
@@ -470,8 +473,68 @@ function renderRotationProviders(rotationKey) {
         container.appendChild(providerDiv);
         renderRotationModels(rotationKey, providerIndex);
     });
+    applyProviderToggleStates();
+}
+
+// --- Manual provider disable/enable -------------------------------------
+let providerManualStatus = {};
+
+async function loadProviderManualStatus() {
+    try {
+        const r = await fetch(`${BASE_PATH}/dashboard/api/providers/manual-status`);
+        if (!r.ok) return;
+        const data = await r.json();
+        if (data && data.success) {
+            providerManualStatus = data.statuses || {};
+            applyProviderToggleStates();
+        }
+    } catch (e) { /* non-fatal */ }
 }
 
+function applyProviderToggleStates() {
+    document.querySelectorAll('.provider-toggle-btn').forEach(btn => {
+        const pid = btn.getAttribute('data-provider');
+        const st = providerManualStatus[pid] || {};
+        _setToggleBtn(btn, !!st.manual_disabled);
+    });
+}
+
+function _setToggleBtn(btn, disabled) {
+    btn.dataset.disabled = disabled ? '1' : '0';
+    if (disabled) {
+        btn.textContent = window.i18n.t('rotations.enable') || 'Enable';
+        btn.style.background = '#198754';  // green = click to re-enable
+    } else {
+        btn.textContent = window.i18n.t('rotations.disable') || 'Disable';
+        btn.style.background = '';
+    }
+}
+
+async function toggleProviderDisabled(btn) {
+    const pid = btn.getAttribute('data-provider');
+    if (!pid) return;
+    const currentlyDisabled = btn.dataset.disabled === '1';
+    const action = currentlyDisabled ? 'manual-enable' : 'manual-disable';
+    btn.disabled = true;
+    try {
+        const r = await fetch(`${BASE_PATH}/dashboard/api/provider/${encodeURIComponent(pid)}/${action}`, { method: 'POST' });
+        const data = await r.json();
+        if (data && data.success) {
+            providerManualStatus[pid] = { ...(providerManualStatus[pid] || {}), manual_disabled: !!data.manual_disabled };
+            // Update every toggle button for this provider (it may appear in several rotations).
+            document.querySelectorAll(`.provider-toggle-btn[data-provider="${CSS.escape(pid)}"]`).forEach(b => _setToggleBtn(b, !!data.manual_disabled));
+        } else {
+            alert((data && data.error) || 'Action failed');
+        }
+    } catch (e) {
+        alert('Request failed: ' + e);
+    } finally {
+        btn.disabled = false;
+    }
+}
+
+document.addEventListener('DOMContentLoaded', () => setTimeout(loadProviderManualStatus, 600));
+
 function renderRotationModels(rotationKey, providerIndex) {
     const container = document.getElementById(`models-${rotationKey}-${providerIndex}`);
     const provider = rotationsConfig.rotations[rotationKey].providers[providerIndex];

templates/dashboard/user_rotations.html

@@ -470,7 +470,10 @@ function renderRotationProviders(rotationKey) {
                     <strong>${window.i18n.t('rotations.provider_label')} ${providerIndex + 1}</strong>
                     ${_usageBadge}
                 </div>
-                <button type="button" class="btn btn-secondary" onclick="removeRotationProvider('${safeKey}', ${providerIndex})" style="background: #dc3545; padding: 5px 10px; font-size: 12px;">${window.i18n.t('rotations.remove')}</button>
+                <div style="display:flex;align-items:center;gap:6px;">
+                    ${(_pid && !META_PROVIDERS.includes(_pid)) ? `<button type="button" class="btn btn-secondary provider-toggle-btn" data-provider="${escHtmlAttr(_pid)}" onclick="toggleProviderDisabled(this)" style="padding:5px 10px;font-size:12px;">${window.i18n.t('rotations.disable') || 'Disable'}</button>` : ''}
+                    <button type="button" class="btn btn-secondary" onclick="removeRotationProvider('${safeKey}', ${providerIndex})" style="background: #dc3545; padding: 5px 10px; font-size: 12px;">${window.i18n.t('rotations.remove')}</button>
+                </div>
             </div>
 
             <div class="form-group">
@@ -493,8 +496,67 @@ function renderRotationProviders(rotationKey) {
         container.appendChild(providerDiv);
         renderRotationModels(rotationKey, providerIndex);
     });
+    applyProviderToggleStates();
+}
+
+// --- Manual provider disable/enable -------------------------------------
+let providerManualStatus = {};
+
+async function loadProviderManualStatus() {
+    try {
+        const r = await fetch(`${BASE_PATH}/dashboard/api/providers/manual-status`);
+        if (!r.ok) return;
+        const data = await r.json();
+        if (data && data.success) {
+            providerManualStatus = data.statuses || {};
+            applyProviderToggleStates();
+        }
+    } catch (e) { /* non-fatal */ }
 }
 
+function applyProviderToggleStates() {
+    document.querySelectorAll('.provider-toggle-btn').forEach(btn => {
+        const pid = btn.getAttribute('data-provider');
+        const st = providerManualStatus[pid] || {};
+        _setToggleBtn(btn, !!st.manual_disabled);
+    });
+}
+
+function _setToggleBtn(btn, disabled) {
+    btn.dataset.disabled = disabled ? '1' : '0';
+    if (disabled) {
+        btn.textContent = window.i18n.t('rotations.enable') || 'Enable';
+        btn.style.background = '#198754';
+    } else {
+        btn.textContent = window.i18n.t('rotations.disable') || 'Disable';
+        btn.style.background = '';
+    }
+}
+
+async function toggleProviderDisabled(btn) {
+    const pid = btn.getAttribute('data-provider');
+    if (!pid) return;
+    const currentlyDisabled = btn.dataset.disabled === '1';
+    const action = currentlyDisabled ? 'manual-enable' : 'manual-disable';
+    btn.disabled = true;
+    try {
+        const r = await fetch(`${BASE_PATH}/dashboard/api/provider/${encodeURIComponent(pid)}/${action}`, { method: 'POST' });
+        const data = await r.json();
+        if (data && data.success) {
+            providerManualStatus[pid] = { ...(providerManualStatus[pid] || {}), manual_disabled: !!data.manual_disabled };
+            document.querySelectorAll(`.provider-toggle-btn[data-provider="${CSS.escape(pid)}"]`).forEach(b => _setToggleBtn(b, !!data.manual_disabled));
+        } else {
+            alert((data && data.error) || 'Action failed');
+        }
+    } catch (e) {
+        alert('Request failed: ' + e);
+    } finally {
+        btn.disabled = false;
+    }
+}
+
+document.addEventListener('DOMContentLoaded', () => setTimeout(loadProviderManualStatus, 600));
+
 function renderRotationModels(rotationKey, providerIndex) {
     const container = document.getElementById(`models-${rotationKey}-${providerIndex}`);
     const provider = rotationsConfig.rotations[rotationKey].providers[providerIndex];