"""
Copyleft (C) 2026 Stefy Lanza <stefy@nexlab.net>

AISBF - AI Service Broker Framework || AI Should Be Free

Configuration management for AISBF.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.

Why did the programmer quit his job? Because he didn't get arrays!

Configuration management for AISBF.
"""
from typing import Any, Dict, List, Literal, Optional, Union
from pydantic import BaseModel, Field, field_validator
import json
import shutil
import os
from pathlib import Path

class ProviderModelConfig(BaseModel):
    """Model configuration within a provider"""
    name: str
    rate_limit: Optional[float] = None
    max_request_tokens: Optional[int] = None
    max_tokens: Optional[int] = None  # Upper bound on generated output tokens (max completion tokens)
    error_cooldown: Optional[int] = None  # Cooldown period in seconds after 3 consecutive failures
    # OpenRouter-style extended fields
    description: Optional[str] = None
    context_length: Optional[int] = None
    architecture: Optional[Dict] = None  # modality, input_modalities, output_modalities, tokenizer, instruct_type
    pricing: Optional[Dict] = None  # prompt, completion, input_cache_read
    top_provider: Optional[Dict] = None  # context_length, max_completion_tokens, is_moderated
    supported_parameters: Optional[List[str]] = None
    default_parameters: Optional[Dict] = None
    # Content classification flags
    nsfw: bool = False  # Model can handle NSFW content
    privacy: bool = False  # Model can handle privacy-sensitive content
    # Tri-state feature overrides
    enable_nsfw_classification: Optional[bool] = None  # None = inherit provider/global default
    enable_privacy_classification: Optional[bool] = None  # None = inherit provider/global default
    enable_context_condensation: Optional[bool] = None  # None = inherit provider/global default
    # Response caching control
    enable_response_cache: Optional[bool] = None  # Enable/disable response caching for this model (None = use provider default)
    enable_prompt_batching: Optional[bool] = None  # None = inherit provider/global default
    enable_prompt_security: Optional[bool] = None  # None = inherit provider/global default
    enable_context_lens: Optional[bool] = None  # None = inherit provider/global default
    block_high_risk_prompts: Optional[bool] = None  # None = inherit provider/global default
    # Studio capability metadata
    studio_capabilities: Optional[List[str]] = None
    studio_capability_source: Optional[str] = None
    studio_capability_unknown: Optional[bool] = None
    studio_capability_notes: Optional[List[str]] = None


class CondensationConfig(BaseModel):
    """Configuration for context condensation"""
    provider_id: Optional[str] = None
    model: Optional[str] = None
    rotation_id: Optional[str] = None
    enabled: bool = True
    max_context: Optional[int] = None  # Maximum context size for condensation model


class FeatureModeConfig(BaseModel):
    """Tri-state feature mode. None/absent means inherit at parent scope."""
    mode: Literal['inherit', 'enabled', 'disabled'] = 'inherit'


class PromptFeatureConfig(BaseModel):
    """Global prompt security and context analytics defaults."""
    security_scan: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    context_lens: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    block_high_risk_prompts: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    persist_prompt_text: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    redact_before_persist: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('enabled'))
    risk_threshold: str = 'high'


class FeatureControlsConfig(BaseModel):
    """Global tri-state defaults shared across request features."""
    nsfw_classification: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    privacy_classification: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    context_condensation: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    response_cache: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    prompt_batching: FeatureModeConfig = Field(default_factory=lambda: _feature_mode_default('disabled'))
    prompt_security: PromptFeatureConfig = Field(default_factory=PromptFeatureConfig)


def _feature_mode_default(mode: str) -> FeatureModeConfig:
    return FeatureModeConfig(mode=mode)


class ProviderConfig(BaseModel):
    id: str
    name: str
    endpoint: str = ""  # May be empty for pre-configured handler types that supply their own default
    type: str
    api_key_required: bool
    rate_limit: float = 0.0
    api_key: Optional[str] = None  # Optional API key in provider config
    models: Optional[List[ProviderModelConfig]] = None  # Optional list of models with their configs
    auth_config: Optional[Dict] = None  # Unified provider authentication configuration (for all provider types)
    kiro_config: Optional[Dict] = None  # Optional Kiro-specific configuration (credentials, region, etc.) - DEPRECATED
    kilo_config: Optional[Dict] = None  # Optional Kilo-specific configuration (credentials file path) - DEPRECATED
    claude_config: Optional[Dict] = None  # Optional Claude-specific configuration (credentials file path) - DEPRECATED
    codex_config: Optional[Dict] = None  # Optional Codex-specific configuration - DEPRECATED
    qwen_config: Optional[Dict] = None  # Optional Qwen-specific configuration - DEPRECATED
    coderai_config: Optional[Dict] = None  # Optional CoderAI-specific configuration
    runpod_config: Optional[Dict] = None  # Optional RunPod-specific configuration
    # Default settings for models in this provider
    default_rate_limit: Optional[float] = None
    default_max_request_tokens: Optional[int] = None
    default_max_tokens: Optional[int] = None  # Default upper bound on generated output tokens (max completion tokens)
    default_rate_limit_TPM: Optional[int] = None
    default_rate_limit_TPH: Optional[int] = None
    default_rate_limit_TPD: Optional[int] = None
    default_context_size: Optional[int] = None
    default_condense_context: Optional[int] = None
    default_condense_method: Optional[Union[str, List[str]]] = None
    default_error_cooldown: Optional[int] = None  # Default cooldown period in seconds after 3 consecutive failures (default: 300)
    # Provider-native caching configuration
    enable_native_caching: bool = False  # Enable provider-native caching (Anthropic cache_control, Google Context Caching, OpenAI and Kilo-compatible APIs)
    cache_ttl: Optional[int] = None  # Cache TTL in seconds for Google Context Caching API
    min_cacheable_tokens: Optional[int] = 1024  # Minimum token count for content to be cacheable (default matches OpenAI)
    prompt_cache_key: Optional[str] = None  # Optional cache key for OpenAI's load balancer routing optimization
    free_tier_limit: Optional[int] = None  # Upstream provider free-tier quota amount
    free_tier_period: Optional[str] = None  # Upstream free-tier period: day, week, month
    free_tier_limit_type: Optional[str] = None  # requests or tokens
    premium_reference_monthly_cost: Optional[float] = None  # Equivalent premium value for one extra upstream free tier
    free_tier_description: Optional[str] = None  # Human description for upstream free tier
    pro_tier_requests_daily: Optional[int] = None  # Explicit request quota for one pro tier per day
    pro_tier_requests_weekly: Optional[int] = None  # Explicit request quota for one pro tier per week
    pro_tier_requests_monthly: Optional[int] = None  # Explicit request quota for one pro tier per month
    pro_tier_tokens_daily: Optional[int] = None  # Explicit token quota for one pro tier per day
    pro_tier_tokens_weekly: Optional[int] = None  # Explicit token quota for one pro tier per week
    pro_tier_tokens_monthly: Optional[int] = None  # Explicit token quota for one pro tier per month
    # Tri-state feature overrides
    enable_nsfw_classification: Optional[bool] = None  # None = use global default
    enable_privacy_classification: Optional[bool] = None  # None = use global default
    enable_context_condensation: Optional[bool] = None  # None = use global default
    # Response caching control
    enable_response_cache: Optional[bool] = None  # Enable/disable response caching for this provider (None = use global default)
    enable_prompt_batching: Optional[bool] = None  # None = use global default
    enable_prompt_security: Optional[bool] = None  # None = use global default
    enable_context_lens: Optional[bool] = None  # None = use global default
    block_high_risk_prompts: Optional[bool] = None  # None = use global default

class RotationConfig(BaseModel):
    model_name: str
    providers: List[Dict]
    notifyerrors: bool = False
    notify_on_failover: bool = False  # Notify the client (banner/metadata/headers) when a failover changes provider/model
    capabilities: Optional[List[str]] = None  # Capabilities for this rotation
    # Content classification flags
    nsfw: bool = False  # Model can handle NSFW content
    privacy: bool = False  # Model can handle privacy-sensitive content
    # OpenRouter-style extended fields
    description: Optional[str] = None
    context_length: Optional[int] = None
    architecture: Optional[Dict] = None
    pricing: Optional[Dict] = None
    supported_parameters: Optional[List[str]] = None
    default_parameters: Optional[Dict] = None
    # Default settings for models in this rotation
    default_rate_limit: Optional[float] = None
    default_max_request_tokens: Optional[int] = None
    default_max_tokens: Optional[int] = None  # Default upper bound on generated output tokens (max completion tokens)
    default_rate_limit_TPM: Optional[int] = None
    default_rate_limit_TPH: Optional[int] = None
    default_rate_limit_TPD: Optional[int] = None
    default_context_size: Optional[int] = None
    default_condense_context: Optional[int] = None
    default_condense_method: Optional[Union[str, List[str]]] = None
    default_error_cooldown: Optional[int] = None  # Default cooldown period in seconds after 3 consecutive failures (default: 300)
    # Tri-state feature overrides
    enable_nsfw_classification: Optional[bool] = None  # None = use global default
    enable_privacy_classification: Optional[bool] = None  # None = use global default
    enable_context_condensation: Optional[bool] = None  # None = use global default
    # Response caching control
    enable_response_cache: Optional[bool] = None  # Enable/disable response caching for this rotation (None = use global default)
    enable_prompt_batching: Optional[bool] = None  # None = use global default
    enable_prompt_security: Optional[bool] = None  # None = use global default
    enable_context_lens: Optional[bool] = None  # None = use global default
    block_high_risk_prompts: Optional[bool] = None  # None = use global default

class AutoselectModelInfo(BaseModel):
    model_id: str
    description: str
    nsfw: bool = False  # Model can handle NSFW content
    privacy: bool = False  # Model can handle privacy-sensitive content
    priority: int = 0  # Escalation tier: higher = more capable, used last
    max_tokens: Optional[int] = None  # Per-model override of max output tokens for this autoselect entry

    @field_validator('model_id')
    @classmethod
    def model_id_must_not_be_empty(cls, v):
        if not v or not v.strip():
            raise ValueError('model_id must not be empty')
        return v

class AutoselectConfig(BaseModel):
    model_name: str
    description: str
    selection_model: str = "internal"
    fallback: str
    available_models: List[AutoselectModelInfo]
    capabilities: Optional[List[str]] = None  # Capabilities for this autoselect configuration
    # Content classification flags
    nsfw: bool = False  # Model can handle NSFW content
    privacy: bool = False  # Model can handle privacy-sensitive content
    classify_nsfw: bool = False  # Enable NSFW classification for this autoselect
    classify_privacy: bool = False  # Enable privacy classification for this autoselect
    classify_semantic: bool = False  # Enable semantic classification for this autoselect
    # OpenRouter-style extended fields
    context_length: Optional[int] = None
    architecture: Optional[Dict] = None
    pricing: Optional[Dict] = None
    supported_parameters: Optional[List[str]] = None
    default_parameters: Optional[Dict] = None
    # Default settings for models in this autoselect
    default_rate_limit: Optional[float] = None
    default_max_request_tokens: Optional[int] = None
    default_max_tokens: Optional[int] = None  # Default upper bound on generated output tokens (max completion tokens)
    default_rate_limit_TPM: Optional[int] = None
    default_rate_limit_TPH: Optional[int] = None
    default_rate_limit_TPD: Optional[int] = None
    default_context_size: Optional[int] = None
    default_condense_context: Optional[int] = None
    default_condense_method: Optional[Union[str, List[str]]] = None
    default_error_cooldown: Optional[int] = None  # Default cooldown period in seconds after 3 consecutive failures (default: 300)
    # Tri-state feature overrides
    enable_nsfw_classification: Optional[bool] = None  # None = use global default
    enable_privacy_classification: Optional[bool] = None  # None = use global default
    enable_context_condensation: Optional[bool] = None  # None = use global default
    # Response caching control
    enable_response_cache: Optional[bool] = None  # Enable/disable response caching for this autoselect (None = use global default)
    enable_prompt_batching: Optional[bool] = None  # None = use global default
    enable_prompt_security: Optional[bool] = None  # None = use global default
    enable_context_lens: Optional[bool] = None  # None = use global default
    block_high_risk_prompts: Optional[bool] = None  # None = use global default

class ResponseCacheConfig(BaseModel):
    """Configuration for response caching with semantic deduplication"""
    enabled: bool = True
    backend: str = "memory"  # 'redis', 'sqlite', 'mysql', or 'memory'
    ttl: int = 600  # Default TTL in seconds (10 minutes)
    max_memory_cache: int = 1000  # Max items for memory cache
    # Redis configuration
    redis_host: str = "localhost"
    redis_port: int = 6379
    redis_db: int = 0
    redis_password: Optional[str] = None
    redis_key_prefix: str = "aisbf:response:"
    # SQLite configuration
    sqlite_path: str = "~/.aisbf/response_cache.db"
    # MySQL configuration
    mysql_host: str = "localhost"
    mysql_port: int = 3306
    mysql_user: str = "aisbf"
    mysql_password: str = ""
    mysql_database: str = "aisbf_response_cache"


class TorConfig(BaseModel):
    """Configuration for TOR hidden service"""
    enabled: bool = False
    control_port: int = 9051
    control_host: str = "127.0.0.1"
    control_password: Optional[str] = None
    hidden_service_dir: Optional[str] = None
    hidden_service_port: int = 80
    socks_port: int = 9050
    socks_host: str = "127.0.0.1"

class BatchingConfig(BaseModel):
    """Configuration for request batching"""
    enabled: bool = False
    window_ms: int = 100  # Batching window in milliseconds
    max_batch_size: int = 8  # Maximum number of requests per batch
    provider_settings: Optional[Dict[str, Dict]] = None  # Provider-specific settings


class ClientRateLimitConfig(BaseModel):
    """Per-category client rate limit thresholds."""
    requests_per_minute: int = 0   # 0 = unlimited
    requests_per_hour: int = 0     # 0 = unlimited

class ClientRateLimitingConfig(BaseModel):
    """Inbound client rate limiting to protect against request flooding."""
    enabled: bool = False
    api: ClientRateLimitConfig = ClientRateLimitConfig(requests_per_minute=60, requests_per_hour=1000)
    general: ClientRateLimitConfig = ClientRateLimitConfig(requests_per_minute=120, requests_per_hour=3000)

class AdaptiveRateLimitingConfig(BaseModel):
    """Configuration for adaptive rate limiting"""
    enabled: bool = True  # Enable adaptive rate limiting
    initial_rate_limit: float = 0.0  # Initial rate limit in seconds (0 = no rate limiting)
    learning_rate: float = 0.1  # How fast to learn from 429s (0.1 = 10% adjustment)
    headroom_percent: int = 10  # Percentage to stay below learned limit (10 = 10% headroom)
    recovery_rate: float = 0.05  # Rate of recovery after successful requests (0.05 = 5% per success)
    max_rate_limit: float = 60.0  # Maximum rate limit in seconds
    min_rate_limit: float = 0.1  # Minimum rate limit in seconds
    backoff_base: float = 2.0  # Base for exponential backoff
    jitter_factor: float = 0.25  # Jitter factor for backoff (0.25 = 25%)
    history_window: int = 3600  # History window in seconds (1 hour)
    consecutive_successes_for_recovery: int = 10  # Successes needed before recovery starts

class SignupConfig(BaseModel):
    """Configuration for user signup functionality"""
    enabled: bool = False
    require_email_verification: bool = True
    verification_token_expiry_hours: int = 24
    
class PaymentGatewayConfig(BaseModel):
    """Configuration for payment gateways"""
    enabled: bool = False
    public_key: Optional[str] = None
    secret_key: Optional[str] = None
    webhook_secret: Optional[str] = None
    api_url: Optional[str] = None
    wallet_address: Optional[str] = None
    minimum_amount: Optional[float] = None
    additional_config: Optional[Dict] = None

class CurrencyConfig(BaseModel):
    """Global currency configuration"""
    code: str = "USD"
    symbol: str = "$"
    decimal_places: int = 2
    position: str = "left"

class SMTPConfig(BaseModel):
    """Configuration for SMTP email sending"""
    enabled: bool = False
    host: str = "localhost"
    port: int = 587
    username: str = ""
    password: str = ""
    use_tls: bool = True
    use_ssl: bool = False
    from_email: str = ""
    from_name: str = "AISBF"

class OAuth2ProviderConfig(BaseModel):
    """Configuration for an OAuth2 provider"""
    enabled: bool = False
    client_id: str = ""
    client_secret: str = ""
    scopes: List[str] = []

class OAuth2Config(BaseModel):
    """Configuration for OAuth2 authentication providers"""
    google: Optional[OAuth2ProviderConfig] = None
    github: Optional[OAuth2ProviderConfig] = None

class AISBFConfig(BaseModel):
    """Global AISBF configuration from aisbf.json"""
    classify_nsfw: bool = False
    classify_privacy: bool = False
    classify_semantic: bool = False
    server: Optional[Dict] = None
    auth: Optional[Dict] = None
    mcp: Optional[Dict] = None
    dashboard: Optional[Dict] = None
    internal_model: Optional[Dict] = None
    tor: Optional[Dict] = None
    database: Optional[Dict] = None
    cache: Optional[Dict] = None
    response_cache: Optional[ResponseCacheConfig] = None
    batching: Optional[BatchingConfig] = None
    feature_controls: Optional[FeatureControlsConfig] = None
    adaptive_rate_limiting: Optional[AdaptiveRateLimitingConfig] = None
    client_rate_limiting: Optional[ClientRateLimitingConfig] = None
    signup: Optional[SignupConfig] = None
    smtp: Optional[SMTPConfig] = None
    oauth2: Optional[OAuth2Config] = None
    currency: Optional[CurrencyConfig] = None
    payment_gateways: Optional[Dict[str, PaymentGatewayConfig]] = None


class AppConfig(BaseModel):
    providers: Dict[str, ProviderConfig]
    rotations: Dict[str, RotationConfig]
    autoselect: Dict[str, AutoselectConfig]
    condensation: Optional[CondensationConfig] = None
    error_tracking: Dict[str, Dict]
    tor: Optional[TorConfig] = None
    aisbf: Optional[AISBFConfig] = None  # Global AISBF config

class Config:
    def __init__(self):
        self._custom_config_dir = None
        # Check for custom config directory from environment variable
        custom_dir = os.environ.get('AISBF_CONFIG_DIR')
        if custom_dir:
            self._custom_config_dir = Path(custom_dir)
        
        # Track loaded file paths for summary
        self._loaded_files = {}
        
        self._ensure_config_directory()
        self._load_providers()
        self._load_rotations()
        self._load_condensation()
        self._load_tor()
        self._load_aisbf_config()
        self._load_autoselect()  # Load autoselect after aisbf config so cache is available
        self._initialize_error_tracking()
        self._log_configuration_summary()

    def reload(self):
        """Reload all configuration files from disk"""
        import logging
        logger = logging.getLogger(__name__)
        logger.info("=== Config.reload() START ===")

        # Clear existing config
        self.providers.clear()
        self.rotations.clear()
        self.autoselect.clear()
        self.error_tracking.clear()
        self._loaded_files.clear()

        # Re-load everything
        self._load_providers()
        self._load_rotations()
        self._load_condensation()
        self._load_tor()
        self._load_aisbf_config()
        self._load_autoselect()
        self._initialize_error_tracking()
        self._log_configuration_summary()

        logger.info("=== Config.reload() END ===")

    def _get_config_source_dir(self):
        """Get the directory containing default config files"""
        # If custom config directory is set, use it first
        if self._custom_config_dir and self._custom_config_dir.exists():
            if (self._custom_config_dir / 'providers.json').exists():
                return self._custom_config_dir
        
        # Try installed locations in order of preference.
        # Check both share_dir/config/ (current layout) and share_dir/ (legacy).
        share_roots = [
            Path.home() / '.local' / 'share' / 'aisbf',
            Path('/usr/local/share/aisbf'),
            Path('/usr/share/aisbf'),
        ]

        for root in share_roots:
            for candidate in (root / 'config', root):
                if candidate.exists() and (candidate / 'providers.json').exists():
                    return candidate
        
        # Fallback to source tree config directory
        # This is for development mode
        source_dir = Path(__file__).parent.parent / 'config'
        if source_dir.exists() and (source_dir / 'providers.json').exists():
            return source_dir
        
        # Last resort: try the old location in the package directory
        package_dir = Path(__file__).parent
        if (package_dir / 'providers.json').exists():
            return package_dir
        
        raise FileNotFoundError("Could not find configuration files")

    def _get_aisbf_config_path(self) -> Path:
        """Resolve the active aisbf.json path consistently across startup and reloads."""
        candidates = []
        if self._custom_config_dir and self._custom_config_dir.exists():
            candidates.append(self._custom_config_dir / 'aisbf.json')

        candidates += [
            Path.home() / '.aisbf' / 'aisbf.json',
            Path.home() / '.local' / 'share' / 'aisbf' / 'aisbf.json',
            Path('/usr/local/share/aisbf/aisbf.json'),
            Path('/usr/share/aisbf/aisbf.json'),
            Path(__file__).parent.parent / 'config' / 'aisbf.json',
        ]

        for candidate in candidates:
            if candidate.exists():
                return candidate

        return candidates[-1]

    def _ensure_config_directory(self):
        """Ensure ~/.aisbf/ directory exists and copy default config files if needed"""
        config_dir = Path.home() / '.aisbf'
        
        # Create config directory if it doesn't exist
        config_dir.mkdir(exist_ok=True)
        
        # Get the source directory for default config files
        try:
            source_dir = self._get_config_source_dir()
        except FileNotFoundError:
            print("Warning: Could not find default configuration files")
            return
        
        # Copy default config files if they don't exist
        for config_file in ['providers.json', 'rotations.json', 'autoselect.json', 'aisbf.json']:
            src = source_dir / config_file
            dst = config_dir / config_file
            
            if not dst.exists() and src.exists():
                shutil.copy2(src, dst)
                print(f"Created default config file: {dst}")
        
        # Copy markdown prompt files if they don't exist
        for prompt_file in ['condensation_conversational.md', 'condensation_semantic.md', 'autoselect.md', 'STUDIO_SYSTEM.md']:
            src = source_dir / prompt_file
            dst = config_dir / prompt_file
            
            if not dst.exists() and src.exists():
                shutil.copy2(src, dst)
                print(f"Created default prompt file: {dst}")

    def _load_providers(self):
        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"=== Config._load_providers START ===")
        
        providers_path = Path.home() / '.aisbf' / 'providers.json'
        logger.info(f"Looking for providers at: {providers_path}")
        
        if not providers_path.exists():
            logger.info(f"User config not found, falling back to source config")
            # Fallback to source config if user config doesn't exist
            try:
                source_dir = self._get_config_source_dir()
                providers_path = source_dir / 'providers.json'
                logger.info(f"Using source config at: {providers_path}")
            except FileNotFoundError:
                logger.error("Could not find providers.json configuration file")
                raise FileNotFoundError("Could not find providers.json configuration file")
        
        logger.info(f"Loading providers from: {providers_path}")
        try:
            with open(providers_path) as f:
                data = json.load(f)
                
                # Validate JSON structure
                if not data or 'providers' not in data:
                    logger.error(f"Invalid providers.json: missing 'providers' key")
                    raise ValueError("Invalid providers.json: missing 'providers' key")
                
                if not isinstance(data['providers'], dict):
                    logger.error(f"Invalid providers.json: 'providers' must be a dictionary")
                    raise ValueError("Invalid providers.json: 'providers' must be a dictionary")
                
                self.providers = {k: ProviderConfig(**v) for k, v in data['providers'].items()}
                self._loaded_files['providers'] = str(providers_path.absolute())
                logger.info(f"Loaded {len(self.providers)} providers: {list(self.providers.keys())}")
                for provider_id, provider_config in self.providers.items():
                    logger.info(f"  - {provider_id}: type={provider_config.type}, endpoint={provider_config.endpoint}")
        except json.JSONDecodeError as e:
            logger.error(f"Failed to parse providers.json: {e}")
            raise ValueError(f"Invalid JSON in providers.json: {e}")
        except Exception as e:
            logger.error(f"Failed to load providers.json: {e}")
            raise
        
        logger.info(f"=== Config._load_providers END ===")

    def _load_rotations(self):
        import logging
        import time
        logger = logging.getLogger(__name__)
        logger.info(f"=== Config._load_rotations START ===")
        
        rotations_path = Path.home() / '.aisbf' / 'rotations.json'
        logger.info(f"Looking for rotations at: {rotations_path}")
        
        if not rotations_path.exists():
            logger.info(f"User config not found, falling back to source config")
            # Fallback to source config if user config doesn't exist
            try:
                source_dir = self._get_config_source_dir()
                rotations_path = source_dir / 'rotations.json'
                logger.info(f"Using source config at: {rotations_path}")
            except FileNotFoundError:
                logger.error("Could not find rotations.json configuration file")
                raise FileNotFoundError("Could not find rotations.json configuration file")
        
        logger.info(f"Loading rotations from: {rotations_path}")
        self.rotations = {}
        try:
            with open(rotations_path) as f:
                data = json.load(f)
                
                # Validate JSON structure
                if not data or 'rotations' not in data:
                    logger.error(f"Invalid rotations.json: missing 'rotations' key")
                    raise ValueError("Invalid rotations.json: missing 'rotations' key")
                
                if not isinstance(data['rotations'], dict):
                    logger.error(f"Invalid rotations.json: 'rotations' must be a dictionary")
                    raise ValueError("Invalid rotations.json: 'rotations' must be a dictionary")
                
                self._loaded_files['rotations'] = str(rotations_path.absolute())
                
                # Extract global notifyerrors setting (top-level, outside rotations)
                self.global_notifyerrors = data.get('notifyerrors', False)
                logger.info(f"Global notifyerrors setting: {self.global_notifyerrors}")

                # Extract global notify_on_failover setting (top-level, outside rotations)
                self.global_notify_on_failover = data.get('notify_on_failover', False)
                logger.info(f"Global notify_on_failover setting: {self.global_notify_on_failover}")

                # Load rotations, merging global notifyerrors with rotation-specific settings
                self.rotations = {}
                for k, v in data['rotations'].items():
                    # If rotation doesn't have its own notifyerrors, use global setting
                    if 'notifyerrors' not in v:
                        v['notifyerrors'] = self.global_notifyerrors
                        logger.info(f"Rotation '{k}' using global notifyerrors: {self.global_notifyerrors}")
                    else:
                        logger.info(f"Rotation '{k}' has own notifyerrors: {v['notifyerrors']}")
                    # If rotation doesn't have its own notify_on_failover, use global setting
                    if 'notify_on_failover' not in v:
                        v['notify_on_failover'] = self.global_notify_on_failover
                        logger.info(f"Rotation '{k}' using global notify_on_failover: {self.global_notify_on_failover}")
                    else:
                        logger.info(f"Rotation '{k}' has own notify_on_failover: {v['notify_on_failover']}")
                    self.rotations[k] = RotationConfig(**v)
                
                logger.info(f"Loaded {len(self.rotations)} rotations: {list(self.rotations.keys())}")
                
                # Validate that all providers referenced in rotations exist
                logger.info(f"=== VALIDATING ROTATION PROVIDERS ===")
                available_providers = list(self.providers.keys())
                logger.info(f"Available providers: {available_providers}")
                
                server_mode = os.environ.get('AISBF_SERVER_MODE')
                warned = set()
                for rotation_id, rotation_config in self.rotations.items():
                    logger.info(f"Validating rotation: {rotation_id}")
                    for provider in rotation_config.providers:
                        provider_id = provider['provider_id']
                        if provider_id not in self.providers:
                            if server_mode and (rotation_id, provider_id) not in warned:
                                warned.add((rotation_id, provider_id))
                                logger.warning(f"!!! CONFIGURATION WARNING !!!")
                                logger.warning(f"Rotation '{rotation_id}' references provider '{provider_id}' which is NOT defined in providers.json")
                                logger.warning(f"Available providers: {available_providers}")
                                logger.warning(f"This provider will be SKIPPED during rotation requests")
                                logger.warning(f"Please add the provider to providers.json or remove it from the rotation configuration")
                                logger.warning(f"!!! END WARNING !!!")
                        else:
                            logger.info(f"  ✓ Provider '{provider_id}' is available")
        except json.JSONDecodeError as e:
            logger.error(f"Failed to parse rotations.json: {e}")
            if rotations_path != Path.home() / '.aisbf' / 'rotations.json':
                # Not user config, just raise
                raise ValueError(f"Invalid JSON in rotations.json: {e}")
            
            # This is the user's config file - we need to recover
            logger.warning("⚠️  Detected corrupted user config file!")
            
            # Backup corrupted file
            backup_path = rotations_path.with_suffix(f".json.corrupted.{int(time.time())}")
            shutil.copy2(rotations_path, backup_path)
            logger.warning(f"📋  Corrupted config backed up to: {backup_path}")
            
            # Copy fresh default config
            try:
                source_dir = self._get_config_source_dir()
                src = source_dir / 'rotations.json'
                shutil.copy2(src, rotations_path)
                logger.warning(f"✅  Replaced corrupted rotations.json with default valid file")
                logger.warning("⚠️  Your previous settings were backed up - you can restore them from the backup file")
                
                # Load the fresh config now
                # Don't call recursively - Uvicorn reloader will restart after file change
                pass
                
            except Exception as backup_err:
                logger.error(f"Failed to recover corrupted config: {backup_err}")
                raise ValueError(f"Invalid JSON in rotations.json: {e}")
                
        except Exception as e:
            logger.error(f"Failed to load rotations.json: {e}")
            raise
        
        logger.info(f"=== Config._load_rotations END ===")

    def _load_autoselect(self):
        """Load autoselect configuration and build model embeddings for semantic matching."""
        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"=== Config._load_autoselect START ===")
        
        autoselect_path = Path.home() / '.aisbf' / 'autoselect.json'
        logger.info(f"Looking for autoselect at: {autoselect_path}")
        
        if not autoselect_path.exists():
            logger.info(f"User config not found, falling back to source config")
            # Fallback to source config if user config doesn't exist
            try:
                source_dir = self._get_config_source_dir()
                autoselect_path = source_dir / 'autoselect.json'
                logger.info(f"Using source config at: {autoselect_path}")
            except FileNotFoundError:
                logger.error("Could not find autoselect.json configuration file")
                raise FileNotFoundError("Could not find autoselect.json configuration file")
        
        logger.info(f"Loading autoselect from: {autoselect_path}")
        try:
            with open(autoselect_path) as f:
                data = json.load(f)
                
                # Validate JSON structure
                if not data:
                    logger.error(f"Invalid autoselect.json: empty file")
                    raise ValueError("Invalid autoselect.json: empty file")
                
                self.autoselect = {}
                for k, v in data.items():
                    # Strip available_models entries with empty/invalid model_id before validation
                    if 'available_models' in v:
                        valid = [m for m in v['available_models'] if m.get('model_id', '').strip()]
                        skipped = len(v['available_models']) - len(valid)
                        if skipped:
                            logger.warning(f"Autoselect '{k}': skipped {skipped} available_model(s) with empty model_id")
                        v = dict(v, available_models=valid)
                    # Default selection_model to "internal" when blank
                    if not v.get('selection_model', '').strip():
                        v = dict(v, selection_model='internal')
                    self.autoselect[k] = AutoselectConfig(**v)
                self._loaded_files['autoselect'] = str(autoselect_path.absolute())
                logger.info(f"Loaded {len(self.autoselect)} autoselect configurations: {list(self.autoselect.keys())}")
                
                # Build and cache model embeddings for semantic matching
                self._build_model_embeddings()
        except json.JSONDecodeError as e:
            logger.error(f"Failed to parse autoselect.json: {e}")
            raise ValueError(f"Invalid JSON in autoselect.json: {e}")
        except Exception as e:
            logger.error(f"Failed to load autoselect.json: {e}")
            raise
            
        logger.info(f"=== Config._load_autoselect END ===")
    
    def _build_model_embeddings(self):
        """
        Build and cache vectorized versions of model descriptions for semantic matching.
        Uses the configured cache backend (Redis, file, or memory).
        """
        import logging
        logger = logging.getLogger(__name__)

        # Collect all model descriptions from all autoselect configs
        model_library = {}
        for autoselect_id, autoselect_config in self.autoselect.items():
            for model_info in autoselect_config.available_models:
                model_library[model_info.model_id] = model_info.description

        if not model_library:
            logger.info("No models to vectorize")
            self._model_embeddings = None
            self._model_embeddings_meta = []
            return

        # Get cache manager
        from .cache import get_cache_manager
        cache_config = self.aisbf.cache if self.aisbf and self.aisbf.cache else None
        cache_manager = get_cache_manager(cache_config)

        # Cache key for embeddings
        embeddings_key = "model_embeddings"

        # Check if embeddings exist in cache and are up-to-date
        rebuild_needed = True
        cached_meta = cache_manager.get(f"{embeddings_key}_meta")

        if cached_meta and cached_meta == list(model_library.keys()):
            # Try to load from numpy file cache (always file-based for large arrays)
            embeddings, _ = cache_manager.load_numpy_array(embeddings_key)
            if embeddings is not None:
                logger.info(f"Loading cached model embeddings from cache")
                self._model_embeddings = embeddings
                self._model_embeddings_meta = cached_meta
                rebuild_needed = False
                logger.info(f"Loaded {len(self._model_embeddings)} model embeddings")
            else:
                logger.warning("Cached embeddings metadata exists but array not found, rebuilding")

        if rebuild_needed:
            logger.info(f"Building model embeddings for {len(model_library)} models...")

            try:
                from sentence_transformers import SentenceTransformer
                import numpy as np

                # Use CPU-friendly model from config
                model_id = "sentence-transformers/all-MiniLM-L6-v2"

                # Check if custom model is configured in aisbf.json
                if self.aisbf and self.aisbf.internal_model:
                    custom_model = self.aisbf.internal_model.get('semantic_vectorization')
                    if custom_model:
                        model_id = custom_model

                logger.info(f"Using embedding model: {model_id}")
                embedder = SentenceTransformer(model_id)

                names = list(model_library.keys())
                descriptions = list(model_library.values())

                logger.info(f"Vectorizing {len(names)} model descriptions on CPU...")
                embeddings = embedder.encode(descriptions, show_progress_bar=True)

                # Save to numpy file cache
                cache_manager.save_numpy_array(embeddings_key, embeddings)

                # Save metadata to cache
                cache_manager.set(f"{embeddings_key}_meta", names)

                self._model_embeddings = embeddings
                self._model_embeddings_meta = names

                logger.info(f"Saved embeddings to cache")
                logger.info(f"Embedding shape: {embeddings.shape}")

            except ImportError as e:
                logger.warning(f"sentence-transformers not installed, skipping embeddings: {e}")
                self._model_embeddings = None
                self._model_embeddings_meta = []
            except Exception as e:
                logger.warning(f"Failed to build model embeddings: {e}")
                self._model_embeddings = None
                self._model_embeddings_meta = []
    
    def find_similar_models(self, query: str, top_k: int = 3) -> List[str]:
        """
        Find the most similar models to a query based on embeddings.
        
        Args:
            query: The user query/description to match against
            top_k: Number of top matches to return
            
        Returns:
            List of model_ids sorted by similarity (best match first)
        """
        import logging
        logger = logging.getLogger(__name__)
        
        if self._model_embeddings is None or self._model_embeddings_meta is None:
            logger.debug("No embeddings available, returning empty list")
            return []
        
        try:
            from sentence_transformers import SentenceTransformer
            import numpy as np
            
            # Load embedder
            model_id = "sentence-transformers/all-MiniLM-L6-v2"
            if self.aisbf and self.aisbf.internal_model:
                custom_model = self.aisbf.internal_model.get('semantic_vectorization')
                if custom_model:
                    model_id = custom_model
            
            embedder = SentenceTransformer(model_id)
            
            # Encode query
            query_embedding = embedder.encode([query])
            
            # Calculate cosine similarity
            query_norm = query_embedding / np.linalg.norm(query_embedding, axis=1, keepdims=True)
            embeddings_norm = self._model_embeddings / np.linalg.norm(self._model_embeddings, axis=1, keepdims=True)
            
            # Compute similarities
            similarities = np.dot(query_norm, embeddings_norm.T)[0]
            
            # Get top_k indices
            top_indices = np.argsort(similarities)[::-1][:top_k]
            
            # Return model_ids in order of similarity
            results = [self._model_embeddings_meta[i] for i in top_indices]
            logger.debug(f"Found similar models: {results}")
            
            return results
            
        except Exception as e:
            logger.warning(f"Error finding similar models: {e}")
            return []
    
    def _load_condensation(self):
        """Load condensation configuration from providers.json"""
        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"=== Config._load_condensation START ===")
        
        providers_path = Path.home() / '.aisbf' / 'providers.json'
        logger.info(f"Looking for condensation config in: {providers_path}")
        
        if not providers_path.exists():
            logger.info(f"User config not found, falling back to source config")
            # Fallback to source config if user config doesn't exist
            try:
                source_dir = self._get_config_source_dir()
                providers_path = source_dir / 'providers.json'
                logger.info(f"Using source config at: {providers_path}")
            except FileNotFoundError:
                logger.warning("Could not find providers.json for condensation config")
                self.condensation = CondensationConfig()
                return
        
        logger.info(f"Loading condensation config from: {providers_path}")
        with open(providers_path) as f:
            data = json.load(f)
            condensation_data = data.get('condensation', {})
            self.condensation = CondensationConfig(**condensation_data)
            self._loaded_files['condensation'] = str(providers_path.absolute())
            logger.info(f"Loaded condensation config: provider_id={self.condensation.provider_id}, model={self.condensation.model}, enabled={self.condensation.enabled}")
            logger.info(f"=== Config._load_condensation END ===")
    
    def _load_tor(self):
        """Load TOR configuration from aisbf.json"""
        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"=== Config._load_tor START ===")
        
        aisbf_path = Path.home() / '.aisbf' / 'aisbf.json'
        logger.info(f"Looking for TOR config in: {aisbf_path}")
        
        if not aisbf_path.exists():
            logger.info(f"User config not found, falling back to source config")
            # Fallback to source config if user config doesn't exist
            try:
                source_dir = self._get_config_source_dir()
                aisbf_path = source_dir / 'aisbf.json'
                logger.info(f"Using source config at: {aisbf_path}")
            except FileNotFoundError:
                logger.warning("Could not find aisbf.json for TOR config")
                self.tor = TorConfig()
                return
        
        logger.info(f"Loading TOR config from: {aisbf_path}")
        with open(aisbf_path) as f:
            data = json.load(f)
            tor_data = data.get('tor', {})
            self.tor = TorConfig(**tor_data)
            self._loaded_files['tor'] = str(aisbf_path.absolute())
            logger.info(f"Loaded TOR config: enabled={self.tor.enabled}, control_port={self.tor.control_port}, hidden_service_port={self.tor.hidden_service_port}")
            logger.info(f"=== Config._load_tor END ===")
    
    def _load_aisbf_config(self):
        """Load global AISBF configuration from aisbf.json"""
        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"=== Config._load_aisbf_config START ===")
        
        aisbf_path = self._get_aisbf_config_path()
        logger.info(f"Looking for AISBF config in: {aisbf_path}")
        
        if not aisbf_path.exists():
            logger.info(f"User config not found, falling back to source config")
            try:
                source_dir = self._get_config_source_dir()
                aisbf_path = source_dir / 'aisbf.json'
                logger.info(f"Using source config at: {aisbf_path}")
            except FileNotFoundError:
                logger.warning("Could not find aisbf.json for AISBF config")
                self.aisbf = AISBFConfig()
                return
        
        logger.info(f"Loading AISBF config from: {aisbf_path}")
        with open(aisbf_path) as f:
            data = json.load(f)
            # Parse response_cache separately if present
            response_cache_data = data.get('response_cache')
            if response_cache_data:
                data['response_cache'] = ResponseCacheConfig(**response_cache_data)
            # Parse batching separately if present
            batching_data = data.get('batching')
            if batching_data:
                data['batching'] = BatchingConfig(**batching_data)
            feature_controls_data = data.get('feature_controls')
            if feature_controls_data:
                data['feature_controls'] = FeatureControlsConfig(**feature_controls_data)
            # Parse adaptive_rate_limiting separately if present
            adaptive_data = data.get('adaptive_rate_limiting')
            if adaptive_data:
                data['adaptive_rate_limiting'] = AdaptiveRateLimitingConfig(**adaptive_data)
            # Parse client_rate_limiting separately if present
            client_rl_data = data.get('client_rate_limiting')
            if client_rl_data:
                api_data = client_rl_data.get('api', {})
                general_data = client_rl_data.get('general', {})
                data['client_rate_limiting'] = ClientRateLimitingConfig(
                    enabled=client_rl_data.get('enabled', False),
                    api=ClientRateLimitConfig(**api_data) if api_data else ClientRateLimitConfig(),
                    general=ClientRateLimitConfig(**general_data) if general_data else ClientRateLimitConfig()
                )
            # Parse signup separately if present
            signup_data = data.get('signup')
            if signup_data:
                data['signup'] = SignupConfig(**signup_data)
            # Parse smtp separately if present
            smtp_data = data.get('smtp')
            if smtp_data:
                data['smtp'] = SMTPConfig(**smtp_data)
            # Parse currency separately if present
            currency_data = data.get('currency')
            if currency_data:
                data['currency'] = CurrencyConfig(**currency_data)
            # Parse payment gateways separately if present
            payment_gateways_data = data.get('payment_gateways')
            if payment_gateways_data:
                data['payment_gateways'] = {k: PaymentGatewayConfig(**v) for k, v in payment_gateways_data.items()}
            self.aisbf = AISBFConfig(**data)
            self._loaded_files['aisbf'] = str(aisbf_path.absolute())
            logger.info(f"Loaded AISBF config: classify_nsfw={self.aisbf.classify_nsfw}, classify_privacy={self.aisbf.classify_privacy}")
            if self.aisbf.response_cache:
                logger.info(f"Response cache config: enabled={self.aisbf.response_cache.enabled}, backend={self.aisbf.response_cache.backend}, ttl={self.aisbf.response_cache.ttl}")
            if self.aisbf.batching:
                logger.info(f"Batching config: enabled={self.aisbf.batching.enabled}, window_ms={self.aisbf.batching.window_ms}, max_batch_size={self.aisbf.batching.max_batch_size}")
            if self.aisbf.adaptive_rate_limiting:
                logger.info(f"Adaptive rate limiting: enabled={self.aisbf.adaptive_rate_limiting.enabled}, initial_rate_limit={self.aisbf.adaptive_rate_limiting.initial_rate_limit}")
            if self.aisbf.signup:
                logger.info(f"Signup config: enabled={self.aisbf.signup.enabled}, require_email_verification={self.aisbf.signup.require_email_verification}")
            if self.aisbf.smtp:
                logger.info(f"SMTP config: host={self.aisbf.smtp.host}, port={self.aisbf.smtp.port}, from_email={self.aisbf.smtp.from_email}")
            logger.info(f"=== Config._load_aisbf_config END ===")

    def _initialize_error_tracking(self):
        self.error_tracking = {}
        for provider_id in self.providers:
            self.error_tracking[provider_id] = {
                'failures': 0,
                'last_failure': None,
                'disabled_until': None
            }
    
    def _log_configuration_summary(self):
        """Log a summary of all loaded configuration files"""
        import logging
        logger = logging.getLogger(__name__)
        
        logger.info("")
        logger.info("=" * 80)
        logger.info("=== CONFIGURATION FILES LOADED ===")
        logger.info("=" * 80)
        
        if 'providers' in self._loaded_files:
            logger.info(f"Providers:    {self._loaded_files['providers']}")
        
        if 'rotations' in self._loaded_files:
            logger.info(f"Rotations:    {self._loaded_files['rotations']}")
        
        if 'autoselect' in self._loaded_files:
            logger.info(f"Autoselect:   {self._loaded_files['autoselect']}")
        
        if 'condensation' in self._loaded_files:
            logger.info(f"Condensation: {self._loaded_files['condensation']}")
        
        logger.info("=" * 80)
        logger.info("")

    def get_provider(self, provider_id: str, warn: bool = True) -> ProviderConfig:
        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"Config.get_provider called with provider_id: {provider_id}")
        logger.info(f"Available providers: {list(self.providers.keys())}")
        result = self.providers.get(provider_id)
        if result:
            logger.info(f"Found provider: {result}")
        else:
            if warn:
                logger.warning(f"Provider {provider_id} not found!")
        return result

    def get_rotation(self, rotation_id: str) -> RotationConfig:
        return self.rotations.get(rotation_id)

    def get_autoselect(self, autoselect_id: str) -> AutoselectConfig:
        return self.autoselect.get(autoselect_id)
    
    def get_condensation(self) -> CondensationConfig:
        return self.condensation
    
    def get_tor(self) -> TorConfig:
        return self.tor
    
    def get_aisbf_config(self) -> AISBFConfig:
        return self.aisbf

    @staticmethod
    def _feature_mode_to_bool(value: Any) -> Optional[bool]:
        if value is None:
            return None
        if isinstance(value, dict):
            value = value.get('mode')
        else:
            value = getattr(value, 'mode', value)
        if value in ('enabled', True):
            return True
        if value in ('disabled', False):
            return False
        return None

    def resolve_feature_enabled(
        self,
        feature_name: str,
        *,
        model_config: Any = None,
        provider_config: Any = None,
        rotation_config: Any = None,
        autoselect_config: Any = None,
        global_default: Optional[bool] = None,
    ) -> bool:
        field_name = f'enable_{feature_name}'
        if feature_name == 'block_high_risk_prompts':
            field_name = 'block_high_risk_prompts'

        def _value_from(scope: Any) -> Optional[bool]:
            if scope is None:
                return None
            if isinstance(scope, dict):
                return scope.get(field_name)
            return getattr(scope, field_name, None)

        for scope in (model_config, provider_config, rotation_config, autoselect_config):
            resolved = _value_from(scope)
            if resolved is not None:
                return bool(resolved)

        aisbf_config = self.get_aisbf_config()
        feature_controls = getattr(aisbf_config, 'feature_controls', None) if aisbf_config else None
        if feature_controls is not None:
            control = getattr(feature_controls, feature_name, None)
            resolved = self._feature_mode_to_bool(control)
            if resolved is not None:
                return resolved

            prompt_security = getattr(feature_controls, 'prompt_security', None)
            if feature_name == 'prompt_security' and prompt_security is not None:
                resolved = self._feature_mode_to_bool(getattr(prompt_security, 'security_scan', None))
                if resolved is not None:
                    return resolved
            if prompt_security is not None and hasattr(prompt_security, feature_name):
                resolved = self._feature_mode_to_bool(getattr(prompt_security, feature_name, None))
                if resolved is not None:
                    return resolved

        if feature_name == 'response_cache' and aisbf_config and aisbf_config.response_cache:
            return bool(aisbf_config.response_cache.enabled)
        if feature_name == 'prompt_batching' and aisbf_config and aisbf_config.batching:
            return bool(aisbf_config.batching.enabled)
        if feature_name == 'context_condensation':
            condensation = self.get_condensation()
            if condensation is not None:
                return bool(condensation.enabled)

        return bool(global_default) if global_default is not None else False

config = Config()

def get_config():
    return config
