"""
REST API Client - Configurable HTTP client with retry logic and failure handling
"""

import time
import logging
import json
import threading
import os
from pathlib import Path
from datetime import datetime, timedelta
from typing import Dict, Any, Optional, List, Union
from urllib.parse import urljoin, urlparse
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

from ..core.thread_manager import ThreadedComponent
from ..core.message_bus import MessageBus, Message, MessageType, MessageBuilder
from ..config.settings import ApiConfig
from ..config.manager import ConfigManager
from ..database.manager import DatabaseManager
from ..database.models import MatchModel, MatchOutcomeModel
from ..config.settings import get_user_data_dir

logger = logging.getLogger(__name__)


class APIEndpoint:
    """Configuration for a single API endpoint"""
    
    def __init__(self, name: str, config: Dict[str, Any]):
        self.name = name
        self.url = config.get('url', '')
        self.method = config.get('method', 'GET').upper()
        self.headers = config.get('headers', {})
        self.params = config.get('params', {})
        self.data = config.get('data', {})
        self.interval = config.get('interval', 300)  # 5 minutes default
        self.enabled = config.get('enabled', True)
        self.timeout = config.get('timeout', 30)
        self.retry_attempts = config.get('retry_attempts', 3)
        self.retry_delay = config.get('retry_delay', 5)
        self.auth = config.get('auth', None)
        self.response_handler = config.get('response_handler', 'default')
        
        # Runtime state
        self.last_request = None
        self.last_success = None
        self.last_error = None
        self.consecutive_failures = 0
        self.total_requests = 0
        self.successful_requests = 0
    
    def should_execute(self) -> bool:
        """Check if endpoint should be executed based on interval"""
        if not self.enabled:
            return False
        
        if self.last_request is None:
            return True
        
        next_execution = self.last_request + timedelta(seconds=self.interval)
        return datetime.utcnow() >= next_execution
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization"""
        return {
            'name': self.name,
            'url': self.url,
            'method': self.method,
            'headers': self.headers,
            'params': self.params,
            'data': self.data,
            'interval': self.interval,
            'enabled': self.enabled,
            'timeout': self.timeout,
            'retry_attempts': self.retry_attempts,
            'retry_delay': self.retry_delay,
            'auth': self.auth,
            'response_handler': self.response_handler,
            'last_request': self.last_request.isoformat() if self.last_request else None,
            'last_success': self.last_success.isoformat() if self.last_success else None,
            'last_error': self.last_error,
            'consecutive_failures': self.consecutive_failures,
            'total_requests': self.total_requests,
            'successful_requests': self.successful_requests
        }


class ResponseHandler:
    """Base class for response handlers"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        """Handle API response and return processed data"""
        raise NotImplementedError
    
    def handle_error(self, endpoint: APIEndpoint, error: Exception) -> Optional[Dict[str, Any]]:
        """Handle API error and return error data"""
        return {
            'error': str(error),
            'endpoint': endpoint.name,
            'timestamp': datetime.utcnow().isoformat()
        }


class DefaultResponseHandler(ResponseHandler):
    """Default response handler - processes JSON responses"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            if response.headers.get('content-type', '').startswith('application/json'):
                return response.json()
            else:
                return {
                    'status_code': response.status_code,
                    'content': response.text,
                    'headers': dict(response.headers)
                }
        except json.JSONDecodeError:
            return {
                'status_code': response.status_code,
                'content': response.text,
                'headers': dict(response.headers)
            }


class NewsResponseHandler(ResponseHandler):
    """Response handler for news APIs - extracts headline and ticker data"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            # Extract news data for overlay templates
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'raw_data': data
            }
            
            # Try to extract common news fields
            if 'articles' in data:
                # NewsAPI format
                articles = data['articles'][:5]  # Top 5 articles
                processed_data['headlines'] = [article.get('title', '') for article in articles]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            elif 'items' in data:
                # RSS format
                items = data['items'][:5]
                processed_data['headlines'] = [item.get('title', '') for item in items]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            elif isinstance(data, list):
                # Direct array format
                processed_data['headlines'] = [item.get('title', '') for item in data[:5]]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            
            return processed_data
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.error(f"Failed to process news response: {e}")
            return self.handle_error(endpoint, e)


class SportsResponseHandler(ResponseHandler):
    """Response handler for sports APIs - extracts scores and game data"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'raw_data': data
            }
            
            # Extract sports data
            if 'games' in data:
                # Generic sports API format
                games = data['games'][:3]  # Top 3 games
                game_data = []
                for game in games:
                    game_data.append({
                        'team1': game.get('home_team', 'Home'),
                        'team2': game.get('away_team', 'Away'),
                        'score1': game.get('home_score', 0),
                        'score2': game.get('away_score', 0),
                        'status': game.get('status', 'Live')
                    })
                processed_data['games'] = game_data
            
            return processed_data
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.error(f"Failed to process sports response: {e}")
            return self.handle_error(endpoint, e)


class UpdatesResponseHandler(ResponseHandler):
    """Response handler for /api/updates endpoint - synchronizes match data"""
    
    def __init__(self, db_manager, user_data_dir, api_client=None):
        self.db_manager = db_manager
        self.user_data_dir = user_data_dir
        self.api_client = api_client  # Reference to parent API client for token access
        self.zip_storage_dir = Path(user_data_dir) / "zip_files"
        self.zip_storage_dir.mkdir(parents=True, exist_ok=True)
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'synchronized_matches': 0,
                'downloaded_zips': 0,
                'errors': []
            }
            
            # Extract fixtures from response - new API format has fixtures containing matches
            fixtures = []
            if isinstance(data, dict):
                if 'fixtures' in data:
                    fixtures = data.get('fixtures', [])
                elif 'matches' in data:
                    # Fallback for old format - treat as single fixture with matches
                    fixtures = [{'matches': data.get('matches', [])}]
            elif isinstance(data, list):
                # Direct array of matches - treat as single fixture
                fixtures = [{'matches': data}]
            
            if not fixtures:
                processed_data['message'] = "No fixtures in response"
                return processed_data
            
            session = self.db_manager.get_session()
            try:
                for fixture_data in fixtures:
                    matches = fixture_data.get('matches', [])
                    for match_data in matches:
                        try:
                            # Add fixture-level data to match if available
                            if 'fixture_id' in fixture_data:
                                match_data['fixture_id'] = fixture_data['fixture_id']
                            if 'fixture_active_time' in fixture_data:
                                match_data['fixture_active_time'] = fixture_data['fixture_active_time']
                            
                            # Synchronize match data to database
                            self._synchronize_match(session, match_data)
                            processed_data['synchronized_matches'] += 1
                            
                            # Download ZIP file if available (check match-level zip_download_url)
                            if 'zip_download_url' in match_data:
                                match_data['zip_url'] = match_data['zip_download_url']
                            if self._download_zip_file(match_data):
                                processed_data['downloaded_zips'] += 1
                                
                        except Exception as e:
                            error_msg = f"Failed to process match {match_data.get('match_number', 'unknown')} in fixture {fixture_data.get('fixture_id', 'unknown')}: {e}"
                            logger.error(error_msg)
                            processed_data['errors'].append(error_msg)
                            # Continue processing other matches even if this one fails
                            continue
                
                session.commit()
                
            finally:
                session.close()
            
            logger.info(f"Synchronized {processed_data['synchronized_matches']} matches, downloaded {processed_data['downloaded_zips']} ZIP files")
            return processed_data
            
        except Exception as e:
            logger.error(f"Failed to process updates response: {e}")
            return self.handle_error(endpoint, e)
    
    def _synchronize_match(self, session, match_data: Dict[str, Any]):
        """Synchronize match data to database"""
        try:
            match_number = match_data.get('match_number')
            fixture_id = match_data.get('fixture_id')
            
            if not match_number or not fixture_id:
                logger.warning(f"Skipping match with missing match_number ({match_number}) or fixture_id ({fixture_id})")
                return
            
            # Check if match already exists by both match_number AND fixture_id
            existing_match = session.query(MatchModel).filter_by(
                match_number=match_number,
                fixture_id=fixture_id
            ).first()
            
            if existing_match:
                # Update existing match
                for key, value in match_data.items():
                    if hasattr(existing_match, key) and key not in ['id', 'created_at']:
                        setattr(existing_match, key, value)
                existing_match.updated_at = datetime.utcnow()
                match = existing_match
                logger.debug(f"Updated existing match {match_number} for fixture {fixture_id}")
            else:
                # Create new match
                match = MatchModel(
                    match_number=match_data.get('match_number'),
                    fighter1_township=match_data.get('fighter1_township', ''),
                    fighter2_township=match_data.get('fighter2_township', ''),
                    venue_kampala_township=match_data.get('venue_kampala_township', ''),
                    start_time=self._parse_datetime(match_data.get('start_time')),
                    end_time=self._parse_datetime(match_data.get('end_time')),
                    result=match_data.get('result'),
                    filename=match_data.get('filename', ''),
                    file_sha1sum=match_data.get('file_sha1sum', ''),
                    fixture_id=fixture_id,
                    active_status=match_data.get('active_status', False),
                    zip_filename=match_data.get('zip_filename'),
                    zip_sha1sum=match_data.get('zip_sha1sum'),
                    zip_upload_status=match_data.get('zip_upload_status', 'pending'),
                    zip_upload_progress=match_data.get('zip_upload_progress', 0.0),
                    done=match_data.get('done', False),
                    running=match_data.get('running', False),
                    fixture_active_time=match_data.get('fixture_active_time')
                )
                session.add(match)
                logger.debug(f"Created new match {match_number} for fixture {fixture_id}")
            
            # Flush to get the match ID for outcomes
            session.flush()
            
            # Handle match outcomes
            outcomes_data = match_data.get('outcomes', {})
            if outcomes_data:
                # Remove existing outcomes
                session.query(MatchOutcomeModel).filter_by(match_id=match.id).delete()
                
                # Add new outcomes
                for column_name, float_value in outcomes_data.items():
                    outcome = MatchOutcomeModel(
                        match_id=match.id,
                        column_name=column_name,
                        float_value=float(float_value)
                    )
                    session.add(outcome)
                    
        except Exception as e:
            logger.error(f"Failed to synchronize match: {e}")
            # Rollback this session to clean state
            session.rollback()
            raise
    
    def _download_zip_file(self, match_data: Dict[str, Any]) -> bool:
        """Download ZIP file to persistent storage with API token authentication"""
        try:
            zip_url = match_data.get('zip_url')
            zip_filename = match_data.get('zip_filename')
            
            if not zip_url or not zip_filename:
                return False
            
            # Prepare headers with API token authentication
            headers = {}
            
            # Get API token from the API client's fastapi_main endpoint
            if self.api_client and hasattr(self.api_client, 'endpoints'):
                fastapi_endpoint = self.api_client.endpoints.get('fastapi_main')
                if fastapi_endpoint and fastapi_endpoint.auth:
                    if fastapi_endpoint.auth.get('type') == 'bearer':
                        token = fastapi_endpoint.auth.get('token')
                        if token:
                            headers['Authorization'] = f"Bearer {token}"
                            logger.debug(f"Using API token for ZIP download: {zip_filename}")
            
            if not headers:
                logger.warning(f"No API token available for ZIP download: {zip_filename}")
            
            # Download ZIP file with authentication
            response = requests.get(zip_url, stream=True, timeout=30, headers=headers)
            response.raise_for_status()
            
            # Save to persistent storage
            zip_path = self.zip_storage_dir / zip_filename
            
            with open(zip_path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    f.write(chunk)
            
            logger.info(f"Downloaded ZIP file: {zip_filename}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to download ZIP file: {e}")
            return False
    
    def _parse_datetime(self, datetime_str) -> Optional[datetime]:
        """Parse datetime string to datetime object"""
        if not datetime_str:
            return None
        
        try:
            # Try ISO format first
            return datetime.fromisoformat(datetime_str.replace('Z', '+00:00'))
        except:
            try:
                # Try common formats
                return datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S')
            except:
                return None


class APIClient(ThreadedComponent):
    """REST API Client component"""
    
    def __init__(self, message_bus: MessageBus, db_manager: DatabaseManager,
                 config_manager: ConfigManager, settings: ApiConfig):
        super().__init__("api_client", message_bus)
        self.db_manager = db_manager
        self.config_manager = config_manager
        self.settings = settings
        
        # HTTP session with retry logic
        self.session = requests.Session()
        self._setup_session()
        
        # API endpoints configuration
        self.endpoints: Dict[str, APIEndpoint] = {}
        self._load_endpoints()
        
        # Response handlers
        self.response_handlers = {
            'default': DefaultResponseHandler(),
            'news': NewsResponseHandler(),
            'sports': SportsResponseHandler(),
            'updates': UpdatesResponseHandler(self.db_manager, get_user_data_dir(), self)
        }
        
        # Statistics
        self.stats = {
            'total_requests': 0,
            'successful_requests': 0,
            'failed_requests': 0,
            'start_time': datetime.utcnow()
        }
        
        # Register message queue
        self.message_queue = self.message_bus.register_component(self.name)
        
        logger.info("APIClient initialized")
    
    def initialize(self) -> bool:
        """Initialize API client"""
        try:
            # Subscribe to messages
            self.message_bus.subscribe(self.name, MessageType.CONFIG_UPDATE, self._handle_config_update)
            self.message_bus.subscribe(self.name, MessageType.API_REQUEST, self._handle_api_request)
            
            logger.info("APIClient initialized successfully")
            return True
            
        except Exception as e:
            logger.error(f"APIClient initialization failed: {e}")
            return False
    
    def _setup_session(self):
        """Setup HTTP session with retry logic"""
        retry_strategy = Retry(
            total=self.settings.retry_attempts,
            backoff_factor=self.settings.retry_delay_seconds,
            status_forcelist=[429, 500, 502, 503, 504],
        )
        
        adapter = HTTPAdapter(max_retries=retry_strategy)
        self.session.mount("http://", adapter)
        self.session.mount("https://", adapter)
        
        # Set default headers
        self.session.headers.update({
            'User-Agent': f'MbetterClient/{self.settings.user_agent}',
            'Accept': 'application/json, text/plain, */*',
            'Accept-Encoding': 'gzip, deflate'
        })
        
        # Set timeout
        self.session.timeout = self.settings.timeout_seconds
    
    def _load_endpoints(self):
        """Load API endpoints from configuration"""
        try:
            endpoints_config = self.config_manager.get_section_config("api_endpoints")
            
            if not endpoints_config:
                # Create default configuration
                default_endpoints = self._get_default_endpoints()
                self.config_manager.update_section("api_endpoints", default_endpoints)
                endpoints_config = default_endpoints
            
            # Load endpoints
            for name, config in endpoints_config.items():
                self.endpoints[name] = APIEndpoint(name, config)
            
            logger.info(f"Loaded {len(self.endpoints)} API endpoints")
            
        except Exception as e:
            logger.error(f"Failed to load API endpoints: {e}")
    
    def _get_default_endpoints(self) -> Dict[str, Dict[str, Any]]:
        """Get default API endpoints configuration"""
        # Get FastAPI server URL, token, and interval from configuration
        fastapi_url = "https://mbetter.nexlab.net/api/updates"
        api_token = ""
        api_interval = 1800  # 30 minutes default
        try:
            api_config = self.config_manager.get_section_config("api") or {}
            fastapi_url = api_config.get("fastapi_url", fastapi_url)
            api_token = api_config.get("api_token", api_token)
            api_interval = api_config.get("api_interval", api_interval)
        except Exception as e:
            logger.warning(f"Could not load API configuration, using defaults: {e}")
        
        # Prepare authentication if token is provided
        auth_config = None
        headers = {"Content-Type": "application/json"}
        
        # Configure based on token availability
        if api_token and api_token.strip():
            auth_config = {
                "type": "bearer",
                "token": api_token.strip()
            }
            # When token is provided: use configurable interval, 10 retries every 30 seconds
            interval = api_interval
            retry_attempts = 10
            retry_delay = 30  # 30 seconds
            enabled = True
        else:
            # When no token: use configurable interval or default, fewer retries, disabled by default
            interval = api_interval if api_interval != 1800 else 600  # Use configured or 10 minutes default
            retry_attempts = 3
            retry_delay = 5
            enabled = False  # Disabled when no authentication
        
        return {
            "fastapi_main": {
                "url": fastapi_url,  # Use the exact URL provided, don't modify it
                "method": "POST",  # Use POST for /api/updates endpoint
                "headers": headers,
                "auth": auth_config,
                "interval": interval,
                "enabled": enabled,
                "timeout": 30,
                "retry_attempts": retry_attempts,
                "retry_delay": retry_delay,
                "response_handler": "updates"  # Use updates handler for match synchronization
            },
            "news_example": {
                "url": "https://newsapi.org/v2/top-headlines",
                "method": "GET",
                "headers": {
                    "X-API-Key": "your-api-key-here"
                },
                "params": {
                    "country": "us",
                    "pageSize": 5
                },
                "interval": 300,  # 5 minutes
                "enabled": False,  # Disabled by default until API key is configured
                "timeout": 30,
                "retry_attempts": 3,
                "retry_delay": 5,
                "response_handler": "news"
            },
            "sports_example": {
                "url": "https://api.sportsdata.io/v3/nfl/scores/json/LiveBoxScores",
                "method": "GET",
                "headers": {
                    "Ocp-Apim-Subscription-Key": "your-api-key-here"
                },
                "interval": 60,  # 1 minute during games
                "enabled": False,
                "timeout": 30,
                "retry_attempts": 3,
                "retry_delay": 5,
                "response_handler": "sports"
            },
            "weather_example": {
                "url": "https://api.openweathermap.org/data/2.5/weather",
                "method": "GET",
                "params": {
                    "q": "New York",
                    "appid": "your-api-key-here",
                    "units": "metric"
                },
                "interval": 1800,  # 30 minutes
                "enabled": False,
                "timeout": 15,
                "retry_attempts": 2,
                "retry_delay": 3,
                "response_handler": "default"
            }
        }
    
    def run(self):
        """Main run loop"""
        try:
            logger.info("APIClient thread started")
            
            # Send ready status
            ready_message = MessageBuilder.system_status(
                sender=self.name,
                status="ready",
                details={
                    "endpoints": len(self.endpoints),
                    "enabled_endpoints": sum(1 for ep in self.endpoints.values() if ep.enabled)
                }
            )
            self.message_bus.publish(ready_message)
            
            # Main execution loop
            while self.running:
                try:
                    # Process messages
                    message = self.message_bus.get_message(self.name, timeout=1.0)
                    if message:
                        self._process_message(message)
                    
                    # Execute scheduled API requests
                    self._execute_scheduled_requests()
                    
                    # Update heartbeat
                    self.heartbeat()
                    
                    time.sleep(1.0)
                    
                except Exception as e:
                    logger.error(f"APIClient run loop error: {e}")
                    time.sleep(5.0)
                    
        except Exception as e:
            logger.error(f"APIClient run failed: {e}")
        finally:
            logger.info("APIClient thread ended")
    
    def _execute_scheduled_requests(self):
        """Execute API requests that are due"""
        for endpoint in self.endpoints.values():
            if endpoint.should_execute():
                self._execute_endpoint_request(endpoint)
    
    def _get_last_fixture_timestamp(self) -> Optional[str]:
        """Get the server activation timestamp of the last active fixture in the database"""
        try:
            session = self.db_manager.get_session()
            try:
                # Get the most recent match with fixture_active_time set
                last_active_match = session.query(MatchModel).filter(
                    MatchModel.fixture_active_time.isnot(None)
                ).order_by(MatchModel.fixture_active_time.desc()).first()
                
                if last_active_match and last_active_match.fixture_active_time:
                    # Return Unix timestamp as string (long integer number)
                    return str(last_active_match.fixture_active_time)
                else:
                    # No fixtures with activation time found - don't send 'from' parameter
                    return None
                    
            finally:
                session.close()
                
        except Exception as e:
            logger.error(f"Failed to get last fixture activation timestamp: {e}")
            return None
    
    def _execute_endpoint_request(self, endpoint: APIEndpoint):
        """Execute a single API request with custom retry logic for token-based endpoints"""
        try:
            endpoint.last_request = datetime.utcnow()
            endpoint.total_requests += 1
            self.stats['total_requests'] += 1
            
            logger.debug(f"Executing API request: {endpoint.name} -> {endpoint.url}")
            
            # Prepare request parameters
            request_kwargs = {
                'method': endpoint.method,
                'url': endpoint.url,
                'headers': endpoint.headers.copy(),
                'timeout': endpoint.timeout
            }
            
            # Prepare data/params based on method
            request_data = endpoint.params.copy() if endpoint.method == 'GET' else endpoint.data.copy()
            
            # For FastAPI /api/updates endpoint, add 'from' parameter if we have fixtures
            if endpoint.name == 'fastapi_main' and 'updates' in endpoint.url.lower():
                last_timestamp = self._get_last_fixture_timestamp()
                if last_timestamp:
                    if endpoint.method == 'GET':
                        request_data['from'] = last_timestamp
                    else:
                        request_data['from'] = last_timestamp
                    
                    logger.debug(f"Adding 'from' parameter to {endpoint.name} request: {last_timestamp}")
                else:
                    # When no fixtures exist, send empty request to get all data
                    logger.debug(f"No fixtures found, sending empty request to {endpoint.name}")
            
            if endpoint.method == 'GET':
                request_kwargs['params'] = request_data
            else:
                # For POST requests, always send JSON data (even if empty)
                request_kwargs['json'] = request_data if request_data else {}
            
            # Debug log the complete request data
            logger.debug(f"Request method: {endpoint.method}")
            logger.debug(f"Request headers: {request_kwargs['headers']}")
            if endpoint.method == 'GET' and request_data:
                logger.debug(f"Request params: {request_data}")
            elif endpoint.method != 'GET' and request_data:
                logger.debug(f"Request JSON data: {request_data}")
            else:
                logger.debug("Request data: (empty)")
            
            # Add authentication if configured
            if endpoint.auth:
                if endpoint.auth.get('type') == 'bearer':
                    request_kwargs['headers']['Authorization'] = f"Bearer {endpoint.auth.get('token')}"
                elif endpoint.auth.get('type') == 'basic':
                    request_kwargs['auth'] = (endpoint.auth.get('username'), endpoint.auth.get('password'))
            
            # Generate curl command for easy debugging/replication (after auth is added)
            curl_cmd = self._generate_curl_command(endpoint.method, endpoint.url, request_kwargs['headers'], request_data)
            logger.debug(f"Curl equivalent: {curl_cmd}")
            
            # Check if this is a token-based endpoint that needs custom retry logic
            is_token_endpoint = (endpoint.auth and
                               endpoint.auth.get('type') == 'bearer' and
                               endpoint.auth.get('token'))
            
            if is_token_endpoint and endpoint.consecutive_failures > 0:
                # For token-based endpoints with failures, use custom retry logic
                success = self._execute_with_custom_retry(endpoint, request_kwargs)
                if not success:
                    return  # Custom retry logic handles error reporting
            else:
                # Standard execution for first attempt or non-token endpoints
                response = self.session.request(**request_kwargs)
                response.raise_for_status()
                
                # Debug log the complete response
                logger.debug(f"Response status code: {response.status_code}")
                logger.debug(f"Response headers: {dict(response.headers)}")
                try:
                    response_text = response.text
                    if len(response_text) > 1000:
                        logger.debug(f"Response body (truncated): {response_text[:1000]}...")
                    else:
                        logger.debug(f"Response body: {response_text}")
                except Exception as e:
                    logger.debug(f"Could not read response body: {e}")
                
                # Handle successful response
                handler = self.response_handlers.get(endpoint.response_handler, self.response_handlers['default'])
                processed_data = handler.handle_response(endpoint, response)
                
                # Update endpoint status
                endpoint.last_success = datetime.utcnow()
                endpoint.last_error = None
                endpoint.consecutive_failures = 0
                endpoint.successful_requests += 1
                self.stats['successful_requests'] += 1
                
                # Send response data via message bus
                if processed_data:
                    response_message = Message(
                        type=MessageType.API_RESPONSE,
                        sender=self.name,
                        data={
                            'endpoint': endpoint.name,
                            'success': True,
                            'data': processed_data,
                            'timestamp': datetime.utcnow().isoformat()
                        }
                    )
                    self.message_bus.publish(response_message)
                
                logger.debug(f"API request successful: {endpoint.name}")
            
        except Exception as e:
            # Handle request failure
            self._handle_request_failure(endpoint, e)
    
    def _execute_with_custom_retry(self, endpoint: APIEndpoint, request_kwargs: dict) -> bool:
        """Execute request with custom retry logic for token-based endpoints"""
        max_retries = min(endpoint.retry_attempts, 10)  # Cap at 10 retries as requested
        retry_delay = 30  # 30 seconds between retries as requested
        
        for attempt in range(max_retries):
            try:
                logger.info(f"Retry attempt {attempt + 1}/{max_retries} for endpoint {endpoint.name}")
                
                response = self.session.request(**request_kwargs)
                response.raise_for_status()
                
                # Debug log the complete response (retry scenario)
                logger.debug(f"Retry response status code: {response.status_code}")
                logger.debug(f"Retry response headers: {dict(response.headers)}")
                try:
                    response_text = response.text
                    if len(response_text) > 1000:
                        logger.debug(f"Retry response body (truncated): {response_text[:1000]}...")
                    else:
                        logger.debug(f"Retry response body: {response_text}")
                except Exception as e:
                    logger.debug(f"Could not read retry response body: {e}")
                
                # Handle successful response
                handler = self.response_handlers.get(endpoint.response_handler, self.response_handlers['default'])
                processed_data = handler.handle_response(endpoint, response)
                
                # Update endpoint status - success!
                endpoint.last_success = datetime.utcnow()
                endpoint.last_error = None
                endpoint.consecutive_failures = 0
                endpoint.successful_requests += 1
                self.stats['successful_requests'] += 1
                
                # Send response data via message bus
                if processed_data:
                    response_message = Message(
                        type=MessageType.API_RESPONSE,
                        sender=self.name,
                        data={
                            'endpoint': endpoint.name,
                            'success': True,
                            'data': processed_data,
                            'timestamp': datetime.utcnow().isoformat(),
                            'retry_attempt': attempt + 1
                        }
                    )
                    self.message_bus.publish(response_message)
                
                logger.info(f"API request successful on retry {attempt + 1}: {endpoint.name}")
                return True
                
            except Exception as e:
                endpoint.last_error = str(e)
                endpoint.consecutive_failures += 1
                self.stats['failed_requests'] += 1
                
                if attempt < max_retries - 1:
                    logger.warning(f"API retry {attempt + 1} failed for {endpoint.name}: {e}. Waiting {retry_delay}s before next retry.")
                    time.sleep(retry_delay)
                else:
                    logger.error(f"All {max_retries} retries failed for {endpoint.name}: {e}")
        
        # All retries failed
        self._handle_request_failure(endpoint, Exception(f"All {max_retries} retries failed. Last error: {endpoint.last_error}"))
        return False
    
    def _handle_request_failure(self, endpoint: APIEndpoint, error: Exception):
        """Handle request failure and send error message"""
        endpoint.last_error = str(error)
        endpoint.consecutive_failures += 1
        self.stats['failed_requests'] += 1
        
        logger.error(f"API request failed: {endpoint.name} - {error}")
        
        # Send error message
        error_message = Message(
            type=MessageType.API_RESPONSE,
            sender=self.name,
            data={
                'endpoint': endpoint.name,
                'success': False,
                'error': str(error),
                'consecutive_failures': endpoint.consecutive_failures,
                'timestamp': datetime.utcnow().isoformat()
            }
        )
        self.message_bus.publish(error_message)
        
        # For token-based endpoints, don't disable after failures - keep retrying per user requirements
        is_token_endpoint = (endpoint.auth and
                           endpoint.auth.get('type') == 'bearer' and
                           endpoint.auth.get('token'))
        
        if not is_token_endpoint and endpoint.consecutive_failures >= self.settings.max_consecutive_failures:
            endpoint.enabled = False
            logger.warning(f"Endpoint disabled due to consecutive failures: {endpoint.name}")
    
    def _process_message(self, message: Message):
        """Process received message"""
        try:
            # Messages are handled by subscribed handlers
            pass
        except Exception as e:
            logger.error(f"Failed to process message: {e}")
    
    def _handle_config_update(self, message: Message):
        """Handle configuration update message"""
        try:
            config_section = message.data.get("config_section")
            config_data = message.data.get("config_data", {})
            
            if config_section == "api_endpoints":
                logger.info("API endpoints configuration updated")
                self._load_endpoints()
            elif config_section == "api":
                logger.info("API configuration updated, reloading endpoints")
                
                # Check for token changes to enable/disable dynamic scheduling
                new_token = config_data.get("api_token", "").strip()
                old_token = ""
                
                try:
                    old_api_config = self.config_manager.get_section_config("api") or {}
                    old_token = old_api_config.get("api_token", "").strip()
                except Exception:
                    pass
                
                # Reload endpoints to pick up new configuration
                self._load_endpoints()
                
                # Handle dynamic timer start/stop based on token changes
                self._handle_token_change(old_token, new_token)
            
        except Exception as e:
            logger.error(f"Failed to handle config update: {e}")
    
    def _handle_token_change(self, old_token: str, new_token: str):
        """Handle dynamic timer start/stop based on token changes"""
        try:
            fastapi_endpoint = self.endpoints.get("fastapi_main")
            if not fastapi_endpoint:
                return
            
            old_has_token = bool(old_token and old_token.strip())
            new_has_token = bool(new_token and new_token.strip())
            
            # Get current interval setting
            current_api_config = self.config_manager.get_section_config("api") or {}
            current_interval = current_api_config.get("api_interval", 1800)  # 30 minutes default
            
            if not old_has_token and new_has_token:
                # Token was added - start timer
                fastapi_endpoint.enabled = True
                fastapi_endpoint.interval = current_interval
                fastapi_endpoint.retry_attempts = 10
                fastapi_endpoint.retry_delay = 30
                fastapi_endpoint.consecutive_failures = 0  # Reset failure count
                fastapi_endpoint.last_request = None  # Reset to trigger immediate first request
                
                logger.info(f"FastAPI timer started - token configured, {current_interval} second intervals enabled")
                
                # Send immediate status update
                status_message = Message(
                    type=MessageType.SYSTEM_STATUS,
                    sender=self.name,
                    data={
                        "status": "timer_started",
                        "endpoint": "fastapi_main",
                        "reason": "token_configured",
                        "interval_seconds": current_interval
                    }
                )
                self.message_bus.publish(status_message)
                
            elif old_has_token and not new_has_token:
                # Token was removed - stop timer
                fastapi_endpoint.enabled = False
                fastapi_endpoint.interval = 600  # Reset to 10 minutes (default)
                fastapi_endpoint.retry_attempts = 3
                fastapi_endpoint.retry_delay = 5
                
                logger.info("FastAPI timer stopped - token removed, automatic requests disabled")
                
                # Send status update
                status_message = Message(
                    type=MessageType.SYSTEM_STATUS,
                    sender=self.name,
                    data={
                        "status": "timer_stopped",
                        "endpoint": "fastapi_main",
                        "reason": "token_removed"
                    }
                )
                self.message_bus.publish(status_message)
                
            elif old_has_token and new_has_token and old_token != new_token:
                # Token was changed - keep timer running but reset failure count
                fastapi_endpoint.consecutive_failures = 0
                fastapi_endpoint.last_request = None  # Trigger immediate request with new token
                
                logger.info("FastAPI token updated - timer continues with new authentication")
                
        except Exception as e:
            logger.error(f"Failed to handle token change: {e}")
    
    def update_fastapi_url(self, new_url: str) -> bool:
        """Update FastAPI server URL and reload endpoints"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            api_config["fastapi_url"] = new_url
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new URL
            self._load_endpoints()
            
            logger.info(f"FastAPI URL updated to: {new_url}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update FastAPI URL: {e}")
            return False
    
    def update_api_token(self, new_token: str) -> bool:
        """Update API authentication token and reload endpoints"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            api_config["api_token"] = new_token
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new token
            self._load_endpoints()
            
            logger.info("API token updated successfully")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update API token: {e}")
            return False
    
    def update_api_config(self, new_url: str = None, new_token: str = None) -> bool:
        """Update FastAPI URL and/or token configuration"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            
            if new_url is not None:
                api_config["fastapi_url"] = new_url
                
            if new_token is not None:
                api_config["api_token"] = new_token
            
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new configuration
            self._load_endpoints()
            
            logger.info(f"API configuration updated - URL: {new_url}, Token: {'*' * len(new_token) if new_token else 'unchanged'}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update API configuration: {e}")
            return False
    
    def _handle_api_request(self, message: Message):
        """Handle manual API request message"""
        try:
            endpoint_name = message.data.get("endpoint")
            
            if endpoint_name in self.endpoints:
                endpoint = self.endpoints[endpoint_name]
                self._execute_endpoint_request(endpoint)
            else:
                logger.warning(f"Unknown endpoint requested: {endpoint_name}")
            
        except Exception as e:
            logger.error(f"Failed to handle API request: {e}")
    
    def get_endpoint_status(self, endpoint_name: str) -> Optional[Dict[str, Any]]:
        """Get status of a specific endpoint"""
        endpoint = self.endpoints.get(endpoint_name)
        if endpoint:
            return endpoint.to_dict()
        return None
    
    def get_all_endpoints_status(self) -> Dict[str, Dict[str, Any]]:
        """Get status of all endpoints"""
        return {name: endpoint.to_dict() for name, endpoint in self.endpoints.items()}
    
    def get_client_stats(self) -> Dict[str, Any]:
        """Get client statistics"""
        uptime = datetime.utcnow() - self.stats['start_time']
        return {
            'total_requests': self.stats['total_requests'],
            'successful_requests': self.stats['successful_requests'],
            'failed_requests': self.stats['failed_requests'],
            'success_rate': (self.stats['successful_requests'] / max(1, self.stats['total_requests'])) * 100,
            'uptime_seconds': uptime.total_seconds(),
            'endpoints_count': len(self.endpoints),
            'enabled_endpoints': sum(1 for ep in self.endpoints.values() if ep.enabled)
        }
    
    def update_endpoint(self, endpoint_name: str, config: Dict[str, Any]) -> bool:
        """Update endpoint configuration"""
        try:
            if endpoint_name in self.endpoints:
                # Update existing endpoint
                endpoint = self.endpoints[endpoint_name]
                for key, value in config.items():
                    if hasattr(endpoint, key):
                        setattr(endpoint, key, value)
            else:
                # Create new endpoint
                self.endpoints[endpoint_name] = APIEndpoint(endpoint_name, config)
            
            # Save to configuration
            all_endpoints = {name: ep.to_dict() for name, ep in self.endpoints.items()}
            self.config_manager.update_section("api_endpoints", all_endpoints)
            
            logger.info(f"Updated endpoint configuration: {endpoint_name}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update endpoint: {e}")
            return False
    
    def shutdown(self):
        """Shutdown API client"""
        try:
            logger.info("Shutting down APIClient...")
            
            if self.session:
                self.session.close()
            
        except Exception as e:
            logger.error(f"APIClient shutdown error: {e}")
    
    def _generate_curl_command(self, method, url, headers, data=None):
        """Generate a curl command equivalent to the current request for debugging purposes."""
        import json
        import shlex
        
        # Start with basic curl command
        cmd_parts = ['curl', '-X', method]
        
        # Add headers
        for key, value in headers.items():
            cmd_parts.extend(['-H', f'{key}: {value}'])
        
        # Add data based on method
        if method == 'GET' and data:
            # For GET requests, params are added to URL
            from urllib.parse import urlencode
            query_string = urlencode(data)
            url = f"{url}?{query_string}"
        elif method == 'POST':
            # For POST requests, always add JSON data (even if empty)
            json_str = json.dumps(data if data else {}, separators=(',', ':'))
            cmd_parts.extend(['-d', json_str])
        
        # Add URL (always last)
        cmd_parts.append(url)
        
        # Use shlex.join for proper shell escaping (Python 3.8+)
        try:
            return shlex.join(cmd_parts)
        except AttributeError:
            # Fallback for Python < 3.8
            return ' '.join(shlex.quote(arg) for arg in cmd_parts)