"""
REST API Client - Configurable HTTP client with retry logic and failure handling
"""

import time
import logging
import json
import threading
import os
import hashlib
from pathlib import Path
from datetime import datetime, timedelta
from typing import Dict, Any, Optional, List, Union
from urllib.parse import urljoin, urlparse
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

from ..core.thread_manager import ThreadedComponent
from ..core.message_bus import MessageBus, Message, MessageType, MessageBuilder
from ..config.settings import ApiConfig
from ..config.manager import ConfigManager
from ..database.manager import DatabaseManager
from ..database.models import MatchModel, MatchOutcomeModel, MatchTemplateModel, MatchOutcomeTemplateModel
from ..config.settings import get_user_data_dir
from ..utils.ssl_utils import create_requests_session_with_ssl_support

logger = logging.getLogger(__name__)


class APIEndpoint:
    """Configuration for a single API endpoint"""
    
    def __init__(self, name: str, config: Dict[str, Any]):
        self.name = name
        self.url = config.get('url', '')
        self.method = config.get('method', 'GET').upper()
        self.headers = config.get('headers', {})
        self.params = config.get('params', {})
        self.data = config.get('data', {})
        self.interval = config.get('interval', 300)  # 5 minutes default
        self.enabled = config.get('enabled', True)
        self.timeout = config.get('timeout', 30)
        self.retry_attempts = config.get('retry_attempts', 3)
        self.retry_delay = config.get('retry_delay', 5)
        self.auth = config.get('auth', None)
        self.response_handler = config.get('response_handler', 'default')
        
        # Runtime state
        self.last_request = None
        self.last_success = None
        self.last_error = None
        self.consecutive_failures = 0
        self.total_requests = 0
        self.successful_requests = 0
    
    def should_execute(self) -> bool:
        """Check if endpoint should be executed based on interval"""
        if not self.enabled:
            return False
        
        if self.last_request is None:
            return True
        
        next_execution = self.last_request + timedelta(seconds=self.interval)
        return datetime.utcnow() >= next_execution
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization"""
        return {
            'name': self.name,
            'url': self.url,
            'method': self.method,
            'headers': self.headers,
            'params': self.params,
            'data': self.data,
            'interval': self.interval,
            'enabled': self.enabled,
            'timeout': self.timeout,
            'retry_attempts': self.retry_attempts,
            'retry_delay': self.retry_delay,
            'auth': self.auth,
            'response_handler': self.response_handler,
            'last_request': self.last_request.isoformat() if self.last_request else None,
            'last_success': self.last_success.isoformat() if self.last_success else None,
            'last_error': self.last_error,
            'consecutive_failures': self.consecutive_failures,
            'total_requests': self.total_requests,
            'successful_requests': self.successful_requests
        }


class ResponseHandler:
    """Base class for response handlers"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        """Handle API response and return processed data"""
        raise NotImplementedError
    
    def handle_error(self, endpoint: APIEndpoint, error: Exception) -> Optional[Dict[str, Any]]:
        """Handle API error and return error data"""
        return {
            'error': str(error),
            'endpoint': endpoint.name,
            'timestamp': datetime.utcnow().isoformat()
        }


class DefaultResponseHandler(ResponseHandler):
    """Default response handler - processes JSON responses"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            if response.headers.get('content-type', '').startswith('application/json'):
                return response.json()
            else:
                return {
                    'status_code': response.status_code,
                    'content': response.text,
                    'headers': dict(response.headers)
                }
        except json.JSONDecodeError:
            return {
                'status_code': response.status_code,
                'content': response.text,
                'headers': dict(response.headers)
            }


class NewsResponseHandler(ResponseHandler):
    """Response handler for news APIs - extracts headline and ticker data"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            # Extract news data for overlay templates
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'raw_data': data
            }
            
            # Try to extract common news fields
            if 'articles' in data:
                # NewsAPI format
                articles = data['articles'][:5]  # Top 5 articles
                processed_data['headlines'] = [article.get('title', '') for article in articles]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            elif 'items' in data:
                # RSS format
                items = data['items'][:5]
                processed_data['headlines'] = [item.get('title', '') for item in items]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            elif isinstance(data, list):
                # Direct array format
                processed_data['headlines'] = [item.get('title', '') for item in data[:5]]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            
            return processed_data
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.error(f"Failed to process news response: {e}")
            return self.handle_error(endpoint, e)


class SportsResponseHandler(ResponseHandler):
    """Response handler for sports APIs - extracts scores and game data"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'raw_data': data
            }
            
            # Extract sports data
            if 'games' in data:
                # Generic sports API format
                games = data['games'][:3]  # Top 3 games
                game_data = []
                for game in games:
                    game_data.append({
                        'team1': game.get('home_team', 'Home'),
                        'team2': game.get('away_team', 'Away'),
                        'score1': game.get('home_score', 0),
                        'score2': game.get('away_score', 0),
                        'status': game.get('status', 'Live')
                    })
                processed_data['games'] = game_data
            
            return processed_data
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.error(f"Failed to process sports response: {e}")
            return self.handle_error(endpoint, e)


class UpdatesResponseHandler(ResponseHandler):
    """Response handler for /api/updates endpoint - synchronizes match data"""

    def __init__(self, db_manager, user_data_dir, api_client=None, message_bus=None):
        self.db_manager = db_manager
        self.user_data_dir = user_data_dir
        self.api_client = api_client  # Reference to parent API client for token access
        self.message_bus = message_bus  # Reference to message bus for progress updates
        self.zip_storage_dir = Path(user_data_dir) / "zip_files"
        self.zip_storage_dir.mkdir(parents=True, exist_ok=True)
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'synchronized_matches': 0,
                'downloaded_zips': 0,
                'expected_zips': 0,
                'errors': []
            }
            
            # Extract fixtures from response - new API format has fixtures containing matches
            fixtures = []
            if isinstance(data, dict):
                if 'fixtures' in data:
                    fixtures = data.get('fixtures', [])
                elif 'matches' in data:
                    # Fallback for old format - treat as single fixture with matches
                    fixtures = [{'matches': data.get('matches', [])}]
            elif isinstance(data, list):
                # Direct array of matches - treat as single fixture
                fixtures = [{'matches': data}]
            
            if not fixtures:
                processed_data['message'] = "No fixtures in response"
                return processed_data
            
            # First pass: synchronize all match data to database
            logger.debug(f"Starting fixture synchronization for {len(fixtures)} fixtures")
            session = self.db_manager.get_session()
            try:
                for fixture_data in fixtures:
                    fixture_id = fixture_data.get('fixture_id', 'unknown')
                    matches = fixture_data.get('matches', [])
                    logger.debug(f"Synchronizing fixture {fixture_id} with {len(matches)} matches")

                    # Handle both list and dict formats for matches
                    if isinstance(matches, dict):
                        # Convert dict to list of match data
                        matches = [match_data for match_data in matches.values()]
                        logger.debug(f"Converted matches dict to list with {len(matches)} items")
                    elif not isinstance(matches, list):
                        logger.warning(f"Unexpected matches format: {type(matches)}, skipping fixture {fixture_id}")
                        continue

                    for match_data in matches:
                        match_number = match_data.get('match_number', 'unknown')
                        try:
                            # Update heartbeat during processing to prevent health check failures
                            if self.api_client:
                                self.api_client.heartbeat()

                            # Add fixture-level data to match if available
                            if 'fixture_id' in fixture_data:
                                match_data['fixture_id'] = fixture_data['fixture_id']
                            if 'fixture_active_time' in fixture_data:
                                match_data['fixture_active_time'] = fixture_data['fixture_active_time']

                            logger.debug(f"Synchronizing match {match_number} in fixture {fixture_id}")

                            # Synchronize match data to database
                            self._synchronize_match(session, match_data)
                            processed_data['synchronized_matches'] += 1
                            logger.debug(f"Successfully synchronized match {match_number}")

                        except Exception as e:
                            error_msg = f"Failed to process match {match_number} in fixture {fixture_id}: {e}"
                            logger.error(error_msg)
                            processed_data['errors'].append(error_msg)
                            # Continue processing other matches even if this one fails
                            continue

                # Update heartbeat before potentially slow commit operation
                if self.api_client:
                    self.api_client.heartbeat()

                logger.debug(f"Committing {processed_data['synchronized_matches']} synchronized matches to database")
                session.commit()
                logger.debug("Database commit completed successfully")

                # Clean up old fixture templates and keep only the new ones
                self._cleanup_old_fixture_templates(session, fixtures)
            finally:
                session.close()

            # Second pass: download ZIP files without holding database session
            # This prevents database locking during potentially slow downloads
            logger.debug(f"Starting ZIP file downloads for {len(fixtures)} fixtures")
            total_matches_with_zips = 0

            # Send initial progress update - starting downloads
            self._send_download_progress(0, processed_data['expected_zips'], "Starting downloads...")

            for fixture_data in fixtures:
                fixture_id = fixture_data.get('fixture_id', 'unknown')
                matches = fixture_data.get('matches', [])
                logger.debug(f"Processing fixture {fixture_id} with {len(matches)} matches")

                for match_data in matches:
                    match_number = match_data.get('match_number', 'unknown')
                    zip_filename = match_data.get('zip_filename')

                    try:
                        # Update heartbeat before each download
                        if self.api_client:
                            self.api_client.heartbeat()

                        # Download ZIP file if available (check match-level zip_download_url)
                        if 'zip_download_url' in match_data:
                            processed_data['expected_zips'] += 1
                            total_matches_with_zips += 1
                            match_data['zip_url'] = match_data['zip_download_url']
                            logger.debug(f"Found ZIP file to download: {zip_filename} for match {match_number} in fixture {fixture_id}")

                            # Attempt download with retries
                            download_success = False
                            max_retries = 3
                            for attempt in range(max_retries):
                                try:
                                    # Send progress update - starting/retrying download
                                    attempt_desc = f" (attempt {attempt + 1}/{max_retries})" if attempt > 0 else ""
                                    progress_percent = int((processed_data['downloaded_zips'] / max(1, processed_data['expected_zips'])) * 100)
                                    self._send_download_progress(processed_data['downloaded_zips'], processed_data['expected_zips'],
                                                                f"Downloading {zip_filename}{attempt_desc}...")

                                    download_success = self._download_zip_file(match_data)
                                    if download_success:
                                        processed_data['downloaded_zips'] += 1
                                        logger.debug(f"Successfully downloaded ZIP file: {zip_filename} for match {match_number}")

                                        # Send progress update - download completed
                                        progress_percent = int((processed_data['downloaded_zips'] / max(1, processed_data['expected_zips'])) * 100)
                                        self._send_download_progress(processed_data['downloaded_zips'], processed_data['expected_zips'],
                                                                    f"Downloaded {zip_filename}")
                                        break
                                    else:
                                        if attempt < max_retries - 1:
                                            logger.warning(f"ZIP download attempt {attempt + 1} failed for {zip_filename}, retrying...")
                                            # Small delay before retry
                                            import time
                                            time.sleep(1)
                                        else:
                                            logger.error(f"All {max_retries} download attempts failed for {zip_filename}")
                                except Exception as retry_e:
                                    if attempt < max_retries - 1:
                                        logger.warning(f"ZIP download attempt {attempt + 1} failed for {zip_filename}: {retry_e}, retrying...")
                                        import time
                                        time.sleep(1)
                                    else:
                                        logger.error(f"All {max_retries} download attempts failed for {zip_filename}: {retry_e}")

                            if not download_success:
                                logger.debug(f"ZIP file download failed after retries: {zip_filename} for match {match_number}")

                    except requests.exceptions.HTTPError as http_err:
                        # Check if this is a "fixture no longer available" error (404)
                        if hasattr(http_err.response, 'status_code') and http_err.response.status_code == 404:
                            logger.debug(f"ZIP file no longer available on server (404): {zip_filename or 'unknown'} for match {match_number} - continuing with normal operations")
                            # Don't add to errors list, just skip this download
                        else:
                            # Other HTTP errors are actual failures
                            error_msg = f"HTTP error downloading ZIP for match {match_number}: {http_err}"
                            logger.error(error_msg)
                            processed_data['errors'].append(error_msg)
                    except Exception as e:
                        error_msg = f"Failed to download ZIP for match {match_number}: {e}"
                        logger.error(error_msg)
                        processed_data['errors'].append(error_msg)
                        # Continue with other downloads even if this one fails
                        continue

            logger.debug(f"ZIP download summary: {processed_data['downloaded_zips']}/{processed_data['expected_zips']} ZIP files downloaded successfully from {total_matches_with_zips} matches")

            # Send final progress update - downloads completed
            if processed_data['expected_zips'] > 0:
                self._send_download_progress(processed_data['downloaded_zips'], processed_data['expected_zips'],
                                           f"Downloads completed - {processed_data['downloaded_zips']}/{processed_data['expected_zips']} files")
            else:
                self._send_download_progress(0, 0, "No downloads needed")

            logger.info(f"Synchronized {processed_data['synchronized_matches']} matches, downloaded {processed_data['downloaded_zips']} ZIP files")
            return processed_data
            
        except Exception as e:
            logger.error(f"Failed to process updates response: {e}")
            return self.handle_error(endpoint, e)
    
    def _synchronize_match(self, session, match_data: Dict[str, Any]):
        """Synchronize match data to template database tables"""
        try:
            match_number = match_data.get('match_number')
            fixture_id = match_data.get('fixture_id')

            if not match_number or not fixture_id:
                logger.warning(f"Skipping match with missing match_number ({match_number}) or fixture_id ({fixture_id})")
                return

            # Check if match already exists by both match_number AND fixture_id in template tables
            existing_match = session.query(MatchTemplateModel).filter_by(
                match_number=match_number,
                fixture_id=fixture_id
            ).first()

            if existing_match:
                # Update existing match template
                for key, value in match_data.items():
                    if hasattr(existing_match, key) and key not in ['id', 'created_at']:
                        setattr(existing_match, key, value)
                existing_match.updated_at = datetime.utcnow()
                match = existing_match
                logger.debug(f"Updated existing match template {match_number} for fixture {fixture_id}")
            else:
                # Create new match template
                match = MatchTemplateModel(
                    match_number=match_data.get('match_number'),
                    fighter1_township=match_data.get('fighter1_township', ''),
                    fighter2_township=match_data.get('fighter2_township', ''),
                    venue_kampala_township=match_data.get('venue_kampala_township', ''),
                    start_time=self._parse_datetime(match_data.get('start_time')),
                    end_time=self._parse_datetime(match_data.get('end_time')),
                    result=match_data.get('result'),
                    filename=match_data.get('filename', ''),
                    file_sha1sum=match_data.get('file_sha1sum', ''),
                    fixture_id=fixture_id,
                    active_status=match_data.get('active_status', False),
                    zip_filename=match_data.get('zip_filename'),
                    zip_sha1sum=match_data.get('zip_sha1sum'),
                    zip_upload_status=match_data.get('zip_upload_status', 'pending'),
                    zip_upload_progress=match_data.get('zip_upload_progress', 0.0),
                    done=match_data.get('done', False),
                    running=match_data.get('running', False),
                    fixture_active_time=match_data.get('fixture_active_time'),
                    created_by=1  # Default admin user
                )
                session.add(match)
                logger.debug(f"Created new match template {match_number} for fixture {fixture_id}")

            # Flush to get the match ID for outcomes
            session.flush()

            # Handle match outcomes in template table
            outcomes_data = match_data.get('outcomes', {})
            if outcomes_data:
                # Remove existing outcomes
                session.query(MatchOutcomeTemplateModel).filter_by(match_id=match.id).delete()

                # Normalize outcomes_data to dict format
                normalized_outcomes = {}
                try:
                    if isinstance(outcomes_data, dict):
                        # Check if values are dicts with 'value' key or direct float values
                        for key, value in outcomes_data.items():
                            if isinstance(value, dict) and 'value' in value:
                                normalized_outcomes[key] = value['value']
                            elif isinstance(value, (int, float)):
                                normalized_outcomes[key] = value
                            else:
                                logger.warning(f"Unexpected outcome value type for {key}: {type(value)}, skipping")
                                continue
                    elif isinstance(outcomes_data, list):
                        # Handle list format [{"name": "WIN1", "value": 2.5}, ...]
                        for item in outcomes_data:
                            if isinstance(item, dict) and 'name' in item and 'value' in item:
                                normalized_outcomes[item['name']] = item['value']
                            else:
                                logger.warning(f"Unexpected list item format: {item}, skipping")
                                continue
                    else:
                        logger.warning(f"Unexpected outcomes_data type: {type(outcomes_data)}, expected dict or list")
                except Exception as e:
                    logger.error(f"Failed to normalize outcomes_data: {e}")
                    # Continue without outcomes if normalization fails

                # Add new outcomes
                for column_name, float_value in normalized_outcomes.items():
                    try:
                        outcome = MatchOutcomeTemplateModel(
                            match_id=match.id,
                            column_name=str(column_name),
                            float_value=float(float_value)
                        )
                        session.add(outcome)
                    except (ValueError, TypeError) as e:
                        logger.warning(f"Failed to process outcome {column_name}: {float_value} - {e}")
                        continue

        except Exception as e:
            logger.error(f"Failed to synchronize match template: {e}")
            # Rollback this session to clean state
            session.rollback()
            raise
    
    def _download_zip_file(self, match_data: Dict[str, Any]) -> bool:
        """Download ZIP file to persistent storage with API token authentication"""
        zip_url = match_data.get('zip_url')
        zip_filename = match_data.get('zip_filename')

        if not zip_url or not zip_filename:
            logger.debug(f"Skipping ZIP download - missing URL or filename: url={zip_url}, filename={zip_filename}")
            return False

        logger.debug(f"Starting ZIP file download: {zip_filename} from {zip_url}")

        try:
            # Prepare headers with API token authentication
            headers = {}

            # Get API token from the API client's fastapi_main endpoint
            if self.api_client and hasattr(self.api_client, 'endpoints'):
                fastapi_endpoint = self.api_client.endpoints.get('fastapi_main')
                if fastapi_endpoint and fastapi_endpoint.auth:
                    if fastapi_endpoint.auth.get('type') == 'bearer':
                        token = fastapi_endpoint.auth.get('token')
                        if token:
                            headers['Authorization'] = f"Bearer {token}"
                            logger.debug(f"Using API token for ZIP download: {zip_filename}")
                        else:
                            logger.debug(f"No API token available for ZIP download: {zip_filename}")

            if not headers:
                logger.warning(f"No API token available for ZIP download: {zip_filename}")

            # Download ZIP file with authentication using session with SSL support
            logger.debug(f"Initiating HTTP request for ZIP file: {zip_filename}")
            response = self.api_client.session.get(zip_url, stream=True, timeout=30, headers=headers)
            response.raise_for_status()
            logger.debug(f"HTTP request successful for {zip_filename}, status: {response.status_code}")

            # Save to persistent storage
            zip_path = self.zip_storage_dir / zip_filename
            logger.debug(f"Saving ZIP file to: {zip_path}")

            # Update heartbeat before potentially slow file write
            if self.api_client:
                self.api_client.heartbeat()

            bytes_downloaded = 0
            with open(zip_path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:  # Filter out keep-alive new chunks
                        f.write(chunk)
                        bytes_downloaded += len(chunk)

                        # Update heartbeat every few chunks for large files
                        if self.api_client and f.tell() % (1024 * 1024) == 0:  # Every MB
                            logger.debug(f"Downloaded {f.tell() // (1024 * 1024)}MB for {zip_filename}")
                            self.api_client.heartbeat()

            logger.debug(f"File download completed: {zip_filename}, total bytes: {bytes_downloaded}")

            # Validate downloaded file integrity
            logger.debug(f"Starting validation for downloaded ZIP file: {zip_filename}")
            if not self._validate_downloaded_zip(zip_path, match_data):
                logger.error(f"ZIP file validation failed: {zip_filename}")
                # Update database validation status to 'invalid'
                try:
                    session = self.db_manager.get_session()
                    try:
                        # Find the match template and update validation status
                        match_template = session.query(MatchTemplateModel).filter_by(
                            zip_filename=zip_filename
                        ).first()
                        if match_template:
                            match_template.zip_validation_status = 'invalid'
                            session.commit()
                            logger.debug(f"Updated database validation status to 'invalid' for {zip_filename}")
                    finally:
                        session.close()
                except Exception as db_e:
                    logger.warning(f"Failed to update database validation status for {zip_filename}: {db_e}")

                # Remove corrupted file
                try:
                    zip_path.unlink()
                    logger.debug(f"Removed corrupted ZIP file: {zip_filename}")
                except Exception as cleanup_e:
                    logger.warning(f"Failed to remove corrupted ZIP file {zip_filename}: {cleanup_e}")
                return False

            # Update database validation status to 'valid' after successful validation
            try:
                session = self.db_manager.get_session()
                try:
                    # Find the match template and update validation status
                    match_template = session.query(MatchTemplateModel).filter_by(
                        zip_filename=zip_filename
                    ).first()
                    if match_template:
                        match_template.zip_validation_status = 'valid'
                        session.commit()
                        logger.debug(f"Updated database validation status to 'valid' for {zip_filename}")
                finally:
                    session.close()
            except Exception as db_e:
                logger.warning(f"Failed to update database validation status for {zip_filename}: {db_e}")

            # Start detailed ZIP validation (checking for video files) asynchronously
            logger.debug(f"Starting detailed ZIP validation for {zip_filename}")
            try:
                # Find the match template to get its ID for validation
                session = self.db_manager.get_session()
                try:
                    match_template = session.query(MatchTemplateModel).filter_by(
                        zip_filename=zip_filename
                    ).first()
                    if match_template:
                        # Start detailed validation asynchronously
                        self._validate_single_zip_async(match_template.id, session, MatchTemplateModel)
                        logger.debug(f"Started detailed validation for match template {match_template.id}")
                    else:
                        logger.warning(f"Could not find match template for ZIP file {zip_filename} to start detailed validation")
                finally:
                    session.close()
            except Exception as val_e:
                logger.warning(f"Failed to start detailed ZIP validation for {zip_filename}: {val_e}")

            logger.info(f"Successfully downloaded and validated ZIP file: {zip_filename}")
            return True

        except requests.exceptions.HTTPError as http_err:
            # Check if this is a "fixture no longer available" error (404)
            if hasattr(http_err.response, 'status_code') and http_err.response.status_code == 404:
                logger.debug(f"ZIP file no longer available on server (404): {zip_filename} - continuing with normal operations")
                return False  # Don't treat as error, just skip this download
            else:
                # Other HTTP errors are actual failures
                logger.error(f"HTTP error downloading ZIP file {zip_filename}: {http_err}")
                raise
        except requests.exceptions.RequestException as req_err:
            logger.error(f"Network error downloading ZIP file {zip_filename}: {req_err}")
            return False
        except IOError as io_err:
            logger.error(f"File I/O error downloading ZIP file {zip_filename}: {io_err}")
            return False
        except Exception as e:
            logger.error(f"Unexpected error downloading ZIP file {zip_filename}: {e}")
            return False
    
    def _parse_datetime(self, datetime_str) -> Optional[datetime]:
        """Parse datetime string to datetime object"""
        if not datetime_str:
            return None

        try:
            # Try ISO format first
            return datetime.fromisoformat(datetime_str.replace('Z', '+00:00'))
        except:
            try:
                # Try common formats
                return datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S')
            except:
                return None

    def _calculate_sha1(self, file_path: Path) -> str:
        """Calculate SHA1 checksum of a file"""
        try:
            sha1 = hashlib.sha1()
            with open(file_path, 'rb') as f:
                # Read in chunks to handle large files
                for chunk in iter(lambda: f.read(8192), b""):
                    sha1.update(chunk)
            return sha1.hexdigest()
        except Exception as e:
            logger.error(f"Failed to calculate SHA1 for {file_path}: {e}")
            return ""

    def _validate_downloaded_zip(self, zip_path: Path, match_data: Dict[str, Any]) -> bool:
        """Validate downloaded ZIP file integrity using checksum and size"""
        try:
            zip_filename = match_data.get('zip_filename', '')
            expected_sha1 = match_data.get('zip_sha1sum', '')
            expected_size = match_data.get('zip_size', None)  # Assuming size might be provided

            logger.debug(f"Starting validation for ZIP file: {zip_filename} at path: {zip_path}")

            # Check if file exists and has content
            if not zip_path.exists():
                logger.error(f"ZIP file does not exist: {zip_path}")
                return False

            actual_size = zip_path.stat().st_size
            logger.debug(f"ZIP file exists with size: {actual_size} bytes")

            if actual_size == 0:
                logger.error(f"ZIP file is empty: {zip_filename}")
                return False

            # Validate SHA1 checksum if provided
            if expected_sha1 and expected_sha1.strip():
                logger.debug(f"Validating SHA1 checksum for {zip_filename} (expected: {expected_sha1})")
                try:
                    actual_sha1 = self._calculate_sha1(zip_path)
                    logger.debug(f"Calculated SHA1: {actual_sha1}")
                    if actual_sha1 != expected_sha1.lower():
                        logger.error(f"SHA1 checksum mismatch for {zip_filename}: expected {expected_sha1}, got {actual_sha1}")
                        return False
                    logger.debug(f"SHA1 checksum validated successfully for {zip_filename}")
                except Exception as sha1_err:
                    logger.error(f"Failed to calculate SHA1 for {zip_filename}: {sha1_err}")
                    return False
            else:
                logger.debug(f"No SHA1 checksum provided for {zip_filename}, skipping checksum validation")

            # Validate file size if provided (as additional check)
            if expected_size is not None and isinstance(expected_size, (int, float)) and expected_size > 0:
                logger.debug(f"Validating file size for {zip_filename} (expected: {expected_size}, actual: {actual_size})")
                if actual_size != expected_size:
                    logger.warning(f"File size mismatch for {zip_filename}: expected {expected_size}, got {actual_size}")
                    # Size mismatch is a warning but not necessarily a failure for ZIP files
                    # as compression can vary, but we'll log it
                else:
                    logger.debug(f"File size validation passed for {zip_filename}")
            else:
                logger.debug(f"No expected size provided for {zip_filename}, skipping size validation")

            logger.debug(f"ZIP file validation successful: {zip_filename} (size: {actual_size} bytes)")
            return True

        except Exception as e:
            logger.error(f"Failed to validate ZIP file {zip_path}: {e}")
            return False

    def _cleanup_old_fixture_templates(self, session, fixtures):
        """Clean up old fixture templates and keep only the new ones"""
        try:
            # Get fixture IDs from the current fixtures that were just synchronized
            current_fixture_ids = set()
            for fixture_data in fixtures:
                fixture_id = fixture_data.get('fixture_id')
                if fixture_id:
                    current_fixture_ids.add(fixture_id)

            if not current_fixture_ids:
                logger.warning("No current fixture IDs found - skipping cleanup")
                return

            logger.info(f"Cleaning up old fixture templates - keeping fixtures: {current_fixture_ids}")

            # Get all fixture IDs currently in the template tables
            existing_fixture_ids = set()
            existing_fixtures = session.query(MatchTemplateModel.fixture_id).distinct().all()
            for row in existing_fixtures:
                existing_fixture_ids.add(row.fixture_id)

            # Find fixture IDs to remove (existing ones not in current fixtures)
            fixtures_to_remove = existing_fixture_ids - current_fixture_ids

            if not fixtures_to_remove:
                logger.info("No old fixture templates to remove")
                return

            logger.info(f"Removing old fixture templates: {fixtures_to_remove}")

            # Delete matches from old fixtures (CASCADE will delete outcomes)
            deleted_matches = 0
            for fixture_id in fixtures_to_remove:
                deleted_count = session.query(MatchTemplateModel).filter(
                    MatchTemplateModel.fixture_id == fixture_id
                ).delete()
                deleted_matches += deleted_count
                logger.info(f"Removed fixture template {fixture_id} with {deleted_count} matches")

            session.commit()
            logger.info(f"Successfully cleaned up {len(fixtures_to_remove)} old fixture templates, removed {deleted_matches} matches total")

        except Exception as e:
            logger.error(f"Failed to cleanup old fixture templates: {e}")
            session.rollback()

    def _send_download_progress(self, downloaded: int, total: int, message: str):
        """Send download progress update via message bus"""
        try:
            if self.message_bus:
                from ..core.message_bus import Message, MessageType
                progress_data = {
                    'downloaded': downloaded,
                    'total': total,
                    'percentage': int((downloaded / max(1, total)) * 100) if total > 0 else 0,
                    'message': message,
                    'timestamp': time.time()
                }

                progress_message = Message(
                    type=MessageType.CUSTOM,
                    sender='api_client',
                    data={
                        'download_progress': progress_data
                    }
                )
                self.message_bus.publish(progress_message)
                logger.debug(f"Sent download progress: {downloaded}/{total} - {message}")
        except Exception as e:
            logger.error(f"Failed to send download progress: {e}")

    def _validate_existing_fixtures(self):
        """Validate existing fixtures on startup and remove invalid ones"""
        try:
            logger.info("Validating existing fixtures on startup...")

            session = self.db_manager.get_session()
            try:
                # Get all fixtures with active matches
                fixtures_to_check = session.query(MatchModel.fixture_id).filter(
                    MatchModel.active_status == True
                ).distinct().all()

                logger.debug(f"Found {len(fixtures_to_check)} fixtures to validate")
                fixtures_to_remove = []

                for fixture_row in fixtures_to_check:
                    fixture_id = fixture_row.fixture_id
                    logger.debug(f"Validating fixture: {fixture_id}")

                    # Skip validation for recycled fixtures (copies of already verified fixtures)
                    if fixture_id.startswith('recycle_'):
                        logger.debug(f"Skipping ZIP validation for recycled fixture: {fixture_id}")
                        continue

                    # Skip validation for fixtures not from today (only today's fixtures need verification)
                    # Get fixture_active_time from any match in this fixture
                    fixture_active_time = None
                    sample_match = session.query(MatchModel).filter(
                        MatchModel.fixture_id == fixture_id
                    ).first()

                    if sample_match and sample_match.fixture_active_time:
                        fixture_active_time = sample_match.fixture_active_time

                    if fixture_active_time:
                        # Convert Unix timestamp to date
                        from datetime import datetime, date
                        fixture_date = datetime.fromtimestamp(fixture_active_time).date()
                        today = date.today()

                        if fixture_date != today:
                            logger.debug(f"Skipping ZIP validation for fixture {fixture_id} from {fixture_date} (not today)")
                            continue

                    # Get all matches for this fixture that have ZIP files
                    matches_with_zips = session.query(MatchModel).filter(
                        MatchModel.fixture_id == fixture_id,
                        MatchModel.active_status == True,
                        MatchModel.zip_filename.isnot(None)
                    ).all()

                    logger.debug(f"Fixture {fixture_id} has {len(matches_with_zips)} matches with ZIP files")

                    if not matches_with_zips:
                        logger.debug(f"Fixture {fixture_id} has no ZIP files to validate")
                        continue

                    # Check if all required ZIP files exist and are valid
                    fixture_valid = True
                    missing_or_invalid_zips = []

                    for match in matches_with_zips:
                        zip_filename = match.zip_filename
                        if not zip_filename:
                            continue

                        # Check if ZIP has already been validated successfully
                        if match.zip_validation_status == 'valid':
                            logger.debug(f"ZIP file already validated: {zip_filename} for fixture {fixture_id}")
                            continue
                        elif match.zip_validation_status == 'validating':
                            logger.debug(f"ZIP file validation in progress: {zip_filename} for fixture {fixture_id}")
                            continue

                        logger.debug(f"Validating ZIP file: {zip_filename} for fixture {fixture_id}")
                        zip_path = self.zip_storage_dir / zip_filename

                        # Create match data dict for validation
                        match_data = {
                            'zip_filename': zip_filename,
                            'zip_sha1sum': match.zip_sha1sum,
                            'zip_size': getattr(match, 'zip_size', None)  # If size field exists
                        }

                        if not self._validate_downloaded_zip(zip_path, match_data):
                            logger.debug(f"ZIP file validation failed: {zip_filename}")
                            missing_or_invalid_zips.append(zip_filename)
                            fixture_valid = False
                        else:
                            logger.debug(f"ZIP file validation passed: {zip_filename}")
                            # Mark as validated in database
                            match.zip_validation_status = 'valid'
                            session.commit()

                    if not fixture_valid:
                        logger.warning(f"Fixture {fixture_id} has invalid/missing ZIP files: {missing_or_invalid_zips}")
                        fixtures_to_remove.append(fixture_id)
                    else:
                        logger.debug(f"Fixture {fixture_id} validation passed")

                # Remove invalid fixtures
                if fixtures_to_remove:
                    logger.info(f"Removing {len(fixtures_to_remove)} invalid fixtures: {fixtures_to_remove}")

                    for fixture_id in fixtures_to_remove:
                        logger.debug(f"Deleting all matches for invalid fixture: {fixture_id}")
                        # Delete all matches for this fixture
                        deleted_count = session.query(MatchModel).filter(
                            MatchModel.fixture_id == fixture_id
                        ).delete()

                        logger.info(f"Removed fixture {fixture_id} with {deleted_count} matches")

                    session.commit()
                    logger.info(f"Successfully removed {len(fixtures_to_remove)} invalid fixtures")
                else:
                    logger.info("All existing fixtures are valid")

            finally:
                session.close()

        except Exception as e:
            logger.error(f"Failed to validate existing fixtures: {e}")
            # Don't fail initialization due to validation errors


class APIClient(ThreadedComponent):
    """REST API Client component"""
    
    def __init__(self, message_bus: MessageBus, db_manager: DatabaseManager,
                 config_manager: ConfigManager, settings: ApiConfig):
        super().__init__("api_client", message_bus)
        self.db_manager = db_manager
        self.config_manager = config_manager
        self.settings = settings

        logger.debug("API CLIENT INIT: rustdesk_id from settings: '{}' (type: {})".format(
            self.settings.rustdesk_id, type(self.settings.rustdesk_id)))
        
        # HTTP session with retry logic
        self.session = requests.Session()
        self._setup_session()
        
        # API endpoints configuration
        self.endpoints: Dict[str, APIEndpoint] = {}
        self._load_endpoints()
        
        # Response handlers
        self.response_handlers = {
            'default': DefaultResponseHandler(),
            'news': NewsResponseHandler(),
            'sports': SportsResponseHandler(),
            'updates': UpdatesResponseHandler(self.db_manager, get_user_data_dir(), self, self.message_bus)
        }
        
        # Statistics
        self.stats = {
            'total_requests': 0,
            'successful_requests': 0,
            'failed_requests': 0,
            'start_time': datetime.utcnow()
        }
        
        # Register message queue
        self.message_queue = self.message_bus.register_component(self.name)
        
        logger.info("APIClient initialized")
    
    def initialize(self) -> bool:
        """Initialize API client"""
        try:
            # Subscribe to messages
            self.message_bus.subscribe(self.name, MessageType.CONFIG_UPDATE, self._handle_config_update)
            self.message_bus.subscribe(self.name, MessageType.API_REQUEST, self._handle_api_request)

            # Validate existing fixtures on startup
            updates_handler = self.response_handlers.get('updates')
            if updates_handler and hasattr(updates_handler, '_validate_existing_fixtures'):
                updates_handler._validate_existing_fixtures()

            logger.info("APIClient initialized successfully")
            return True

        except Exception as e:
            logger.error(f"APIClient initialization failed: {e}")
            return False
    
    def _setup_session(self):
        """Setup HTTP session with retry logic and SSL support"""
        try:
            # Use SSL-aware session that handles self-signed certificates
            self.session = create_requests_session_with_ssl_support(
                verify_ssl=self.settings.verify_ssl
            )
            
            # Set default headers
            self.session.headers.update({
                'User-Agent': f'MbetterClient/{self.settings.user_agent}',
                'Accept': 'application/json, text/plain, */*',
                'Accept-Encoding': 'gzip, deflate'
            })
            
            # Set timeout
            self.session.timeout = self.settings.timeout_seconds
            
            if not self.settings.verify_ssl:
                logger.info("API client configured to accept self-signed certificates")
            else:
                logger.info("API client configured with SSL verification enabled")
                
        except Exception as e:
            logger.error(f"Failed to setup SSL session, falling back to basic session: {e}")
            # Fallback to original session setup
            retry_strategy = Retry(
                total=self.settings.retry_attempts,
                backoff_factor=self.settings.retry_delay_seconds,
                status_forcelist=[429, 500, 502, 503, 504],
                allowed_methods=None,  # Allow all methods (replaces deprecated method_whitelist)
            )
            
            adapter = HTTPAdapter(max_retries=retry_strategy)
            self.session.mount("http://", adapter)
            self.session.mount("https://", adapter)
            
            # Set default headers
            self.session.headers.update({
                'User-Agent': f'MbetterClient/{self.settings.user_agent}',
                'Accept': 'application/json, text/plain, */*',
                'Accept-Encoding': 'gzip, deflate'
            })
            
            # Set timeout
            self.session.timeout = self.settings.timeout_seconds
    
    def _load_endpoints(self):
        """Load API endpoints from configuration"""
        try:
            endpoints_config = self.config_manager.get_section_config("api_endpoints")
            
            if not endpoints_config:
                # Create default configuration
                default_endpoints = self._get_default_endpoints()
                self.config_manager.update_section("api_endpoints", default_endpoints)
                endpoints_config = default_endpoints
            
            # Load endpoints
            for name, config in endpoints_config.items():
                self.endpoints[name] = APIEndpoint(name, config)
            
            logger.info(f"Loaded {len(self.endpoints)} API endpoints")
            
        except Exception as e:
            logger.error(f"Failed to load API endpoints: {e}")
    
    def _get_default_endpoints(self) -> Dict[str, Dict[str, Any]]:
        """Get default API endpoints configuration"""
        # Get FastAPI server URL, token, and interval from configuration
        fastapi_url = "https://mbetter.nexlab.net"
        api_token = ""
        api_interval = 1800  # 30 minutes default
        try:
            api_config = self.config_manager.get_section_config("api") or {}
            logger.debug("RAW API CONFIG: {}".format(api_config))

            # Load fastapi_url
            config_fastapi_url = api_config.get("fastapi_url")
            logger.debug("CONFIG FASTAPI_URL: '{}' (type: {})".format(config_fastapi_url, type(config_fastapi_url)))

            if config_fastapi_url and config_fastapi_url.strip():
                fastapi_url = config_fastapi_url.strip()
                logger.debug("USING CONFIG URL: {}".format(fastapi_url))
            else:
                logger.debug("CONFIG URL EMPTY, USING DEFAULT: {}".format(fastapi_url))

            # Load api_token
            config_api_token = api_config.get("api_token")
            logger.debug("CONFIG API_TOKEN: '{}' (type: {}, length: {})".format(
                config_api_token, type(config_api_token), len(config_api_token) if config_api_token else 0))

            if config_api_token and config_api_token.strip():
                api_token = config_api_token.strip()
                logger.debug("USING CONFIG TOKEN: {}...{}".format(api_token[:10], api_token[-10:] if len(api_token) > 20 else api_token))
            else:
                logger.debug("CONFIG TOKEN EMPTY, USING DEFAULT: {}".format(api_token))

            api_interval = api_config.get("api_interval", api_interval)
            logger.debug("API CONFIG LOADED - fastapi_url: {}, api_token: {}, api_interval: {}".format(
                fastapi_url, '*' * len(api_token) if api_token else 'empty', api_interval))
        except Exception as e:
            logger.warning("Could not load API configuration, using defaults: {}".format(e))
            logger.debug("CONFIG LOAD ERROR DETAILS: {}".format(str(e)))
        
        # Prepare authentication if token is provided
        auth_config = None
        headers = {"Content-Type": "application/json"}
        
        # Configure based on token availability
        if api_token and api_token.strip():
            auth_config = {
                "type": "bearer",
                "token": api_token.strip()
            }
            logger.debug("AUTH CONFIG - Token available, length: {}".format(len(api_token.strip())))
            # When token is provided: use configurable interval, 10 retries every 30 seconds
            interval = api_interval
            retry_attempts = 10
            retry_delay = 30  # 30 seconds
            enabled = True
        else:
            auth_config = None
            logger.debug("AUTH CONFIG - No token available")
            # When no token: use configurable interval or default, fewer retries, disabled by default
            interval = api_interval if api_interval != 1800 else 600  # Use configured or 10 minutes default
            retry_attempts = 3
            retry_delay = 5
            enabled = False  # Disabled when no authentication
        
        # Construct full URLs by appending paths to base URL
        fastapi_main_url = fastapi_url.rstrip('/') + "/api/updates"
        endpoints = {
            "fastapi_main": {
                "url": fastapi_main_url,
                "method": "POST",  # Use POST for /api/updates endpoint
                "headers": headers,
                "auth": auth_config,
                "interval": interval,
                "enabled": enabled,
                "timeout": 30,
                "retry_attempts": retry_attempts,
                "retry_delay": retry_delay,
                "response_handler": "updates"  # Use updates handler for match synchronization
            },
        }

        logger.debug("ENDPOINT URLS - base_url: {}, fastapi_main: {}".format(fastapi_url, fastapi_main_url))

        return endpoints
    
    def run(self):
        """Main run loop"""
        try:
            logger.info("APIClient thread started")
            
            # Send ready status
            ready_message = MessageBuilder.system_status(
                sender=self.name,
                status="ready",
                details={
                    "endpoints": len(self.endpoints),
                    "enabled_endpoints": sum(1 for ep in self.endpoints.values() if ep.enabled)
                }
            )
            self.message_bus.publish(ready_message)
            
            # Main execution loop
            while self.running and not self.shutdown_event.is_set():
                try:
                    # Update heartbeat at the start of each loop
                    self.heartbeat()

                    # Process messages
                    message = self.message_bus.get_message(self.name, timeout=1.0)
                    if message:
                        self._process_message(message)

                    # Update heartbeat before potentially long operations
                    self.heartbeat()

                    # Execute scheduled API requests
                    self._execute_scheduled_requests()

                    # Update heartbeat after operations
                    self.heartbeat()

                    time.sleep(1.0)

                except Exception as e:
                    logger.error(f"APIClient run loop error: {e}")
                    # Update heartbeat even in error cases
                    self.heartbeat()
                    time.sleep(5.0)
                    
        except Exception as e:
            logger.error(f"APIClient run failed: {e}")
        finally:
            logger.info("APIClient thread ended")
    
    def _execute_scheduled_requests(self):
        """Execute API requests that are due"""
        for endpoint in self.endpoints.values():
            if endpoint.should_execute():
                self._execute_endpoint_request(endpoint)
    
    def _get_last_fixture_timestamp(self) -> Optional[str]:
        """Get the server activation timestamp of the last active fixture in the database"""
        try:
            # Update heartbeat before database operation
            self.heartbeat()
            
            session = self.db_manager.get_session()
            try:
                # Get the most recent match with fixture_active_time set
                last_active_match = session.query(MatchModel).filter(
                    MatchModel.fixture_active_time.isnot(None)
                ).order_by(MatchModel.fixture_active_time.desc()).first()
                
                # Update heartbeat after database query
                self.heartbeat()
                
                if last_active_match and last_active_match.fixture_active_time:
                    # Return Unix timestamp as string (long integer number)
                    return str(last_active_match.fixture_active_time)
                else:
                    # No fixtures with activation time found - don't send 'from' parameter
                    return None
                    
            finally:
                session.close()
                
        except Exception as e:
            logger.error(f"Failed to get last fixture activation timestamp: {e}")
            # Update heartbeat even on error
            self.heartbeat()
            return None
    
    def _execute_endpoint_request(self, endpoint: APIEndpoint):
        """Execute a single API request with custom retry logic for token-based endpoints"""
        try:
            # Update heartbeat before starting potentially long operation
            self.heartbeat()
            
            endpoint.last_request = datetime.utcnow()
            endpoint.total_requests += 1
            self.stats['total_requests'] += 1
            
            logger.debug("Executing API request: {} -> {}".format(endpoint.name, endpoint.url))
            
            # Prepare request parameters
            request_kwargs = {
                'method': endpoint.method,
                'url': endpoint.url,
                'headers': endpoint.headers.copy(),
                'timeout': endpoint.timeout
            }
            
            # Prepare data/params based on method
            request_data = endpoint.params.copy() if endpoint.method == 'GET' else endpoint.data.copy()
            
            # For FastAPI /api/updates endpoint, add 'from' parameter and rustdesk_id if provided
            if endpoint.name == 'fastapi_main' and 'updates' in endpoint.url.lower():
                last_timestamp = self._get_last_fixture_timestamp()
                if last_timestamp:
                    request_data['from'] = last_timestamp
                    logger.debug("Adding 'from' parameter to {} request: {}".format(endpoint.name, last_timestamp))
                else:
                    # When no fixtures exist, send empty request to get all data
                    logger.debug("No fixtures found, sending empty request to {}".format(endpoint.name))

                # Add rustdesk_id to the updates call if provided
                logger.debug("DEBUG: Checking rustdesk_id - value: '{}' (type: {}), bool: {}".format(
                    self.settings.rustdesk_id, type(self.settings.rustdesk_id), bool(self.settings.rustdesk_id)))
                if self.settings.rustdesk_id:
                    request_data['rustdesk_id'] = self.settings.rustdesk_id
                    logger.debug("Adding rustdesk_id to {} request: {}".format(endpoint.name, self.settings.rustdesk_id))
                else:
                    logger.debug("rustdesk_id not set, skipping addition to request")
            
            if endpoint.method == 'GET':
                request_kwargs['params'] = request_data
            else:
                # For POST requests, always send JSON data (even if empty)
                request_kwargs['json'] = request_data if request_data else {}
            
            # Debug log the complete request data
            logger.debug(f"Request method: {endpoint.method}")
            logger.debug(f"Request headers: {request_kwargs['headers']}")
            if endpoint.method == 'GET' and request_data:
                logger.debug(f"Request params: {request_data}")
            elif endpoint.method != 'GET' and request_data:
                logger.debug(f"Request JSON data: {request_data}")
            else:
                logger.debug("Request data: (empty)")
            
            # Add authentication if configured
            if endpoint.auth:
                logger.debug("AUTH DEBUG - endpoint.auth: {}".format(endpoint.auth))
                logger.debug("AUTH DEBUG - auth type: {}".format(endpoint.auth.get('type')))
                if endpoint.auth.get('type') == 'bearer':
                    token = endpoint.auth.get('token')
                    logger.debug("AUTH DEBUG - raw token value: '{}' (repr: {})".format(token, repr(token)))
                    if token and token.strip():
                        request_kwargs['headers']['Authorization'] = "Bearer {}".format(token.strip())
                        logger.debug("AUTH - Added Bearer token for {}: {}...{}".format(endpoint.name, token.strip()[:10], token.strip()[-10:] if len(token.strip()) > 20 else token.strip()))
                    else:
                        logger.debug("AUTH - Bearer token configured but empty for {}".format(endpoint.name))
                elif endpoint.auth.get('type') == 'basic':
                    request_kwargs['auth'] = (endpoint.auth.get('username'), endpoint.auth.get('password'))
                    logger.debug("AUTH - Added Basic auth for {}".format(endpoint.name))
            else:
                logger.debug("AUTH - No authentication configured for {}".format(endpoint.name))
            
            # Generate curl command for easy debugging/replication (after auth is added)
            curl_cmd = self._generate_curl_command(endpoint.method, endpoint.url, request_kwargs['headers'], request_data)
            logger.debug(f"Curl equivalent: {curl_cmd}")
            
            # Check if this is a token-based endpoint that needs custom retry logic
            is_token_endpoint = (endpoint.auth and
                               endpoint.auth.get('type') == 'bearer' and
                               endpoint.auth.get('token'))
            
            if is_token_endpoint and endpoint.consecutive_failures > 0:
                # For token-based endpoints with failures, use custom retry logic
                success = self._execute_with_custom_retry(endpoint, request_kwargs)
                if not success:
                    return  # Custom retry logic handles error reporting
            else:
                # Standard execution for first attempt or non-token endpoints
                response = self.session.request(**request_kwargs)
                response.raise_for_status()
                
                # Update heartbeat after HTTP request completes
                self.heartbeat()
                
                # Debug log the complete response
                logger.debug(f"Response status code: {response.status_code}")
                logger.debug(f"Response headers: {dict(response.headers)}")
                try:
                    response_text = response.text
                    if len(response_text) > 1000:
                        logger.debug(f"Response body (truncated): {response_text[:1000]}...")
                    else:
                        logger.debug(f"Response body: {response_text}")
                except Exception as e:
                    logger.debug(f"Could not read response body: {e}")
                
                # Handle successful response
                handler = self.response_handlers.get(endpoint.response_handler, self.response_handlers['default'])
                processed_data = handler.handle_response(endpoint, response)
                
                # Update heartbeat after response processing
                self.heartbeat()
                
                # Update endpoint status
                endpoint.last_success = datetime.utcnow()
                endpoint.last_error = None
                endpoint.consecutive_failures = 0
                endpoint.successful_requests += 1
                self.stats['successful_requests'] += 1
                
                # Send response data via message bus
                if processed_data:
                    response_message = Message(
                        type=MessageType.API_RESPONSE,
                        sender=self.name,
                        data={
                            'endpoint': endpoint.name,
                            'success': True,
                            'data': processed_data,
                            'timestamp': datetime.utcnow().isoformat()
                        }
                    )
                    self.message_bus.publish(response_message)

                    # Check if this is a successful fixture update that might trigger game start
                    synchronized_matches = processed_data.get('synchronized_matches', 0)
                    downloaded_zips = processed_data.get('downloaded_zips', 0)
                    expected_zips = processed_data.get('expected_zips', 0)

                    if endpoint.name == 'fastapi_main' and synchronized_matches > 0:
                        if downloaded_zips == expected_zips:
                            logger.info(f"Fixture update completed successfully - {synchronized_matches} matches synchronized, {downloaded_zips}/{expected_zips} ZIPs downloaded")
                            logger.debug("All expected ZIP files downloaded - fixtures are ready for games")

                            # Send a message to trigger game start check
                            game_start_check_message = Message(
                                type=MessageType.SYSTEM_STATUS,
                                sender=self.name,
                                data={
                                    'status': 'fixture_update_completed',
                                    'synchronized_matches': synchronized_matches,
                                    'downloaded_zips': downloaded_zips,
                                    'expected_zips': expected_zips,
                                    'timestamp': datetime.utcnow().isoformat()
                                }
                            )
                            self.message_bus.publish(game_start_check_message)
                            logger.debug("Sent fixture_update_completed message to trigger game start check")
                        else:
                            logger.info(f"Fixture update partially completed - {synchronized_matches} matches synchronized, {downloaded_zips}/{expected_zips} ZIPs downloaded")
                            logger.debug(f"Waiting for remaining {expected_zips - downloaded_zips} ZIP files before fixtures are ready for games")
                
                logger.debug(f"API request successful: {endpoint.name}")
            
        except Exception as e:
            # Handle request failure
            self.heartbeat()  # Update heartbeat even on failure
            self._handle_request_failure(endpoint, e)
    
    def _execute_with_custom_retry(self, endpoint: APIEndpoint, request_kwargs: dict) -> bool:
        """Execute request with custom retry logic for token-based endpoints"""
        max_retries = min(endpoint.retry_attempts, 10)  # Cap at 10 retries as requested
        retry_delay = 30  # 30 seconds between retries as requested
        
        for attempt in range(max_retries):
            try:
                logger.info(f"Retry attempt {attempt + 1}/{max_retries} for endpoint {endpoint.name}")
                
                response = self.session.request(**request_kwargs)
                response.raise_for_status()
                
                # Update heartbeat after HTTP retry request completes
                self.heartbeat()
                
                # Debug log the complete response (retry scenario)
                logger.debug(f"Retry response status code: {response.status_code}")
                logger.debug(f"Retry response headers: {dict(response.headers)}")
                try:
                    response_text = response.text
                    if len(response_text) > 1000:
                        logger.debug(f"Retry response body (truncated): {response_text[:1000]}...")
                    else:
                        logger.debug(f"Retry response body: {response_text}")
                except Exception as e:
                    logger.debug(f"Could not read retry response body: {e}")
                
                # Handle successful response
                handler = self.response_handlers.get(endpoint.response_handler, self.response_handlers['default'])
                processed_data = handler.handle_response(endpoint, response)
                
                # Update heartbeat after response processing
                self.heartbeat()
                
                # Update endpoint status - success!
                endpoint.last_success = datetime.utcnow()
                endpoint.last_error = None
                endpoint.consecutive_failures = 0
                endpoint.successful_requests += 1
                self.stats['successful_requests'] += 1
                
                # Send response data via message bus
                if processed_data:
                    response_message = Message(
                        type=MessageType.API_RESPONSE,
                        sender=self.name,
                        data={
                            'endpoint': endpoint.name,
                            'success': True,
                            'data': processed_data,
                            'timestamp': datetime.utcnow().isoformat(),
                            'retry_attempt': attempt + 1
                        }
                    )
                    self.message_bus.publish(response_message)
                
                logger.info(f"API request successful on retry {attempt + 1}: {endpoint.name}")
                return True
                
            except Exception as e:
                endpoint.last_error = str(e)
                endpoint.consecutive_failures += 1
                self.stats['failed_requests'] += 1
                
                if attempt < max_retries - 1:
                    logger.warning(f"API retry {attempt + 1} failed for {endpoint.name}: {e}. Waiting {retry_delay}s before next retry.")
                    # Sleep in smaller chunks to allow heartbeat updates during long delays
                    remaining_delay = retry_delay
                    while remaining_delay > 0 and self.running:
                        sleep_chunk = min(5.0, remaining_delay)  # Sleep in 5-second chunks
                        time.sleep(sleep_chunk)
                        self.heartbeat()  # Update heartbeat during retry delays
                        remaining_delay -= sleep_chunk
                else:
                    logger.error(f"All {max_retries} retries failed for {endpoint.name}: {e}")
        
        # All retries failed
        self._handle_request_failure(endpoint, Exception(f"All {max_retries} retries failed. Last error: {endpoint.last_error}"))
        return False
    
    def _handle_request_failure(self, endpoint: APIEndpoint, error: Exception):
        """Handle request failure and send error message"""
        # Update heartbeat when handling failures
        self.heartbeat()
        
        endpoint.last_error = str(error)
        endpoint.consecutive_failures += 1
        self.stats['failed_requests'] += 1
        
        logger.error(f"API request failed: {endpoint.name} - {error}")
        
        # Log heartbeat status for debugging
        logger.debug(f"API client heartbeat updated during error handling for {endpoint.name}")
        
        # Send error message
        error_message = Message(
            type=MessageType.API_RESPONSE,
            sender=self.name,
            data={
                'endpoint': endpoint.name,
                'success': False,
                'error': str(error),
                'consecutive_failures': endpoint.consecutive_failures,
                'timestamp': datetime.utcnow().isoformat()
            }
        )
        self.message_bus.publish(error_message)
        
        # For token-based endpoints, don't disable after failures - keep retrying per user requirements
        is_token_endpoint = (endpoint.auth and
                           endpoint.auth.get('type') == 'bearer' and
                           endpoint.auth.get('token'))
        
        if not is_token_endpoint and endpoint.consecutive_failures >= self.settings.max_consecutive_failures:
            endpoint.enabled = False
            logger.warning(f"Endpoint disabled due to consecutive failures: {endpoint.name}")
    
    def _process_message(self, message: Message):
        """Process received message"""
        try:
            if message.type == MessageType.CONFIG_UPDATE:
                self._handle_config_update(message)
            elif message.type == MessageType.API_REQUEST:
                self._handle_api_request(message)
            # Add other message types as needed
        except Exception as e:
            logger.error(f"Failed to process message: {e}")
    
    def _handle_config_update(self, message: Message):
        """Handle configuration update message"""
        try:
            config_section = message.data.get("config_section")
            config_data = message.data.get("config_data", {})

            if config_section == "api_endpoints":
                logger.info("API endpoints configuration updated")
                self._load_endpoints()
            elif config_section == "api":
                logger.info("API configuration updated, reloading endpoints")

                # Check for token and URL changes
                new_token = config_data.get("api_token", "").strip()
                new_url = config_data.get("fastapi_url", "").strip()
                old_token = ""
                old_url = ""

                try:
                    old_api_config = self.config_manager.get_section_config("api") or {}
                    old_token = old_api_config.get("api_token", "").strip()
                    old_url = old_api_config.get("fastapi_url", "").strip()
                except Exception:
                    pass

                # Reload endpoints to pick up new configuration
                self._load_endpoints()

                # Handle dynamic timer start/stop based on token changes
                self._handle_token_change(old_token, new_token)

                # Handle URL changes - trigger immediate connection test
                self._handle_url_change(old_url, new_url)

        except Exception as e:
            logger.error(f"Failed to handle config update: {e}")
    
    def _handle_token_change(self, old_token: str, new_token: str):
        """Handle dynamic timer start/stop based on token changes"""
        try:
            fastapi_endpoint = self.endpoints.get("fastapi_main")
            if not fastapi_endpoint:
                return

            old_has_token = bool(old_token and old_token.strip())
            new_has_token = bool(new_token and new_token.strip())

            # Get current interval setting
            current_api_config = self.config_manager.get_section_config("api") or {}
            current_interval = current_api_config.get("api_interval", 1800)  # 30 minutes default

            if not old_has_token and new_has_token:
                # Token was added - start timer and trigger immediate fixture update
                fastapi_endpoint.enabled = True
                fastapi_endpoint.interval = current_interval
                fastapi_endpoint.retry_attempts = 10
                fastapi_endpoint.retry_delay = 30
                fastapi_endpoint.consecutive_failures = 0  # Reset failure count
                fastapi_endpoint.last_request = None  # Reset to trigger immediate first request

                logger.info(f"FastAPI timer started - token configured, {current_interval} second intervals enabled")

                # Trigger immediate fixture update check
                logger.info("Token configured - triggering immediate fixture update check")
                self._execute_endpoint_request(fastapi_endpoint)

                # Send immediate status update
                status_message = Message(
                    type=MessageType.SYSTEM_STATUS,
                    sender=self.name,
                    data={
                        "status": "timer_started",
                        "endpoint": "fastapi_main",
                        "reason": "token_configured",
                        "interval_seconds": current_interval,
                        "immediate_update_triggered": True
                    }
                )
                self.message_bus.publish(status_message)

            elif old_has_token and not new_has_token:
                # Token was removed - stop timer
                fastapi_endpoint.enabled = False
                fastapi_endpoint.interval = 600  # Reset to 10 minutes (default)
                fastapi_endpoint.retry_attempts = 3
                fastapi_endpoint.retry_delay = 5

                logger.info("FastAPI timer stopped - token removed, automatic requests disabled")

                # Send status update
                status_message = Message(
                    type=MessageType.SYSTEM_STATUS,
                    sender=self.name,
                    data={
                        "status": "timer_stopped",
                        "endpoint": "fastapi_main",
                        "reason": "token_removed"
                    }
                )
                self.message_bus.publish(status_message)

            elif old_has_token and new_has_token and old_token != new_token:
                # Token was changed - keep timer running but reset failure count and trigger immediate update
                fastapi_endpoint.consecutive_failures = 0
                fastapi_endpoint.last_request = None  # Trigger immediate request with new token

                logger.info("FastAPI token updated - timer continues with new authentication")

                # Trigger immediate fixture update check with new token
                logger.info("Token updated - triggering immediate fixture update check with new token")
                self._execute_endpoint_request(fastapi_endpoint)

        except Exception as e:
            logger.error(f"Failed to handle token change: {e}")

    def _handle_url_change(self, old_url: str, new_url: str):
        """Handle URL changes - trigger immediate connection test"""
        try:
            if old_url != new_url and new_url.strip():
                fastapi_endpoint = self.endpoints.get("fastapi_main")
                if fastapi_endpoint:
                    logger.info(f"FastAPI URL changed from '{old_url}' to '{new_url}' - triggering immediate connection test")

                    # Temporarily enable endpoint for connection test if it was disabled
                    was_enabled = fastapi_endpoint.enabled
                    if not was_enabled:
                        fastapi_endpoint.enabled = True
                        logger.debug("Temporarily enabled fastapi_main endpoint for connection test")

                    # Reset failure count and last request to ensure immediate execution
                    fastapi_endpoint.consecutive_failures = 0
                    fastapi_endpoint.last_request = None

                    # Trigger immediate request to test new URL
                    logger.info("Triggering immediate connection test to new FastAPI endpoint")
                    self._execute_endpoint_request(fastapi_endpoint)

                    # Restore original enabled state after test
                    if not was_enabled:
                        fastapi_endpoint.enabled = False
                        logger.debug("Restored fastapi_main endpoint to disabled state after test")

                    # Send status update about URL change
                    status_message = Message(
                        type=MessageType.SYSTEM_STATUS,
                        sender=self.name,
                        data={
                            "status": "url_changed",
                            "endpoint": "fastapi_main",
                            "old_url": old_url,
                            "new_url": new_url,
                            "connection_test_triggered": True,
                            "endpoint_was_enabled": was_enabled
                        }
                    )
                    self.message_bus.publish(status_message)

        except Exception as e:
            logger.error(f"Failed to handle URL change: {e}")

    def update_fastapi_url(self, new_url: str) -> bool:
        """Update FastAPI server URL and reload endpoints"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            api_config["fastapi_url"] = new_url
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new URL
            self._load_endpoints()
            
            logger.info(f"FastAPI URL updated to: {new_url}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update FastAPI URL: {e}")
            return False
    
    def update_api_token(self, new_token: str) -> bool:
        """Update API authentication token and reload endpoints"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            api_config["api_token"] = new_token
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new token
            self._load_endpoints()
            
            logger.info("API token updated successfully")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update API token: {e}")
            return False
    
    def update_api_config(self, new_url: str = None, new_token: str = None) -> bool:
        """Update FastAPI URL and/or token configuration"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            
            if new_url is not None:
                api_config["fastapi_url"] = new_url
                
            if new_token is not None:
                api_config["api_token"] = new_token
            
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new configuration
            self._load_endpoints()
            
            logger.info(f"API configuration updated - URL: {new_url}, Token: {'*' * len(new_token) if new_token else 'unchanged'}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update API configuration: {e}")
            return False
    
    def _handle_api_request(self, message: Message):
        """Handle manual API request message"""
        try:
            endpoint_name = message.data.get("endpoint")
            
            if endpoint_name in self.endpoints:
                endpoint = self.endpoints[endpoint_name]
                self._execute_endpoint_request(endpoint)
            else:
                logger.warning(f"Unknown endpoint requested: {endpoint_name}")
            
        except Exception as e:
            logger.error(f"Failed to handle API request: {e}")
    
    def get_endpoint_status(self, endpoint_name: str) -> Optional[Dict[str, Any]]:
        """Get status of a specific endpoint"""
        endpoint = self.endpoints.get(endpoint_name)
        if endpoint:
            return endpoint.to_dict()
        return None
    
    def get_all_endpoints_status(self) -> Dict[str, Dict[str, Any]]:
        """Get status of all endpoints"""
        return {name: endpoint.to_dict() for name, endpoint in self.endpoints.items()}
    
    def get_client_stats(self) -> Dict[str, Any]:
        """Get client statistics"""
        uptime = datetime.utcnow() - self.stats['start_time']
        return {
            'total_requests': self.stats['total_requests'],
            'successful_requests': self.stats['successful_requests'],
            'failed_requests': self.stats['failed_requests'],
            'success_rate': (self.stats['successful_requests'] / max(1, self.stats['total_requests'])) * 100,
            'uptime_seconds': uptime.total_seconds(),
            'endpoints_count': len(self.endpoints),
            'enabled_endpoints': sum(1 for ep in self.endpoints.values() if ep.enabled)
        }
    
    def update_endpoint(self, endpoint_name: str, config: Dict[str, Any]) -> bool:
        """Update endpoint configuration"""
        try:
            if endpoint_name in self.endpoints:
                # Update existing endpoint
                endpoint = self.endpoints[endpoint_name]
                for key, value in config.items():
                    if hasattr(endpoint, key):
                        setattr(endpoint, key, value)
            else:
                # Create new endpoint
                self.endpoints[endpoint_name] = APIEndpoint(endpoint_name, config)
            
            # Save to configuration
            all_endpoints = {name: ep.to_dict() for name, ep in self.endpoints.items()}
            self.config_manager.update_section("api_endpoints", all_endpoints)
            
            logger.info(f"Updated endpoint configuration: {endpoint_name}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update endpoint: {e}")
            return False
    
    def shutdown(self):
        """Shutdown API client"""
        try:
            logger.info("Shutting down APIClient...")
            
            if self.session:
                self.session.close()
            
        except Exception as e:
            logger.error(f"APIClient shutdown error: {e}")
    
    def _generate_curl_command(self, method, url, headers, data=None):
        """Generate a curl command equivalent to the current request for debugging purposes."""
        import json
        import shlex
        
        # Start with basic curl command
        cmd_parts = ['curl', '-X', method]
        
        # Add headers
        for key, value in headers.items():
            cmd_parts.extend(['-H', f'{key}: {value}'])
        
        # Add data based on method
        if method == 'GET' and data:
            # For GET requests, params are added to URL
            from urllib.parse import urlencode
            query_string = urlencode(data)
            url = f"{url}?{query_string}"
        elif method == 'POST':
            # For POST requests, always add JSON data (even if empty)
            json_str = json.dumps(data if data else {}, separators=(',', ':'))
            cmd_parts.extend(['-d', json_str])
        
        # Add URL (always last)
        cmd_parts.append(url)
        
        # Use shlex.join for proper shell escaping (Python 3.8+)
        try:
            return shlex.join(cmd_parts)
        except AttributeError:
            # Fallback for Python < 3.8
            return ' '.join(shlex.quote(arg) for arg in cmd_parts)