"""
REST API Client - Configurable HTTP client with retry logic and failure handling
"""

import time
import logging
import json
import threading
import os
import hashlib
from pathlib import Path
from datetime import datetime, timedelta
from typing import Dict, Any, Optional, List, Union
from urllib.parse import urljoin, urlparse
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

from ..core.thread_manager import ThreadedComponent
from ..core.message_bus import MessageBus, Message, MessageType, MessageBuilder
from ..config.settings import ApiConfig
from ..config.manager import ConfigManager
from ..database.manager import DatabaseManager
from ..database.models import (
    MatchModel, MatchOutcomeModel, MatchTemplateModel, MatchOutcomeTemplateModel,
    ReportsSyncQueueModel
)
from ..config.settings import get_user_data_dir
from ..utils.ssl_utils import create_requests_session_with_ssl_support

logger = logging.getLogger(__name__)


class APIEndpoint:
    """Configuration for a single API endpoint"""
    
    def __init__(self, name: str, config: Dict[str, Any]):
        self.name = name
        self.url = config.get('url', '')
        self.method = config.get('method', 'GET').upper()
        self.headers = config.get('headers', {})
        self.params = config.get('params', {})
        self.data = config.get('data', {})
        self.interval = config.get('interval', 300)  # 5 minutes default
        self.enabled = config.get('enabled', True)
        self.timeout = config.get('timeout', 30)
        self.retry_attempts = config.get('retry_attempts', 3)
        self.retry_delay = config.get('retry_delay', 5)
        self.auth = config.get('auth', None)
        self.response_handler = config.get('response_handler', 'default')
        
        # Runtime state
        self.last_request = None
        self.last_success = None
        self.last_error = None
        self.consecutive_failures = 0
        self.total_requests = 0
        self.successful_requests = 0
    
    def should_execute(self) -> bool:
        """Check if endpoint should be executed based on interval"""
        if not self.enabled:
            return False
        
        if self.last_request is None:
            return True
        
        next_execution = self.last_request + timedelta(seconds=self.interval)
        return datetime.utcnow() >= next_execution
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization"""
        return {
            'name': self.name,
            'url': self.url,
            'method': self.method,
            'headers': self.headers,
            'params': self.params,
            'data': self.data,
            'interval': self.interval,
            'enabled': self.enabled,
            'timeout': self.timeout,
            'retry_attempts': self.retry_attempts,
            'retry_delay': self.retry_delay,
            'auth': self.auth,
            'response_handler': self.response_handler,
            'last_request': self.last_request.isoformat() if self.last_request else None,
            'last_success': self.last_success.isoformat() if self.last_success else None,
            'last_error': self.last_error,
            'consecutive_failures': self.consecutive_failures,
            'total_requests': self.total_requests,
            'successful_requests': self.successful_requests
        }


class ResponseHandler:
    """Base class for response handlers"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        """Handle API response and return processed data"""
        raise NotImplementedError
    
    def handle_error(self, endpoint: APIEndpoint, error: Exception) -> Optional[Dict[str, Any]]:
        """Handle API error and return error data"""
        return {
            'error': str(error),
            'endpoint': endpoint.name,
            'timestamp': datetime.utcnow().isoformat()
        }


class DefaultResponseHandler(ResponseHandler):
    """Default response handler - processes JSON responses"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            if response.headers.get('content-type', '').startswith('application/json'):
                return response.json()
            else:
                return {
                    'status_code': response.status_code,
                    'content': response.text,
                    'headers': dict(response.headers)
                }
        except json.JSONDecodeError:
            return {
                'status_code': response.status_code,
                'content': response.text,
                'headers': dict(response.headers)
            }


class NewsResponseHandler(ResponseHandler):
    """Response handler for news APIs - extracts headline and ticker data"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            # Extract news data for overlay templates
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'raw_data': data
            }
            
            # Try to extract common news fields
            if 'articles' in data:
                # NewsAPI format
                articles = data['articles'][:5]  # Top 5 articles
                processed_data['headlines'] = [article.get('title', '') for article in articles]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            elif 'items' in data:
                # RSS format
                items = data['items'][:5]
                processed_data['headlines'] = [item.get('title', '') for item in items]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            elif isinstance(data, list):
                # Direct array format
                processed_data['headlines'] = [item.get('title', '') for item in data[:5]]
                processed_data['ticker_text'] = ' • '.join(processed_data['headlines'])
            
            return processed_data
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.error(f"Failed to process news response: {e}")
            return self.handle_error(endpoint, e)


class SportsResponseHandler(ResponseHandler):
    """Response handler for sports APIs - extracts scores and game data"""
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'raw_data': data
            }
            
            # Extract sports data
            if 'games' in data:
                # Generic sports API format
                games = data['games'][:3]  # Top 3 games
                game_data = []
                for game in games:
                    game_data.append({
                        'team1': game.get('home_team', 'Home'),
                        'team2': game.get('away_team', 'Away'),
                        'score1': game.get('home_score', 0),
                        'score2': game.get('away_score', 0),
                        'status': game.get('status', 'Live')
                    })
                processed_data['games'] = game_data
            
            return processed_data
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.error(f"Failed to process sports response: {e}")
            return self.handle_error(endpoint, e)


class UpdatesResponseHandler(ResponseHandler):
    """Response handler for /api/updates endpoint - synchronizes match data"""

    def __init__(self, db_manager, user_data_dir, api_client=None, message_bus=None):
        self.db_manager = db_manager
        self.user_data_dir = user_data_dir
        self.api_client = api_client  # Reference to parent API client for token access
        self.message_bus = message_bus  # Reference to message bus for progress updates
        self.zip_storage_dir = Path(user_data_dir) / "zip_files"
        self.zip_storage_dir.mkdir(parents=True, exist_ok=True)
    
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response) -> Optional[Dict[str, Any]]:
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'synchronized_matches': 0,
                'downloaded_zips': 0,
                'expected_zips': 0,
                'errors': []
            }
            
            # Extract fixtures from response - new API format has fixtures containing matches
            fixtures = []
            if isinstance(data, dict):
                if 'fixtures' in data:
                    fixtures = data.get('fixtures', [])
                elif 'matches' in data:
                    # Fallback for old format - treat as single fixture with matches
                    fixtures = [{'matches': data.get('matches', [])}]
            elif isinstance(data, list):
                # Direct array of matches - treat as single fixture
                fixtures = [{'matches': data}]
            
            if not fixtures:
                processed_data['message'] = "No fixtures in response"
                return processed_data
            
            # First pass: synchronize all match data to database
            logger.debug(f"Starting fixture synchronization for {len(fixtures)} fixtures")
            session = self.db_manager.get_session()
            try:
                for fixture_data in fixtures:
                    fixture_id = fixture_data.get('fixture_id', 'unknown')
                    matches = fixture_data.get('matches', [])
                    logger.debug(f"Synchronizing fixture {fixture_id} with {len(matches)} matches")

                    # Handle both list and dict formats for matches
                    if isinstance(matches, dict):
                        # Convert dict to list of match data
                        matches = [match_data for match_data in matches.values()]
                        logger.debug(f"Converted matches dict to list with {len(matches)} items")
                    elif not isinstance(matches, list):
                        logger.warning(f"Unexpected matches format: {type(matches)}, skipping fixture {fixture_id}")
                        continue

                    for match_data in matches:
                        match_number = match_data.get('match_number', 'unknown')
                        try:
                            # Update heartbeat during processing to prevent health check failures
                            if self.api_client:
                                self.api_client.heartbeat()

                            # Add fixture-level data to match if available
                            if 'fixture_id' in fixture_data:
                                match_data['fixture_id'] = fixture_data['fixture_id']
                            if 'fixture_active_time' in fixture_data:
                                match_data['fixture_active_time'] = fixture_data['fixture_active_time']

                            logger.debug(f"Synchronizing match {match_number} in fixture {fixture_id}")

                            # Synchronize match data to database
                            self._synchronize_match(session, match_data)
                            processed_data['synchronized_matches'] += 1
                            logger.debug(f"Successfully synchronized match {match_number}")

                        except Exception as e:
                            error_msg = f"Failed to process match {match_number} in fixture {fixture_id}: {e}"
                            logger.error(error_msg)
                            processed_data['errors'].append(error_msg)
                            # Continue processing other matches even if this one fails
                            continue

                # Update heartbeat before potentially slow commit operation
                if self.api_client:
                    self.api_client.heartbeat()

                logger.debug(f"Committing {processed_data['synchronized_matches']} synchronized matches to database")
                session.commit()
                logger.debug("Database commit completed successfully")

                # Clean up old fixture templates and keep only the new ones
                self._cleanup_old_fixture_templates(session, fixtures)
            finally:
                session.close()

            # Second pass: download ZIP files without holding database session
            # This prevents database locking during potentially slow downloads
            logger.debug(f"Starting ZIP file downloads for {len(fixtures)} fixtures")
            total_matches_with_zips = 0

            # Send initial progress update - starting downloads
            self._send_download_progress(0, processed_data['expected_zips'], "Starting downloads...")

            for fixture_data in fixtures:
                fixture_id = fixture_data.get('fixture_id', 'unknown')
                matches = fixture_data.get('matches', [])
                logger.debug(f"Processing fixture {fixture_id} with {len(matches)} matches")

                for match_data in matches:
                    match_number = match_data.get('match_number', 'unknown')
                    zip_filename = match_data.get('zip_filename')

                    try:
                        # Update heartbeat before each download
                        if self.api_client:
                            self.api_client.heartbeat()

                        # Download ZIP file if available (check match-level zip_download_url)
                        if 'zip_download_url' in match_data:
                            processed_data['expected_zips'] += 1
                            total_matches_with_zips += 1
                            match_data['zip_url'] = match_data['zip_download_url']
                            logger.debug(f"Found ZIP file to download: {zip_filename} for match {match_number} in fixture {fixture_id}")

                            # Attempt download with retries
                            download_success = False
                            max_retries = 3
                            for attempt in range(max_retries):
                                try:
                                    # Send progress update - starting/retrying download
                                    attempt_desc = f" (attempt {attempt + 1}/{max_retries})" if attempt > 0 else ""
                                    progress_percent = int((processed_data['downloaded_zips'] / max(1, processed_data['expected_zips'])) * 100)
                                    self._send_download_progress(processed_data['downloaded_zips'], processed_data['expected_zips'],
                                                                f"Downloading {zip_filename}{attempt_desc}...")

                                    download_success = self._download_zip_file(match_data)
                                    if download_success:
                                        processed_data['downloaded_zips'] += 1
                                        logger.debug(f"Successfully downloaded ZIP file: {zip_filename} for match {match_number}")

                                        # Send progress update - download completed
                                        progress_percent = int((processed_data['downloaded_zips'] / max(1, processed_data['expected_zips'])) * 100)
                                        self._send_download_progress(processed_data['downloaded_zips'], processed_data['expected_zips'],
                                                                    f"Downloaded {zip_filename}")
                                        break
                                    else:
                                        if attempt < max_retries - 1:
                                            logger.warning(f"ZIP download attempt {attempt + 1} failed for {zip_filename}, retrying...")
                                            # Small delay before retry
                                            import time
                                            time.sleep(1)
                                        else:
                                            logger.error(f"All {max_retries} download attempts failed for {zip_filename}")
                                except Exception as retry_e:
                                    if attempt < max_retries - 1:
                                        logger.warning(f"ZIP download attempt {attempt + 1} failed for {zip_filename}: {retry_e}, retrying...")
                                        import time
                                        time.sleep(1)
                                    else:
                                        logger.error(f"All {max_retries} download attempts failed for {zip_filename}: {retry_e}")

                            if not download_success:
                                logger.debug(f"ZIP file download failed after retries: {zip_filename} for match {match_number}")

                    except requests.exceptions.HTTPError as http_err:
                        # Check if this is a "fixture no longer available" error (404)
                        if hasattr(http_err.response, 'status_code') and http_err.response.status_code == 404:
                            logger.debug(f"ZIP file no longer available on server (404): {zip_filename or 'unknown'} for match {match_number} - continuing with normal operations")
                            # Don't add to errors list, just skip this download
                        else:
                            # Other HTTP errors are actual failures
                            error_msg = f"HTTP error downloading ZIP for match {match_number}: {http_err}"
                            logger.error(error_msg)
                            processed_data['errors'].append(error_msg)
                    except Exception as e:
                        error_msg = f"Failed to download ZIP for match {match_number}: {e}"
                        logger.error(error_msg)
                        processed_data['errors'].append(error_msg)
                        # Continue with other downloads even if this one fails
                        continue

            logger.debug(f"ZIP download summary: {processed_data['downloaded_zips']}/{processed_data['expected_zips']} ZIP files downloaded successfully from {total_matches_with_zips} matches")

            # Send final progress update - downloads completed
            if processed_data['expected_zips'] > 0:
                self._send_download_progress(processed_data['downloaded_zips'], processed_data['expected_zips'],
                                           f"Downloads completed - {processed_data['downloaded_zips']}/{processed_data['expected_zips']} files")
            else:
                self._send_download_progress(0, 0, "No downloads needed")

            logger.info(f"Synchronized {processed_data['synchronized_matches']} matches, downloaded {processed_data['downloaded_zips']} ZIP files")
            return processed_data
            
        except Exception as e:
            logger.error(f"Failed to process updates response: {e}")
            return self.handle_error(endpoint, e)
    
    def _synchronize_match(self, session, match_data: Dict[str, Any]):
        """Synchronize match data to template database tables"""
        try:
            match_number = match_data.get('match_number')
            fixture_id = match_data.get('fixture_id')

            if not match_number or not fixture_id:
                logger.warning(f"Skipping match with missing match_number ({match_number}) or fixture_id ({fixture_id})")
                return

            # Check if match already exists by both match_number AND fixture_id in template tables
            existing_match = session.query(MatchTemplateModel).filter_by(
                match_number=match_number,
                fixture_id=fixture_id
            ).first()

            if existing_match:
                # Update existing match template
                for key, value in match_data.items():
                    if hasattr(existing_match, key) and key not in ['id', 'created_at']:
                        setattr(existing_match, key, value)
                existing_match.updated_at = datetime.utcnow()
                match = existing_match
                logger.debug(f"Updated existing match template {match_number} for fixture {fixture_id}")
            else:
                # Create new match template
                match = MatchTemplateModel(
                    match_number=match_data.get('match_number'),
                    fighter1_township=match_data.get('fighter1_township', ''),
                    fighter2_township=match_data.get('fighter2_township', ''),
                    venue_kampala_township=match_data.get('venue_kampala_township', ''),
                    start_time=self._parse_datetime(match_data.get('start_time')),
                    end_time=self._parse_datetime(match_data.get('end_time')),
                    result=match_data.get('result'),
                    filename=match_data.get('filename', ''),
                    file_sha1sum=match_data.get('file_sha1sum', ''),
                    fixture_id=fixture_id,
                    active_status=match_data.get('active_status', False),
                    zip_filename=match_data.get('zip_filename'),
                    zip_sha1sum=match_data.get('zip_sha1sum'),
                    zip_upload_status=match_data.get('zip_upload_status', 'pending'),
                    zip_upload_progress=match_data.get('zip_upload_progress', 0.0),
                    done=match_data.get('done', False),
                    running=match_data.get('running', False),
                    fixture_active_time=match_data.get('fixture_active_time'),
                    created_by=1  # Default admin user
                )
                session.add(match)
                logger.debug(f"Created new match template {match_number} for fixture {fixture_id}")

            # Flush to get the match ID for outcomes
            session.flush()

            # Handle match outcomes in template table
            outcomes_data = match_data.get('outcomes', {})
            if outcomes_data:
                # Remove existing outcomes
                session.query(MatchOutcomeTemplateModel).filter_by(match_id=match.id).delete()

                # Normalize outcomes_data to dict format
                normalized_outcomes = {}
                try:
                    if isinstance(outcomes_data, dict):
                        # Check if values are dicts with 'value' key or direct float values
                        for key, value in outcomes_data.items():
                            if isinstance(value, dict) and 'value' in value:
                                normalized_outcomes[key] = value['value']
                            elif isinstance(value, (int, float)):
                                normalized_outcomes[key] = value
                            else:
                                logger.warning(f"Unexpected outcome value type for {key}: {type(value)}, skipping")
                                continue
                    elif isinstance(outcomes_data, list):
                        # Handle list format [{"name": "WIN1", "value": 2.5}, ...]
                        for item in outcomes_data:
                            if isinstance(item, dict) and 'name' in item and 'value' in item:
                                normalized_outcomes[item['name']] = item['value']
                            else:
                                logger.warning(f"Unexpected list item format: {item}, skipping")
                                continue
                    else:
                        logger.warning(f"Unexpected outcomes_data type: {type(outcomes_data)}, expected dict or list")
                except Exception as e:
                    logger.error(f"Failed to normalize outcomes_data: {e}")
                    # Continue without outcomes if normalization fails

                # Add new outcomes
                for column_name, float_value in normalized_outcomes.items():
                    try:
                        outcome = MatchOutcomeTemplateModel(
                            match_id=match.id,
                            column_name=str(column_name),
                            float_value=float(float_value)
                        )
                        session.add(outcome)
                    except (ValueError, TypeError) as e:
                        logger.warning(f"Failed to process outcome {column_name}: {float_value} - {e}")
                        continue

        except Exception as e:
            logger.error(f"Failed to synchronize match template: {e}")
            # Rollback this session to clean state
            session.rollback()
            raise
    
    def _download_zip_file(self, match_data: Dict[str, Any]) -> bool:
        """Download ZIP file to persistent storage with API token authentication"""
        zip_url = match_data.get('zip_url')
        zip_filename = match_data.get('zip_filename')

        if not zip_url or not zip_filename:
            logger.debug(f"Skipping ZIP download - missing URL or filename: url={zip_url}, filename={zip_filename}")
            return False

        logger.debug(f"Starting ZIP file download: {zip_filename} from {zip_url}")

        try:
            # Prepare headers with API token authentication
            headers = {}

            # Get API token from the API client's fastapi_main endpoint
            if self.api_client and hasattr(self.api_client, 'endpoints'):
                fastapi_endpoint = self.api_client.endpoints.get('fastapi_main')
                if fastapi_endpoint and fastapi_endpoint.auth:
                    if fastapi_endpoint.auth.get('type') == 'bearer':
                        token = fastapi_endpoint.auth.get('token')
                        if token:
                            headers['Authorization'] = f"Bearer {token}"
                            logger.debug(f"Using API token for ZIP download: {zip_filename}")
                        else:
                            logger.debug(f"No API token available for ZIP download: {zip_filename}")

            if not headers:
                logger.warning(f"No API token available for ZIP download: {zip_filename}")

            # Download ZIP file with authentication using session with SSL support
            logger.debug(f"Initiating HTTP request for ZIP file: {zip_filename}")
            response = self.api_client.session.get(zip_url, stream=True, timeout=30, headers=headers)
            response.raise_for_status()
            logger.debug(f"HTTP request successful for {zip_filename}, status: {response.status_code}")

            # Save to persistent storage
            zip_path = self.zip_storage_dir / zip_filename
            logger.debug(f"Saving ZIP file to: {zip_path}")

            # Update heartbeat before potentially slow file write
            if self.api_client:
                self.api_client.heartbeat()

            bytes_downloaded = 0
            with open(zip_path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:  # Filter out keep-alive new chunks
                        f.write(chunk)
                        bytes_downloaded += len(chunk)

                        # Update heartbeat every few chunks for large files
                        if self.api_client and f.tell() % (1024 * 1024) == 0:  # Every MB
                            logger.debug(f"Downloaded {f.tell() // (1024 * 1024)}MB for {zip_filename}")
                            self.api_client.heartbeat()

            logger.debug(f"File download completed: {zip_filename}, total bytes: {bytes_downloaded}")

            # Validate downloaded file integrity
            logger.debug(f"Starting validation for downloaded ZIP file: {zip_filename}")
            if not self._validate_downloaded_zip(zip_path, match_data):
                logger.error(f"ZIP file validation failed: {zip_filename}")
                # Update database validation status to 'invalid'
                try:
                    session = self.db_manager.get_session()
                    try:
                        # Find the match template and update validation status
                        match_template = session.query(MatchTemplateModel).filter_by(
                            zip_filename=zip_filename
                        ).first()
                        if match_template:
                            match_template.zip_validation_status = 'invalid'
                            session.commit()
                            logger.debug(f"Updated database validation status to 'invalid' for {zip_filename}")
                    finally:
                        session.close()
                except Exception as db_e:
                    logger.warning(f"Failed to update database validation status for {zip_filename}: {db_e}")

                # Remove corrupted file
                try:
                    zip_path.unlink()
                    logger.debug(f"Removed corrupted ZIP file: {zip_filename}")
                except Exception as cleanup_e:
                    logger.warning(f"Failed to remove corrupted ZIP file {zip_filename}: {cleanup_e}")
                return False

            # Update database validation status to 'valid' after successful validation
            try:
                session = self.db_manager.get_session()
                try:
                    # Find the match template and update validation status
                    match_template = session.query(MatchTemplateModel).filter_by(
                        zip_filename=zip_filename
                    ).first()
                    if match_template:
                        match_template.zip_validation_status = 'valid'
                        session.commit()
                        logger.debug(f"Updated database validation status to 'valid' for {zip_filename}")
                finally:
                    session.close()
            except Exception as db_e:
                logger.warning(f"Failed to update database validation status for {zip_filename}: {db_e}")

            # Start detailed ZIP validation (checking for video files) asynchronously
            logger.debug(f"Starting detailed ZIP validation for {zip_filename}")
            try:
                # Find the match template to get its ID for validation
                session = self.db_manager.get_session()
                try:
                    match_template = session.query(MatchTemplateModel).filter_by(
                        zip_filename=zip_filename
                    ).first()
                    if match_template:
                        # Start detailed validation asynchronously
                        self._validate_single_zip_async(match_template.id, session, MatchTemplateModel)
                        logger.debug(f"Started detailed validation for match template {match_template.id}")
                    else:
                        logger.warning(f"Could not find match template for ZIP file {zip_filename} to start detailed validation")
                finally:
                    session.close()
            except Exception as val_e:
                logger.warning(f"Failed to start detailed ZIP validation for {zip_filename}: {val_e}")

            logger.info(f"Successfully downloaded and validated ZIP file: {zip_filename}")
            return True

        except requests.exceptions.HTTPError as http_err:
            # Check if this is a "fixture no longer available" error (404)
            if hasattr(http_err.response, 'status_code') and http_err.response.status_code == 404:
                logger.debug(f"ZIP file no longer available on server (404): {zip_filename} - continuing with normal operations")
                return False  # Don't treat as error, just skip this download
            else:
                # Other HTTP errors are actual failures
                logger.error(f"HTTP error downloading ZIP file {zip_filename}: {http_err}")
                raise
        except requests.exceptions.RequestException as req_err:
            logger.error(f"Network error downloading ZIP file {zip_filename}: {req_err}")
            return False
        except IOError as io_err:
            logger.error(f"File I/O error downloading ZIP file {zip_filename}: {io_err}")
            return False
        except Exception as e:
            logger.error(f"Unexpected error downloading ZIP file {zip_filename}: {e}")
            return False
    
    def _parse_datetime(self, datetime_str) -> Optional[datetime]:
        """Parse datetime string to datetime object"""
        if not datetime_str:
            return None

        try:
            # Try ISO format first
            return datetime.fromisoformat(datetime_str.replace('Z', '+00:00'))
        except:
            try:
                # Try common formats
                return datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S')
            except:
                return None

    def _calculate_sha1(self, file_path: Path) -> str:
        """Calculate SHA1 checksum of a file"""
        try:
            sha1 = hashlib.sha1()
            with open(file_path, 'rb') as f:
                # Read in chunks to handle large files
                for chunk in iter(lambda: f.read(8192), b""):
                    sha1.update(chunk)
            return sha1.hexdigest()
        except Exception as e:
            logger.error(f"Failed to calculate SHA1 for {file_path}: {e}")
            return ""

    def _validate_downloaded_zip(self, zip_path: Path, match_data: Dict[str, Any]) -> bool:
        """Validate downloaded ZIP file integrity using checksum and size"""
        try:
            zip_filename = match_data.get('zip_filename', '')
            expected_sha1 = match_data.get('zip_sha1sum', '')
            expected_size = match_data.get('zip_size', None)  # Assuming size might be provided

            logger.debug(f"Starting validation for ZIP file: {zip_filename} at path: {zip_path}")

            # Check if file exists and has content
            if not zip_path.exists():
                logger.error(f"ZIP file does not exist: {zip_path}")
                return False

            actual_size = zip_path.stat().st_size
            logger.debug(f"ZIP file exists with size: {actual_size} bytes")

            if actual_size == 0:
                logger.error(f"ZIP file is empty: {zip_filename}")
                return False

            # Validate SHA1 checksum if provided
            if expected_sha1 and expected_sha1.strip():
                logger.debug(f"Validating SHA1 checksum for {zip_filename} (expected: {expected_sha1})")
                try:
                    actual_sha1 = self._calculate_sha1(zip_path)
                    logger.debug(f"Calculated SHA1: {actual_sha1}")
                    if actual_sha1 != expected_sha1.lower():
                        logger.error(f"SHA1 checksum mismatch for {zip_filename}: expected {expected_sha1}, got {actual_sha1}")
                        return False
                    logger.debug(f"SHA1 checksum validated successfully for {zip_filename}")
                except Exception as sha1_err:
                    logger.error(f"Failed to calculate SHA1 for {zip_filename}: {sha1_err}")
                    return False
            else:
                logger.debug(f"No SHA1 checksum provided for {zip_filename}, skipping checksum validation")

            # Validate file size if provided (as additional check)
            if expected_size is not None and isinstance(expected_size, (int, float)) and expected_size > 0:
                logger.debug(f"Validating file size for {zip_filename} (expected: {expected_size}, actual: {actual_size})")
                if actual_size != expected_size:
                    logger.warning(f"File size mismatch for {zip_filename}: expected {expected_size}, got {actual_size}")
                    # Size mismatch is a warning but not necessarily a failure for ZIP files
                    # as compression can vary, but we'll log it
                else:
                    logger.debug(f"File size validation passed for {zip_filename}")
            else:
                logger.debug(f"No expected size provided for {zip_filename}, skipping size validation")

            logger.debug(f"ZIP file validation successful: {zip_filename} (size: {actual_size} bytes)")
            return True

        except Exception as e:
            logger.error(f"Failed to validate ZIP file {zip_path}: {e}")
            return False

    def _cleanup_old_fixture_templates(self, session, fixtures):
        """Clean up old fixture templates and keep only the new ones"""
        try:
            # Get fixture IDs from the current fixtures that were just synchronized
            current_fixture_ids = set()
            for fixture_data in fixtures:
                fixture_id = fixture_data.get('fixture_id')
                if fixture_id:
                    current_fixture_ids.add(fixture_id)

            if not current_fixture_ids:
                logger.warning("No current fixture IDs found - skipping cleanup")
                return

            logger.info(f"Cleaning up old fixture templates - keeping fixtures: {current_fixture_ids}")

            # Get all fixture IDs currently in the template tables
            existing_fixture_ids = set()
            existing_fixtures = session.query(MatchTemplateModel.fixture_id).distinct().all()
            for row in existing_fixtures:
                existing_fixture_ids.add(row.fixture_id)

            # Find fixture IDs to remove (existing ones not in current fixtures)
            fixtures_to_remove = existing_fixture_ids - current_fixture_ids

            if not fixtures_to_remove:
                logger.info("No old fixture templates to remove")
                return

            logger.info(f"Removing old fixture templates: {fixtures_to_remove}")

            # Delete matches from old fixtures (CASCADE will delete outcomes)
            deleted_matches = 0
            for fixture_id in fixtures_to_remove:
                deleted_count = session.query(MatchTemplateModel).filter(
                    MatchTemplateModel.fixture_id == fixture_id
                ).delete()
                deleted_matches += deleted_count
                logger.info(f"Removed fixture template {fixture_id} with {deleted_count} matches")

            session.commit()
            logger.info(f"Successfully cleaned up {len(fixtures_to_remove)} old fixture templates, removed {deleted_matches} matches total")

        except Exception as e:
            logger.error(f"Failed to cleanup old fixture templates: {e}")
            session.rollback()

    def _send_download_progress(self, downloaded: int, total: int, message: str):
        """Send download progress update via message bus"""
        try:
            if self.message_bus:
                from ..core.message_bus import Message, MessageType
                progress_data = {
                    'downloaded': downloaded,
                    'total': total,
                    'percentage': int((downloaded / max(1, total)) * 100) if total > 0 else 0,
                    'message': message,
                    'timestamp': time.time()
                }

                progress_message = Message(
                    type=MessageType.CUSTOM,
                    sender='api_client',
                    data={
                        'download_progress': progress_data
                    }
                )
                self.message_bus.publish(progress_message)
                logger.debug(f"Sent download progress: {downloaded}/{total} - {message}")
        except Exception as e:
            logger.error(f"Failed to send download progress: {e}")

    def _validate_existing_fixtures(self):
        """Validate existing fixtures on startup and remove invalid ones"""
        try:
            logger.info("Validating existing fixtures on startup...")

            session = self.db_manager.get_session()
            try:
                # Get all fixtures with active matches
                fixtures_to_check = session.query(MatchModel.fixture_id).filter(
                    MatchModel.active_status == True
                ).distinct().all()

                logger.debug(f"Found {len(fixtures_to_check)} fixtures to validate")
                fixtures_to_remove = []

                for fixture_row in fixtures_to_check:
                    fixture_id = fixture_row.fixture_id
                    logger.debug(f"Validating fixture: {fixture_id}")

                    # Skip validation for recycled fixtures (copies of already verified fixtures)
                    if fixture_id.startswith('recycle_'):
                        logger.debug(f"Skipping ZIP validation for recycled fixture: {fixture_id}")
                        continue

                    # Skip validation for fixtures not from today (only today's fixtures need verification)
                    # Get fixture_active_time from any match in this fixture
                    fixture_active_time = None
                    sample_match = session.query(MatchModel).filter(
                        MatchModel.fixture_id == fixture_id
                    ).first()

                    if sample_match and sample_match.fixture_active_time:
                        fixture_active_time = sample_match.fixture_active_time

                    if fixture_active_time:
                        # Convert Unix timestamp to date
                        from datetime import datetime, date
                        fixture_date = datetime.fromtimestamp(fixture_active_time).date()
                        today = date.today()

                        if fixture_date != today:
                            logger.debug(f"Skipping ZIP validation for fixture {fixture_id} from {fixture_date} (not today)")
                            continue

                    # Get all matches for this fixture that have ZIP files
                    matches_with_zips = session.query(MatchModel).filter(
                        MatchModel.fixture_id == fixture_id,
                        MatchModel.active_status == True,
                        MatchModel.zip_filename.isnot(None)
                    ).all()

                    logger.debug(f"Fixture {fixture_id} has {len(matches_with_zips)} matches with ZIP files")

                    if not matches_with_zips:
                        logger.debug(f"Fixture {fixture_id} has no ZIP files to validate")
                        continue

                    # Check if all required ZIP files exist and are valid
                    fixture_valid = True
                    missing_or_invalid_zips = []

                    for match in matches_with_zips:
                        zip_filename = match.zip_filename
                        if not zip_filename:
                            continue

                        # Check if ZIP has already been validated successfully
                        if match.zip_validation_status == 'valid':
                            logger.debug(f"ZIP file already validated: {zip_filename} for fixture {fixture_id}")
                            continue
                        elif match.zip_validation_status == 'validating':
                            logger.debug(f"ZIP file validation in progress: {zip_filename} for fixture {fixture_id}")
                            continue

                        logger.debug(f"Validating ZIP file: {zip_filename} for fixture {fixture_id}")
                        zip_path = self.zip_storage_dir / zip_filename

                        # Create match data dict for validation
                        match_data = {
                            'zip_filename': zip_filename,
                            'zip_sha1sum': match.zip_sha1sum,
                            'zip_size': getattr(match, 'zip_size', None)  # If size field exists
                        }

                        if not self._validate_downloaded_zip(zip_path, match_data):
                            logger.debug(f"ZIP file validation failed: {zip_filename}")
                            missing_or_invalid_zips.append(zip_filename)
                            fixture_valid = False
                        else:
                            logger.debug(f"ZIP file validation passed: {zip_filename}")
                            # Mark as validated in database
                            match.zip_validation_status = 'valid'
                            session.commit()

                    if not fixture_valid:
                        logger.warning(f"Fixture {fixture_id} has invalid/missing ZIP files: {missing_or_invalid_zips}")
                        fixtures_to_remove.append(fixture_id)
                    else:
                        logger.debug(f"Fixture {fixture_id} validation passed")

                # Remove invalid fixtures
                if fixtures_to_remove:
                    logger.info(f"Removing {len(fixtures_to_remove)} invalid fixtures: {fixtures_to_remove}")

                    for fixture_id in fixtures_to_remove:
                        logger.debug(f"Deleting all matches for invalid fixture: {fixture_id}")
                        # Delete all matches for this fixture
                        deleted_count = session.query(MatchModel).filter(
                            MatchModel.fixture_id == fixture_id
                        ).delete()

                        logger.info(f"Removed fixture {fixture_id} with {deleted_count} matches")

                    session.commit()
                    logger.info(f"Successfully removed {len(fixtures_to_remove)} invalid fixtures")
                else:
                    logger.info("All existing fixtures are valid")

            finally:
                session.close()

        except Exception as e:
            logger.error(f"Failed to validate existing fixtures: {e}")
            # Don't fail initialization due to validation errors


class ReportsSyncResponseHandler(ResponseHandler):
    """Response handler for synchronizing report data to server with offline support and incremental updates"""

    def __init__(self, db_manager, user_data_dir, api_client=None, message_bus=None):
        self.db_manager = db_manager
        self.user_data_dir = user_data_dir
        self.api_client = api_client
        self.message_bus = message_bus
        self.max_queue_size = 1000  # Maximum number of pending sync items
        self.max_retries = 5  # Maximum retry attempts for failed syncs
        self.retry_backoff_base = 60  # Base backoff time in seconds (exponential)
        
        # Import tracking model
        from ..database.models import ReportsSyncTrackingModel
        self.ReportsSyncTrackingModel = ReportsSyncTrackingModel
        
    def handle_response(self, endpoint: APIEndpoint, response: requests.Response, report_data: Dict[str, Any] = None) -> Optional[Dict[str, Any]]:
        """Handle server response to report sync submission"""
        try:
            data = response.json()
            
            processed_data = {
                'source': endpoint.name,
                'timestamp': datetime.utcnow().isoformat(),
                'sync_status': 'success',
                'synced_items': 0,
                'failed_items': 0,
                'errors': []
            }
            
            # Process server response
            if data.get('success'):
                # Server accepted the sync data
                synced_count = data.get('synced_count', 0)
                processed_data['synced_items'] = synced_count
                logger.info(f"Successfully synced {synced_count} report items to server")
                
                # Update tracking records for successfully synced items
                if report_data:
                    self._update_sync_tracking(report_data)
                
                # Clear successfully synced items from queue
                self._clear_synced_items()
            else:
                # Server rejected the sync data
                error_msg = data.get('error', 'Unknown error')
                processed_data['sync_status'] = 'failed'
                processed_data['errors'].append(error_msg)
                logger.error(f"Server rejected report sync: {error_msg}")
                
                # Re-queue failed items for retry
                self._requeue_failed_items()
            
            return processed_data
            
        except json.JSONDecodeError as e:
            logger.error(f"Failed to parse server response: {e}")
            return self.handle_error(endpoint, e)
        except Exception as e:
            logger.error(f"Failed to process reports sync response: {e}")
            return self.handle_error(endpoint, e)
    
    def handle_error(self, endpoint: APIEndpoint, error: Exception) -> Optional[Dict[str, Any]]:
        """Handle sync errors and queue for retry"""
        error_data = {
            'error': str(error),
            'endpoint': endpoint.name,
            'timestamp': datetime.utcnow().isoformat(),
            'sync_status': 'error'
        }
        
        # Queue failed sync for retry
        self._queue_for_retry(error_data)
        
        return error_data
    
    def collect_report_data(self, date_range: str = 'today') -> Dict[str, Any]:
        """Collect report data from database for synchronization (incremental - only new/changed data)"""
        try:
            # NEW: Query server for last sync information before collecting data
            client_id = self._get_client_id()
            server_info = self.query_server_last_sync(client_id)
            
            # Check if recovery is needed (returns tuple: (needs_recovery, is_full_resync))
            recovery_info = self.needs_recovery(server_info)
            needs_recovery, is_full_resync = recovery_info
            
            if needs_recovery and not is_full_resync:
                # Only recover if not a full resync scenario
                logger.warning("Local tracking mismatch detected, recovering from server...")
                if server_info and self.recover_local_tracking(server_info):
                    logger.info("Successfully recovered local tracking from server")
                else:
                    logger.error("Failed to recover local tracking, proceeding with caution")
            
            # Check if server indicated full resync is needed
            if is_full_resync:
                logger.warning("Server indicated full resync is needed - performing full sync")
                # Force full sync by setting start_date to datetime.min
                date_range = 'all'
            
            from ..database.models import (
                BetModel, BetDetailModel, ExtractionStatsModel, MatchModel,
                ReportsSyncTrackingModel, PersistentRedistributionAdjustmentModel
            )
            from datetime import datetime, timedelta, date
            import hashlib
            
            # Determine date range
            now = datetime.utcnow()
            if date_range == 'today':
                start_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
                end_date = now
            elif date_range == 'yesterday':
                start_date = (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
                end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
            elif date_range == 'week':
                start_date = now - timedelta(days=7)
                end_date = now
            else:  # 'all' or custom
                start_date = datetime.min
                end_date = now
            
            session = self.db_manager.get_session()
            try:
                # Get last sync time for incremental sync
                last_sync_time = session.query(
                    self.ReportsSyncTrackingModel.last_synced_at
                ).filter(
                    self.ReportsSyncTrackingModel.entity_type == 'sync'
                ).order_by(
                    self.ReportsSyncTrackingModel.last_synced_at.desc()
                ).first()
                
                # If we have a previous sync, only collect data since then
                if last_sync_time:
                    start_date = last_sync_time.last_synced_at
                    logger.info(f"Incremental sync: collecting data since {start_date}")
                else:
                    logger.info(f"Full sync: collecting all data (no previous sync found)")
                
                # Collect bets data (incremental - only new/updated bets)
                bets_query = session.query(BetModel).filter(
                    BetModel.bet_datetime >= start_date,
                    BetModel.bet_datetime <= end_date
                )
                bets = bets_query.all()
                
                # Filter bets to only include those not yet synced or updated since last sync
                bets_to_sync = []
                for bet in bets:
                    # Check if this bet has been synced before
                    tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                        entity_type='bet',
                        entity_id=bet.uuid
                    ).first()
                    
                    if tracking:
                        # Check if bet has been updated since last sync
                        if bet.updated_at > tracking.last_synced_at:
                            bets_to_sync.append(bet)
                            logger.debug(f"Bet {bet.uuid} updated since last sync, including in sync")
                        else:
                            logger.debug(f"Bet {bet.uuid} already synced and not updated, skipping")
                    else:
                        # New bet, include in sync
                        bets_to_sync.append(bet)
                        logger.debug(f"Bet {bet.uuid} is new, including in sync")
                
                # Collect extraction stats (incremental - only new/updated stats)
                stats_query = session.query(ExtractionStatsModel).filter(
                    ExtractionStatsModel.match_datetime >= start_date,
                    ExtractionStatsModel.match_datetime <= end_date
                )
                stats = stats_query.all()
                
                # Filter stats to only include those not yet synced or updated since last sync
                stats_to_sync = []
                for stat in stats:
                    # Check if this stat has been synced before
                    tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                        entity_type='extraction_stat',
                        entity_id=str(stat.match_id)
                    ).first()
                    
                    if tracking:
                        # Check if stat has been updated since last sync
                        if stat.updated_at > tracking.last_synced_at:
                            stats_to_sync.append(stat)
                            logger.debug(f"Extraction stat {stat.match_id} updated since last sync, including in sync")
                        else:
                            logger.debug(f"Extraction stat {stat.match_id} already synced and not updated, skipping")
                    else:
                        # New stat, include in sync
                        stats_to_sync.append(stat)
                        logger.debug(f"Extraction stat {stat.match_id} is new, including in sync")
                
                # Get cap compensation balance (accumulated shortfall)
                # Use the global record (date 1970-01-01) to match dashboard display
                cap_compensation_balance = 0.0
                try:
                    from datetime import date
                    global_date = date(1970, 1, 1)
                    global_record = session.query(PersistentRedistributionAdjustmentModel)\
                        .filter_by(date=global_date)\
                        .first()
                    if global_record:
                        cap_compensation_balance = float(global_record.accumulated_shortfall)
                        logger.debug(f"Cap compensation balance (global record): {cap_compensation_balance}")
                except Exception as e:
                    logger.warning(f"Failed to get cap compensation balance: {e}")
                
                # Build report data payload (incremental - only new/changed data)
                report_data = {
                    'sync_id': self._generate_sync_id(),
                    'client_id': self._get_client_id(),
                    'sync_timestamp': datetime.utcnow().isoformat(),
                    'date_range': date_range,
                    'start_date': start_date.isoformat(),
                    'end_date': end_date.isoformat(),
                    'bets': [],
                    'extraction_stats': [],
                    'cap_compensation_balance': cap_compensation_balance,
                    'summary': self._calculate_summary(bets_to_sync, stats_to_sync),
                    'is_incremental': True,  # Flag to indicate this is an incremental sync
                    'sync_type': 'incremental' if last_sync_time else 'full'
                }
                
                # Add bets data (only new/updated bets)
                for bet in bets_to_sync:
                    bet_details = session.query(BetDetailModel).filter_by(
                        bet_id=bet.uuid
                    ).filter(BetDetailModel.result != 'cancelled').all()
                    
                    if bet_details:  # Only include bets with non-cancelled details
                        bet_data = {
                            'uuid': bet.uuid,
                            'fixture_id': bet.fixture_id,
                            'bet_datetime': bet.bet_datetime.isoformat(),
                            'paid': bet.paid,
                            'paid_out': bet.paid_out,
                            'total_amount': float(sum(detail.amount for detail in bet_details)),
                            'bet_count': len(bet_details),
                            'details': []
                        }
                        
                        for detail in bet_details:
                            match = session.query(MatchModel).filter_by(id=detail.match_id).first()
                            detail_data = {
                                'match_id': detail.match_id,
                                'match_number': match.match_number if match else None,
                                'outcome': detail.outcome,
                                'amount': float(detail.amount),
                                'win_amount': float(detail.win_amount),
                                'result': detail.result
                            }
                            bet_data['details'].append(detail_data)
                        
                        report_data['bets'].append(bet_data)
                
                # Add extraction stats (only new/updated stats)
                for stat in stats_to_sync:
                    # Get the match to retrieve the stored accumulated shortfall and cap_percent at completion time
                    match = session.query(MatchModel).filter_by(id=stat.match_id).first()
                    accumulated_shortfall = float(match.accumulated_shortfall) if match and match.accumulated_shortfall is not None else 0.0
                    cap_percent = float(match.cap_percent) if match and match.cap_percent is not None else 70.0
                    
                    stat_data = {
                        'match_id': stat.match_id,
                        'fixture_id': stat.fixture_id,
                        'match_datetime': stat.match_datetime.isoformat(),
                        'total_bets': stat.total_bets,
                        'total_amount_collected': float(stat.total_amount_collected),
                        'total_redistributed': float(stat.total_redistributed),
                        'actual_result': stat.actual_result,
                        'extraction_result': stat.extraction_result,
                        'cap_applied': stat.cap_applied,
                        'cap_percentage': float(stat.cap_percentage) if stat.cap_percentage else None,
                        'under_bets': stat.under_bets,
                        'under_amount': float(stat.under_amount),
                        'over_bets': stat.over_bets,
                        'over_amount': float(stat.over_amount),
                        'result_breakdown': stat.result_breakdown,
                        'accumulated_shortfall': accumulated_shortfall,  # Historical value at match completion
                        'cap_percent': cap_percent  # CAP percentage configured at match completion time
                    }
                    report_data['extraction_stats'].append(stat_data)
                
                logger.info(f"Collected report data: {len(report_data['bets'])} bets, {len(report_data['extraction_stats'])} stats")
                return report_data
                
            finally:
                session.close()
                
        except Exception as e:
            logger.error(f"Failed to collect report data: {e}")
            raise
    
    def queue_report_sync(self, date_range: str = 'today') -> bool:
        """Queue report data for synchronization in database"""
        try:
            # Collect report data
            report_data = self.collect_report_data(date_range)
            
            session = self.db_manager.get_session()
            try:
                # Check queue size limit
                queue_count = session.query(ReportsSyncQueueModel).filter(
                    ReportsSyncQueueModel.status.in_(['pending', 'syncing'])
                ).count()
                
                if queue_count >= self.max_queue_size:
                    # Remove oldest completed items
                    session.query(ReportsSyncQueueModel).filter(
                        ReportsSyncQueueModel.status == 'completed'
                    ).order_by(ReportsSyncQueueModel.created_at.asc()).limit(
                        queue_count - self.max_queue_size + 1
                    ).delete(synchronize_session=False)
                
                # Create new queue item
                queue_item = ReportsSyncQueueModel(
                    sync_id=report_data['sync_id'],
                    client_id=report_data['client_id'],
                    status='pending',
                    retry_count=0,
                    sync_data=report_data,
                    synced_items=0,
                    failed_items=0
                )
                session.add(queue_item)
                session.commit()
                
                logger.info(f"Queued report sync {report_data['sync_id']} for date range {date_range}")
                return True
            except Exception as e:
                logger.error(f"Failed to queue report sync: {e}")
                session.rollback()
                return False
            finally:
                session.close()
            
        except Exception as e:
            logger.error(f"Failed to queue report sync: {e}")
            return False
    
    def process_sync_queue(self) -> Dict[str, Any]:
        """Process pending sync items with retry logic from database"""
        results = {
            'processed': 0,
            'succeeded': 0,
            'failed': 0,
            'remaining': 0,
            'errors': []
        }
        
        session = self.db_manager.get_session()
        try:
            # Get pending items that should be retried now
            pending_items = session.query(ReportsSyncQueueModel).filter(
                ReportsSyncQueueModel.status == 'pending'
            ).order_by(ReportsSyncQueueModel.created_at.asc()).all()
            
            if not pending_items:
                logger.debug("No items in sync queue")
                return results
            
            logger.info(f"Processing sync queue with {len(pending_items)} items")
            
            for item in pending_items:
                # Check retry limit
                if not item.can_retry(self.max_retries):
                    logger.warning(f"Sync item {item.sync_id} exceeded max retries, marking as failed")
                    item.status = 'failed'
                    results['failed'] += 1
                    continue
                
                # Check if we should retry based on backoff
                if not item.should_retry_now():
                    backoff_time = self._calculate_backoff_time(item.retry_count)
                    logger.debug(f"Sync item {item.sync_id} waiting for backoff ({backoff_time}s)")
                    continue
                
                # Attempt sync
                try:
                    item.mark_syncing()
                    session.commit()
                    
                    success = self._sync_to_server(item.sync_data)
                    
                    if success:
                        item.mark_completed()
                        results['succeeded'] += 1
                        logger.info(f"Successfully synced {item.sync_id}")
                    else:
                        # Calculate next retry time
                        next_retry = datetime.utcnow() + timedelta(
                            seconds=self._calculate_backoff_time(item.retry_count + 1)
                        )
                        item.mark_failed("Sync failed", item.retry_count + 1, next_retry)
                        results['failed'] += 1
                        logger.warning(f"Failed to sync {item.sync_id}, will retry (attempt {item.retry_count})")
                    
                    results['processed'] += 1
                    session.commit()
                    
                except Exception as e:
                    # Calculate next retry time
                    next_retry = datetime.utcnow() + timedelta(
                        seconds=self._calculate_backoff_time(item.retry_count + 1)
                    )
                    item.mark_failed(str(e), item.retry_count + 1, next_retry)
                    results['failed'] += 1
                    error_msg = f"Error syncing {item.sync_id}: {e}"
                    results['errors'].append(error_msg)
                    logger.error(error_msg)
                    session.commit()
            
            # Get remaining count
            results['remaining'] = session.query(ReportsSyncQueueModel).filter(
                ReportsSyncQueueModel.status.in_(['pending', 'syncing'])
            ).count()
            
            logger.info(f"Sync queue processed: {results['succeeded']} succeeded, {results['failed']} failed, {results['remaining']} remaining")
            return results
            
        except Exception as e:
            logger.error(f"Failed to process sync queue: {e}")
            session.rollback()
            return results
        finally:
            session.close()
    
    def _sync_to_server(self, report_data: Dict[str, Any]) -> bool:
        """Send report data to server with retry logic"""
        try:
            if not self.api_client or not hasattr(self.api_client, 'session'):
                logger.error("API client not available for sync")
                return False
            
            # Get sync endpoint configuration
            sync_endpoint = self.api_client.endpoints.get('reports_sync')
            if not sync_endpoint:
                logger.error("Reports sync endpoint not configured")
                return False
            
            # Prepare headers with authentication
            headers = {}
            if sync_endpoint.auth and sync_endpoint.auth.get('type') == 'bearer':
                token = sync_endpoint.auth.get('token')
                if token:
                    headers['Authorization'] = f"Bearer {token}"
            
            # Send data with retry logic
            max_attempts = 3
            for attempt in range(max_attempts):
                try:
                    logger.debug(f"Syncing report data to server (attempt {attempt + 1}/{max_attempts})")
                    
                    response = self.api_client.session.post(
                        sync_endpoint.url,
                        json=report_data,
                        headers=headers,
                        timeout=sync_endpoint.timeout
                    )
                    
                    if response.status_code == 200:
                        logger.info(f"Successfully synced report data to server")
                        return True
                    elif response.status_code == 401:
                        logger.error("Authentication failed for reports sync")
                        return False
                    elif response.status_code == 429:
                        # Rate limited - wait and retry
                        wait_time = 2 ** attempt
                        logger.warning(f"Rate limited, waiting {wait_time}s before retry")
                        time.sleep(wait_time)
                        continue
                    else:
                        logger.warning(f"Server returned status {response.status_code}")
                        if attempt < max_attempts - 1:
                            time.sleep(2 ** attempt)
                            continue
                        return False
                        
                except requests.exceptions.Timeout:
                    logger.warning(f"Timeout during sync attempt {attempt + 1}")
                    if attempt < max_attempts - 1:
                        time.sleep(2 ** attempt)
                        continue
                    return False
                except requests.exceptions.ConnectionError:
                    logger.warning(f"Connection error during sync attempt {attempt + 1}")
                    if attempt < max_attempts - 1:
                        time.sleep(2 ** attempt)
                        continue
                    return False
            
            return False
            
        except Exception as e:
            logger.error(f"Unexpected error during sync: {e}")
            return False
    
    def query_server_last_sync(self, client_id: str = None) -> Optional[Dict[str, Any]]:
        """Query server for last sync information"""
        try:
            if not self.api_client or not hasattr(self.api_client, 'session'):
                logger.error("API client not available for last sync query")
                return None
            
            # Get sync endpoint configuration to extract base URL and auth
            sync_endpoint = self.api_client.endpoints.get('reports_sync')
            if not sync_endpoint:
                logger.error("Reports sync endpoint not configured")
                return None
            
            # Construct last-sync URL from sync endpoint URL
            base_url = sync_endpoint.url.replace('/sync', '/last-sync')
            
            # Prepare headers with authentication
            headers = {}
            if sync_endpoint.auth and sync_endpoint.auth.get('type') == 'bearer':
                token = sync_endpoint.auth.get('token')
                if token:
                    headers['Authorization'] = f"Bearer {token}"
            
            # Only send client_id parameter if rustdesk_id is configured
            params = {}
            if self.api_client.settings and self.api_client.settings.rustdesk_id:
                client_id = str(self.api_client.settings.rustdesk_id)
                params = {"client_id": client_id}
                logger.debug(f"Querying server last sync with client_id: {client_id}")
            else:
                logger.debug("Querying server last sync without client_id (rustdesk_id not configured)")
            
            response = self.api_client.session.get(
                base_url,
                headers=headers,
                params=params,
                timeout=30
            )
            
            if response.status_code == 200:
                data = response.json()
                if data.get('success'):
                    # Check if server indicates full resync is needed
                    last_sync_id = data.get('last_sync_id')
                    needs_full_resync = data.get('needs_full_resync', False)
                    
                    # Server indicates full resync is needed if:
                    # 1. needs_full_resync flag is True
                    # 2. last_sync_id is null
                    if needs_full_resync or last_sync_id is None:
                        logger.warning(f"Server indicates full resync is needed (needs_full_resync={needs_full_resync}, last_sync_id={last_sync_id})")
                        # Mark this response as requiring full resync
                        data['needs_full_resync'] = True
                        return data
                    
                    logger.info(f"Successfully queried server last sync: {last_sync_id}")
                    return data
                else:
                    logger.warning(f"Server returned unsuccessful response: {data.get('error')}")
                    return None
            elif response.status_code == 404:
                # No sync record found on server (first sync scenario) - needs full resync
                logger.info("No previous sync found on server (first sync) - full resync needed")
                # Return a dict indicating full resync is needed
                return {
                    'success': True,
                    'needs_full_resync': True,
                    'last_sync_id': None,
                    'last_sync_timestamp': None,
                    'total_syncs': 0
                }
            else:
                logger.error(f"Failed to query server last sync: status {response.status_code}")
                return None
                
        except Exception as e:
            logger.error(f"Error querying server last sync: {e}")
            return None
    
    def recover_local_tracking(self, server_info: Dict[str, Any]) -> bool:
        """Recover local tracking from server state"""
        try:
            session = self.db_manager.get_session()
            try:
                # Parse server timestamps
                last_sync_timestamp = server_info.get('last_sync_timestamp')
                if last_sync_timestamp:
                    sync_time = datetime.fromisoformat(last_sync_timestamp.replace('Z', '+00:00'))
                else:
                    sync_time = datetime.utcnow()
                
                # Update or create sync tracking record
                sync_tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                    entity_type='sync',
                    entity_id='latest'
                ).first()
                
                if sync_tracking:
                    sync_tracking.last_synced_at = sync_time
                    sync_tracking.sync_count = server_info.get('total_syncs', 1)
                    sync_tracking.updated_at = sync_time
                    logger.info(f"Recovered sync tracking: last sync at {sync_time}")
                else:
                    sync_tracking = self.ReportsSyncTrackingModel(
                        entity_type='sync',
                        entity_id='latest',
                        last_synced_at=sync_time,
                        sync_count=server_info.get('total_syncs', 1)
                    )
                    session.add(sync_tracking)
                    logger.info(f"Created sync tracking from server: first sync at {sync_time}")
                
                session.commit()
                logger.info("Successfully recovered local tracking from server")
                return True
                
            except Exception as e:
                logger.error(f"Failed to recover local tracking: {e}")
                session.rollback()
                return False
            finally:
                session.close()
                
        except Exception as e:
            logger.error(f"Error recovering local tracking: {e}")
            return False
    
    def needs_recovery(self, server_info: Optional[Dict[str, Any]]) -> tuple[bool, bool]:
        """Check if local tracking needs recovery from server
        
        Returns:
            tuple: (needs_recovery, is_full_resync)
                - needs_recovery: True if recovery is needed
                - is_full_resync: True if full resync is needed (don't recover, just clear)
        """
        if not server_info:
            # No server info available, can't determine recovery need
            return (False, False)
        
        try:
            # Check if server explicitly indicates full resync is needed
            needs_full_resync = server_info.get('needs_full_resync', False)
            last_sync_id = server_info.get('last_sync_id')
            
            # Server indicates full resync is needed if:
            # 1. needs_full_resync flag is True
            # 2. last_sync_id is null
            if needs_full_resync or last_sync_id is None:
                logger.warning(f"Server indicates full resync is needed (needs_full_resync={needs_full_resync}, last_sync_id={last_sync_id}) - clearing local tracking")
                # Clear local tracking to force full resync
                self._clear_local_tracking()
                # Return (True, True) - needs recovery but it's a full resync scenario
                return (True, True)
            
            session = self.db_manager.get_session()
            try:
                # Check if local tracking exists
                local_tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                    entity_type='sync',
                    entity_id='latest'
                ).first()
                
                if not local_tracking:
                    # No local tracking exists - need recovery
                    logger.info("No local tracking found - recovery needed")
                    return True
                
                # Compare sync IDs if available
                server_sync_id = server_info.get('last_sync_id')
                if server_sync_id:
                    # If server has sync info but local doesn't match, need recovery
                    # We can't directly compare sync IDs since they're generated differently
                    # Instead, compare timestamps
                    server_timestamp = server_info.get('last_sync_timestamp')
                    if server_timestamp:
                        server_time = datetime.fromisoformat(server_timestamp.replace('Z', '+00:00'))
                        
                        # If server time is significantly newer than local, need recovery
                        time_diff = abs((server_time - local_tracking.last_synced_at).total_seconds())
                        if time_diff > 60:  # More than 1 minute difference
                            logger.warning(f"Sync time mismatch detected: server={server_time}, local={local_tracking.last_synced_at}")
                            return True
                
                return False
                
            finally:
                session.close()
                
        except Exception as e:
            logger.error(f"Error checking recovery need: {e}")
            return False
    
    def _clear_local_tracking(self) -> bool:
        """Clear all local sync tracking records to force full resync"""
        try:
            session = self.db_manager.get_session()
            try:
                # Delete all sync tracking records
                deleted_count = session.query(self.ReportsSyncTrackingModel).delete()
                session.commit()
                logger.info(f"Cleared {deleted_count} local sync tracking records to force full resync")
                return True
            except Exception as e:
                logger.error(f"Failed to clear local tracking: {e}")
                session.rollback()
                return False
            finally:
                session.close()
        except Exception as e:
            logger.error(f"Error clearing local tracking: {e}")
            return False
    
    def _generate_sync_id(self) -> str:
        """Generate unique sync ID"""
        import uuid
        return f"sync_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
    
    def _get_client_id(self) -> str:
        """Get unique client identifier"""
        try:
            # Try to get from settings
            if self.api_client and hasattr(self.api_client, 'settings'):
                client_id = getattr(self.api_client.settings, 'rustdesk_id', None)
                if client_id:
                    return str(client_id)
        except:
            pass
        
        # Fallback to machine ID
        import uuid
        import platform
        machine_id = f"{platform.node()}_{uuid.getnode()}"
        return hashlib.sha256(machine_id.encode()).hexdigest()[:16]
    
    def _calculate_summary(self, bets: List, stats: List) -> Dict[str, Any]:
        """Calculate summary statistics"""
        total_payin = 0.0
        total_bets = 0
        
        for bet in bets:
            bet_details = [d for d in bet.bet_details if d.result != 'cancelled']
            if bet_details:
                total_payin += sum(d.amount for d in bet_details)
                total_bets += len(bet_details)
        
        total_payout = sum(stat.total_redistributed for stat in stats)
        
        return {
            'total_payin': float(total_payin),
            'total_payout': float(total_payout),
            'net_profit': float(total_payin - total_payout),
            'total_bets': total_bets,
            'total_matches': len(stats)
        }
    
    def _calculate_backoff_time(self, retry_count: int) -> int:
        """Calculate exponential backoff time for retry"""
        return self.retry_backoff_base * (2 ** retry_count)
    
    def _queue_for_retry(self, error_data: Dict[str, Any]):
        """Queue failed sync for retry in database"""
        session = self.db_manager.get_session()
        try:
            sync_id = self._generate_sync_id()
            client_id = self._get_client_id()
            
            # Check queue size limit
            queue_count = session.query(ReportsSyncQueueModel).filter(
                ReportsSyncQueueModel.status.in_(['pending', 'syncing'])
            ).count()
            
            if queue_count >= self.max_queue_size:
                # Remove oldest completed items
                session.query(ReportsSyncQueueModel).filter(
                    ReportsSyncQueueModel.status == 'completed'
                ).order_by(ReportsSyncQueueModel.created_at.asc()).limit(
                    queue_count - self.max_queue_size + 1
                ).delete(synchronize_session=False)
            
            # Create new queue item
            queue_item = ReportsSyncQueueModel(
                sync_id=sync_id,
                client_id=client_id,
                status='pending',
                retry_count=0,
                sync_data=error_data,
                synced_items=0,
                failed_items=0
            )
            session.add(queue_item)
            session.commit()
            
            logger.warning(f"Queued failed sync for retry: {sync_id}")
        except Exception as e:
            logger.error(f"Failed to queue sync for retry: {e}")
            session.rollback()
        finally:
            session.close()
    
    def _clear_synced_items(self):
        """Remove successfully synced items from database queue"""
        session = self.db_manager.get_session()
        try:
            session.query(ReportsSyncQueueModel).filter(
                ReportsSyncQueueModel.status == 'completed'
            ).delete(synchronize_session=False)
            session.commit()
            logger.info("Cleared completed sync items from queue")
        except Exception as e:
            logger.error(f"Failed to clear synced items: {e}")
            session.rollback()
        finally:
            session.close()
    
    def _update_sync_tracking(self, report_data: Dict[str, Any] = None):
        """Update sync tracking records after successful sync"""
        session = self.db_manager.get_session()
        try:
            # Get the sync timestamp for this sync
            sync_time = datetime.utcnow()
            
            # Update or create tracking record for this sync operation
            sync_tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                entity_type='sync',
                entity_id='latest'
            ).first()
            
            if sync_tracking:
                # Update existing tracking record
                sync_tracking.last_synced_at = sync_time
                sync_tracking.sync_count += 1
                sync_tracking.updated_at = sync_time
                logger.info(f"Updated sync tracking: last sync at {sync_time}, total syncs: {sync_tracking.sync_count}")
            else:
                # Create new tracking record
                sync_tracking = self.ReportsSyncTrackingModel(
                    entity_type='sync',
                    entity_id='latest',
                    last_synced_at=sync_time,
                    sync_count=1
                )
                session.add(sync_tracking)
                logger.info(f"Created sync tracking: first sync at {sync_time}")
            
            # Track individual bets that were synced
            if report_data and 'bets' in report_data:
                for bet_data in report_data['bets']:
                    bet_uuid = bet_data.get('uuid')
                    if bet_uuid:
                        bet_tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                            entity_type='bet',
                            entity_id=bet_uuid
                        ).first()
                        
                        if bet_tracking:
                            # Update existing bet tracking
                            bet_tracking.last_synced_at = sync_time
                            bet_tracking.sync_count += 1
                            bet_tracking.updated_at = sync_time
                        else:
                            # Create new bet tracking
                            bet_tracking = self.ReportsSyncTrackingModel(
                                entity_type='bet',
                                entity_id=bet_uuid,
                                last_synced_at=sync_time,
                                sync_count=1
                            )
                            session.add(bet_tracking)
            
            # Track individual extraction stats that were synced
            if report_data and 'extraction_stats' in report_data:
                for stat_data in report_data['extraction_stats']:
                    match_id = stat_data.get('match_id')
                    if match_id:
                        stat_tracking = session.query(self.ReportsSyncTrackingModel).filter_by(
                            entity_type='extraction_stat',
                            entity_id=str(match_id)
                        ).first()
                        
                        if stat_tracking:
                            # Update existing stat tracking
                            stat_tracking.last_synced_at = sync_time
                            stat_tracking.sync_count += 1
                            stat_tracking.updated_at = sync_time
                        else:
                            # Create new stat tracking
                            stat_tracking = self.ReportsSyncTrackingModel(
                                entity_type='extraction_stat',
                                entity_id=str(match_id),
                                last_synced_at=sync_time,
                                sync_count=1
                            )
                            session.add(stat_tracking)
            
            session.commit()
            if report_data:
                logger.info(f"Updated tracking for {len(report_data.get('bets', []))} bets and {len(report_data.get('extraction_stats', []))} extraction stats")
        except Exception as e:
            logger.error(f"Failed to update sync tracking: {e}")
            session.rollback()
        finally:
            session.close()
    
    def _requeue_failed_items(self):
        """Re-queue failed items for retry in database"""
        session = self.db_manager.get_session()
        try:
            failed_items = session.query(ReportsSyncQueueModel).filter(
                ReportsSyncQueueModel.status == 'failed'
            ).all()
            
            for item in failed_items:
                item.status = 'pending'
                item.retry_count = 0
                item.next_retry_at = None
                item.error_message = None
            
            session.commit()
            logger.info(f"Re-queued {len(failed_items)} failed sync items")
        except Exception as e:
            logger.error(f"Failed to requeue failed items: {e}")
            session.rollback()
        finally:
            session.close()
    
    def get_queue_status(self) -> Dict[str, Any]:
        """Get current sync queue status from database"""
        session = self.db_manager.get_session()
        try:
            pending = session.query(ReportsSyncQueueModel).filter_by(status='pending').count()
            syncing = session.query(ReportsSyncQueueModel).filter_by(status='syncing').count()
            completed = session.query(ReportsSyncQueueModel).filter_by(status='completed').count()
            failed = session.query(ReportsSyncQueueModel).filter_by(status='failed').count()
            total = session.query(ReportsSyncQueueModel).count()
            
            return {
                'total': total,
                'pending': pending,
                'syncing': syncing,
                'completed': completed,
                'failed': failed,
                'max_queue_size': self.max_queue_size
            }
        except Exception as e:
            logger.error(f"Failed to get queue status: {e}")
            return {
                'total': 0,
                'pending': 0,
                'syncing': 0,
                'completed': 0,
                'failed': 0,
                'max_queue_size': self.max_queue_size
            }
        finally:
            session.close()


class APIClient(ThreadedComponent):
    """REST API Client component"""
    
    def __init__(self, message_bus: MessageBus, db_manager: DatabaseManager,
                 config_manager: ConfigManager, settings: ApiConfig):
        super().__init__("api_client", message_bus)
        self.db_manager = db_manager
        self.config_manager = config_manager
        self.settings = settings

        logger.debug("API CLIENT INIT: rustdesk_id from settings: '{}' (type: {})".format(
            self.settings.rustdesk_id, type(self.settings.rustdesk_id)))
        
        # HTTP session with retry logic
        self.session = requests.Session()
        self._setup_session()
        
        # API endpoints configuration
        self.endpoints: Dict[str, APIEndpoint] = {}
        self._load_endpoints()
        
        # Response handlers
        self.response_handlers = {
            'default': DefaultResponseHandler(),
            'news': NewsResponseHandler(),
            'sports': SportsResponseHandler(),
            'updates': UpdatesResponseHandler(self.db_manager, get_user_data_dir(), self, self.message_bus),
            'reports_sync': ReportsSyncResponseHandler(self.db_manager, get_user_data_dir(), self, self.message_bus)
        }
        
        # Statistics
        self.stats = {
            'total_requests': 0,
            'successful_requests': 0,
            'failed_requests': 0,
            'start_time': datetime.utcnow()
        }
        
        # Register message queue
        self.message_queue = self.message_bus.register_component(self.name)
        
        logger.info("APIClient initialized")
    
    def initialize(self) -> bool:
        """Initialize API client"""
        try:
            # Subscribe to messages
            self.message_bus.subscribe(self.name, MessageType.CONFIG_UPDATE, self._handle_config_update)
            self.message_bus.subscribe(self.name, MessageType.API_REQUEST, self._handle_api_request)
            self.message_bus.subscribe(self.name, MessageType.FORCE_API_UPDATE, self._handle_force_api_update)

            # Validate existing fixtures on startup
            updates_handler = self.response_handlers.get('updates')
            if updates_handler and hasattr(updates_handler, '_validate_existing_fixtures'):
                updates_handler._validate_existing_fixtures()

            logger.info("APIClient initialized successfully")
            return True

        except Exception as e:
            logger.error(f"APIClient initialization failed: {e}")
            return False
    
    def _setup_session(self):
        """Setup HTTP session with retry logic and SSL support"""
        try:
            # Use SSL-aware session that handles self-signed certificates
            self.session = create_requests_session_with_ssl_support(
                verify_ssl=self.settings.verify_ssl
            )
            
            # Set default headers
            self.session.headers.update({
                'User-Agent': f'MbetterClient/{self.settings.user_agent}',
                'Accept': 'application/json, text/plain, */*',
                'Accept-Encoding': 'gzip, deflate'
            })
            
            # Set timeout
            self.session.timeout = self.settings.timeout_seconds
            
            if not self.settings.verify_ssl:
                logger.info("API client configured to accept self-signed certificates")
            else:
                logger.info("API client configured with SSL verification enabled")
                
        except Exception as e:
            logger.error(f"Failed to setup SSL session, falling back to basic session: {e}")
            # Fallback to original session setup
            retry_strategy = Retry(
                total=self.settings.retry_attempts,
                backoff_factor=self.settings.retry_delay_seconds,
                status_forcelist=[429, 500, 502, 503, 504],
                allowed_methods=None,  # Allow all methods (replaces deprecated method_whitelist)
            )
            
            adapter = HTTPAdapter(max_retries=retry_strategy)
            self.session.mount("http://", adapter)
            self.session.mount("https://", adapter)
            
            # Set default headers
            self.session.headers.update({
                'User-Agent': f'MbetterClient/{self.settings.user_agent}',
                'Accept': 'application/json, text/plain, */*',
                'Accept-Encoding': 'gzip, deflate'
            })
            
            # Set timeout
            self.session.timeout = self.settings.timeout_seconds
    
    def _load_endpoints(self):
        """Load API endpoints from configuration"""
        try:
            endpoints_config = self.config_manager.get_section_config("api_endpoints")
            
            if not endpoints_config:
                # Create default configuration
                default_endpoints = self._get_default_endpoints()
                self.config_manager.update_section("api_endpoints", default_endpoints)
                endpoints_config = default_endpoints
            
            # Load endpoints
            for name, config in endpoints_config.items():
                self.endpoints[name] = APIEndpoint(name, config)
            
            logger.info(f"Loaded {len(self.endpoints)} API endpoints")
            
        except Exception as e:
            logger.error(f"Failed to load API endpoints: {e}")
    
    def _get_default_endpoints(self) -> Dict[str, Dict[str, Any]]:
        """Get default API endpoints configuration"""
        # Get FastAPI server URL, token, and interval from configuration
        fastapi_url = "https://mbetter.nexlab.net"
        api_token = ""
        api_interval = 1800  # 30 minutes default
        try:
            api_config = self.config_manager.get_section_config("api") or {}
            logger.debug("RAW API CONFIG: {}".format(api_config))

            # Load fastapi_url
            config_fastapi_url = api_config.get("fastapi_url")
            logger.debug("CONFIG FASTAPI_URL: '{}' (type: {})".format(config_fastapi_url, type(config_fastapi_url)))

            if config_fastapi_url and config_fastapi_url.strip():
                fastapi_url = config_fastapi_url.strip()
                logger.debug("USING CONFIG URL: {}".format(fastapi_url))
            else:
                logger.debug("CONFIG URL EMPTY, USING DEFAULT: {}".format(fastapi_url))

            # Load api_token
            config_api_token = api_config.get("api_token")
            logger.debug("CONFIG API_TOKEN: '{}' (type: {}, length: {})".format(
                config_api_token, type(config_api_token), len(config_api_token) if config_api_token else 0))

            if config_api_token and config_api_token.strip():
                api_token = config_api_token.strip()
                logger.debug("USING CONFIG TOKEN: {}...{}".format(api_token[:10], api_token[-10:] if len(api_token) > 20 else api_token))
            else:
                logger.debug("CONFIG TOKEN EMPTY, USING DEFAULT: {}".format(api_token))

            api_interval = api_config.get("api_interval", api_interval)
            logger.debug("API CONFIG LOADED - fastapi_url: {}, api_token: {}, api_interval: {}".format(
                fastapi_url, '*' * len(api_token) if api_token else 'empty', api_interval))
        except Exception as e:
            logger.warning("Could not load API configuration, using defaults: {}".format(e))
            logger.debug("CONFIG LOAD ERROR DETAILS: {}".format(str(e)))
        
        # Prepare authentication if token is provided
        auth_config = None
        headers = {"Content-Type": "application/json"}
        
        # Configure based on token availability
        if api_token and api_token.strip():
            auth_config = {
                "type": "bearer",
                "token": api_token.strip()
            }
            logger.debug("AUTH CONFIG - Token available, length: {}".format(len(api_token.strip())))
            # When token is provided: use configurable interval, 10 retries every 30 seconds
            interval = api_interval
            retry_attempts = 10
            retry_delay = 30  # 30 seconds
            enabled = True
        else:
            auth_config = None
            logger.debug("AUTH CONFIG - No token available")
            # When no token: use configurable interval or default, fewer retries, disabled by default
            interval = api_interval if api_interval != 1800 else 600  # Use configured or 10 minutes default
            retry_attempts = 3
            retry_delay = 5
            enabled = False  # Disabled when no authentication
        
        # Construct full URLs by appending paths to base URL
        fastapi_main_url = fastapi_url.rstrip('/') + "/api/updates"
        reports_sync_url = fastapi_url.rstrip('/') + "/api/reports/sync"
        reports_last_sync_url = fastapi_url.rstrip('/') + "/api/reports/last-sync"
        endpoints = {
            "fastapi_main": {
                "url": fastapi_main_url,
                "method": "POST",  # Use POST for /api/updates endpoint
                "headers": headers,
                "auth": auth_config,
                "interval": interval,
                "enabled": enabled,
                "timeout": 30,
                "retry_attempts": retry_attempts,
                "retry_delay": retry_delay,
                "response_handler": "updates"  # Use updates handler for match synchronization
            },
            "reports_sync": {
                "url": reports_sync_url,
                "method": "POST",  # Use POST for reports sync
                "headers": headers,
                "auth": auth_config,
                "interval": 600,  # 10 minutes for incremental reports sync
                "enabled": enabled,  # Enable when token is available
                "timeout": 60,  # Longer timeout for large report data
                "retry_attempts": 5,  # More retries for unreliable connections
                "retry_delay": 60,  # 60 seconds between retries
                "response_handler": "reports_sync"  # Use reports_sync handler
            },
            "reports_last_sync": {
                "url": reports_last_sync_url,
                "method": "GET",
                "headers": headers,
                "auth": auth_config,
                "interval": 600,  # 10 minutes
                "enabled": enabled,
                "timeout": 30,
                "retry_attempts": 3,
                "retry_delay": 30,
                "response_handler": "default"  # Use default handler
            },
        }

        logger.debug("ENDPOINT URLS - base_url: {}, fastapi_main: {}, reports_sync: {}, reports_last_sync: {}".format(
            fastapi_url, fastapi_main_url, reports_sync_url, reports_last_sync_url))

        return endpoints
    
    def run(self):
        """Main run loop"""
        try:
            logger.info("APIClient thread started")
            
            # Send ready status
            ready_message = MessageBuilder.system_status(
                sender=self.name,
                status="ready",
                details={
                    "endpoints": len(self.endpoints),
                    "enabled_endpoints": sum(1 for ep in self.endpoints.values() if ep.enabled)
                }
            )
            self.message_bus.publish(ready_message)
            
            # Main execution loop
            while self.running and not self.shutdown_event.is_set():
                try:
                    # Update heartbeat at the start of each loop
                    self.heartbeat()

                    # Process messages
                    message = self.message_bus.get_message(self.name, timeout=1.0)
                    if message:
                        self._process_message(message)

                    # Update heartbeat before potentially long operations
                    self.heartbeat()

                    # Execute scheduled API requests
                    self._execute_scheduled_requests()

                    # Update heartbeat after operations
                    self.heartbeat()

                    time.sleep(1.0)

                except Exception as e:
                    logger.error(f"APIClient run loop error: {e}")
                    # Update heartbeat even in error cases
                    self.heartbeat()
                    time.sleep(5.0)
                    
        except Exception as e:
            logger.error(f"APIClient run failed: {e}")
        finally:
            logger.info("APIClient thread ended")
    
    def _execute_scheduled_requests(self):
        """Execute API requests that are due"""
        for endpoint in self.endpoints.values():
            if endpoint.should_execute():
                self._execute_endpoint_request(endpoint)
    
    def _get_last_fixture_timestamp(self) -> Optional[str]:
        """Get the server activation timestamp of the last active fixture in the database"""
        try:
            # Update heartbeat before database operation
            self.heartbeat()
            
            session = self.db_manager.get_session()
            try:
                # Get the most recent match with fixture_active_time set
                last_active_match = session.query(MatchModel).filter(
                    MatchModel.fixture_active_time.isnot(None)
                ).order_by(MatchModel.fixture_active_time.desc()).first()
                
                # Update heartbeat after database query
                self.heartbeat()
                
                if last_active_match and last_active_match.fixture_active_time:
                    # Return Unix timestamp as string (long integer number)
                    return str(last_active_match.fixture_active_time)
                else:
                    # No fixtures with activation time found - don't send 'from' parameter
                    return None
                    
            finally:
                session.close()
                
        except Exception as e:
            logger.error(f"Failed to get last fixture activation timestamp: {e}")
            # Update heartbeat even on error
            self.heartbeat()
            return None
    
    def _execute_endpoint_request(self, endpoint: APIEndpoint):
        """Execute a single API request with custom retry logic for token-based endpoints"""
        try:
            # Update heartbeat before starting potentially long operation
            self.heartbeat()
            
            endpoint.last_request = datetime.utcnow()
            endpoint.total_requests += 1
            self.stats['total_requests'] += 1
            
            logger.debug("Executing API request: {} -> {}".format(endpoint.name, endpoint.url))
            
            # Prepare request parameters
            request_kwargs = {
                'method': endpoint.method,
                'url': endpoint.url,
                'headers': endpoint.headers.copy(),
                'timeout': endpoint.timeout
            }
            
            # Prepare data/params based on method
            request_data = endpoint.params.copy() if endpoint.method == 'GET' else endpoint.data.copy()
            
            # For reports_sync endpoint, collect report data before sending
            if endpoint.name == 'reports_sync':
                logger.debug("Collecting report data for reports_sync endpoint")
                reports_handler = self.response_handlers.get('reports_sync')
                if reports_handler and hasattr(reports_handler, 'collect_report_data'):
                    try:
                        # Collect report data for all time (not just today)
                        report_data = reports_handler.collect_report_data(date_range='all')
                        logger.info(f"Collected report data: {len(report_data.get('bets', []))} bets, {len(report_data.get('extraction_stats', []))} stats")
                        
                        # Log the complete report data structure for debugging
                        logger.debug(f"Report data structure: {list(report_data.keys())}")
                        logger.debug(f"Report data keys: {json.dumps({k: type(v).__name__ for k, v in report_data.items()})}")
                        
                        # Log sample data (first bet and first stat if available)
                        if report_data.get('bets'):
                            logger.debug(f"Sample bet data: {json.dumps(report_data['bets'][0], default=str)}")
                        if report_data.get('extraction_stats'):
                            logger.debug(f"Sample extraction stat: {json.dumps(report_data['extraction_stats'][0], default=str)}")
                        
                        # Calculate payload size
                        json_payload = json.dumps(report_data, default=str)
                        payload_size = len(json_payload.encode('utf-8'))
                        logger.info(f"Reports sync payload size: {payload_size} bytes ({payload_size/1024:.2f} KB)")
                        
                        request_data = report_data
                    except Exception as e:
                        logger.error(f"Failed to collect report data: {e}")
                        import traceback
                        logger.error(f"Data collection traceback: {traceback.format_exc()}")
                        # Send empty data if collection fails
                        request_data = {}
            
            # For FastAPI /api/updates endpoint, add 'from' parameter and rustdesk_id if provided
            if endpoint.name == 'fastapi_main' and 'updates' in endpoint.url.lower():
                # Check if forced timestamp is requested (for empty templates table scenario)
                force_timestamp = getattr(endpoint, 'force_timestamp', False)
                
                if force_timestamp:
                    # Force timestamp to 0 to download latest fixture from server
                    request_data['from'] = '0'
                    logger.debug("Forcing timestamp to 0 to download latest fixture from server")
                else:
                    last_timestamp = self._get_last_fixture_timestamp()
                    if last_timestamp:
                        request_data['from'] = last_timestamp
                        logger.debug("Adding 'from' parameter to {} request: {}".format(endpoint.name, last_timestamp))
                    else:
                        # When no fixtures exist, send empty request to get all data
                        logger.debug("No fixtures found, sending empty request to {}".format(endpoint.name))

                # Add rustdesk_id to the updates call if provided
                logger.debug("DEBUG: Checking rustdesk_id - value: '{}' (type: {}), bool: {}".format(
                    self.settings.rustdesk_id, type(self.settings.rustdesk_id), bool(self.settings.rustdesk_id)))
                if self.settings.rustdesk_id:
                    request_data['rustdesk_id'] = self.settings.rustdesk_id
                    logger.debug("Adding rustdesk_id to {} request: {}".format(endpoint.name, self.settings.rustdesk_id))
                else:
                    logger.debug("rustdesk_id not set, skipping addition to request")
            
            if endpoint.method == 'GET':
                request_kwargs['params'] = request_data
            else:
                # For POST requests, always send JSON data (even if empty)
                request_kwargs['json'] = request_data if request_data else {}
            
            # Debug log the complete request data
            logger.debug(f"Request method: {endpoint.method}")
            logger.debug(f"Request headers: {request_kwargs['headers']}")
            if endpoint.method == 'GET' and request_data:
                logger.debug(f"Request params: {request_data}")
            elif endpoint.method != 'GET' and request_data:
                logger.debug(f"Request JSON data: {request_data}")
            else:
                logger.debug("Request data: (empty)")
            
            # Add authentication if configured
            if endpoint.auth:
                logger.debug("AUTH DEBUG - endpoint.auth: {}".format(endpoint.auth))
                logger.debug("AUTH DEBUG - auth type: {}".format(endpoint.auth.get('type')))
                if endpoint.auth.get('type') == 'bearer':
                    token = endpoint.auth.get('token')
                    logger.debug("AUTH DEBUG - raw token value: '{}' (repr: {})".format(token, repr(token)))
                    if token and token.strip():
                        request_kwargs['headers']['Authorization'] = "Bearer {}".format(token.strip())
                        logger.debug("AUTH - Added Bearer token for {}: {}...{}".format(endpoint.name, token.strip()[:10], token.strip()[-10:] if len(token.strip()) > 20 else token.strip()))
                    else:
                        logger.debug("AUTH - Bearer token configured but empty for {}".format(endpoint.name))
                elif endpoint.auth.get('type') == 'basic':
                    request_kwargs['auth'] = (endpoint.auth.get('username'), endpoint.auth.get('password'))
                    logger.debug("AUTH - Added Basic auth for {}".format(endpoint.name))
            else:
                logger.debug("AUTH - No authentication configured for {}".format(endpoint.name))
            
            # Generate curl command for easy debugging/replication (after auth is added)
            curl_cmd = self._generate_curl_command(endpoint.method, endpoint.url, request_kwargs['headers'], request_data)
            logger.debug(f"Curl equivalent: {curl_cmd}")
            
            # Check if this is a token-based endpoint that needs custom retry logic
            is_token_endpoint = (endpoint.auth and
                               endpoint.auth.get('type') == 'bearer' and
                               endpoint.auth.get('token'))
            
            if is_token_endpoint and endpoint.consecutive_failures > 0:
                # For token-based endpoints with failures, use custom retry logic
                success = self._execute_with_custom_retry(endpoint, request_kwargs)
                if not success:
                    return  # Custom retry logic handles error reporting
            else:
                # Standard execution for first attempt or non-token endpoints
                response = self.session.request(**request_kwargs)
                
                # Update heartbeat after HTTP request completes
                self.heartbeat()
                
                # Debug log the complete response
                logger.debug(f"Response status code: {response.status_code}")
                logger.debug(f"Response headers: {dict(response.headers)}")
                try:
                    response_text = response.text
                    if len(response_text) > 1000:
                        logger.debug(f"Response body (truncated): {response_text[:1000]}...")
                    else:
                        logger.debug(f"Response body: {response_text}")
                    
                    # For reports_sync endpoint, log detailed error information on 500 errors
                    if endpoint.name == 'reports_sync' and response.status_code >= 500:
                        logger.error(f"Reports sync server error - Status: {response.status_code}")
                        logger.error(f"Server response: {response_text}")
                        logger.error(f"Request URL: {request_kwargs.get('url')}")
                        logger.error(f"Request method: {request_kwargs.get('method')}")
                        logger.error(f"Request headers: {request_kwargs.get('headers')}")
                        if 'json' in request_kwargs:
                            json_data = request_kwargs['json']
                            logger.error(f"Request JSON keys: {list(json_data.keys())}")
                            logger.error(f"Request JSON size: {len(json.dumps(json_data, default=str).encode('utf-8'))} bytes")
                except Exception as e:
                    logger.debug(f"Could not read response body: {e}")
                
                # Now raise for status to trigger error handling
                response.raise_for_status()
                
                # Handle successful response
                handler = self.response_handlers.get(endpoint.response_handler, self.response_handlers['default'])
                # Pass report_data to handler if this is reports_sync endpoint
                if endpoint.name == 'reports_sync' and 'request_data' in locals():
                    processed_data = handler.handle_response(endpoint, response, request_data)
                else:
                    processed_data = handler.handle_response(endpoint, response)
                
                # Update heartbeat after response processing
                self.heartbeat()
                
                # Update endpoint status
                endpoint.last_success = datetime.utcnow()
                endpoint.last_error = None
                endpoint.consecutive_failures = 0
                endpoint.successful_requests += 1
                self.stats['successful_requests'] += 1
                
                # Send response data via message bus
                if processed_data:
                    response_message = Message(
                        type=MessageType.API_RESPONSE,
                        sender=self.name,
                        data={
                            'endpoint': endpoint.name,
                            'success': True,
                            'data': processed_data,
                            'timestamp': datetime.utcnow().isoformat()
                        }
                    )
                    self.message_bus.publish(response_message)

                    # Check if this is a successful fixture update that might trigger game start
                    synchronized_matches = processed_data.get('synchronized_matches', 0)
                    downloaded_zips = processed_data.get('downloaded_zips', 0)
                    expected_zips = processed_data.get('expected_zips', 0)

                    if endpoint.name == 'fastapi_main' and synchronized_matches > 0:
                        if downloaded_zips == expected_zips:
                            logger.info(f"Fixture update completed successfully - {synchronized_matches} matches synchronized, {downloaded_zips}/{expected_zips} ZIPs downloaded")
                            logger.debug("All expected ZIP files downloaded - fixtures are ready for games")

                            # Send a message to trigger game start check
                            game_start_check_message = Message(
                                type=MessageType.SYSTEM_STATUS,
                                sender=self.name,
                                data={
                                    'status': 'fixture_update_completed',
                                    'synchronized_matches': synchronized_matches,
                                    'downloaded_zips': downloaded_zips,
                                    'expected_zips': expected_zips,
                                    'timestamp': datetime.utcnow().isoformat()
                                }
                            )
                            self.message_bus.publish(game_start_check_message)
                            logger.debug("Sent fixture_update_completed message to trigger game start check")
                        else:
                            logger.info(f"Fixture update partially completed - {synchronized_matches} matches synchronized, {downloaded_zips}/{expected_zips} ZIPs downloaded")
                            logger.debug(f"Waiting for remaining {expected_zips - downloaded_zips} ZIP files before fixtures are ready for games")
                
                logger.debug(f"API request successful: {endpoint.name}")
            
        except Exception as e:
            # Handle request failure
            self.heartbeat()  # Update heartbeat even on failure
            self._handle_request_failure(endpoint, e)
    
    def _execute_with_custom_retry(self, endpoint: APIEndpoint, request_kwargs: dict) -> bool:
        """Execute request with custom retry logic for token-based endpoints"""
        max_retries = min(endpoint.retry_attempts, 10)  # Cap at 10 retries as requested
        retry_delay = 30  # 30 seconds between retries as requested
        
        for attempt in range(max_retries):
            try:
                logger.info(f"Retry attempt {attempt + 1}/{max_retries} for endpoint {endpoint.name}")
                
                response = self.session.request(**request_kwargs)
                response.raise_for_status()
                
                # Update heartbeat after HTTP retry request completes
                self.heartbeat()
                
                # Debug log the complete response (retry scenario)
                logger.debug(f"Retry response status code: {response.status_code}")
                logger.debug(f"Retry response headers: {dict(response.headers)}")
                try:
                    response_text = response.text
                    if len(response_text) > 1000:
                        logger.debug(f"Retry response body (truncated): {response_text[:1000]}...")
                    else:
                        logger.debug(f"Retry response body: {response_text}")
                except Exception as e:
                    logger.debug(f"Could not read retry response body: {e}")
                
                # Handle successful response
                handler = self.response_handlers.get(endpoint.response_handler, self.response_handlers['default'])
                processed_data = handler.handle_response(endpoint, response)
                
                # Update heartbeat after response processing
                self.heartbeat()
                
                # Update endpoint status - success!
                endpoint.last_success = datetime.utcnow()
                endpoint.last_error = None
                endpoint.consecutive_failures = 0
                endpoint.successful_requests += 1
                self.stats['successful_requests'] += 1
                
                # Send response data via message bus
                if processed_data:
                    response_message = Message(
                        type=MessageType.API_RESPONSE,
                        sender=self.name,
                        data={
                            'endpoint': endpoint.name,
                            'success': True,
                            'data': processed_data,
                            'timestamp': datetime.utcnow().isoformat(),
                            'retry_attempt': attempt + 1
                        }
                    )
                    self.message_bus.publish(response_message)
                
                logger.info(f"API request successful on retry {attempt + 1}: {endpoint.name}")
                return True
                
            except Exception as e:
                endpoint.last_error = str(e)
                endpoint.consecutive_failures += 1
                self.stats['failed_requests'] += 1
                
                if attempt < max_retries - 1:
                    logger.warning(f"API retry {attempt + 1} failed for {endpoint.name}: {e}. Waiting {retry_delay}s before next retry.")
                    # Sleep in smaller chunks to allow heartbeat updates during long delays
                    remaining_delay = retry_delay
                    while remaining_delay > 0 and self.running:
                        sleep_chunk = min(5.0, remaining_delay)  # Sleep in 5-second chunks
                        time.sleep(sleep_chunk)
                        self.heartbeat()  # Update heartbeat during retry delays
                        remaining_delay -= sleep_chunk
                else:
                    logger.error(f"All {max_retries} retries failed for {endpoint.name}: {e}")
        
        # All retries failed
        self._handle_request_failure(endpoint, Exception(f"All {max_retries} retries failed. Last error: {endpoint.last_error}"))
        return False
    
    def _handle_request_failure(self, endpoint: APIEndpoint, error: Exception):
        """Handle request failure and send error message"""
        # Update heartbeat when handling failures
        self.heartbeat()
        
        endpoint.last_error = str(error)
        endpoint.consecutive_failures += 1
        self.stats['failed_requests'] += 1
        
        logger.error(f"API request failed: {endpoint.name} - {error}")
        
        # Log heartbeat status for debugging
        logger.debug(f"API client heartbeat updated during error handling for {endpoint.name}")
        
        # Send error message
        error_message = Message(
            type=MessageType.API_RESPONSE,
            sender=self.name,
            data={
                'endpoint': endpoint.name,
                'success': False,
                'error': str(error),
                'consecutive_failures': endpoint.consecutive_failures,
                'timestamp': datetime.utcnow().isoformat()
            }
        )
        self.message_bus.publish(error_message)
        
        # For token-based endpoints, don't disable after failures - keep retrying per user requirements
        is_token_endpoint = (endpoint.auth and
                           endpoint.auth.get('type') == 'bearer' and
                           endpoint.auth.get('token'))
        
        if not is_token_endpoint and endpoint.consecutive_failures >= self.settings.max_consecutive_failures:
            endpoint.enabled = False
            logger.warning(f"Endpoint disabled due to consecutive failures: {endpoint.name}")
    
    def _process_message(self, message: Message):
        """Process received message"""
        try:
            if message.type == MessageType.CONFIG_UPDATE:
                self._handle_config_update(message)
            elif message.type == MessageType.API_REQUEST:
                self._handle_api_request(message)
            elif message.type == MessageType.FORCE_API_UPDATE:
                self._handle_force_api_update(message)
            # Add other message types as needed
        except Exception as e:
            logger.error(f"Failed to process message: {e}")
    
    def _handle_config_update(self, message: Message):
        """Handle configuration update message"""
        try:
            config_section = message.data.get("config_section")
            config_data = message.data.get("config_data", {})

            if config_section == "api_endpoints":
                logger.info("API endpoints configuration updated")
                self._load_endpoints()
            elif config_section == "api":
                logger.info("API configuration updated, reloading endpoints")

                # Check for token and URL changes
                new_token = config_data.get("api_token", "").strip()
                new_url = config_data.get("fastapi_url", "").strip()
                old_token = ""
                old_url = ""

                try:
                    old_api_config = self.config_manager.get_section_config("api") or {}
                    old_token = old_api_config.get("api_token", "").strip()
                    old_url = old_api_config.get("fastapi_url", "").strip()
                except Exception:
                    pass

                # Reload endpoints to pick up new configuration
                self._load_endpoints()

                # Handle dynamic timer start/stop based on token changes
                self._handle_token_change(old_token, new_token)

                # Handle URL changes - trigger immediate connection test
                self._handle_url_change(old_url, new_url)

        except Exception as e:
            logger.error(f"Failed to handle config update: {e}")
    
    def _handle_token_change(self, old_token: str, new_token: str):
        """Handle dynamic timer start/stop based on token changes"""
        try:
            fastapi_endpoint = self.endpoints.get("fastapi_main")
            if not fastapi_endpoint:
                return

            old_has_token = bool(old_token and old_token.strip())
            new_has_token = bool(new_token and new_token.strip())

            # Get current interval setting
            current_api_config = self.config_manager.get_section_config("api") or {}
            current_interval = current_api_config.get("api_interval", 1800)  # 30 minutes default

            if not old_has_token and new_has_token:
                # Token was added - start timer and trigger immediate fixture update
                fastapi_endpoint.enabled = True
                fastapi_endpoint.interval = current_interval
                fastapi_endpoint.retry_attempts = 10
                fastapi_endpoint.retry_delay = 30
                fastapi_endpoint.consecutive_failures = 0  # Reset failure count
                fastapi_endpoint.last_request = None  # Reset to trigger immediate first request

                logger.info(f"FastAPI timer started - token configured, {current_interval} second intervals enabled")

                # Trigger immediate fixture update check
                logger.info("Token configured - triggering immediate fixture update check")
                self._execute_endpoint_request(fastapi_endpoint)

                # Send immediate status update
                status_message = Message(
                    type=MessageType.SYSTEM_STATUS,
                    sender=self.name,
                    data={
                        "status": "timer_started",
                        "endpoint": "fastapi_main",
                        "reason": "token_configured",
                        "interval_seconds": current_interval,
                        "immediate_update_triggered": True
                    }
                )
                self.message_bus.publish(status_message)

            elif old_has_token and not new_has_token:
                # Token was removed - stop timer
                fastapi_endpoint.enabled = False
                fastapi_endpoint.interval = 600  # Reset to 10 minutes (default)
                fastapi_endpoint.retry_attempts = 3
                fastapi_endpoint.retry_delay = 5

                logger.info("FastAPI timer stopped - token removed, automatic requests disabled")

                # Send status update
                status_message = Message(
                    type=MessageType.SYSTEM_STATUS,
                    sender=self.name,
                    data={
                        "status": "timer_stopped",
                        "endpoint": "fastapi_main",
                        "reason": "token_removed"
                    }
                )
                self.message_bus.publish(status_message)

            elif old_has_token and new_has_token and old_token != new_token:
                # Token was changed - keep timer running but reset failure count and trigger immediate update
                fastapi_endpoint.consecutive_failures = 0
                fastapi_endpoint.last_request = None  # Trigger immediate request with new token

                logger.info("FastAPI token updated - timer continues with new authentication")

                # Trigger immediate fixture update check with new token
                logger.info("Token updated - triggering immediate fixture update check with new token")
                self._execute_endpoint_request(fastapi_endpoint)

        except Exception as e:
            logger.error(f"Failed to handle token change: {e}")

    def _handle_url_change(self, old_url: str, new_url: str):
        """Handle URL changes - trigger immediate connection test"""
        try:
            if old_url != new_url and new_url.strip():
                fastapi_endpoint = self.endpoints.get("fastapi_main")
                if fastapi_endpoint:
                    logger.info(f"FastAPI URL changed from '{old_url}' to '{new_url}' - triggering immediate connection test")

                    # Temporarily enable endpoint for connection test if it was disabled
                    was_enabled = fastapi_endpoint.enabled
                    if not was_enabled:
                        fastapi_endpoint.enabled = True
                        logger.debug("Temporarily enabled fastapi_main endpoint for connection test")

                    # Reset failure count and last request to ensure immediate execution
                    fastapi_endpoint.consecutive_failures = 0
                    fastapi_endpoint.last_request = None

                    # Trigger immediate request to test new URL
                    logger.info("Triggering immediate connection test to new FastAPI endpoint")
                    self._execute_endpoint_request(fastapi_endpoint)

                    # Restore original enabled state after test
                    if not was_enabled:
                        fastapi_endpoint.enabled = False
                        logger.debug("Restored fastapi_main endpoint to disabled state after test")

                    # Send status update about URL change
                    status_message = Message(
                        type=MessageType.SYSTEM_STATUS,
                        sender=self.name,
                        data={
                            "status": "url_changed",
                            "endpoint": "fastapi_main",
                            "old_url": old_url,
                            "new_url": new_url,
                            "connection_test_triggered": True,
                            "endpoint_was_enabled": was_enabled
                        }
                    )
                    self.message_bus.publish(status_message)

        except Exception as e:
            logger.error(f"Failed to handle URL change: {e}")

    def update_fastapi_url(self, new_url: str) -> bool:
        """Update FastAPI server URL and reload endpoints"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            api_config["fastapi_url"] = new_url
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new URL
            self._load_endpoints()
            
            logger.info(f"FastAPI URL updated to: {new_url}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update FastAPI URL: {e}")
            return False
    
    def update_api_token(self, new_token: str) -> bool:
        """Update API authentication token and reload endpoints"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            api_config["api_token"] = new_token
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new token
            self._load_endpoints()
            
            logger.info("API token updated successfully")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update API token: {e}")
            return False
    
    def update_api_config(self, new_url: str = None, new_token: str = None) -> bool:
        """Update FastAPI URL and/or token configuration"""
        try:
            # Update configuration
            api_config = self.config_manager.get_section_config("api") or {}
            
            if new_url is not None:
                api_config["fastapi_url"] = new_url
                
            if new_token is not None:
                api_config["api_token"] = new_token
            
            self.config_manager.update_section("api", api_config)
            
            # Reload endpoints to pick up new configuration
            self._load_endpoints()
            
            logger.info(f"API configuration updated - URL: {new_url}, Token: {'*' * len(new_token) if new_token else 'unchanged'}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update API configuration: {e}")
            return False
    
    def _handle_api_request(self, message: Message):
        """Handle manual API request message"""
        try:
            endpoint_name = message.data.get("endpoint")
            
            if endpoint_name in self.endpoints:
                endpoint = self.endpoints[endpoint_name]
                self._execute_endpoint_request(endpoint)
            else:
                logger.warning(f"Unknown endpoint requested: {endpoint_name}")
            
        except Exception as e:
            logger.error(f"Failed to handle API request: {e}")
    
    def _handle_force_api_update(self, message: Message):
        """Handle forced API update request from games thread"""
        try:
            logger.info("Received forced API update request: {}".format(message.data))
            force_timestamp = message.data.get("force_timestamp", False)
            reason = message.data.get("reason", "unknown")
            
            logger.info("Forced API update triggered - reason: {}, force_timestamp: {}".format(reason, force_timestamp))
            
            # Get fastapi_main endpoint
            fastapi_endpoint = self.endpoints.get('fastapi_main')
            if fastapi_endpoint:
                # Reset last_request to trigger immediate execution
                fastapi_endpoint.last_request = None
                
                # Store force_timestamp flag for use in request
                fastapi_endpoint.force_timestamp = force_timestamp
                
                # Execute immediate request
                logger.info("Executing immediate forced API update to synchronize match templates")
                self._execute_endpoint_request(fastapi_endpoint)
                
                # Clear the force_timestamp flag after execution
                if hasattr(fastapi_endpoint, 'force_timestamp'):
                    delattr(fastapi_endpoint, 'force_timestamp')
                
                logger.info("Forced API update completed successfully")
                
                # Send response back to games thread
                response_message = Message(
                    type=MessageType.FORCE_API_UPDATE,
                    sender=self.name,
                    recipient=message.sender,
                    data={
                        "status": "completed",
                        "reason": reason,
                        "timestamp": datetime.utcnow().isoformat()
                    }
                )
                self.message_bus.publish(response_message)
            else:
                logger.error("fastapi_main endpoint not found - cannot execute forced API update")
                
        except Exception as e:
            logger.error("Failed to handle forced API update: {}".format(e))
    
    def get_endpoint_status(self, endpoint_name: str) -> Optional[Dict[str, Any]]:
        """Get status of a specific endpoint"""
        endpoint = self.endpoints.get(endpoint_name)
        if endpoint:
            return endpoint.to_dict()
        return None
    
    def get_all_endpoints_status(self) -> Dict[str, Dict[str, Any]]:
        """Get status of all endpoints"""
        return {name: endpoint.to_dict() for name, endpoint in self.endpoints.items()}
    
    def get_client_stats(self) -> Dict[str, Any]:
        """Get client statistics"""
        uptime = datetime.utcnow() - self.stats['start_time']
        return {
            'total_requests': self.stats['total_requests'],
            'successful_requests': self.stats['successful_requests'],
            'failed_requests': self.stats['failed_requests'],
            'success_rate': (self.stats['successful_requests'] / max(1, self.stats['total_requests'])) * 100,
            'uptime_seconds': uptime.total_seconds(),
            'endpoints_count': len(self.endpoints),
            'enabled_endpoints': sum(1 for ep in self.endpoints.values() if ep.enabled)
        }
    
    def update_endpoint(self, endpoint_name: str, config: Dict[str, Any]) -> bool:
        """Update endpoint configuration"""
        try:
            if endpoint_name in self.endpoints:
                # Update existing endpoint
                endpoint = self.endpoints[endpoint_name]
                for key, value in config.items():
                    if hasattr(endpoint, key):
                        setattr(endpoint, key, value)
            else:
                # Create new endpoint
                self.endpoints[endpoint_name] = APIEndpoint(endpoint_name, config)
            
            # Save to configuration
            all_endpoints = {name: ep.to_dict() for name, ep in self.endpoints.items()}
            self.config_manager.update_section("api_endpoints", all_endpoints)
            
            logger.info(f"Updated endpoint configuration: {endpoint_name}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to update endpoint: {e}")
            return False
    
    def shutdown(self):
        """Shutdown API client"""
        try:
            logger.info("Shutting down APIClient...")
            
            if self.session:
                self.session.close()
            
        except Exception as e:
            logger.error(f"APIClient shutdown error: {e}")
    
    def _generate_curl_command(self, method, url, headers, data=None):
        """Generate a curl command equivalent to the current request for debugging purposes."""
        import json
        import shlex
        
        # Start with basic curl command
        cmd_parts = ['curl', '-X', method]
        
        # Add headers
        for key, value in headers.items():
            cmd_parts.extend(['-H', f'{key}: {value}'])
        
        # Add data based on method
        if method == 'GET' and data:
            # For GET requests, params are added to URL
            from urllib.parse import urlencode
            query_string = urlencode(data)
            url = f"{url}?{query_string}"
        elif method == 'POST':
            # For POST requests, always add JSON data (even if empty)
            json_str = json.dumps(data if data else {}, separators=(',', ':'))
            cmd_parts.extend(['-d', json_str])
        
        # Add URL (always last)
        cmd_parts.append(url)
        
        # Use shlex.join for proper shell escaping (Python 3.8+)
        try:
            return shlex.join(cmd_parts)
        except AttributeError:
            # Fallback for Python < 3.8
            return ' '.join(shlex.quote(arg) for arg in cmd_parts)