Implement database-based queue for reports synchronization

- Add ReportsSyncQueueModel database model for persistent sync queue storage
- Update ReportsSyncResponseHandler to use database instead of JSON files
- Add transaction safety with rollback support for queue operations
- Implement automatic cleanup of completed items
- Add database indexes for efficient querying
- Create comprehensive test suite for database-based queue (test_reports_sync_db.py)
- Update documentation to reflect database implementation

This change improves performance, data integrity, and scalability of the
reports synchronization feature by migrating from JSON file storage to SQLite database.
parent 6e4ccd92
This diff is collapsed.
This diff is collapsed.
...@@ -1157,3 +1157,91 @@ class MatchOutcomeTemplateModel(BaseModel): ...@@ -1157,3 +1157,91 @@ class MatchOutcomeTemplateModel(BaseModel):
def __repr__(self): def __repr__(self):
return f'<MatchOutcomeTemplate {self.column_name}={self.float_value} for MatchTemplate {self.match_id}>' return f'<MatchOutcomeTemplate {self.column_name}={self.float_value} for MatchTemplate {self.match_id}>'
class ReportsSyncQueueModel(BaseModel):
"""Queue for storing failed report synchronization operations"""
__tablename__ = 'reports_sync_queue'
__table_args__ = (
Index('ix_reports_sync_queue_sync_id', 'sync_id'),
Index('ix_reports_sync_queue_status', 'status'),
Index('ix_reports_sync_queue_retry_count', 'retry_count'),
Index('ix_reports_sync_queue_next_retry_at', 'next_retry_at'),
Index('ix_reports_sync_queue_created_at', 'created_at'),
)
sync_id = Column(String(255), nullable=False, unique=True, comment='Unique sync identifier')
client_id = Column(String(255), nullable=False, comment='Client identifier')
status = Column(String(20), default='pending', nullable=False, comment='Queue status: pending, syncing, completed, failed')
retry_count = Column(Integer, default=0, nullable=False, comment='Number of retry attempts')
next_retry_at = Column(DateTime, comment='Next retry timestamp')
error_message = Column(Text, comment='Error message if sync failed')
sync_data = Column(JSON, nullable=False, comment='Sync data payload')
synced_items = Column(Integer, default=0, comment='Number of items successfully synced')
failed_items = Column(Integer, default=0, comment='Number of items that failed to sync')
completed_at = Column(DateTime, comment='Timestamp when sync was completed')
def is_pending(self) -> bool:
"""Check if sync is pending"""
return self.status == 'pending'
def is_syncing(self) -> bool:
"""Check if sync is in progress"""
return self.status == 'syncing'
def is_completed(self) -> bool:
"""Check if sync is completed"""
return self.status == 'completed'
def is_failed(self) -> bool:
"""Check if sync failed"""
return self.status == 'failed'
def can_retry(self, max_retries: int = 5) -> bool:
"""Check if sync can be retried"""
return self.retry_count < max_retries
def should_retry_now(self) -> bool:
"""Check if sync should be retried now"""
if self.status != 'pending':
return False
if self.next_retry_at is None:
return True
return datetime.utcnow() >= self.next_retry_at
def mark_syncing(self):
"""Mark sync as in progress"""
self.status = 'syncing'
self.updated_at = datetime.utcnow()
def mark_completed(self, synced_items: int = 0, failed_items: int = 0):
"""Mark sync as completed"""
self.status = 'completed'
self.synced_items = synced_items
self.failed_items = failed_items
self.completed_at = datetime.utcnow()
self.updated_at = datetime.utcnow()
def mark_failed(self, error_message: str, retry_count: int = None, next_retry_at: DateTime = None):
"""Mark sync as failed and schedule retry"""
self.status = 'pending' # Reset to pending for retry
self.error_message = error_message
if retry_count is not None:
self.retry_count = retry_count
if next_retry_at is not None:
self.next_retry_at = next_retry_at
self.updated_at = datetime.utcnow()
def to_dict(self, exclude_fields: Optional[List[str]] = None) -> Dict[str, Any]:
"""Convert to dictionary"""
result = super().to_dict(exclude_fields)
result['is_pending'] = self.is_pending()
result['is_syncing'] = self.is_syncing()
result['is_completed'] = self.is_completed()
result['is_failed'] = self.is_failed()
result['can_retry'] = self.can_retry()
result['should_retry_now'] = self.should_retry_now()
return result
def __repr__(self):
return f'<ReportsSyncQueue {self.sync_id}: status={self.status}, retries={self.retry_count}>'
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment