fix: align reports and repair historical stats

mbetterclient/database/migrations.py

@@ -7541,6 +7541,122 @@ class Migration_068_ResetRedistributionBalance(DatabaseMigration):
         return True
 
 
+class Migration_069_RepairExtractionStatsFinancials(DatabaseMigration):
+    """Repair historical extraction stats financial fields from authoritative bet details"""
+
+    def __init__(self):
+        super().__init__("069", "Repair extraction stats financial fields")
+
+    def _snapshot_financial_fields(self, stat_row):
+        return {
+            'total_bets': getattr(stat_row, 'total_bets', 0),
+            'total_amount_collected': getattr(stat_row, 'total_amount_collected', 0.0),
+            'total_redistributed': getattr(stat_row, 'total_redistributed', 0.0),
+            'under_bets': getattr(stat_row, 'under_bets', 0),
+            'under_amount': getattr(stat_row, 'under_amount', 0.0),
+            'over_bets': getattr(stat_row, 'over_bets', 0),
+            'over_amount': getattr(stat_row, 'over_amount', 0.0),
+            'result_breakdown': getattr(stat_row, 'result_breakdown', None),
+        }
+
+    def _build_delta_log(self, match_id, before_values, after_values):
+        changes = []
+        for field_name, before_value in before_values.items():
+            after_value = after_values[field_name]
+            if before_value != after_value:
+                changes.append(f"{field_name}: {before_value} -> {after_value}")
+
+        if not changes:
+            return None
+
+        return f"Repaired extraction stats row for match_id={match_id}: " + ", ".join(changes)
+
+    def _rebuild_result_breakdown(self, stat_row, active_details):
+        breakdown = {}
+
+        for detail in active_details:
+            outcome = getattr(detail, 'outcome', None)
+            if not outcome:
+                continue
+
+            if outcome not in breakdown:
+                breakdown[outcome] = {'bets': 0, 'amount': 0.0}
+
+            breakdown[outcome]['bets'] += 1
+            breakdown[outcome]['amount'] += float(getattr(detail, 'amount', 0) or 0)
+
+        stat_row.result_breakdown = breakdown
+
+    def _recompute_stat_row(self, stat_row, detail_rows):
+        active_details = [detail for detail in detail_rows if getattr(detail, 'result', None) != 'cancelled']
+
+        stat_row.total_bets = len(active_details)
+        stat_row.total_amount_collected = sum(float(getattr(detail, 'amount', 0) or 0) for detail in active_details)
+        stat_row.total_redistributed = sum(
+            float(getattr(detail, 'win_amount', 0) or 0)
+            for detail in active_details
+            if getattr(detail, 'result', None) == 'win'
+        )
+        stat_row.under_bets = sum(1 for detail in active_details if getattr(detail, 'outcome', None) == 'UNDER')
+        stat_row.under_amount = sum(
+            float(getattr(detail, 'amount', 0) or 0)
+            for detail in active_details
+            if getattr(detail, 'outcome', None) == 'UNDER'
+        )
+        stat_row.over_bets = sum(1 for detail in active_details if getattr(detail, 'outcome', None) == 'OVER')
+        stat_row.over_amount = sum(
+            float(getattr(detail, 'amount', 0) or 0)
+            for detail in active_details
+            if getattr(detail, 'outcome', None) == 'OVER'
+        )
+        self._rebuild_result_breakdown(stat_row, active_details)
+        return stat_row
+
+    def up(self, db_manager) -> bool:
+        session = None
+        try:
+            from .models import ExtractionStatsModel, BetDetailModel
+
+            session = db_manager.get_session()
+            stats_rows = session.query(ExtractionStatsModel).all()
+            modified_rows = 0
+            match_counts = {}
+
+            for stat_row in stats_rows:
+                match_counts[stat_row.match_id] = match_counts.get(stat_row.match_id, 0) + 1
+
+            for match_id, row_count in match_counts.items():
+                if row_count > 1:
+                    logger.warning(f"Duplicate extraction stats rows detected for match_id {match_id}: {row_count} rows")
+
+            for stat_row in stats_rows:
+                detail_rows = session.query(BetDetailModel).filter_by(match_id=stat_row.match_id).all()
+                before_values = self._snapshot_financial_fields(stat_row)
+                self._recompute_stat_row(stat_row, detail_rows)
+                after_values = self._snapshot_financial_fields(stat_row)
+                delta_log = self._build_delta_log(stat_row.match_id, before_values, after_values)
+                if delta_log:
+                    modified_rows += 1
+                    logger.info(delta_log)
+
+            session.commit()
+            logger.info(f"Modified {modified_rows} extraction stats rows out of {len(stats_rows)} processed")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to repair extraction stats financial fields: {e}")
+            if session:
+                session.rollback()
+            return False
+        finally:
+            if session:
+                session.close()
+
+    def down(self, db_manager) -> bool:
+        """Cannot rollback this migration because prior incorrect values are not preserved"""
+        logger.warning("Cannot rollback Migration_069_RepairExtractionStatsFinancials - original financial values not preserved")
+        return True
+
+
 MIGRATIONS: List[DatabaseMigration] = [
     Migration_001_InitialSchema(),
     Migration_002_AddIndexes(),
@@ -7610,6 +7726,7 @@ MIGRATIONS: List[DatabaseMigration] = [
     Migration_066_AddWalletTables(),
     Migration_067_FixMatchResults(),
     Migration_068_ResetRedistributionBalance(),
+    Migration_069_RepairExtractionStatsFinancials(),
 ]
 
 
@@ -7821,4 +7938,4 @@ def get_migration_status(db_manager) -> Dict[str, Any]:
             'total_migrations': 0,
             'applied_migrations': 0,
             'pending_migrations': 0
-        }
\ No newline at end of file
+        }

mbetterclient/web_dashboard/routes.py

@@ -7944,6 +7944,8 @@ def get_match_reports():
                         'payout': 0.0
                     }
 
+                match = session.query(MatchModel).filter_by(id=match_id).first()
+
                 match_stats[match_id]['bets_count'] += 1
                 match_stats[match_id]['payin'] += float(detail.amount)
 
@@ -8176,11 +8178,22 @@ def download_excel_report():
                 total_payin += bet_total
                 total_bets += len(bet_details)
 
-            extraction_stats = session.query(ExtractionStatsModel).filter(
-                ExtractionStatsModel.match_datetime >= start_datetime,
-                ExtractionStatsModel.match_datetime <= end_datetime
-            ).all()
-            total_payout = sum(float(stat.total_redistributed) for stat in extraction_stats)
+            bet_details_query = session.query(BetDetailModel).join(
+                BetModel, BetDetailModel.bet_id == BetModel.uuid
+            ).filter(
+                BetModel.bet_datetime >= start_datetime,
+                BetModel.bet_datetime <= end_datetime,
+                BetDetailModel.result != 'cancelled'
+            )
+            bet_details = bet_details_query.all()
+
+            total_payout = 0.0
+            for detail in bet_details:
+                match = session.query(MatchModel).filter_by(id=detail.match_id).first()
+                if is_bet_detail_winning(detail, match, session):
+                    total_payout += float(detail.win_amount or 0)
+
+            unique_match_ids = {detail.match_id for detail in bet_details}
 
             # Summary headers
             ws_summary['A3'] = "Metric"
@@ -8196,7 +8209,7 @@ def download_excel_report():
                 ("Total Payout", f"{total_payout:.2f}"),
                 ("Net Profit", f"{total_payin - total_payout:.2f}"),
                 ("Total Bets", str(total_bets)),
-                ("Total Matches", str(len(extraction_stats)))
+                ("Total Matches", str(len(unique_match_ids)))
             ]
 
             for i, (metric, value) in enumerate(summary_data, 4):

test_reports_stats_repair_migration.py

+import os
+import sys
+from datetime import datetime
+from types import SimpleNamespace
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from mbetterclient.database.migrations import Migration_069_RepairExtractionStatsFinancials
+from mbetterclient.database.models import Base, BetDetailModel, ExtractionStatsModel, MatchModel
+
+
+class StatRow:
+    def __init__(self):
+        self.match_id = 10
+        self.total_bets = 99
+        self.total_amount_collected = 9999.0
+        self.total_redistributed = 8888.0
+        self.under_bets = 9
+        self.under_amount = 999.0
+        self.over_bets = 8
+        self.over_amount = 888.0
+        self.actual_result = 'WIN1'
+        self.extraction_result = 'WIN1'
+        self.match_datetime = 'keep-me'
+        self.result_breakdown = {'WIN1': {'bets': 2, 'amount': 50.0}}
+
+
+class DetailRow:
+    def __init__(self, outcome, amount, win_amount, result):
+        self.outcome = outcome
+        self.amount = amount
+        self.win_amount = win_amount
+        self.result = result
+
+
+class SessionStub:
+    def __init__(self, stats_rows, detail_rows_by_match_id):
+        self.stats_rows = list(stats_rows)
+        self.detail_rows_by_match_id = dict(detail_rows_by_match_id)
+        self.committed = False
+        self.rolled_back = False
+        self.closed = False
+
+    def query(self, model):
+        model_name = getattr(model, '__name__', str(model))
+        if model_name == 'ExtractionStatsModel':
+            return QueryStub(self.stats_rows)
+        if model_name == 'BetDetailModel':
+            return DetailQueryStub(self.detail_rows_by_match_id)
+        raise AssertionError(f'unexpected model: {model_name}')
+
+    def commit(self):
+        self.committed = True
+
+    def rollback(self):
+        self.rolled_back = True
+
+    def close(self):
+        self.closed = True
+
+
+class QueryStub:
+    def __init__(self, rows):
+        self.rows = rows
+
+    def all(self):
+        return list(self.rows)
+
+
+class DetailQueryStub:
+    def __init__(self, detail_rows_by_match_id):
+        self.detail_rows_by_match_id = detail_rows_by_match_id
+
+    def filter_by(self, **kwargs):
+        match_id = kwargs['match_id']
+        return QueryStub(self.detail_rows_by_match_id.get(match_id, []))
+
+
+class DummyDbManager:
+    def __init__(self, session):
+        self._session = session
+
+    def get_session(self):
+        return self._session
+
+
+def test_repair_migration_recomputes_financial_fields_from_bet_details():
+    migration = Migration_069_RepairExtractionStatsFinancials()
+    stat = StatRow()
+    details = [
+        DetailRow('UNDER', 100.0, 0.0, 'lost'),
+        DetailRow('WIN1', 50.0, 125.0, 'win'),
+        DetailRow('OVER', 75.0, 0.0, 'cancelled'),
+    ]
+
+    repaired = migration._recompute_stat_row(stat, details)
+
+    assert repaired.total_bets == 2
+    assert repaired.total_amount_collected == 150.0
+    assert repaired.total_redistributed == 125.0
+    assert repaired.under_bets == 1
+    assert repaired.under_amount == 100.0
+    assert repaired.over_bets == 0
+    assert repaired.over_amount == 0.0
+    assert repaired.actual_result == 'WIN1'
+    assert repaired.extraction_result == 'WIN1'
+    assert repaired.match_datetime == 'keep-me'
+    assert repaired.result_breakdown == {
+        'UNDER': {'bets': 1, 'amount': 100.0},
+        'WIN1': {'bets': 1, 'amount': 50.0},
+    }
+
+
+def test_repair_migration_ignores_unsupported_non_enum_result_values():
+    migration = Migration_069_RepairExtractionStatsFinancials()
+    stat = StatRow()
+    details = [
+        DetailRow('WIN1', 50.0, 125.0, 'won'),
+    ]
+
+    repaired = migration._recompute_stat_row(stat, details)
+
+    assert repaired.total_bets == 1
+    assert repaired.total_amount_collected == 50.0
+    assert repaired.total_redistributed == 0.0
+    assert repaired.result_breakdown == {
+        'WIN1': {'bets': 1, 'amount': 50.0},
+    }
+
+
+def test_repair_migration_excludes_cancelled_bets_from_all_totals():
+    migration = Migration_069_RepairExtractionStatsFinancials()
+    stat = StatRow()
+    details = [
+        DetailRow('OVER', 40.0, 500.0, 'cancelled'),
+    ]
+
+    repaired = migration._recompute_stat_row(stat, details)
+
+    assert repaired.total_bets == 0
+    assert repaired.total_amount_collected == 0.0
+    assert repaired.total_redistributed == 0.0
+    assert repaired.over_bets == 0
+    assert repaired.over_amount == 0.0
+    assert repaired.result_breakdown == {}
+
+
+def test_repair_migration_updates_rows_logs_duplicates_and_change_deltas(monkeypatch):
+    migration = Migration_069_RepairExtractionStatsFinancials()
+    stat_one = StatRow()
+    stat_one.match_id = 10
+    stat_two = StatRow()
+    stat_two.match_id = 10
+    stat_three = StatRow()
+    stat_three.match_id = 11
+    session = SessionStub(
+        [stat_one, stat_two, stat_three],
+        {
+            10: [DetailRow('UNDER', 25.0, 0.0, 'lost')],
+            11: [DetailRow('OVER', 40.0, 120.0, 'win')],
+        },
+    )
+
+    models_module = SimpleNamespace(
+        ExtractionStatsModel=type('ExtractionStatsModel', (), {}),
+        BetDetailModel=type('BetDetailModel', (), {}),
+    )
+    monkeypatch.setitem(sys.modules, 'mbetterclient.database.models', models_module)
+
+    info_logs = []
+    warning_logs = []
+    monkeypatch.setattr('mbetterclient.database.migrations.logger.info', lambda message: info_logs.append(message))
+    monkeypatch.setattr('mbetterclient.database.migrations.logger.warning', lambda message: warning_logs.append(message))
+
+    success = migration.up(DummyDbManager(session))
+
+    assert success is True
+    assert stat_one.total_bets == 1
+    assert stat_one.total_amount_collected == 25.0
+    assert stat_one.total_redistributed == 0.0
+    assert stat_two.total_bets == 1
+    assert stat_two.total_amount_collected == 25.0
+    assert stat_two.total_redistributed == 0.0
+    assert stat_three.total_bets == 1
+    assert stat_three.total_amount_collected == 40.0
+    assert stat_three.total_redistributed == 120.0
+    assert stat_three.result_breakdown == {'OVER': {'bets': 1, 'amount': 40.0}}
+    assert session.committed is True
+    assert session.rolled_back is False
+    assert session.closed is True
+    assert any('Duplicate extraction stats rows detected for match_id 10' in message for message in warning_logs)
+    assert any('Modified 3 extraction stats rows' in message for message in info_logs)
+    assert any('match_id=10' in message and 'total_bets: 99 -> 1' in message for message in info_logs)
+    assert any('match_id=10' in message and 'result_breakdown:' in message for message in info_logs)
+    assert any('match_id=11' in message and 'total_redistributed: 8888.0 -> 120.0' in message for message in info_logs)
+
+
+def test_repair_migration_runs_against_real_sqlalchemy_models(monkeypatch):
+    engine = create_engine('sqlite:///:memory:')
+    Base.metadata.create_all(engine)
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    match = MatchModel(
+        id=77,
+        match_number=1,
+        fighter1_township='Town A',
+        fighter2_township='Town B',
+        venue_kampala_township='Kampala',
+        filename='fixture.json',
+        file_sha1sum='sha1-fixture',
+        fixture_id='fixture-77',
+        start_time=datetime(2026, 5, 1, 12, 0, 0),
+        status='done',
+        result='WIN1',
+        winning_outcomes=['WIN1'],
+        active_status=False,
+    )
+    stat = ExtractionStatsModel(
+        match_id=77,
+        fixture_id='fixture-77',
+        match_datetime=datetime(2026, 5, 1, 12, 0, 0),
+        total_bets=9,
+        total_amount_collected=900.0,
+        total_redistributed=800.0,
+        actual_result='WIN1',
+        result_breakdown='{"LEGACY":{"bets":9,"amount":900.0,"coefficient":2.0}}',
+        under_bets=4,
+        under_amount=400.0,
+        over_bets=5,
+        over_amount=500.0,
+        extraction_result='WIN1',
+        cap_applied=False,
+    )
+    detail_win = BetDetailModel(match_id=77, bet_id='bet-1', outcome='WIN1', amount=50.0, win_amount=125.0, result='win')
+    detail_under = BetDetailModel(match_id=77, bet_id='bet-2', outcome='UNDER', amount=25.0, win_amount=0.0, result='lost')
+    detail_cancelled = BetDetailModel(match_id=77, bet_id='bet-3', outcome='OVER', amount=100.0, win_amount=999.0, result='cancelled')
+    session.add_all([match, stat, detail_win, detail_under, detail_cancelled])
+    session.commit()
+
+    info_logs = []
+    warning_logs = []
+    monkeypatch.setattr('mbetterclient.database.migrations.logger.info', lambda message: info_logs.append(message))
+    monkeypatch.setattr('mbetterclient.database.migrations.logger.warning', lambda message: warning_logs.append(message))
+
+    migration = Migration_069_RepairExtractionStatsFinancials()
+    success = migration.up(DummyDbManager(session))
+    session.expire_all()
+    repaired = session.query(ExtractionStatsModel).filter_by(match_id=77).one()
+
+    assert success is True
+    assert repaired.total_bets == 2
+    assert repaired.total_amount_collected == 75.0
+    assert repaired.total_redistributed == 125.0
+    assert repaired.under_bets == 1
+    assert repaired.under_amount == 25.0
+    assert repaired.over_bets == 0
+    assert repaired.over_amount == 0.0
+    assert repaired.get_result_breakdown() == {
+        'UNDER': {'bets': 1, 'amount': 25.0},
+        'WIN1': {'bets': 1, 'amount': 50.0},
+    }
+    assert any('Modified 1 extraction stats rows out of 1 processed' in message for message in info_logs)
+    assert any('result_breakdown:' in message for message in info_logs)
+    assert not warning_logs
+
+    session.close()
+    engine.dispose()