Prepare for AI generated matches

app/__init__.py

@@ -38,7 +38,11 @@ def create_app(config_name=None):
         """Make script_root available in all templates for proxy support"""
         return {
             'script_root': request.script_root if request else '',
-            'url_root': request.url_root if request else ''
+            'url_root': request.url_root if request else '',
+            # Expose Python builtins used in templates (e.g. pagination summaries).
+            # Jinja2/Flask do not provide these as globals by default.
+            'min': min,
+            'max': max
         }
     
     # Initialize extensions

app/database/migrations.py

@@ -1349,6 +1349,184 @@ class Migration_016_AllowNullMatchNumber(Migration):
         return True
 
 
+class Migration_017_AddForceFullResync(Migration):
+    """Add force_full_resync column to client_activity table"""
+
+    def __init__(self):
+        super().__init__("017", "Add force_full_resync column to client_activity table")
+
+    def up(self):
+        """Add force_full_resync column"""
+        try:
+            inspector = inspect(db.engine)
+
+            # Skip if the table doesn't exist yet (fresh db will get it via create_all)
+            if 'client_activity' not in inspector.get_table_names():
+                logger.info("client_activity table does not exist yet, skipping migration 017")
+                return True
+
+            columns = [col['name'] for col in inspector.get_columns('client_activity')]
+            if 'force_full_resync' in columns:
+                logger.info("force_full_resync column already exists, skipping creation")
+                return True
+
+            with db.engine.connect() as conn:
+                conn.execute(text("""
+                    ALTER TABLE client_activity
+                    ADD COLUMN force_full_resync TINYINT(1) NOT NULL DEFAULT 0
+                """))
+                conn.commit()
+
+            logger.info("Added force_full_resync column successfully")
+            return True
+
+        except Exception as e:
+            logger.error(f"Migration 017 failed: {str(e)}")
+            raise
+
+    def down(self):
+        """Drop force_full_resync column (MySQL has no DROP COLUMN IF EXISTS, so guard it)"""
+        try:
+            inspector = inspect(db.engine)
+            if 'client_activity' not in inspector.get_table_names():
+                return True
+            columns = [col['name'] for col in inspector.get_columns('client_activity')]
+            with db.engine.connect() as conn:
+                if 'force_full_resync' in columns:
+                    conn.execute(text("ALTER TABLE client_activity DROP COLUMN force_full_resync"))
+                conn.commit()
+            logger.info("Dropped force_full_resync column")
+            return True
+        except Exception as e:
+            logger.error(f"Rollback of migration 017 failed: {str(e)}")
+            raise
+
+    def can_rollback(self) -> bool:
+        return True
+
+
+class Migration_018_AddResyncProgress(Migration):
+    """Add resync_in_progress and resync_remaining columns to client_activity table"""
+
+    def __init__(self):
+        super().__init__("018", "Add resync progress columns to client_activity table")
+
+    def up(self):
+        """Add resync_in_progress and resync_remaining columns"""
+        try:
+            inspector = inspect(db.engine)
+
+            if 'client_activity' not in inspector.get_table_names():
+                logger.info("client_activity table does not exist yet, skipping migration 018")
+                return True
+
+            columns = [col['name'] for col in inspector.get_columns('client_activity')]
+
+            with db.engine.connect() as conn:
+                if 'resync_in_progress' not in columns:
+                    conn.execute(text("""
+                        ALTER TABLE client_activity
+                        ADD COLUMN resync_in_progress TINYINT(1) NOT NULL DEFAULT 0
+                    """))
+                    logger.info("Added resync_in_progress column successfully")
+                else:
+                    logger.info("resync_in_progress column already exists, skipping")
+
+                if 'resync_remaining' not in columns:
+                    conn.execute(text("""
+                        ALTER TABLE client_activity
+                        ADD COLUMN resync_remaining INT NULL
+                    """))
+                    logger.info("Added resync_remaining column successfully")
+                else:
+                    logger.info("resync_remaining column already exists, skipping")
+
+                conn.commit()
+
+            return True
+
+        except Exception as e:
+            logger.error(f"Migration 018 failed: {str(e)}")
+            raise
+
+    def down(self):
+        """Drop resync progress columns (MySQL has no DROP COLUMN IF EXISTS, so guard it)"""
+        try:
+            inspector = inspect(db.engine)
+            if 'client_activity' not in inspector.get_table_names():
+                return True
+            columns = [col['name'] for col in inspector.get_columns('client_activity')]
+            with db.engine.connect() as conn:
+                if 'resync_in_progress' in columns:
+                    conn.execute(text("ALTER TABLE client_activity DROP COLUMN resync_in_progress"))
+                if 'resync_remaining' in columns:
+                    conn.execute(text("ALTER TABLE client_activity DROP COLUMN resync_remaining"))
+                conn.commit()
+            logger.info("Dropped resync progress columns")
+            return True
+        except Exception as e:
+            logger.error(f"Rollback of migration 018 failed: {str(e)}")
+            raise
+
+    def can_rollback(self) -> bool:
+        return True
+
+
+class Migration_019_AddFixtureSourceFlag(Migration):
+    """Add is_fixture_source column to api_tokens table"""
+
+    def __init__(self):
+        super().__init__("019", "Add is_fixture_source column to api_tokens table")
+
+    def up(self):
+        """Add is_fixture_source column"""
+        try:
+            inspector = inspect(db.engine)
+
+            if 'api_tokens' not in inspector.get_table_names():
+                logger.info("api_tokens table does not exist yet, skipping migration 019")
+                return True
+
+            columns = [col['name'] for col in inspector.get_columns('api_tokens')]
+            if 'is_fixture_source' in columns:
+                logger.info("is_fixture_source column already exists, skipping creation")
+                return True
+
+            with db.engine.connect() as conn:
+                conn.execute(text("""
+                    ALTER TABLE api_tokens
+                    ADD COLUMN is_fixture_source TINYINT(1) NOT NULL DEFAULT 0
+                """))
+                conn.commit()
+
+            logger.info("Added is_fixture_source column successfully")
+            return True
+
+        except Exception as e:
+            logger.error(f"Migration 019 failed: {str(e)}")
+            raise
+
+    def down(self):
+        """Drop is_fixture_source column (MySQL has no DROP COLUMN IF EXISTS, so guard it)"""
+        try:
+            inspector = inspect(db.engine)
+            if 'api_tokens' not in inspector.get_table_names():
+                return True
+            columns = [col['name'] for col in inspector.get_columns('api_tokens')]
+            with db.engine.connect() as conn:
+                if 'is_fixture_source' in columns:
+                    conn.execute(text("ALTER TABLE api_tokens DROP COLUMN is_fixture_source"))
+                conn.commit()
+            logger.info("Dropped is_fixture_source column")
+            return True
+        except Exception as e:
+            logger.error(f"Rollback of migration 019 failed: {str(e)}")
+            raise
+
+    def can_rollback(self) -> bool:
+        return True
+
+
 class MigrationManager:
     """Manages database migrations and versioning"""
     
@@ -1370,6 +1548,9 @@ class MigrationManager:
             Migration_014_AddAccumulatedShortfallAndCapPercentage(),
             Migration_015_AllowNullResults(),
             Migration_016_AllowNullMatchNumber(),
+            Migration_017_AddForceFullResync(),
+            Migration_018_AddResyncProgress(),
+            Migration_019_AddFixtureSourceFlag(),
         ]
     
     def ensure_version_table(self):

app/main/routes.py

@@ -1606,10 +1606,12 @@ def create_api_token():
         if existing_token:
             return jsonify({'error': 'A token with this name already exists'}), 400
         
-        # Generate token
-        api_token = current_user.generate_api_token(token_name)
-        
-        logger.info(f"API token '{token_name}' created by user {current_user.username}")
+        # Generate token (optionally as a fixture/match source provider token)
+        is_fixture_source = bool(data.get('is_fixture_source', False))
+        api_token = current_user.generate_api_token(token_name, is_fixture_source=is_fixture_source)
+
+        logger.info(f"API token '{token_name}' created by user {current_user.username} "
+                    f"(fixture_source={is_fixture_source})")
         
         return jsonify({
             'message': 'API token created successfully',
@@ -1977,7 +1979,10 @@ def clients():
                 'last_seen_ago': last_seen_ago,
                 'ip_address': client_activity.ip_address,
                 'user_agent': client_activity.user_agent,
-                'remote_domain': remote_domain
+                'remote_domain': remote_domain,
+                'force_full_resync': bool(getattr(client_activity, 'force_full_resync', False)),
+                'resync_in_progress': bool(getattr(client_activity, 'resync_in_progress', False)),
+                'resync_remaining': getattr(client_activity, 'resync_remaining', None)
             })
 
         # Apply search filter
@@ -2010,6 +2015,82 @@ def clients():
         flash('Error loading clients', 'error')
         return render_template('main/clients.html', clients=[])
 
+
+def _accessible_client_token_ids():
+    """Return the set of API token ids whose clients the current user may manage."""
+    from app.models import APIToken
+    if current_user.is_admin:
+        return None  # None == no restriction (all clients)
+    return [t.id for t in APIToken.query.filter_by(user_id=current_user.id).all()]
+
+
+@csrf.exempt
+@bp.route('/clients/<rustdesk_id>/force-resync', methods=['POST'])
+@login_required
+@require_active_user
+def force_resync_client(rustdesk_id):
+    """Ask a single client to perform a one-time full resync of all historical reports.
+
+    Sets the force_full_resync flag on the client's activity record(s). The client
+    picks it up on its next /api/reports/last-sync poll and lazily re-sends all
+    history in the background; the flag is cleared automatically once served.
+    """
+    try:
+        from app.models import ClientActivity
+
+        query = ClientActivity.query.filter_by(rustdesk_id=rustdesk_id)
+        token_ids = _accessible_client_token_ids()
+        if token_ids is not None:
+            query = query.filter(ClientActivity.api_token_id.in_(token_ids)) if token_ids else query.filter(db.false())
+
+        activities = query.all()
+        if not activities:
+            flash('Client not found or access denied', 'error')
+            return redirect(url_for('main.clients'))
+
+        for activity in activities:
+            activity.force_full_resync = True
+        db.session.commit()
+
+        flash(f'Full resync requested for client {rustdesk_id}. It will resync in the background on its next check-in.', 'success')
+        return redirect(url_for('main.clients'))
+
+    except Exception as e:
+        db.session.rollback()
+        logger.error(f"Force resync client error: {str(e)}")
+        flash('Error requesting full resync', 'error')
+        return redirect(url_for('main.clients'))
+
+
+@csrf.exempt
+@bp.route('/clients/force-resync-all', methods=['POST'])
+@login_required
+@require_active_user
+def force_resync_all():
+    """Ask all accessible clients to perform a one-time full resync."""
+    try:
+        from app.models import ClientActivity
+
+        query = ClientActivity.query
+        token_ids = _accessible_client_token_ids()
+        if token_ids is not None:
+            query = query.filter(ClientActivity.api_token_id.in_(token_ids)) if token_ids else query.filter(db.false())
+
+        count = 0
+        for activity in query.all():
+            activity.force_full_resync = True
+            count += 1
+        db.session.commit()
+
+        flash(f'Full resync requested for {count} client(s). They will resync in the background on their next check-in.', 'success')
+        return redirect(url_for('main.clients'))
+
+    except Exception as e:
+        db.session.rollback()
+        logger.error(f"Force resync all error: {str(e)}")
+        flash('Error requesting full resync', 'error')
+        return redirect(url_for('main.clients'))
+
 # Reports Routes
 
 @csrf.exempt
@@ -2650,9 +2731,19 @@ def sync_logs():
                              })
     
     except Exception as e:
-        logger.error(f"Sync logs page error: {str(e)}")
+        logger.error(f"Sync logs page error: {str(e)}", exc_info=True)
         flash('Error loading sync logs', 'error')
-        return render_template('main/sync_logs.html', logs=[], pagination=None, client_ids=[])
+        return render_template('main/sync_logs.html', logs=[], pagination=None, client_ids=[],
+                             filters={
+                                 'client_id': '',
+                                 'operation_type': '',
+                                 'status': '',
+                                 'search': '',
+                                 'start_date': '',
+                                 'end_date': '',
+                                 'sort_by': 'created_at',
+                                 'sort_order': 'desc'
+                             })
 
 def export_bet_detail(client_id, match_id, bet_uuid, export_format):
     """Export bet detail to CSV, XLSX, or PDF"""

app/models.py

@@ -43,25 +43,26 @@ class User(UserMixin, db.Model):
         self.last_login = datetime.utcnow()
         db.session.commit()
     
-    def generate_api_token(self, name, expires_in=None):
+    def generate_api_token(self, name, expires_in=None, is_fixture_source=False):
         """Generate a new API token for this user"""
         from flask import current_app
-        
+
         # Generate a secure random token
         token_value = secrets.token_urlsafe(32)
-        
+
         # Set expiration (default to 50 years if not specified)
         if expires_in is None:
             expires_at = datetime.utcnow() + timedelta(days=365*50)  # 50 years
         else:
             expires_at = datetime.utcnow() + expires_in
-        
+
         # Create the token record
         api_token = APIToken(
             user_id=self.id,
             name=name,
             token_hash=APIToken.hash_token(token_value),
-            expires_at=expires_at
+            expires_at=expires_at,
+            is_fixture_source=is_fixture_source
         )
         
         db.session.add(api_token)
@@ -207,7 +208,32 @@ class Match(db.Model):
             import logging
             logger = logging.getLogger(__name__)
             logger.error(f"Failed to update fixture active time for {self.fixture_id}: {str(e)}")
-    
+
+    @classmethod
+    def bump_fixture_active_time(cls, fixture_id):
+        """Force-refresh a fixture's active time so clients re-fetch it.
+
+        Unlike update_fixture_active_time (which only fills NULLs once), this overwrites
+        fixture_active_time on every match of the fixture with the current time. Used when a
+        new record is added to an already-live fixture so the client's `from`-query
+        (fixture_active_time > from) returns it again and pulls only the new match.
+        """
+        import time
+        import logging
+        logger = logging.getLogger(__name__)
+        try:
+            current_time = int(time.time())
+            matches = cls.query.filter_by(fixture_id=fixture_id).all()
+            for match in matches:
+                match.fixture_active_time = current_time
+            db.session.commit()
+            logger.info(f"Bumped fixture {fixture_id} active time to {current_time} ({len(matches)} matches)")
+            return current_time
+        except Exception as e:
+            db.session.rollback()
+            logger.error(f"Failed to bump fixture active time for {fixture_id}: {str(e)}")
+            return None
+
     @classmethod
     def get_fixtures_with_active_time(cls, from_timestamp=None, limit=None):
         """Get fixtures ordered by active time, optionally from a specific timestamp with limit"""
@@ -613,11 +639,15 @@ class APIToken(db.Model):
     expires_at = db.Column(db.DateTime, nullable=False, index=True)
     last_used_at = db.Column(db.DateTime)
     last_used_ip = db.Column(db.String(45))
-    
+    # When True this token identifies a fixture/match *source* (a provider that pushes
+    # fixtures/matches to the server), not an mbetterc player client. Source tokens may
+    # only call the fixture-source endpoints and are rejected by the client endpoints.
+    is_fixture_source = db.Column(db.Boolean, nullable=False, default=False)
+
     # Metadata
     created_at = db.Column(db.DateTime, default=datetime.utcnow)
     updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
-    
+
     def __init__(self, **kwargs):
         super(APIToken, self).__init__(**kwargs)
     
@@ -682,7 +712,8 @@ class APIToken(db.Model):
             'last_used_ip': self.last_used_ip,
             'created_at': self.created_at.isoformat() if self.created_at else None,
             'is_expired': self.is_expired(),
-            'is_valid': self.is_valid()
+            'is_valid': self.is_valid(),
+            'is_fixture_source': self.is_fixture_source
         }
         
         # Only include the plain token when explicitly requested (e.g., during creation)
@@ -831,7 +862,17 @@ class ClientActivity(db.Model):
     last_seen = db.Column(db.DateTime, default=datetime.utcnow, index=True)
     ip_address = db.Column(db.String(45))
     user_agent = db.Column(db.Text)
-    
+    # When True, the server asks this client to perform a one-time full resync of
+    # all historical reports. The flag is cleared the first time the client's
+    # /api/reports/last-sync poll is served, so the resync happens only once.
+    force_full_resync = db.Column(db.Boolean, nullable=False, default=False)
+    # True while the client is actively sending its resync backlog in batches.
+    # Set when the resync is requested/served and cleared when the client reports
+    # the backlog is drained (resync_in_progress=false on its final batch).
+    resync_in_progress = db.Column(db.Boolean, nullable=False, default=False)
+    # Approximate number of records the client still has left to send.
+    resync_remaining = db.Column(db.Integer, nullable=True)
+
     # Relationships
     api_token = db.relationship('APIToken', backref='client_activity', lazy='select')
     

app/templates/main/clients.html

@@ -5,9 +5,20 @@
 {% block content %}
 <div class="d-flex justify-content-between align-items-center mb-4">
     <h1>Connected Clients</h1>
-    <div class="d-flex gap-1">
-        <span class="badge badge-success" style="background-color: #28a745; color: white;">Online</span>
-        <span class="badge badge-secondary" style="background-color: #6c757d; color: white;">Offline</span>
+    <div class="d-flex align-items-center gap-2">
+        <div class="d-flex gap-1">
+            <span class="badge badge-success" style="background-color: #28a745; color: white;">Online</span>
+            <span class="badge badge-secondary" style="background-color: #6c757d; color: white;">Offline</span>
+        </div>
+        {% if clients %}
+        <form method="POST" action="{{ url_for('main.force_resync_all') }}"
+              onsubmit="return confirm('Ask ALL listed clients to perform a one-time full resync of their historical reports? They will resync lazily in the background.');"
+              class="m-0">
+            <button type="submit" class="btn btn-sm btn-outline-warning">
+                <i class="fas fa-sync"></i> Request full resync (all)
+            </button>
+        </form>
+        {% endif %}
     </div>
 </div>
 
@@ -64,6 +75,7 @@
                 <th>Remote Link</th>
                 <th>IP Address</th>
                 <th>User Agent</th>
+                <th>Resync</th>
             </tr>
         </thead>
         <tbody>
@@ -113,11 +125,31 @@
                                 <span class="text-muted">Unknown</span>
                             {% endif %}
                         </td>
+                        <td>
+                            {% if client.resync_in_progress %}
+                                <span class="badge bg-info text-dark" title="The client is sending its backlog in batches">
+                                    <i class="fas fa-sync fa-spin"></i> Resyncing{% if client.resync_remaining %} ({{ client.resync_remaining }} left){% endif %}
+                                </span>
+                            {% elif client.force_full_resync %}
+                                <span class="badge bg-warning text-dark" title="The client will perform a full resync on its next check-in">
+                                    <i class="fas fa-hourglass-half"></i> Resync pending
+                                </span>
+                            {% else %}
+                                <form method="POST"
+                                      action="{{ url_for('main.force_resync_client', rustdesk_id=client.rustdesk_id) }}"
+                                      onsubmit="return confirm('Ask this client to perform a one-time full resync of its historical reports?');"
+                                      class="m-0">
+                                    <button type="submit" class="btn btn-sm btn-outline-warning" title="Request a one-time full resync">
+                                        <i class="fas fa-sync"></i> Full resync
+                                    </button>
+                                </form>
+                            {% endif %}
+                        </td>
                     </tr>
                 {% endfor %}
             {% else %}
                 <tr>
-                    <td colspan="7" class="text-center text-muted">
+                    <td colspan="8" class="text-center text-muted">
                         No clients found. Clients will appear here when they connect to the API.
                     </td>
                 </tr>

app/templates/main/user_tokens.html

@@ -375,6 +375,9 @@
                             <tr data-token-id="{{ token.id }}">
                                 <td>
                                     <strong>{{ token.name }}</strong>
+                                    {% if token.is_fixture_source %}
+                                        <span class="badge bg-info text-dark" title="This token is a fixture/match source provider, not a player client">Fixture source</span>
+                                    {% endif %}
                                 </td>
                                 <td>
                                     {% if token.is_valid() %}
@@ -465,6 +468,11 @@
                            placeholder="e.g., Mobile App, Dashboard Integration">
                     <div class="form-text">Choose a descriptive name to identify this token</div>
                 </div>
+                <div class="mb-3 form-check">
+                    <input type="checkbox" class="form-check-input" id="tokenFixtureSource" name="is_fixture_source">
+                    <label class="form-check-label" for="tokenFixtureSource">Fixture source (provider) token</label>
+                    <div class="form-text">For systems that push fixtures/matches to the server. Such tokens can only add fixtures/matches and cannot be used by an mbetterc player client.</div>
+                </div>
                 <div class="warning-message">
                     ⚠️ <strong>Important:</strong> The token will only be shown once after creation. Make sure to copy and store it securely.
                 </div>
@@ -578,12 +586,13 @@ document.getElementById('createTokenForm').addEventListener('submit', async func
     
     const formData = new FormData(this);
     const tokenName = formData.get('name');
-    
+    const isFixtureSource = formData.get('is_fixture_source') !== null;
+
     if (!tokenName.trim()) {
         showAlert('Token name is required', 'danger');
         return;
     }
-    
+
     try {
         // Use fetch directly since url_for() already includes the script root
         const response = await fetch('{{ url_for("main.create_api_token") }}', {
@@ -591,7 +600,7 @@ document.getElementById('createTokenForm').addEventListener('submit', async func
             headers: {
                 'Content-Type': 'application/json'
             },
-            body: JSON.stringify({ name: tokenName })
+            body: JSON.stringify({ name: tokenName, is_fixture_source: isFixtureSource })
         });
         
         const data = await response.json();

app/upload/routes.py

@@ -16,6 +16,80 @@ from app.upload.forms import FixtureUploadForm, ZipUploadForm
 
 logger = logging.getLogger(__name__)
 
+
+class ChunkedAssembledFile:
+    """A lazy file-like wrapper over an assembled chunked upload.
+
+    Mirrors the FileLike used in finalize_upload so file_handler.process_upload can treat an
+    assembled-on-disk file like an uploaded file (read/seek/tell/save/close).
+    """
+
+    def __init__(self, path, filename):
+        self.path = path
+        self.filename = filename
+        self.name = filename
+        self.content_type = 'application/zip'
+        self._closed = False
+        self._file = None
+
+    def _fh(self):
+        if self._file is None:
+            self._file = open(self.path, 'rb')
+        return self._file
+
+    def read(self, size=-1):
+        if self._closed:
+            raise ValueError("I/O operation on closed file")
+        return self._fh().read() if size == -1 else self._fh().read(size)
+
+    def seek(self, pos, whence=0):
+        if self._closed:
+            raise ValueError("I/O operation on closed file")
+        self._fh().seek(pos, whence)
+
+    def tell(self):
+        if self._closed:
+            raise ValueError("I/O operation on closed file")
+        return self._fh().tell()
+
+    def close(self):
+        if self._file:
+            self._file.close()
+        self._closed = True
+
+    def save(self, dst):
+        if self._closed:
+            raise ValueError("I/O operation on closed file")
+        shutil.move(self.path, dst)
+
+
+def save_upload_chunk(temp_dir, chunk_index, total_chunks, file_name, chunk_file, match_id=None):
+    """Save one chunk of a chunked upload into temp_dir; write metadata when complete.
+
+    Returns the number of chunks received so far. Shared by the session-auth web upload and
+    the token-auth fixture-source API so both speak the same proxy-friendly chunk protocol.
+    """
+    os.makedirs(temp_dir, exist_ok=True)
+    chunk_path = os.path.join(temp_dir, f'chunk_{int(chunk_index):06d}')
+    chunk_file.save(chunk_path)
+
+    received = len([f for f in os.listdir(temp_dir) if f.startswith('chunk_')])
+    if received == int(total_chunks):
+        with open(os.path.join(temp_dir, 'metadata.txt'), 'w') as f:
+            f.write(f'{file_name}\n{match_id or ""}\n')
+    return received
+
+
+def assemble_chunks(temp_dir, file_name):
+    """Concatenate chunk_* files in temp_dir into a single file; return its path."""
+    final_path = os.path.join(temp_dir, secure_filename(file_name))
+    with open(final_path, 'wb') as outfile:
+        for chunk_file in sorted(f for f in os.listdir(temp_dir) if f.startswith('chunk_')):
+            with open(os.path.join(temp_dir, chunk_file), 'rb') as infile:
+                shutil.copyfileobj(infile, outfile)
+    return final_path
+
+
 @csrf.exempt
 @bp.route('/fixture', methods=['GET', 'POST'])
 @login_required