Skip to content

A
aisbf
Commit d1787cd5 authored by Stefy Lanza (nextime / spora )'s avatar Stefy Lanza (nextime / spora )
Browse files
Options

fix: harden prompt analytics and savings estimates

parent 26a52d9f
Hide whitespace changes
Inline Side-by-side
Showing with 1786 additions and 678 deletions
aisbf/analytics.py
View file @ d1787cd5
......@@ -200,6 +200,12 @@ class Analytics:
'limit': int(getattr(provider_config, 'free_tier_limit', 0) or 0),
'premium_monthly_cost': float(getattr(provider_config, 'premium_reference_monthly_cost', 0) or 0),
'description': getattr(provider_config, 'free_tier_description', None) or f'{provider_id} free tier',
'pro_tier_requests_daily': getattr(provider_config, 'pro_tier_requests_daily', None),
'pro_tier_requests_weekly': getattr(provider_config, 'pro_tier_requests_weekly', None),
'pro_tier_requests_monthly': getattr(provider_config, 'pro_tier_requests_monthly', None),
'pro_tier_tokens_daily': getattr(provider_config, 'pro_tier_tokens_daily', None),
'pro_tier_tokens_weekly': getattr(provider_config, 'pro_tier_tokens_weekly', None),
'pro_tier_tokens_monthly': getattr(provider_config, 'pro_tier_tokens_monthly', None),
'provider_family': provider_id,
'source': 'config'
}
......@@ -212,6 +218,30 @@ class Analytics:
return {**info, 'provider_family': key, 'source': 'default'}
return None
def _resolve_pro_tier_capacity(self, tier_info: Dict[str, Any], limit_type: str, period: str) -> tuple[Optional[int], str]:
period = (period or 'month').lower()
limit_type = (limit_type or 'requests').lower()
configured_keys = {
('requests', 'day'): 'pro_tier_requests_daily',
('requests', 'week'): 'pro_tier_requests_weekly',
('requests', 'month'): 'pro_tier_requests_monthly',
('tokens', 'day'): 'pro_tier_tokens_daily',
('tokens', 'week'): 'pro_tier_tokens_weekly',
('tokens', 'month'): 'pro_tier_tokens_monthly',
}
config_key = configured_keys.get((limit_type, period))
if config_key:
configured_value = tier_info.get(config_key)
if configured_value:
return int(configured_value), 'configured'
free_limit = int(tier_info.get('limit', 0) or 0)
if free_limit > 0:
return free_limit * 5, 'derived_5x_free_tier'
return 1, 'single_pro_tier_estimate'
def _derive_quota_from_usage(self, provider_id: str, usage_data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
if not usage_data or not isinstance(usage_data, dict):
return None
......@@ -1246,6 +1276,12 @@ class Analytics:
) -> Dict[str, Any]:
start, end, _ = self._normalize_time_window('custom' if from_datetime or to_datetime else '24h', from_datetime, to_datetime)
normalized_provider_filter = (provider_filter or '').strip().lower()
normalized_model_filter = (model_filter or '').strip().lower()
explicit_single_provider_scope = bool(normalized_provider_filter)
explicit_single_model_scope = bool(normalized_model_filter)
provider_stats = self.get_all_providers_stats(
start,
end,
......@@ -1294,9 +1330,6 @@ class Analytics:
limit_type = (runtime_quota or {}).get('limit_type') or tier_info.get('limit_type') or 'requests'
period = (runtime_quota or {}).get('period') or tier_info.get('period') or 'month'
if free_limit <= 0:
continue
premium_price = float(tier_info.get('premium_monthly_cost', 0) or 0)
if premium_price <= 0:
continue
......@@ -1306,53 +1339,150 @@ class Analytics:
else:
usage_amount = usage.get('request_count', 0) or 0
if usage_amount <= free_limit:
if usage_amount <= 0:
continue
estimated_payg_cost = self._estimate_aggregate_cost({
provider_id: {
'tokens_used': usage.get('tokens_used', 0),
'prompt_tokens': usage.get('prompt_tokens', 0),
'completion_tokens': usage.get('completion_tokens', 0),
'actual_cost': usage.get('actual_cost', 0.0),
'model_name': model_filter if explicit_single_model_scope else None,
}
})
if estimated_payg_cost <= 0:
continue
extra_free_tiers = max((usage_amount - 1) // free_limit, 0)
if extra_free_tiers <= 0:
if explicit_single_provider_scope or explicit_single_model_scope:
if premium_price <= estimated_payg_cost:
continue
if free_limit <= 0:
continue
if tier_info.get('source') == 'default' and usage_amount > free_limit * 10:
covered_usage_amount = min(usage_amount, free_limit)
if covered_usage_amount <= 0:
continue
coverage_ratio = min(max(float(covered_usage_amount) / float(usage_amount), 0.0), 1.0)
saved_tokens = int((usage['tokens_used'] or 0) * coverage_ratio)
if saved_tokens <= 0:
continue
saved_tokens = usage['tokens_used'] * extra_free_tiers
equivalent_token_savings += saved_tokens
premium_equivalent_cost = premium_price * extra_free_tiers
premium_equivalent_cost = min(max(estimated_payg_cost, 0.0) * coverage_ratio, premium_price)
provider_equivalents.append({
'provider_id': provider_id,
'tokens_used': usage['tokens_used'],
'request_count': usage.get('request_count', 0) or 0,
'usage_amount': usage_amount,
'covered_usage_amount': covered_usage_amount,
'coverage_ratio': coverage_ratio,
'free_tier_limit': free_limit,
'free_tier_period': period,
'free_tier_limit_type': limit_type,
'extra_free_tiers': extra_free_tiers,
'equivalent_saved_tokens': saved_tokens,
'premium_reference_name': tier_info.get('description'),
'premium_reference_monthly_cost': premium_price,
'equivalent_saved_cost': premium_equivalent_cost,
'estimated_payg_cost': estimated_payg_cost,
'quota_source': 'runtime' if runtime_quota else tier_info.get('source', 'default')
})
equivalent_cost_savings += premium_equivalent_cost
feature_savings = {}
provider_cost_rows = self.get_cost_overview(
start,
end,
user_filter=user_filter,
provider_filter=provider_filter,
model_filter=model_filter,
rotation_filter=rotation_filter,
autoselect_filter=autoselect_filter
).get('providers', [])
total_requests = 0
for row in provider_stats:
request_count = row.get('request_count')
if isinstance(request_count, dict):
request_count = request_count.get('total') or 0
elif request_count is None:
request_count = row.get('requests', 0) or 0
if isinstance(request_count, dict):
request_count = request_count.get('total') or 0
total_requests += int(request_count or 0)
total_estimated_cost = sum(float(row.get('estimated_cost', 0) or 0) for row in provider_cost_rows)
avg_tokens_per_request = int(sum((usage.get('tokens_used') or 0) for usage in provider_totals.values()) / total_requests) if total_requests > 0 else 0
avg_cost_per_request = (total_estimated_cost / total_requests) if total_requests > 0 else 0.0
try:
from .cache import get_response_cache
cache = get_response_cache()
cache_stats = cache.get_stats() if user_filter in (None, -1) else cache.get_user_stats(user_filter)
cache_hits = int(cache_stats.get('hits') or 0)
if cache_hits > 0 and avg_tokens_per_request > 0:
feature_savings['response_cache'] = {
'count': cache_hits,
'tokens_saved': cache_hits * avg_tokens_per_request,
'cost_saved': cache_hits * avg_cost_per_request,
'avg_tokens_saved': avg_tokens_per_request,
'max_tokens_saved': avg_tokens_per_request,
}
except Exception:
pass
try:
from .batching import get_request_batcher
batcher = get_request_batcher()
batch_stats = batcher.get_stats() if batcher else {}
batches_formed = int(batch_stats.get('batches_formed') or 0)
requests_batched = int(batch_stats.get('requests_batched') or 0)
if batches_formed > 0 and requests_batched > batches_formed and avg_tokens_per_request > 0:
requests_avoided = max(requests_batched - batches_formed, 0)
feature_savings['batching'] = {
'count': batches_formed,
'tokens_saved': requests_avoided * avg_tokens_per_request,
'cost_saved': requests_avoided * avg_cost_per_request,
'avg_tokens_saved': int((requests_avoided * avg_tokens_per_request) / batches_formed) if batches_formed else 0,
'max_tokens_saved': requests_avoided * avg_tokens_per_request if batches_formed == 1 else max(avg_tokens_per_request, int(avg_tokens_per_request * (batch_stats.get('avg_batch_size') or 1))),
}
except Exception:
pass
savings_by_type = {}
if provider_equivalents:
savings_by_type['free_tier_equivalent'] = {
'count': len(provider_equivalents),
'tokens_saved': equivalent_token_savings,
'cost_saved': equivalent_cost_savings,
'avg_tokens_saved': int(equivalent_token_savings / len(provider_equivalents)) if provider_equivalents else 0,
'max_tokens_saved': max((item['equivalent_saved_tokens'] for item in provider_equivalents), default=0)
}
savings_by_type.update(feature_savings)
total_feature_tokens_saved = sum(int(item.get('tokens_saved') or 0) for item in feature_savings.values())
total_feature_cost_saved = sum(float(item.get('cost_saved') or 0) for item in feature_savings.values())
direct_feature_savings = {
'tokens_saved': total_feature_tokens_saved,
'cost_saved': total_feature_cost_saved,
}
free_tier_equivalent_savings = {
'tokens_saved': equivalent_token_savings,
'cost_saved': equivalent_cost_savings,
}
return {
'total_tokens_saved': equivalent_token_savings,
'total_cost_saved': equivalent_cost_savings,
'total_tokens_saved': equivalent_token_savings + total_feature_tokens_saved,
'total_cost_saved': equivalent_cost_savings + total_feature_cost_saved,
'direct_feature_savings': direct_feature_savings,
'free_tier_equivalent_savings': free_tier_equivalent_savings,
'date_range': {
'start': start.isoformat(),
'end': end.isoformat()
},
'provider_equivalents': provider_equivalents,
'savings_by_type': {
'free_tier_equivalent': {
'count': len(provider_equivalents),
'tokens_saved': equivalent_token_savings,
'cost_saved': equivalent_cost_savings,
'avg_tokens_saved': int(equivalent_token_savings / len(provider_equivalents)) if provider_equivalents else 0,
'max_tokens_saved': max((item['equivalent_saved_tokens'] for item in provider_equivalents), default=0)
}
} if provider_equivalents else {}
'savings_by_type': savings_by_type
}
def get_cost_overview(
......
aisbf/database.py
View file @ d1787cd5
......@@ -115,7 +115,7 @@ class DatabaseManager:
All database operations are non-blocking using asyncio and thread pool executors.
"""
def __init__(self, db_config: Optional[Dict[str, Any]] = None):
def __init__(self, db_config: Optional[Dict[str, Any]] = None, database_type: str = 'config'):
"""
Initialize the database manager.
......@@ -138,6 +138,7 @@ class DatabaseManager:
else:
self.db_config = db_config
self.database_type = database_type
self.db_type = self.db_config.get('type', 'sqlite').lower()
self.executor = get_db_executor()
......@@ -168,6 +169,10 @@ class DatabaseManager:
raise
else:
raise ValueError(f"Unsupported database type: {self.db_type}")
@staticmethod
def _format_dt_db(value: datetime) -> str:
return value.strftime('%Y-%m-%d %H:%M:%S')
@property
def placeholder(self) -> str:
......@@ -2476,6 +2481,608 @@ class DatabaseManager:
''', (user_id, autoselect_name))
conn.commit()
def record_dashboard_event(
self,
event_type: str,
path: Optional[str] = None,
user_id: Optional[int] = None,
username: Optional[str] = None,
session_id: Optional[str] = None,
ip_address: Optional[str] = None,
country_code: Optional[str] = None,
method: Optional[str] = None,
status_code: Optional[int] = None,
provider_id: Optional[str] = None,
rotation_id: Optional[str] = None,
autoselect_id: Optional[str] = None,
listing_id: Optional[int] = None,
target_user_id: Optional[int] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> int:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
payload = json.dumps(metadata or {})
cursor.execute(f'''
INSERT INTO dashboard_events (
event_type, path, user_id, username, session_id, ip_address, country_code,
method, status_code, provider_id, rotation_id, autoselect_id, listing_id,
target_user_id, metadata, created_at
) VALUES (
{placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, CURRENT_TIMESTAMP
)
''', (
event_type, path, user_id, username, session_id, ip_address, country_code,
method, status_code, provider_id, rotation_id, autoselect_id, listing_id,
target_user_id, payload,
))
conn.commit()
return cursor.lastrowid
def get_dashboard_event_summary(
self,
start: datetime,
end: datetime,
event_types: Optional[List[str]] = None,
user_id: Optional[int] = None,
provider_filter: Optional[str] = None,
rotation_filter: Optional[str] = None,
autoselect_filter: Optional[str] = None,
path_filter: Optional[str] = None,
country_filter: Optional[str] = None,
) -> Dict[str, Any]:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
where_clauses = [f"created_at >= {placeholder}", f"created_at <= {placeholder}"]
params: List[Any] = [self._format_dt_db(start), self._format_dt_db(end)]
if event_types:
where_clauses.append(f"event_type IN ({','.join([placeholder] * len(event_types))})")
params.extend(event_types)
if user_id == -1:
where_clauses.append("user_id IS NULL")
elif user_id is not None:
where_clauses.append(f"user_id = {placeholder}")
params.append(user_id)
if provider_filter:
where_clauses.append(f"provider_id = {placeholder}")
params.append(provider_filter)
if rotation_filter:
where_clauses.append(f"rotation_id = {placeholder}")
params.append(rotation_filter)
if autoselect_filter:
where_clauses.append(f"autoselect_id = {placeholder}")
params.append(autoselect_filter)
if path_filter:
where_clauses.append(f"path = {placeholder}")
params.append(path_filter)
if country_filter:
where_clauses.append(f"country_code = {placeholder}")
params.append(country_filter)
where_sql = ' AND '.join(where_clauses)
cursor.execute(f'''SELECT COUNT(*), COUNT(DISTINCT ip_address), COUNT(DISTINCT COALESCE(user_id, -1)), COUNT(DISTINCT COALESCE(session_id, '')) FROM dashboard_events WHERE {where_sql}''', tuple(params))
counts = cursor.fetchone() or (0, 0, 0, 0)
cursor.execute(f'''SELECT event_type, COUNT(*) FROM dashboard_events WHERE {where_sql} GROUP BY event_type ORDER BY COUNT(*) DESC''', tuple(params))
by_type = [{'event_type': row[0], 'count': row[1]} for row in cursor.fetchall()]
cursor.execute(f'''SELECT path, COUNT(*) FROM dashboard_events WHERE {where_sql} GROUP BY path ORDER BY COUNT(*) DESC LIMIT 20''', tuple(params))
pages = [{'path': row[0] or 'unknown', 'count': row[1]} for row in cursor.fetchall()]
cursor.execute(f'''SELECT country_code, COUNT(*) FROM dashboard_events WHERE {where_sql} GROUP BY country_code ORDER BY COUNT(*) DESC LIMIT 20''', tuple(params))
countries = [{'country_code': row[0] or 'unknown', 'count': row[1]} for row in cursor.fetchall()]
cursor.execute(f'''SELECT username, user_id, COUNT(*) FROM dashboard_events WHERE {where_sql} GROUP BY username, user_id ORDER BY COUNT(*) DESC LIMIT 20''', tuple(params))
users = [{'username': row[0] or 'guest', 'user_id': row[1], 'count': row[2]} for row in cursor.fetchall()]
return {
'total_events': counts[0] or 0,
'unique_ips': counts[1] or 0,
'unique_visitors': counts[2] or 0,
'unique_sessions': counts[3] or 0,
'by_type': by_type,
'pages': pages,
'countries': countries,
'users': users,
}
def get_dashboard_event_series(
self,
start: datetime,
end: datetime,
bucket: str = 'hour',
event_types: Optional[List[str]] = None,
user_id: Optional[int] = None,
) -> List[Dict[str, Any]]:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
where_clauses = [f"created_at >= {placeholder}", f"created_at <= {placeholder}"]
params: List[Any] = [self._format_dt_db(start), self._format_dt_db(end)]
if event_types:
where_clauses.append(f"event_type IN ({','.join([placeholder] * len(event_types))})")
params.extend(event_types)
if user_id == -1:
where_clauses.append("user_id IS NULL")
elif user_id is not None:
where_clauses.append(f"user_id = {placeholder}")
params.append(user_id)
if self.db_type == 'sqlite':
bucket_expr = "strftime('%Y-%m-%d %H:00:00', created_at)" if bucket == 'hour' else "strftime('%Y-%m-%d', created_at)"
else:
bucket_expr = "DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00')" if bucket == 'hour' else "DATE_FORMAT(created_at, '%Y-%m-%d')"
where_sql = ' AND '.join(where_clauses)
cursor.execute(f'''
SELECT {bucket_expr} AS bucket, COUNT(*), COUNT(DISTINCT ip_address), COUNT(DISTINCT COALESCE(user_id, -1))
FROM dashboard_events
WHERE {where_sql}
GROUP BY bucket
ORDER BY bucket ASC
''', tuple(params))
return [
{'bucket': row[0], 'events': row[1], 'unique_ips': row[2], 'unique_visitors': row[3]}
for row in cursor.fetchall()
]
def get_dashboard_events(
self,
start: datetime,
end: datetime,
event_types: Optional[List[str]] = None,
user_id: Optional[int] = None,
limit: int = 200,
) -> List[Dict[str, Any]]:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
where_clauses = [f"created_at >= {placeholder}", f"created_at <= {placeholder}"]
params: List[Any] = [self._format_dt_db(start), self._format_dt_db(end)]
if event_types:
where_clauses.append(f"event_type IN ({','.join([placeholder] * len(event_types))})")
params.extend(event_types)
if user_id == -1:
where_clauses.append("user_id IS NULL")
elif user_id is not None:
where_clauses.append(f"user_id = {placeholder}")
params.append(user_id)
params.append(limit)
where_sql = ' AND '.join(where_clauses)
cursor.execute(f'''
SELECT id, event_type, path, user_id, username, session_id, ip_address, country_code,
method, status_code, provider_id, rotation_id, autoselect_id, listing_id,
target_user_id, metadata, created_at
FROM dashboard_events
WHERE {where_sql}
ORDER BY created_at DESC
LIMIT {placeholder}
''', tuple(params))
rows = []
for row in cursor.fetchall():
rows.append({
'id': row[0],
'event_type': row[1],
'path': row[2],
'user_id': row[3],
'username': row[4],
'session_id': row[5],
'ip_address': row[6],
'country_code': row[7],
'method': row[8],
'status_code': row[9],
'provider_id': row[10],
'rotation_id': row[11],
'autoselect_id': row[12],
'listing_id': row[13],
'target_user_id': row[14],
'metadata': _studio_json_loads(row[15], {}),
'created_at': row[16],
})
return rows
def record_prompt_analysis_run(
self,
*,
user_id: Optional[int],
token_id: Optional[int],
provider_id: str,
model_name: str,
rotation_id: Optional[str] = None,
autoselect_id: Optional[str] = None,
request_path: Optional[str] = None,
request_hash: Optional[str] = None,
prompt_tokens: Optional[int] = None,
effective_context: Optional[int] = None,
scan_enabled: bool = True,
analytics_enabled: bool = True,
blocked: bool = False,
risk_level: Optional[str] = None,
risk_score: Optional[int] = None,
findings_count: int = 0,
summary_json: Optional[Dict[str, Any]] = None,
) -> int:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
summary_payload = _studio_json_dumps(summary_json or {})
cursor.execute(f'''
INSERT INTO prompt_analysis_runs (
user_id, token_id, provider_id, model_name, rotation_id, autoselect_id,
request_path, request_hash, prompt_tokens, effective_context, scan_enabled,
analytics_enabled, blocked, risk_level, risk_score, findings_count, summary_json, created_at
) VALUES (
{placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, CURRENT_TIMESTAMP
)
''', (
user_id, token_id, provider_id, model_name, rotation_id, autoselect_id,
request_path, request_hash, prompt_tokens, effective_context, scan_enabled,
analytics_enabled, blocked, risk_level, risk_score, findings_count,
summary_payload,
))
conn.commit()
run_id = cursor.lastrowid
if run_id:
return int(run_id)
if self.db_type == 'mysql':
cursor.execute('SELECT LAST_INSERT_ID()')
row = cursor.fetchone()
if row and row[0]:
return int(row[0])
raise RuntimeError('Failed to resolve prompt analysis run id after insert')
def record_prompt_analysis_findings(self, run_id: int, findings: List[Dict[str, Any]]) -> None:
if not findings:
return
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
for finding in findings:
cursor.execute(f'''
INSERT INTO prompt_analysis_findings (
run_id, category, severity, detector, span_label, message_index,
evidence_preview, metadata, created_at
) VALUES (
{placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, CURRENT_TIMESTAMP
)
''', (
run_id,
finding.get('category'),
finding.get('severity'),
finding.get('detector'),
finding.get('span_label'),
finding.get('message_index'),
finding.get('evidence_preview'),
json.dumps(finding.get('metadata') or {}),
))
conn.commit()
def get_prompt_analysis_summary(
self,
start: datetime,
end: datetime,
user_id: Optional[int] = None,
provider_filter: Optional[str] = None,
model_filter: Optional[str] = None,
rotation_filter: Optional[str] = None,
autoselect_filter: Optional[str] = None,
) -> Dict[str, Any]:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
where_clauses = [f"created_at >= {placeholder}", f"created_at <= {placeholder}"]
params: List[Any] = [self._format_dt_db(start), self._format_dt_db(end)]
if user_id == -1:
where_clauses.append("user_id IS NULL")
elif user_id is not None:
where_clauses.append(f"user_id = {placeholder}")
params.append(user_id)
if provider_filter:
where_clauses.append(f"provider_id = {placeholder}")
params.append(provider_filter)
if model_filter:
where_clauses.append(f"model_name = {placeholder}")
params.append(model_filter)
if rotation_filter:
where_clauses.append(f"rotation_id = {placeholder}")
params.append(rotation_filter)
if autoselect_filter:
where_clauses.append(f"autoselect_id = {placeholder}")
params.append(autoselect_filter)
where_sql = ' AND '.join(where_clauses)
cursor.execute(f'''
SELECT COUNT(*), SUM(CASE WHEN blocked THEN 1 ELSE 0 END),
SUM(CASE WHEN risk_level = 'high' THEN 1 ELSE 0 END),
AVG(COALESCE(prompt_tokens, 0)), AVG(COALESCE(effective_context, 0))
FROM prompt_analysis_runs WHERE {where_sql}
''', tuple(params))
counts = cursor.fetchone() or (0, 0, 0, 0, 0)
cursor.execute(f'''
SELECT category, severity, COUNT(*)
FROM prompt_analysis_findings
WHERE run_id IN (SELECT id FROM prompt_analysis_runs WHERE {where_sql})
GROUP BY category, severity
ORDER BY COUNT(*) DESC
LIMIT 20
''', tuple(params))
findings = [
{'category': row[0], 'severity': row[1], 'count': row[2]}
for row in cursor.fetchall()
]
return {
'total_runs': counts[0] or 0,
'blocked_runs': counts[1] or 0,
'high_risk_runs': counts[2] or 0,
'avg_prompt_tokens': float(counts[3] or 0),
'avg_effective_context': float(counts[4] or 0),
'findings': findings,
}
def get_prompt_analysis_series(
self,
start: datetime,
end: datetime,
bucket: str = 'hour',
user_id: Optional[int] = None,
) -> List[Dict[str, Any]]:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
where_clauses = [f"created_at >= {placeholder}", f"created_at <= {placeholder}"]
params: List[Any] = [self._format_dt_db(start), self._format_dt_db(end)]
if user_id == -1:
where_clauses.append("user_id IS NULL")
elif user_id is not None:
where_clauses.append(f"user_id = {placeholder}")
params.append(user_id)
if self.db_type == 'sqlite':
bucket_expr = "strftime('%Y-%m-%d %H:00:00', created_at)" if bucket == 'hour' else "strftime('%Y-%m-%d', created_at)"
else:
bucket_expr = "DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00')" if bucket == 'hour' else "DATE_FORMAT(created_at, '%Y-%m-%d')"
where_sql = ' AND '.join(where_clauses)
cursor.execute(f'''
SELECT {bucket_expr} AS bucket, COUNT(*),
SUM(CASE WHEN risk_level = 'high' THEN 1 ELSE 0 END),
SUM(CASE WHEN blocked THEN 1 ELSE 0 END)
FROM prompt_analysis_runs
WHERE {where_sql}
GROUP BY bucket
ORDER BY bucket ASC
''', tuple(params))
return [
{'bucket': row[0], 'runs': row[1], 'high_risk': row[2] or 0, 'blocked': row[3] or 0}
for row in cursor.fetchall()
]
def get_prompt_analysis_details(
self,
start: datetime,
end: datetime,
user_id: Optional[int] = None,
provider_filter: Optional[str] = None,
model_filter: Optional[str] = None,
rotation_filter: Optional[str] = None,
autoselect_filter: Optional[str] = None,
limit: int = 25,
) -> Dict[str, Any]:
with self._get_connection() as conn:
cursor = conn.cursor()
placeholder = '?' if self.db_type == 'sqlite' else '%s'
where_clauses = [f"created_at >= {placeholder}", f"created_at <= {placeholder}"]
params: List[Any] = [self._format_dt_db(start), self._format_dt_db(end)]
if user_id == -1:
where_clauses.append("user_id IS NULL")
elif user_id is not None:
where_clauses.append(f"user_id = {placeholder}")
params.append(user_id)
if provider_filter:
where_clauses.append(f"provider_id = {placeholder}")
params.append(provider_filter)
if model_filter:
where_clauses.append(f"model_name = {placeholder}")
params.append(model_filter)
if rotation_filter:
where_clauses.append(f"rotation_id = {placeholder}")
params.append(rotation_filter)
if autoselect_filter:
where_clauses.append(f"autoselect_id = {placeholder}")
params.append(autoselect_filter)
where_sql = ' AND '.join(where_clauses)
def _json_extract(column: str, path: str) -> str:
if self.db_type == 'sqlite':
return f"json_extract({column}, '$.{path}')"
return f"JSON_UNQUOTE(JSON_EXTRACT(CAST({column} AS JSON), '$.{path}'))"
role_expr = _json_extract('summary_json', 'composition.largest_segment_role')
shape_expr = _json_extract('summary_json', 'composition.prompt_shape')
tools_expr = _json_extract('summary_json', 'composition.has_tools')
system_expr = _json_extract('summary_json', 'composition.has_system_prompt')
cursor.execute(f'''
SELECT provider_id, COUNT(*),
SUM(CASE WHEN blocked THEN 1 ELSE 0 END),
SUM(CASE WHEN risk_level = 'high' THEN 1 ELSE 0 END),
AVG(COALESCE(prompt_tokens, 0)),
AVG(COALESCE(effective_context, 0))
FROM prompt_analysis_runs
WHERE {where_sql}
GROUP BY provider_id
ORDER BY COUNT(*) DESC
LIMIT 10
''', tuple(params))
provider_breakdown = [
{
'provider_id': row[0] or 'unknown',
'runs': row[1] or 0,
'blocked': row[2] or 0,
'high_risk': row[3] or 0,
'avg_prompt_tokens': float(row[4] or 0),
'avg_effective_context': float(row[5] or 0),
}
for row in cursor.fetchall()
]
cursor.execute(f'''
SELECT model_name, COUNT(*),
SUM(CASE WHEN blocked THEN 1 ELSE 0 END),
SUM(CASE WHEN risk_level = 'high' THEN 1 ELSE 0 END)
FROM prompt_analysis_runs
WHERE {where_sql}
GROUP BY model_name
ORDER BY COUNT(*) DESC
LIMIT 10
''', tuple(params))
model_breakdown = [
{
'model_name': row[0] or 'unknown',
'runs': row[1] or 0,
'blocked': row[2] or 0,
'high_risk': row[3] or 0,
}
for row in cursor.fetchall()
]
cursor.execute(f'''
SELECT COALESCE(risk_level, 'none') AS risk_level, COUNT(*)
FROM prompt_analysis_runs
WHERE {where_sql}
GROUP BY COALESCE(risk_level, 'none')
ORDER BY COUNT(*) DESC
''', tuple(params))
risk_breakdown = [
{'risk_level': row[0] or 'none', 'count': row[1] or 0}
for row in cursor.fetchall()
]
cursor.execute(f'''
SELECT COALESCE({role_expr}, 'unknown') AS role_name, COUNT(*)
FROM prompt_analysis_runs
WHERE {where_sql}
GROUP BY COALESCE({role_expr}, 'unknown')
ORDER BY COUNT(*) DESC
LIMIT 10
''', tuple(params))
segment_breakdown = [
{'role': row[0] or 'unknown', 'count': row[1] or 0}
for row in cursor.fetchall()
]
cursor.execute(f'''
SELECT COALESCE({shape_expr}, 'empty') AS prompt_shape, COUNT(*)
FROM prompt_analysis_runs
WHERE {where_sql}
GROUP BY COALESCE({shape_expr}, 'empty')
ORDER BY COUNT(*) DESC
LIMIT 10
''', tuple(params))
shape_breakdown = [
{'prompt_shape': row[0] or 'empty', 'count': row[1] or 0}
for row in cursor.fetchall()
]
cursor.execute(f'''
SELECT
SUM(CASE WHEN LOWER(COALESCE({tools_expr}, 'false')) IN ('1', 'true') THEN 1 ELSE 0 END),
SUM(CASE WHEN LOWER(COALESCE({system_expr}, 'false')) IN ('1', 'true') THEN 1 ELSE 0 END),
AVG(COALESCE(findings_count, 0)),
AVG(COALESCE(risk_score, 0))
FROM prompt_analysis_runs
WHERE {where_sql}
''', tuple(params))
posture_row = cursor.fetchone() or (0, 0, 0, 0)
cursor.execute(f'''
SELECT id, created_at, user_id, provider_id, model_name, rotation_id, autoselect_id,
blocked, COALESCE(risk_level, 'none'), COALESCE(risk_score, 0),
COALESCE(prompt_tokens, 0), COALESCE(effective_context, 0),
COALESCE(findings_count, 0), summary_json
FROM prompt_analysis_runs
WHERE {where_sql}
ORDER BY created_at DESC
LIMIT {placeholder}
''', tuple([*params, limit]))
recent_runs = []
run_rows = cursor.fetchall()
for row in run_rows:
summary = _studio_json_loads(row[13], {})
composition = summary.get('composition') if isinstance(summary, dict) else {}
security_summary = summary.get('security_summary') if isinstance(summary, dict) else {}
recent_runs.append({
'id': row[0],
'created_at': row[1],
'user_id': row[2],
'provider_id': row[3],
'model_name': row[4],
'rotation_id': row[5],
'autoselect_id': row[6],
'blocked': bool(row[7]),
'risk_level': row[8] or 'none',
'risk_score': row[9] or 0,
'prompt_tokens': row[10] or 0,
'effective_context': row[11] or 0,
'findings_count': row[12] or 0,
'prompt_shape': composition.get('prompt_shape') or 'empty',
'largest_segment_role': composition.get('largest_segment_role') or 'unknown',
'has_tools': bool(composition.get('has_tools')),
'has_system_prompt': bool(composition.get('has_system_prompt')),
'high_count': security_summary.get('high_count', 0),
'medium_count': security_summary.get('medium_count', 0),
'info_count': security_summary.get('info_count', 0),
})
if run_rows:
run_ids = [row[0] for row in run_rows]
cursor.execute(f'''
SELECT paf.run_id, paf.category, paf.severity, COUNT(*)
FROM prompt_analysis_findings paf
WHERE paf.run_id IN ({','.join([placeholder] * len(run_ids))})
GROUP BY paf.run_id, paf.category, paf.severity
ORDER BY paf.run_id DESC, COUNT(*) DESC
''', tuple(run_ids))
findings_by_run: Dict[int, List[Dict[str, Any]]] = {}
for finding_row in cursor.fetchall():
findings_by_run.setdefault(finding_row[0], []).append({
'category': finding_row[1],
'severity': finding_row[2],
'count': finding_row[3] or 0,
})
for run in recent_runs:
run['findings'] = findings_by_run.get(run['id'], [])[:3]
else:
findings_by_run = {}
return {
'provider_breakdown': provider_breakdown,
'model_breakdown': model_breakdown,
'risk_breakdown': risk_breakdown,
'segment_breakdown': segment_breakdown,
'shape_breakdown': shape_breakdown,
'recent_runs': recent_runs,
'posture': {
'runs_with_tools': posture_row[0] or 0,
'runs_with_system_prompt': posture_row[1] or 0,
'avg_findings_count': float(posture_row[2] or 0),
'avg_risk_score': float(posture_row[3] or 0),
},
}
# User-specific prompt methods
def save_user_prompt(self, user_id: int, prompt_key: str, content: str):
"""
......@@ -5172,6 +5779,168 @@ def DatabaseManager__run_config_migrations(self, cursor, auto_increment, timesta
logger.info("No duplicate cache settings found")
except Exception as e:
logger.warning(f"Migration check for duplicate cache settings: {e}")
try:
if self.db_type == 'sqlite':
cursor.execute("PRAGMA table_info(dashboard_events)")
if not cursor.fetchall():
cursor.execute(f'''
CREATE TABLE dashboard_events (
id INTEGER PRIMARY KEY {auto_increment},
event_type VARCHAR(64) NOT NULL,
path VARCHAR(255),
user_id INTEGER,
username VARCHAR(255),
session_id VARCHAR(255),
ip_address VARCHAR(64),
country_code VARCHAR(16),
method VARCHAR(16),
status_code INTEGER,
provider_id VARCHAR(255),
rotation_id VARCHAR(255),
autoselect_id VARCHAR(255),
listing_id INTEGER,
target_user_id INTEGER,
metadata TEXT,
created_at TIMESTAMP DEFAULT {timestamp_default}
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_dashboard_events_created_at ON dashboard_events(created_at)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_dashboard_events_event_type ON dashboard_events(event_type)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_dashboard_events_user_id ON dashboard_events(user_id)')
logger.info("✅ Migration: Created dashboard_events table")
else:
cursor.execute("""
SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_NAME = 'dashboard_events'
""")
if not cursor.fetchone():
cursor.execute(f'''
CREATE TABLE dashboard_events (
id INTEGER PRIMARY KEY {auto_increment},
event_type VARCHAR(64) NOT NULL,
path VARCHAR(255),
user_id INTEGER,
username VARCHAR(255),
session_id VARCHAR(255),
ip_address VARCHAR(64),
country_code VARCHAR(16),
method VARCHAR(16),
status_code INTEGER,
provider_id VARCHAR(255),
rotation_id VARCHAR(255),
autoselect_id VARCHAR(255),
listing_id INTEGER,
target_user_id INTEGER,
metadata TEXT,
created_at TIMESTAMP DEFAULT {timestamp_default}
)
''')
cursor.execute('CREATE INDEX idx_dashboard_events_created_at ON dashboard_events(created_at)')
cursor.execute('CREATE INDEX idx_dashboard_events_event_type ON dashboard_events(event_type)')
cursor.execute('CREATE INDEX idx_dashboard_events_user_id ON dashboard_events(user_id)')
logger.info("✅ Migration: Created dashboard_events table")
except Exception as e:
logger.warning(f"Migration check for dashboard_events table: {e}")
try:
if self.db_type == 'sqlite':
cursor.execute("PRAGMA table_info(prompt_analysis_runs)")
if not cursor.fetchall():
cursor.execute(f'''
CREATE TABLE prompt_analysis_runs (
id INTEGER PRIMARY KEY {auto_increment},
user_id INTEGER,
token_id INTEGER,
provider_id VARCHAR(255) NOT NULL,
model_name VARCHAR(255) NOT NULL,
rotation_id VARCHAR(255),
autoselect_id VARCHAR(255),
request_path VARCHAR(255),
request_hash VARCHAR(64),
prompt_tokens INTEGER,
effective_context INTEGER,
scan_enabled {boolean_type} DEFAULT 1,
analytics_enabled {boolean_type} DEFAULT 1,
blocked {boolean_type} DEFAULT 0,
risk_level VARCHAR(16),
risk_score INTEGER,
findings_count INTEGER DEFAULT 0,
summary_json TEXT,
created_at TIMESTAMP DEFAULT {timestamp_default}
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_prompt_analysis_runs_created_at ON prompt_analysis_runs(created_at)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_prompt_analysis_runs_user_id ON prompt_analysis_runs(user_id)')
logger.info("✅ Migration: Created prompt_analysis_runs table")
cursor.execute("PRAGMA table_info(prompt_analysis_findings)")
if not cursor.fetchall():
cursor.execute(f'''
CREATE TABLE prompt_analysis_findings (
id INTEGER PRIMARY KEY {auto_increment},
run_id INTEGER NOT NULL,
category VARCHAR(64),
severity VARCHAR(16),
detector VARCHAR(64),
span_label VARCHAR(64),
message_index INTEGER,
evidence_preview TEXT,
metadata TEXT,
created_at TIMESTAMP DEFAULT {timestamp_default}
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_prompt_analysis_findings_run_id ON prompt_analysis_findings(run_id)')
logger.info("✅ Migration: Created prompt_analysis_findings table")
else:
cursor.execute("""
SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_NAME = 'prompt_analysis_runs'
""")
if not cursor.fetchone():
cursor.execute(f'''
CREATE TABLE prompt_analysis_runs (
id INTEGER PRIMARY KEY {auto_increment},
user_id INTEGER,
token_id INTEGER,
provider_id VARCHAR(255) NOT NULL,
model_name VARCHAR(255) NOT NULL,
rotation_id VARCHAR(255),
autoselect_id VARCHAR(255),
request_path VARCHAR(255),
request_hash VARCHAR(64),
prompt_tokens INTEGER,
effective_context INTEGER,
scan_enabled {boolean_type} DEFAULT 1,
analytics_enabled {boolean_type} DEFAULT 1,
blocked {boolean_type} DEFAULT 0,
risk_level VARCHAR(16),
risk_score INTEGER,
findings_count INTEGER DEFAULT 0,
summary_json TEXT,
created_at TIMESTAMP DEFAULT {timestamp_default}
)
''')
cursor.execute("""
SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_NAME = 'prompt_analysis_findings'
""")
if not cursor.fetchone():
cursor.execute(f'''
CREATE TABLE prompt_analysis_findings (
id INTEGER PRIMARY KEY {auto_increment},
run_id INTEGER NOT NULL,
category VARCHAR(64),
severity VARCHAR(16),
detector VARCHAR(64),
span_label VARCHAR(64),
message_index INTEGER,
evidence_preview TEXT,
metadata TEXT,
created_at TIMESTAMP DEFAULT {timestamp_default}
)
''')
except Exception as e:
logger.warning(f"Migration check for prompt analysis tables: {e}")
# Migration: Create account_tiers table if missing
try:
......
templates/dashboard/analytics.html
View file @ d1787cd5
......@@ -36,12 +36,19 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
{% block content %}
<h2 style="margin-bottom: 30px;">Token Usage Analytics</h2>
<!-- Quick Links Navigation -->
<div style="background: var(--bg-page); padding: 15px; border-radius: 8px; margin-bottom: 30px;">
<div style="display: flex; gap: 15px; flex-wrap: wrap;">
<a href="{{ url_for(request, '/dashboard/analytics') }}" class="btn" style="text-decoration: none;">
📊 Analytics
</a>
<a href="{{ url_for(request, '/dashboard/prompt-analytics') }}" class="btn btn-secondary" style="text-decoration: none;">
🧠 Prompt Analytics
</a>
{% if is_config_admin %}
<a href="{{ url_for(request, '/dashboard/traffic-visits') }}" class="btn btn-secondary" style="text-decoration: none;">
👣 Traffic &amp; Visits
</a>
{% endif %}
<a href="{{ url_for(request, '/dashboard/response-cache') }}" class="btn btn-secondary" style="text-decoration: none;">
💾 Response Cache
</a>
......@@ -51,7 +58,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
</div>
</div>
<!-- Date Range Filter Section -->
<div style="background: var(--bg-page); padding: 20px; border-radius: 8px; margin-bottom: 30px;">
<h3 style="margin-bottom: 15px;">Filter by Date Range</h3>
<form method="get" action="{{ url_for(request, "/dashboard/analytics") }}" style="display: flex; flex-wrap: wrap; gap: 15px; align-items: flex-end;">
......@@ -68,33 +74,31 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
<option value="custom" {% if selected_time_range == 'custom' %}selected{% endif %}>Custom Range</option>
</select>
</div>
<div id="customDateRange" style="display: {% if selected_time_range == 'custom' or from_date %}flex{% else %}none{% endif %}; flex-wrap: wrap; gap: 15px; flex: 2;">
<div style="flex: 1; min-width: 200px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">From Date & Time</label>
<input type="datetime-local" name="from_date" value="{{ from_date or '' }}"
style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">From Date &amp; Time</label>
<input type="datetime-local" name="from_date" value="{{ from_date or '' }}" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
</div>
<div style="flex: 1; min-width: 200px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">To Date & Time</label>
<input type="datetime-local" name="to_date" value="{{ to_date or '' }}"
style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">To Date &amp; Time</label>
<input type="datetime-local" name="to_date" value="{{ to_date or '' }}" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
</div>
</div>
<div>
<button type="submit" style="padding: 10px 20px; background: var(--color-primary); color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: bold;">
Apply Filter
</button>
</div>
</form>
{% if from_date or to_date %}
<div style="margin-top: 15px; padding: 10px; background:var(--bg-accent); border-radius: 4px;">
<strong style="color: var(--color-link);">Selected Range: </strong>
<span style="color: var(--color-text);">
{% if from_date %}{{ from_date }}{% else %}Beginning{% endif %}
to
to
{% if to_date %}{{ to_date }}{% else %}Now{% endif %}
</span>
{% if date_range_usage %}
......@@ -106,91 +110,86 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
{% endif %}
</div>
<!-- Filter by Provider/Model/Rotation/Autoselect -->
<div style="background: var(--bg-page); padding: 20px; border-radius: 8px; margin-bottom: 30px;">
<h3 style="margin-bottom: 15px;">Filter by Provider, Model, Rotation, Autoselect, or User</h3>
<form method="get" action="{{ url_for(request, "/dashboard/analytics") }}" style="display: flex; flex-wrap: wrap; gap: 15px; align-items: flex-end;">
<!-- Preserve date filter parameters -->
<input type="hidden" name="time_range" value="{{ selected_time_range }}">
{% if from_date %}<input type="hidden" name="from_date" value="{{ from_date }}">{% endif %}
{% if to_date %}<input type="hidden" name="to_date" value="{{ to_date }}">{% endif %}
<div style="flex: 1; min-width: 150px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">Provider</label>
<select name="provider_filter" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<option value="" data-i18n="analytics_page.all_providers">All Providers</option>
<option value="">All Providers</option>
{% for provider in available_providers %}
<option value="{{ provider }}" {% if selected_provider == provider %}selected{% endif %}>{{ provider }}</option>
{% endfor %}
</select>
</div>
<div style="flex: 1; min-width: 150px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">Model</label>
<select name="model_filter" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<option value="" data-i18n="analytics_page.all_models">All Models</option>
<option value="">All Models</option>
{% for model in available_models %}
<option value="{{ model }}" {% if selected_model == model %}selected{% endif %}>{{ model }}</option>
{% endfor %}
</select>
</div>
<div style="flex: 1; min-width: 150px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">Rotation</label>
<select name="rotation_filter" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<option value="" data-i18n="analytics_page.all_rotations">All Rotations</option>
<option value="">All Rotations</option>
{% for rotation in available_rotations %}
<option value="{{ rotation }}" {% if selected_rotation == rotation %}selected{% endif %}>{{ rotation }}</option>
{% endfor %}
</select>
</div>
<div style="flex: 1; min-width: 150px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">Autoselect</label>
<select name="autoselect_filter" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<option value="" data-i18n="analytics_page.all_autoselects">All Autoselects</option>
<option value="">All Autoselects</option>
{% for autoselect in available_autoselects %}
<option value="{{ autoselect }}" {% if selected_autoselect == autoselect %}selected{% endif %}>{{ autoselect }}</option>
{% endfor %}
</select>
</div>
{% if is_admin %}
<div style="flex: 1; min-width: 150px;">
<label style="display: block; margin-bottom: 5px; color: var(--color-muted); font-size: 14px;">User</label>
{% if available_users|length < 25 %}
<select name="user_filter" id="userFilterSelect" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<option value="" data-i18n="analytics_page.all_users">All Users</option>
<option value="">All Users</option>
{% for user in available_users %}
<option value="{{ user.id }}" {% if selected_user == user.id %}selected{% endif %}>{{ user.username }}{% if user.role == 'admin' %} (admin){% endif %}</option>
{% endfor %}
</select>
{% else %}
<div style="position: relative;">
<input type="text" id="userSearchInput" data-i18n="analytics_page.search_users_placeholder" placeholder="Search users..." autocomplete="off"
style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<input type="text" id="userSearchInput" placeholder="Search users..." autocomplete="off" style="width: 100%; padding: 10px; border-radius: 4px; background:var(--bg-accent); color: white; border: 1px solid #2a4a7a;">
<input type="hidden" name="user_filter" id="userFilterValue" value="{{ selected_user or '' }}">
<div id="userSearchResults" style="display: none; position: absolute; top: 100%; left: 0; right: 0; background:var(--bg-accent); border: 1px solid #2a4a7a; border-top: none; border-radius: 0 0 4px 4px; max-height: 300px; overflow-y: auto; z-index: 1000;">
</div>
<div id="userSearchResults" style="display: none; position: absolute; top: 100%; left: 0; right: 0; background:var(--bg-accent); border: 1px solid #2a4a7a; border-top: none; border-radius: 0 0 4px 4px; max-height: 300px; overflow-y: auto; z-index: 1000;"></div>
</div>
{% endif %}
</div>
<div style="flex: 1; min-width: 150px; display: flex; align-items: center;">
<label style="display: flex; align-items: center; color: var(--color-text); cursor: pointer; user-select: none;">
<input type="checkbox" name="global_only" value="1" id="globalOnlyCheckbox" {% if global_only == '1' %}checked{% endif %}
style="margin-right: 8px; width: 18px; height: 18px; cursor: pointer;">
<input type="checkbox" name="global_only" value="1" id="globalOnlyCheckbox" {% if global_only == '1' %}checked{% endif %} style="margin-right: 8px; width: 18px; height: 18px; cursor: pointer;">
<span style="font-size: 14px;">Global requests only</span>
</label>
</div>
{% endif %}
<div>
<button type="submit" style="padding: 10px 20px; background: var(--color-link); color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: bold;">
Apply Filters
</button>
</div>
{% if selected_provider or selected_model or selected_rotation or selected_autoselect or selected_user %}
<div>
<a href="{{ url_for(request, "/dashboard/analytics?time_range=%s%s%s"|format(selected_time_range, "&from_date=" + from_date if from_date else "", "&to_date=" + to_date if to_date else "")) }}" style="padding: 10px 20px; background: var(--color-secondary); color: var(--color-text); border: none; border-radius: 4px; text-decoration: none; display: inline-block;">
......@@ -199,7 +198,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
</div>
{% endif %}
</form>
{% if selected_provider or selected_model or selected_rotation or selected_autoselect or selected_user %}
<div style="margin-top: 15px; padding: 10px; background:var(--bg-accent); border-radius: 4px;">
<strong style="color: var(--color-link);">Active Filters: </strong>
......@@ -217,44 +216,42 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
<script>
const analyticsData = {
selectedUser: {{ selected_user or 'null' }},
availableUsers: {{ available_users|tojson }}
availableUsers: {{ available_users|tojson }},
isConfigAdmin: {{ 'true' if is_config_admin else 'false' }},
dashboardVisitSeries: {{ dashboard_visit_series|default('[]')|safe }},
promptAnalysisSeries: {{ prompt_analysis_series|default('[]')|safe }}
};
</script>
<script>
const BASE_PATH = {{ request.scope.get('root_path', '') | tojson }};
function escHtml(s) {
return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;').replace(/'/g,'&#39;');
}
document.getElementById('timeRangeSelect').addEventListener('change', function() {
var customRange = document.getElementById('customDateRange');
if (this.value === 'custom') {
customRange.style.display = 'flex';
} else {
customRange.style.display = 'none';
}
const customRange = document.getElementById('customDateRange');
customRange.style.display = this.value === 'custom' ? 'flex' : 'none';
});
// Handle global_only checkbox for select dropdown (when < 25 users)
{% if is_admin and available_users|length < 25 %}
(function() {
const userSelect = document.getElementById('userFilterSelect');
const globalOnlyCheckbox = document.getElementById('globalOnlyCheckbox');
if (globalOnlyCheckbox && userSelect) {
globalOnlyCheckbox.addEventListener('change', function() {
if (this.checked) {
// When global_only is checked, clear user filter and disable select
userSelect.value = '';
userSelect.disabled = true;
userSelect.style.opacity = '0.5';
userSelect.style.cursor = 'not-allowed';
} else {
// Re-enable user select
userSelect.disabled = false;
userSelect.style.opacity = '1';
userSelect.style.cursor = 'pointer';
}
});
// Initialize state on page load
if (globalOnlyCheckbox.checked) {
userSelect.disabled = true;
userSelect.style.opacity = '0.5';
......@@ -264,11 +261,6 @@ document.getElementById('timeRangeSelect').addEventListener('change', function()
})();
{% endif %}
function escHtml(s) {
return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;').replace(/'/g,'&#39;');
}
// User search autocomplete functionality
{% if is_admin and available_users|length >= 25 %}
(function() {
const searchInput = document.getElementById('userSearchInput');
......@@ -277,15 +269,13 @@ function escHtml(s) {
let debounceTimer;
let selectedUserId = analyticsData.selectedUser;
// Set initial display value if a user is selected
if (analyticsData.selectedUser) {
const user = analyticsData.availableUsers.find(u => u.id == analyticsData.selectedUser);
if (user) {
searchInput.value = user.username + (user.role === 'admin' ? ' (admin)' : '');
}
}
// Search users via API
async function searchUsers(query) {
try {
const response = await fetch(`${BASE_PATH}/api/users/search?q=${encodeURIComponent(query)}`);
......@@ -296,36 +286,28 @@ function escHtml(s) {
return [];
}
}
// Display search results
function displayResults(users) {
if (users.length === 0) {
resultsDiv.innerHTML = '<div style="padding: 10px; color: var(--color-muted);">No users found</div>';
resultsDiv.style.display = 'block';
return;
}
resultsDiv.innerHTML = users.map(user => `
<div class="user-result-item" data-user-id="${user.id}" data-username="${escHtml(user.username)}" data-role="${escHtml(user.role)}"
style="padding: 10px; cursor: pointer; border-bottom: 1px solid #2a4a7a;">
<div class="user-result-item" data-user-id="${user.id}" data-username="${escHtml(user.username)}" data-role="${escHtml(user.role)}" style="padding: 10px; cursor: pointer; border-bottom: 1px solid #2a4a7a;">
${escHtml(user.username)}${user.role === 'admin' ? ' <span style="color: var(--color-link);">(admin)</span>' : ''}
</div>
`).join('');
resultsDiv.style.display = 'block';
// Add click handlers
document.querySelectorAll('.user-result-item').forEach(item => {
item.addEventListener('mouseenter', function() {
this.style.background = 'var(--color-secondary-hover)';
});
item.addEventListener('mouseleave', function() {
this.style.background = '';
});
item.addEventListener('mouseenter', function() { this.style.background = 'var(--color-secondary-hover)'; });
item.addEventListener('mouseleave', function() { this.style.background = ''; });
item.addEventListener('click', function() {
const userId = this.dataset.userId;
const username = this.dataset.username;
const role = this.dataset.role;
searchInput.value = username + (role === 'admin' ? ' (admin)' : '');
hiddenInput.value = userId;
selectedUserId = userId;
......@@ -333,24 +315,14 @@ function escHtml(s) {
});
});
}
// Handle input
searchInput.addEventListener('input', function() {
const query = this.value.trim();
clearTimeout(debounceTimer);
if (query.length === 0) {
// Show all users option
resultsDiv.innerHTML = '<div class="user-result-item" data-user-id="" data-username="" style="padding: 10px; cursor: pointer; border-bottom: 1px solid #2a4a7a;">All Users</div>';
resultsDiv.style.display = 'block';
document.querySelector('.user-result-item').addEventListener('mouseenter', function() {
this.style.background = 'var(--color-secondary-hover)';
});
document.querySelector('.user-result-item').addEventListener('mouseleave', function() {
this.style.background = '';
});
document.querySelector('.user-result-item').addEventListener('click', function() {
searchInput.value = '';
hiddenInput.value = '';
......@@ -359,25 +331,17 @@ function escHtml(s) {
});
return;
}
debounceTimer = setTimeout(async () => {
const users = await searchUsers(query);
displayResults(users);
}, 300);
});
// Handle focus
searchInput.addEventListener('focus', function() {
if (this.value.trim().length === 0) {
resultsDiv.innerHTML = '<div class="user-result-item" data-user-id="" data-username="" style="padding: 10px; cursor: pointer; border-bottom: 1px solid #2a4a7a;">All Users</div>';
resultsDiv.style.display = 'block';
document.querySelector('.user-result-item').addEventListener('mouseenter', function() {
this.style.background = 'var(--color-secondary-hover)';
});
document.querySelector('.user-result-item').addEventListener('mouseleave', function() {
this.style.background = '';
});
document.querySelector('.user-result-item').addEventListener('click', function() {
searchInput.value = '';
hiddenInput.value = '';
......@@ -386,20 +350,17 @@ function escHtml(s) {
});
}
});
// Close results when clicking outside
document.addEventListener('click', function(e) {
if (!searchInput.contains(e.target) && !resultsDiv.contains(e.target)) {
resultsDiv.style.display = 'none';
}
});
// Handle global_only checkbox interaction
const globalOnlyCheckbox = document.getElementById('globalOnlyCheckbox');
if (globalOnlyCheckbox) {
globalOnlyCheckbox.addEventListener('change', function() {
if (this.checked) {
// When global_only is checked, clear user filter
searchInput.value = '';
hiddenInput.value = '';
selectedUserId = null;
......@@ -407,14 +368,12 @@ function escHtml(s) {
searchInput.style.opacity = '0.5';
searchInput.style.cursor = 'not-allowed';
} else {
// Re-enable user search
searchInput.disabled = false;
searchInput.style.opacity = '1';
searchInput.style.cursor = 'text';
}
});
// Initialize state on page load
if (globalOnlyCheckbox.checked) {
searchInput.disabled = true;
searchInput.style.opacity = '0.5';
......@@ -423,147 +382,260 @@ function escHtml(s) {
}
})();
{% endif %}
</script>
{% if recommendations %}
<h3 style="margin-bottom: 15px;">Optimization Recommendations</h3>
<div style="margin-bottom: 30px;">
{% for rec in recommendations %}
<div style="background: {% if rec.severity == 'high' %}#3a1a1a{% elif rec.severity == 'medium' %}#3a2a1a{% else %}#1a2a3a{% endif %};
padding: 15px; border-radius: 4px; margin-bottom: 10px;
border: 1px solid {% if rec.severity == 'high' %}#ef4444{% elif rec.severity == 'medium' %}#f39c12{% else %}#3b82f6{% endif %};">
<strong style="color: {% if rec.severity == 'high' %}#f87171{% elif rec.severity == 'medium' %}#fcd34d{% else %}#60a5fa{% endif %};">
{{ rec.type|replace('_', ' ')|title }}
</strong>
<p style="margin: 5px 0;">{{ rec.message }}</p>
<small style="color: var(--color-muted);">{{ rec.action }}</small>
<div>
<h3 style="margin-bottom: 15px;">Provider Statistics</h3>
{% if provider_stats %}
<table>
<tr>
<th>Provider</th>
<th>Model</th>
<th>Rotation</th>
<th>Autoselect</th>
<th>Total Requests</th>
<th>Success</th>
<th>Errors</th>
<th>Error Rate</th>
<th>Avg Latency</th>
<th>Input Tokens</th>
<th>Output Tokens</th>
<th>Total Tokens</th>
</tr>
{% for provider in provider_stats %}
<tr style="cursor: pointer;" onclick="showProviderDetails('{{ provider.provider_id }}', '{{ provider.model_name or '' }}', '{{ provider.rotation_id or '' }}', '{{ provider.autoselect_id or '' }}', {{ provider.tokens.TPM }}, {{ provider.tokens.TPH }}, {{ provider.tokens.TPD }})">
<td><strong>{{ provider.provider_id }}</strong></td>
<td>{{ provider.model_name or '' }}</td>
<td>{{ provider.rotation_id or '' }}</td>
<td>{{ provider.autoselect_id or '' }}</td>
<td>{{ provider.requests.total }}</td>
<td>{{ provider.requests.success }}</td>
<td>{{ provider.requests.error }}</td>
<td {% if provider.error_rate > 0.1 %}style="color: #f87171;"{% endif %}>{{ "%.1f"|format(provider.error_rate * 100) }}%</td>
<td {% if provider.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>{% if provider.avg_latency_ms > 1000 %}{{ "%.1f"|format(provider.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(provider.avg_latency_ms) }}ms{% endif %}</td>
<td><strong>{{ format_tokens(provider.tokens.prompt or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider.tokens.completion or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider.tokens.total or 0) }}</strong></td>
</tr>
{% endfor %}
<tr style="background:var(--bg-accent); font-weight: bold;">
<td><strong>Total</strong></td>
<td></td><td></td><td></td>
<td>{{ provider_stats | sum(attribute='requests.total') }}</td>
<td>{{ provider_stats | sum(attribute='requests.success') }}</td>
<td>{{ provider_stats | sum(attribute='requests.error') }}</td>
<td>
{% set total_requests = provider_stats | sum(attribute='requests.total') %}
{% set total_errors = provider_stats | sum(attribute='requests.error') %}
{% if total_requests > 0 %}{{ "%.1f"|format((total_errors / total_requests) * 100) }}%{% else %}0.0%{% endif %}
</td>
<td>
{% set total_requests = provider_stats | sum(attribute='requests.total') %}
{% if total_requests > 0 %}
{% set weighted_sum = namespace(value=0) %}
{% for provider in provider_stats %}
{% set weighted_sum.value = weighted_sum.value + (provider.avg_latency_ms | float * provider.requests.total) %}
{% endfor %}
{% set avg_latency = weighted_sum.value / total_requests %}
{% if avg_latency > 1000 %}{{ "%.1f"|format(avg_latency / 1000) }}s{% else %}{{ "%.0f"|format(avg_latency) }}ms{% endif %}
{% else %}
N/A
{% endif %}
</td>
<td><strong>{{ format_tokens(provider_stats | sum(attribute='tokens.prompt') or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider_stats | sum(attribute='tokens.completion') or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider_stats | sum(attribute='tokens.total') or 0) }}</strong></td>
</tr>
</table>
<div id="providerModal" style="display: none; position: fixed; z-index: 1000; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.6);">
<div style="background-color: var(--bg-page); margin: 10% auto; padding: 30px; border: 1px solid #888; border-radius: 8px; width: 60%; max-width: 600px;">
<span onclick="closeProviderModal()" style="color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer;">&times;</span>
<h3 style="margin-top: 0;">Provider Rate Details</h3>
<div id="modalContent"></div>
</div>
</div>
{% else %}
<p style="color: var(--color-muted);">No provider statistics available yet. Make API requests to see analytics.</p>
{% endif %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Cost Overview</h3>
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px;">
<div style="background: #2ecc71; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">{% if from_date or to_date %}Selected Period Cost{% else %}Today's Estimated Cost{% endif %}</h4>
<p style="font-size: 28px; font-weight: bold;">{{ currency_symbol }}{{ "%.2f"|format(cost_overview.total_estimated_cost_today) }}</p>
</div>
<div style="background: #3498db; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Combined Savings Estimate</h4>
<p style="font-size: 28px; font-weight: bold;">{{ currency_symbol }}{{ "%.2f"|format((optimization_savings.total_cost_saved if optimization_savings else 0) or 0) }}</p>
</div>
{% for pc in cost_overview.providers %}
<div style="background:var(--bg-accent); padding: 15px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 5px;">{{ pc.provider_id }}</h4>
<p style="font-size: 20px; font-weight: bold;">${{ "%.2f"|format(pc.estimated_cost) }}</p>
<small style="color: var(--color-muted);">{{ format_tokens(pc.tokens_today) }} tokens</small>
</div>
{% endfor %}
</div>
<h3 style="margin-top: 30px; margin-bottom: 15px;">Optimization Savings</h3>
{% if optimization_savings and optimization_savings.total_tokens_saved > 0 %}
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin-bottom: 20px;">
<div style="background: #27ae60; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Total Tokens Saved</h4>
<p style="font-size: 28px; font-weight: bold;">{{ format_tokens(optimization_savings.total_tokens_saved) }}</p>
</div>
<div style="background: #16a085; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Total Combined Estimate</h4>
<p style="font-size: 28px; font-weight: bold;">${{ "%.2f"|format(optimization_savings.total_cost_saved) }}</p>
</div>
<div style="background: #1abc9c; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Cache / Batching Savings</h4>
<p style="font-size: 28px; font-weight: bold;">${{ "%.2f"|format((optimization_savings.direct_feature_savings.cost_saved if optimization_savings else 0) or 0) }}</p>
</div>
<div style="background: #2980b9; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Free-Tier Equivalent</h4>
<p style="font-size: 28px; font-weight: bold;">${{ "%.2f"|format((optimization_savings.free_tier_equivalent_savings.cost_saved if optimization_savings else 0) or 0) }}</p>
</div>
</div>
<p style="color: var(--color-muted); margin-top: -6px; margin-bottom: 18px;">The combined estimate includes direct execution savings from response caching and batching plus free-tier/subscription-equivalent avoided cost estimates.</p>
{% else %}
<p style="color: var(--color-muted);">No optimization savings recorded yet. Increase usage on providers with known upstream free tiers, or enable deduplication, condensation, batching, or caching to see savings.</p>
{% endif %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Model Performance</h3>
{% if model_performance %}
<table>
<tr>
<th>Provider</th>
<th>Model</th>
<th>Type</th>
<th>Context Size</th>
<th>Condense %</th>
<th>Condense Method</th>
<th>Tokens/Day</th>
<th>Error Rate</th>
<th>Avg Latency</th>
</tr>
{% for model in model_performance %}
<tr>
<td>{{ model.provider_id }}</td>
<td>{{ model.model_name }}</td>
<td>{% if model.is_rotation %}<span style="color: #f39c12;">Rotation: {{ model.rotation_id }}</span>{% elif model.is_autoselect %}<span style="color: #9b59b6;">Autoselect: {{ model.autoselect_id }}</span>{% else %}<span style="color: #3498db;">{{ model.provider_type|title }}</span>{% endif %}</td>
<td>{{ model.context_size|default('N/A') }}</td>
<td>{{ model.condense_context|default('N/A') }}%</td>
<td>{{ model.condense_method|default('None') }}</td>
<td>{{ format_tokens(model.tokens_per_day) }}</td>
<td {% if model.error_rate > 0.1 %}style="color: #f87171;"{% endif %}>{{ "%.1f"|format(model.error_rate * 100) }}%</td>
<td {% if model.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>{% if model.avg_latency_ms > 1000 %}{{ "%.1f"|format(model.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(model.avg_latency_ms) }}ms{% endif %}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p style="color: var(--color-muted);">No model performance data available yet.</p>
{% endif %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Token Usage Over Time</h3>
{% if token_over_time != '[]' %}
<div style="background: var(--bg-page); padding: 20px; border-radius: 8px;">
<canvas id="tokenChart" style="width: 100%; height: 300px;"></canvas>
</div>
{% else %}
<p style="color: var(--color-muted);">No token usage data available for the selected period.</p>
{% endif %}
{% if rotation_breakdown %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Rotation Breakdown</h3>
{% for rot in rotation_breakdown %}
<div style="background: var(--bg-page); padding: 15px; border-radius: 8px; margin-bottom: 20px;">
<h4 style="margin: 0 0 10px 0; color: #f39c12;">{{ rot.rotation_id }}</h4>
<table>
<tr><th>Provider</th><th>Model</th><th>Requests</th><th>Hit %</th><th>Tokens</th><th>Token %</th><th>Avg Latency</th></tr>
{% for e in rot.entries %}
<tr>
<td>{{ e.provider_id }}</td>
<td>{{ e.model_name or '' }}</td>
<td>{{ e.requests }}</td>
<td>{{ e.hit_pct }}%</td>
<td>{{ format_tokens(e.tokens) }}</td>
<td>{{ e.token_pct }}%</td>
<td>{% if e.avg_latency_ms > 1000 %}{{ "%.1f"|format(e.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(e.avg_latency_ms) }}ms{% endif %}</td>
</tr>
{% endfor %}
</table>
</div>
{% endfor %}
</div>
{% endif %}
{% endif %}
<h3 style="margin-bottom: 15px;">Provider Statistics</h3>
{% if provider_stats %}
<table>
<tr>
<th data-i18n="analytics_page.col_provider">Provider</th>
<th>Model</th>
<th>Rotation</th>
<th>Autoselect</th>
<th data-i18n="analytics_page.col_total_requests">Total Requests</th>
<th data-i18n="analytics_page.col_success">Success</th>
<th data-i18n="analytics_page.col_errors">Errors</th>
<th data-i18n="analytics_page.col_error_rate">Error Rate</th>
<th data-i18n="analytics_page.col_avg_latency">Avg Latency</th>
<th data-i18n="analytics_page.col_input_tokens">Input Tokens</th>
<th data-i18n="analytics_page.col_output_tokens">Output Tokens</th>
<th data-i18n="analytics_page.col_total_tokens">Total Tokens</th>
</tr>
{% for provider in provider_stats %}
<tr style="cursor: pointer;" onclick="showProviderDetails('{{ provider.provider_id }}', '{{ provider.model_name or '' }}', '{{ provider.rotation_id or '' }}', '{{ provider.autoselect_id or '' }}', {{ provider.tokens.TPM }}, {{ provider.tokens.TPH }}, {{ provider.tokens.TPD }})">
<td><strong>{{ provider.provider_id }}</strong></td>
<td>{{ provider.model_name or '' }}</td>
<td>{{ provider.rotation_id or '' }}</td>
<td>{{ provider.autoselect_id or '' }}</td>
<td>{{ provider.requests.total }}</td>
<td>{{ provider.requests.success }}</td>
<td>{{ provider.requests.error }}</td>
<td {% if provider.error_rate > 0.1 %}style="color: #f87171;"{% endif %}>
{{ "%.1f"|format(provider.error_rate * 100) }}%
</td>
<td {% if provider.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>
{% if provider.avg_latency_ms > 1000 %}{{ "%.1f"|format(provider.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(provider.avg_latency_ms) }}ms{% endif %}
</td>
<td><strong>{{ format_tokens(provider.tokens.prompt or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider.tokens.completion or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider.tokens.total or 0) }}</strong></td>
</tr>
{% if autoselect_breakdown %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Autoselect Breakdown</h3>
{% for asel in autoselect_breakdown %}
<div style="background: var(--bg-page); padding: 15px; border-radius: 8px; margin-bottom: 20px;">
<h4 style="margin: 0 0 10px 0; color: #9b59b6;">{{ asel.autoselect_id }}</h4>
<table>
<tr><th>Selected Model / Rotation</th><th>Requests</th><th>Hit %</th><th>Tokens</th><th>Token %</th><th>Selection Latency</th></tr>
{% for e in asel.entries %}
<tr>
<td><strong>{{ e.model_name or '(unknown)' }}</strong></td>
<td>{{ e.requests }}</td>
<td>{{ e.hit_pct }}%</td>
<td>{{ format_tokens(e.tokens) }}</td>
<td>{{ e.token_pct }}%</td>
<td>{% if e.avg_latency_ms > 1000 %}{{ "%.1f"|format(e.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(e.avg_latency_ms) }}ms{% endif %}</td>
</tr>
{% endfor %}
</table>
</div>
{% endfor %}
{% if provider_stats %}
<tr style="background:var(--bg-accent); font-weight: bold;">
<td><strong data-i18n="analytics_page.total">Total</strong></td>
<td></td><td></td><td></td>
<td>{{ provider_stats | sum(attribute='requests.total') }}</td>
<td>{{ provider_stats | sum(attribute='requests.success') }}</td>
<td>{{ provider_stats | sum(attribute='requests.error') }}</td>
<td>
{% set total_requests = provider_stats | sum(attribute='requests.total') %}
{% set total_errors = provider_stats | sum(attribute='requests.error') %}
{% if total_requests > 0 %}
{{ "%.1f"|format((total_errors / total_requests) * 100) }}%
{% else %}
0.0%
{% endif %}
</td>
<td>
{% set total_requests = provider_stats | sum(attribute='requests.total') %}
{% if total_requests > 0 %}
{% set weighted_sum = namespace(value=0) %}
{% for provider in provider_stats %}
{% set weighted_sum.value = weighted_sum.value + (provider.avg_latency_ms | float * provider.requests.total) %}
{% endfor %}
{% set avg_latency = weighted_sum.value / total_requests %}
{% if avg_latency > 1000 %}{{ "%.1f"|format(avg_latency / 1000) }}s{% else %}{{ "%.0f"|format(avg_latency) }}ms{% endif %}
{% else %}
N/A
{% endif %}
</td>
<td><strong>{{ format_tokens(provider_stats | sum(attribute='tokens.prompt') or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider_stats | sum(attribute='tokens.completion') or 0) }}</strong></td>
<td><strong>{{ format_tokens(provider_stats | sum(attribute='tokens.total') or 0) }}</strong></td>
</tr>
{% endif %}
</table>
<!-- Modal for provider details -->
<div id="providerModal" style="display: none; position: fixed; z-index: 1000; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.6);">
<div style="background-color: var(--bg-page); margin: 10% auto; padding: 30px; border: 1px solid #888; border-radius: 8px; width: 60%; max-width: 600px;">
<span onclick="closeProviderModal()" style="color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer;">&times;</span>
<h3 style="margin-top: 0;">Provider Rate Details</h3>
<div id="modalContent"></div>
{% if is_config_admin %}
<h3 style="margin-top: 40px; margin-bottom: 15px; color: #f87171;">Analytics Management</h3>
<div style="background: var(--bg-page); padding: 20px; border-radius: 8px; border: 1px solid #f87171; margin-bottom: 30px;">
<p style="margin: 0 0 15px 0; color: var(--color-muted);">These actions permanently delete analytics data from the database and cannot be undone.</p>
<div style="display: flex; gap: 15px; flex-wrap: wrap;">
<button onclick="confirmDeleteAnalytics('global')" class="btn" style="background: #e67e22; border: none; cursor: pointer;">Reset Global Analytics</button>
<button onclick="confirmDeleteAnalytics('all')" class="btn" style="background: #e74c3c; border: none; cursor: pointer;">Reset All Analytics (Global + Users)</button>
</div>
</div>
{% endif %}
<div style="margin-top: 30px; display: flex; gap: 10px; flex-wrap: wrap;">
<a href="{{ url_for(request, '/dashboard') }}" class="btn btn-secondary" data-i18n="analytics_page.back">Back to Dashboard</a>
<a href="{{ url_for(request, '/dashboard/analytics/provider-quotas') }}" class="btn btn-secondary">Provider Quota Debug</a>
{% if is_admin %}
<a href="{{ url_for(request, '/dashboard/rate-limits') }}" class="btn">Rate Limits</a>
<a href="{{ url_for(request, '/dashboard/response-cache') }}" class="btn">Response Cache</a>
{% endif %}
</div>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>
<script>
function showProviderDetails(providerId, modelName, rotationId, autoselectId, tpm, tph, tpd) {
const modal = document.getElementById('providerModal');
const content = document.getElementById('modalContent');
if (!modal || !content) return;
let title = '<strong>' + providerId + '</strong>';
if (modelName) title += ' / ' + modelName;
if (rotationId) title += ' (Rotation: ' + rotationId + ')';
if (autoselectId) title += ' (Autoselect: ' + autoselectId + ')';
content.innerHTML = `
<p style="margin-bottom: 20px;">${title}</p>
<table style="width: 100%;">
<tr>
<th style="text-align: left; padding: 10px; background: var(--bg-accent);">Metric</th>
<th style="text-align: right; padding: 10px; background: var(--bg-accent);">Value</th>
</tr>
<tr>
<td style="padding: 10px;">Tokens per Minute</td>
<td style="padding: 10px; text-align: right;"><strong>${formatTokens(tpm)}</strong></td>
</tr>
<tr>
<td style="padding: 10px;">Tokens per Hour</td>
<td style="padding: 10px; text-align: right;"><strong>${formatTokens(tph)}</strong></td>
</tr>
<tr>
<td style="padding: 10px;">Tokens per Day</td>
<td style="padding: 10px; text-align: right;"><strong>${formatTokens(tpd)}</strong></td>
</tr>
<tr><th style="text-align: left; padding: 10px; background: var(--bg-accent);">Metric</th><th style="text-align: right; padding: 10px; background: var(--bg-accent);">Value</th></tr>
<tr><td style="padding: 10px;">Tokens per Minute</td><td style="padding: 10px; text-align: right;"><strong>${formatTokens(tpm)}</strong></td></tr>
<tr><td style="padding: 10px;">Tokens per Hour</td><td style="padding: 10px; text-align: right;"><strong>${formatTokens(tph)}</strong></td></tr>
<tr><td style="padding: 10px;">Tokens per Day</td><td style="padding: 10px; text-align: right;"><strong>${formatTokens(tpd)}</strong></td></tr>
</table>
<p style="margin-top: 20px; color: var(--color-muted); font-size: 14px;">
These rates are calculated based on the selected time range and filters.
</p>
`;
modal.style.display = 'block';
}
function closeProviderModal() {
document.getElementById('providerModal').style.display = 'none';
const modal = document.getElementById('providerModal');
if (modal) modal.style.display = 'none';
}
function formatTokens(value) {
......@@ -575,463 +647,66 @@ function formatTokens(value) {
return value.toString();
}
// Close modal when clicking outside
window.onclick = function(event) {
const modal = document.getElementById('providerModal');
if (event.target == modal) {
if (modal && event.target == modal) {
closeProviderModal();
}
}
</script>
{% else %}
<p style="color: var(--color-muted);">No provider statistics available yet. Make API requests to see analytics.</p>
{% endif %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Cost Overview</h3>
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px;">
<div style="background: #2ecc71; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">
{% if from_date or to_date %}Selected Period Cost{% else %}Today's Estimated Cost{% endif %}
</h4>
<p style="font-size: 28px; font-weight: bold;">{{ currency_symbol }}{{ "%.2f"|format(cost_overview.total_estimated_cost_today) }}</p>
{% if cost_overview.date_range %}
<small style="color: rgba(255,255,255,0.8);">
{{ cost_overview.date_range.start[:10] }} to {{ cost_overview.date_range.end[:10] }}
</small>
{% endif %}
</div>
<div style="background: #3498db; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">💰 Estimated Savings</h4>
<p style="font-size: 28px; font-weight: bold;" id="savings-amount">{{ currency_symbol }}{{ "%.2f"|format((optimization_savings.total_cost_saved if optimization_savings else 0) or 0) }}</p>
<small style="color: rgba(255,255,255,0.8);">
{% if optimization_savings and optimization_savings.provider_equivalents %}
Configured free-tier equivalents in selected period
{% else %}
No configured free-tier savings in selected period
{% endif %}
</small>
</div>
{% for pc in cost_overview.providers %}
<div style="background:var(--bg-accent); padding: 15px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 5px;">{{ pc.provider_id }}</h4>
<p style="font-size: 20px; font-weight: bold;">${{ "%.2f"|format(pc.estimated_cost) }}</p>
<small style="color: var(--color-muted);">{{ format_tokens(pc.tokens_today) }} tokens</small>
</div>
{% endfor %}
</div>
<h3 style="margin-top: 30px; margin-bottom: 15px;">Optimization Savings</h3>
{% if optimization_savings and optimization_savings.total_tokens_saved > 0 %}
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin-bottom: 20px;">
<div style="background: #27ae60; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Total Tokens Saved</h4>
<p style="font-size: 28px; font-weight: bold;">{{ format_tokens(optimization_savings.total_tokens_saved) }}</p>
{% if optimization_savings.date_range %}
<small style="color: rgba(255,255,255,0.8);">
{{ optimization_savings.date_range.start[:10] }} to {{ optimization_savings.date_range.end[:10] }}
</small>
{% endif %}
</div>
<div style="background: #16a085; color: white; padding: 20px; border-radius: 8px;">
<h4 style="font-size: 14px; margin-bottom: 10px;">Total Cost Saved</h4>
<p style="font-size: 28px; font-weight: bold;">${{ "%.2f"|format(optimization_savings.total_cost_saved) }}</p>
<small style="color: rgba(255,255,255,0.8);">From optimizations</small>
</div>
</div>
{% if optimization_savings.provider_equivalents %}
<div style="background: var(--bg-page); padding: 16px; border-radius: 8px; margin-bottom: 20px;">
<h4 style="margin: 0 0 10px 0;">Free-Tier Equivalent Savings</h4>
<p style="margin: 0 0 10px 0; color: var(--color-muted);">
The table below only uses explicitly configured provider free-tier limits. Generic defaults are excluded to avoid overstated savings on subscription or bundled plans.
Extra free-tier equivalents are counted only for complete additional quota blocks beyond the first included tier within the selected period.
</p>
<table>
<tr>
<th>Provider</th>
<th>Used Tokens</th>
<th>Requests</th>
<th>Measured Usage</th>
<th>Free Limit</th>
<th>Extra Free Tiers</th>
<th>Equivalent Saved Tokens</th>
<th>Premium Reference</th>
<th>Equivalent Saved Cost</th>
</tr>
{% for item in optimization_savings.provider_equivalents %}
<tr>
<td><strong>{{ item.provider_id }}</strong></td>
<td>{{ format_tokens(item.tokens_used) }}</td>
<td>{{ item.request_count }}</td>
<td>{{ item.usage_amount }} {{ item.free_tier_limit_type }}</td>
<td>{{ item.free_tier_limit }}/{{ item.free_tier_period }} {{ item.free_tier_limit_type }} <small style="color: var(--color-muted);">({{ item.quota_source }})</small></td>
<td>{{ item.extra_free_tiers }}</td>
<td>{{ format_tokens(item.equivalent_saved_tokens) }}</td>
<td>
{% if item.premium_reference_name %}
{{ item.premium_reference_name }} ({{ currency_symbol }}{{ "%.2f"|format(item.premium_reference_monthly_cost) }}/mo)
{% else %}
N/A
{% endif %}
</td>
<td>{{ currency_symbol }}{{ "%.2f"|format(item.equivalent_saved_cost) }}</td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
{% if optimization_savings.savings_by_type %}
<table>
<tr>
<th data-i18n="analytics_page.col_opt_type">Optimization Type</th>
<th data-i18n="analytics_page.col_count">Count</th>
<th data-i18n="analytics_page.col_tokens_saved">Tokens Saved</th>
<th data-i18n="analytics_page.col_cost_saved">Cost Saved</th>
<th data-i18n="analytics_page.col_avg_tokens_opt">Avg Tokens/Optimization</th>
<th data-i18n="analytics_page.col_max_tokens_saved">Max Tokens Saved</th>
</tr>
{% for opt_type, stats in optimization_savings.savings_by_type.items() %}
<tr>
<td><strong>{{ opt_type.title() }}</strong></td>
<td>{{ stats.count }}</td>
<td>{{ format_tokens(stats.tokens_saved) }}</td>
<td>${{ "%.4f"|format(stats.cost_saved) }}</td>
<td>{{ format_tokens(stats.avg_tokens_saved) }}</td>
<td>{{ format_tokens(stats.max_tokens_saved) }}</td>
</tr>
{% endfor %}
<tr style="background:var(--bg-accent); font-weight: bold;">
<td><strong data-i18n="analytics_page.total">Total</strong></td>
<td>{{ optimization_savings.savings_by_type.values() | sum(attribute='count') }}</td>
<td>{{ format_tokens(optimization_savings.total_tokens_saved) }}</td>
<td>${{ "%.4f"|format(optimization_savings.total_cost_saved) }}</td>
<td>-</td>
<td>-</td>
</tr>
</table>
{% endif %}
{% else %}
<p style="color: var(--color-muted);">No optimization savings recorded yet. Increase usage on providers with known upstream free tiers, or enable deduplication, condensation, batching, or caching to see savings.</p>
{% endif %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Model Performance</h3>
{% if model_performance %}
<table>
<tr>
<th data-i18n="analytics_page.col_provider">Provider</th>
<th data-i18n="analytics_page.col_model">Model</th>
<th data-i18n="analytics_page.col_type">Type</th>
<th data-i18n="analytics_page.col_context_size">Context Size</th>
<th>Condense %</th>
<th data-i18n="analytics_page.col_condense_method">Condense Method</th>
<th data-i18n="analytics_page.col_tpd">Tokens/Day</th>
<th data-i18n="analytics_page.col_error_rate">Error Rate</th>
<th data-i18n="analytics_page.col_avg_latency">Avg Latency</th>
</tr>
{% for model in model_performance %}
<tr>
<td>{{ model.provider_id }}</td>
<td>{{ model.model_name }}</td>
<td>
{% if model.is_rotation %}
<span style="color: #f39c12;">Rotation: {{ model.rotation_id }}</span>
{% elif model.is_autoselect %}
<span style="color: #9b59b6;">Autoselect: {{ model.autoselect_id }}</span>
{% else %}
<span style="color: #3498db;">{{ model.provider_type|title }}</span>
{% endif %}
</td>
<td>{{ model.context_size|default('N/A') }}</td>
<td>{{ model.condense_context|default('N/A') }}%</td>
<td>{{ model.condense_method|default('None') }}</td>
<td>{{ format_tokens(model.tokens_per_day) }}</td>
<td {% if model.error_rate > 0.1 %}style="color: #f87171;"{% endif %}>
{{ "%.1f"|format(model.error_rate * 100) }}%
</td>
<td {% if model.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>
{% if model.avg_latency_ms > 1000 %}{{ "%.1f"|format(model.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(model.avg_latency_ms) }}ms{% endif %}
</td>
</tr>
{% endfor %}
</table>
{% else %}
<p style="color: var(--color-muted);">No model performance data available yet.</p>
{% endif %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">
Token Usage Over Time
{% if from_date or to_date %}
(Custom Range)
{% elif selected_time_range == '1h' %}
(Last 1 Hour)
{% elif selected_time_range == '6h' %}
(Last 6 Hours)
{% elif selected_time_range == '24h' %}
(Last 24 Hours)
{% elif selected_time_range == 'yesterday' %}
(Yesterday)
{% elif selected_time_range == '7d' %}
(Last 7 Days)
{% elif selected_time_range == '30d' %}
(Last 30 Days)
{% elif selected_time_range == '90d' %}
(Last 90 Days)
{% else %}
(Last 24 Hours)
{% endif %}
</h3>
{% if token_over_time != '[]' %}
<div style="background: var(--bg-page); padding: 20px; border-radius: 8px;">
<canvas id="tokenChart" style="width: 100%; height: 300px;"></canvas>
</div>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>
<script>
const tokenData = {{ token_over_time|safe }};
// Group by provider if multiple
const tokenData = {{ token_over_time|safe }};
if (document.getElementById('tokenChart') && tokenData.length) {
const providers = [...new Set(tokenData.map(d => d.provider_id || 'all'))];
if (providers.length > 1) {
// Multiple providers - show stacked
const datasets = providers.map((provider, i) => {
const colors = ['#e94560', '#3498db', '#2ecc71', '#f39c12', '#9b59b6', '#1abc9c'];
const data = tokenData.filter(d => (d.provider_id || 'all') === provider).map(d => ({
x: d.timestamp,
y: d.tokens
}));
return {
label: provider,
data: data,
backgroundColor: colors[i % colors.length],
borderColor: colors[i % colors.length],
fill: false,
tension: 0.1
};
const data = tokenData.filter(d => (d.provider_id || 'all') === provider).map(d => ({ x: d.timestamp, y: d.tokens }));
return { label: provider, data, backgroundColor: colors[i % colors.length], borderColor: colors[i % colors.length], fill: false, tension: 0.1 };
});
new Chart(document.getElementById('tokenChart'), {
type: 'line',
data: { datasets: datasets },
options: {
responsive: true,
scales: {
x: {
type: 'time',
time: {
unit: {% if selected_time_range == '1h' or selected_time_range == '6h' %}'hour'{% elif selected_time_range == '24h' %}'hour'{% elif selected_time_range == '7d' %}'day'{% else %}'day'{% endif %}
},
title: { display: true, text: 'Time' }
},
y: { beginAtZero: true, title: { display: true, text: 'Tokens' } }
}
}
data: { datasets },
options: { responsive: true, scales: { x: { type: 'time', time: { unit: 'hour' }, title: { display: true, text: 'Time' } }, y: { beginAtZero: true, title: { display: true, text: 'Tokens' } } } }
});
} else {
// Single provider
new Chart(document.getElementById('tokenChart'), {
type: 'line',
data: {
datasets: [{
label: 'Tokens Used',
data: tokenData.map(d => ({x: d.timestamp, y: d.tokens})),
borderColor: '#e94560',
backgroundColor: 'rgba(233, 69, 96, 0.1)',
fill: true,
tension: 0.1
}]
},
options: {
responsive: true,
scales: {
x: {
type: 'time',
time: {
unit: {% if selected_time_range == '1h' or selected_time_range == '6h' %}'hour'{% elif selected_time_range == '24h' %}'hour'{% elif selected_time_range == '7d' %}'day'{% else %}'day'{% endif %}
},
title: { display: true, text: 'Time' }
},
y: { beginAtZero: true, title: { display: true, text: 'Tokens' } }
}
}
data: { datasets: [{ label: 'Tokens Used', data: tokenData.map(d => ({x: d.timestamp, y: d.tokens})), borderColor: '#e94560', backgroundColor: 'rgba(233, 69, 96, 0.1)', fill: true, tension: 0.1 }] },
options: { responsive: true, scales: { x: { type: 'time', time: { unit: 'hour' }, title: { display: true, text: 'Time' } }, y: { beginAtZero: true, title: { display: true, text: 'Tokens' } } } }
});
}
</script>
{% else %}
<p style="color: var(--color-muted);">No token usage data available for the selected period.</p>
{% endif %}
<!-- Rotation Breakdown -->
{% if rotation_breakdown %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Rotation Breakdown</h3>
{% for rot in rotation_breakdown %}
<div style="background: var(--bg-page); padding: 15px; border-radius: 8px; margin-bottom: 20px;">
<h4 style="margin: 0 0 10px 0; color: #f39c12;">⟳ {{ rot.rotation_id }}
<span style="font-weight: normal; font-size: 13px; color: var(--color-muted); margin-left: 10px;">
{{ rot.total_requests }} requests · {{ format_tokens(rot.total_tokens) }} tokens
</span>
</h4>
<table>
<tr>
<th>Provider</th>
<th>Model</th>
<th>Requests</th>
<th>Hit %</th>
<th>Tokens</th>
<th>Token %</th>
<th>Avg Latency</th>
</tr>
{% for e in rot.entries %}
<tr>
<td>{{ e.provider_id }}</td>
<td>{{ e.model_name or '' }}</td>
<td>{{ e.requests }}</td>
<td>
<div style="display: flex; align-items: center; gap: 8px;">
<div style="background: #2a4a7a; border-radius: 4px; width: 80px; height: 8px; overflow: hidden;">
<div style="background: #f39c12; width: {{ e.hit_pct }}%; height: 100%;"></div>
</div>
{{ e.hit_pct }}%
</div>
</td>
<td>{{ format_tokens(e.tokens) }}</td>
<td>
<div style="display: flex; align-items: center; gap: 8px;">
<div style="background: #2a4a7a; border-radius: 4px; width: 80px; height: 8px; overflow: hidden;">
<div style="background: #3498db; width: {{ e.token_pct }}%; height: 100%;"></div>
</div>
{{ e.token_pct }}%
</div>
</td>
<td {% if e.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>
{% if e.avg_latency_ms > 1000 %}{{ "%.1f"|format(e.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(e.avg_latency_ms) }}ms{% endif %}
</td>
</tr>
{% endfor %}
</table>
</div>
{% endfor %}
{% endif %}
}
<!-- Autoselect Breakdown -->
{% if autoselect_breakdown %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Autoselect Breakdown</h3>
{% for asel in autoselect_breakdown %}
<div style="background: var(--bg-page); padding: 15px; border-radius: 8px; margin-bottom: 20px;">
<h4 style="margin: 0 0 10px 0; color: #9b59b6;">⚡ {{ asel.autoselect_id }}
<span style="font-weight: normal; font-size: 13px; color: var(--color-muted); margin-left: 10px;">
{{ asel.total_requests }} requests · {{ format_tokens(asel.total_tokens) }} tokens
</span>
</h4>
<table>
<tr>
<th>Selected Model / Rotation</th>
<th>Requests</th>
<th>Hit %</th>
<th>Tokens</th>
<th>Token %</th>
<th>Selection Latency</th>
</tr>
{% for e in asel.entries %}
<tr>
<td><strong>{{ e.model_name or '(unknown)' }}</strong></td>
<td>{{ e.requests }}</td>
<td>
<div style="display: flex; align-items: center; gap: 8px;">
<div style="background: #2a4a7a; border-radius: 4px; width: 80px; height: 8px; overflow: hidden;">
<div style="background: #9b59b6; width: {{ e.hit_pct }}%; height: 100%;"></div>
</div>
{{ e.hit_pct }}%
</div>
</td>
<td>{{ format_tokens(e.tokens) }}</td>
<td>
<div style="display: flex; align-items: center; gap: 8px;">
<div style="background: #2a4a7a; border-radius: 4px; width: 80px; height: 8px; overflow: hidden;">
<div style="background: #3498db; width: {{ e.token_pct }}%; height: 100%;"></div>
</div>
{{ e.token_pct }}%
</div>
</td>
<td {% if e.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>
{% if e.avg_latency_ms > 1000 %}{{ "%.1f"|format(e.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(e.avg_latency_ms) }}ms{% endif %}
</td>
</tr>
{% endfor %}
</table>
</div>
{% endfor %}
{% endif %}
if (document.getElementById('promptAnalysisChart') && analyticsData.promptAnalysisSeries.length) {
new Chart(document.getElementById('promptAnalysisChart'), {
type: 'line',
data: {
datasets: [
{ label: 'Runs', data: analyticsData.promptAnalysisSeries.map(d => ({x: d.bucket, y: d.runs})), borderColor: '#2563eb', backgroundColor: 'rgba(37,99,235,0.10)', fill: true, tension: 0.2 },
{ label: 'High Risk', data: analyticsData.promptAnalysisSeries.map(d => ({x: d.bucket, y: d.high_risk})), borderColor: '#f59e0b', backgroundColor: 'rgba(245,158,11,0.08)', fill: false, tension: 0.2 },
{ label: 'Blocked', data: analyticsData.promptAnalysisSeries.map(d => ({x: d.bucket, y: d.blocked})), borderColor: '#dc2626', backgroundColor: 'rgba(220,38,38,0.08)', fill: false, tension: 0.2 }
]
},
options: { responsive: true, scales: { x: { type: 'time', time: { unit: 'hour' }, title: { display: true, text: 'Time' } }, y: { beginAtZero: true, title: { display: true, text: 'Count' } } } }
});
}
<!-- Internal Models Stats (kiro, kilo, claude CLI, etc.) -->
{% set internal_providers = ['kiro', 'kilo', 'claude', 'codex'] %}
{% set internal_stats = provider_stats | selectattr('provider_id', 'in', internal_providers) | list %}
{% if not internal_stats %}
{# also catch any provider_id that contains these keywords #}
{% set internal_stats = [] %}
{% for p in provider_stats %}
{% if 'kiro' in p.provider_id or 'kilo' in p.provider_id or 'claude' in p.provider_id %}
{% set _ = internal_stats.append(p) %}
{% endif %}
{% endfor %}
{% endif %}
{% if internal_stats %}
<h3 style="margin-top: 30px; margin-bottom: 15px;">Internal / CLI Provider Stats</h3>
<table>
<tr>
<th>Provider</th>
<th>Model</th>
<th>Total Requests</th>
<th>Success</th>
<th>Errors</th>
<th>Error Rate</th>
<th>Avg Latency</th>
<th>Input Tokens</th>
<th>Output Tokens</th>
<th>Total Tokens</th>
</tr>
{% for p in internal_stats %}
<tr>
<td><strong>{{ p.provider_id }}</strong></td>
<td>{{ p.model_name or '' }}</td>
<td>{{ p.requests.total }}</td>
<td>{{ p.requests.success }}</td>
<td>{{ p.requests.error }}</td>
<td {% if p.error_rate > 0.1 %}style="color: #f87171;"{% endif %}>{{ "%.1f"|format(p.error_rate * 100) }}%</td>
<td {% if p.avg_latency_ms > 5000 %}style="color: #fcd34d;"{% endif %}>
{% if p.avg_latency_ms > 1000 %}{{ "%.1f"|format(p.avg_latency_ms / 1000) }}s{% else %}{{ "%.0f"|format(p.avg_latency_ms) }}ms{% endif %}
</td>
<td>{{ format_tokens(p.tokens.prompt or 0) }}</td>
<td>{{ format_tokens(p.tokens.completion or 0) }}</td>
<td><strong>{{ format_tokens(p.tokens.total or 0) }}</strong></td>
</tr>
{% endfor %}
</table>
{% endif %}
if (analyticsData.isConfigAdmin && document.getElementById('dashboardVisitChart') && analyticsData.dashboardVisitSeries.length) {
new Chart(document.getElementById('dashboardVisitChart'), {
type: 'line',
data: {
datasets: [
{ label: 'Events', data: analyticsData.dashboardVisitSeries.map(d => ({x: d.bucket, y: d.events})), borderColor: '#1f6feb', backgroundColor: 'rgba(31,111,235,0.12)', fill: true, tension: 0.2 },
{ label: 'Unique IPs', data: analyticsData.dashboardVisitSeries.map(d => ({x: d.bucket, y: d.unique_ips})), borderColor: '#0f766e', backgroundColor: 'rgba(15,118,110,0.08)', fill: false, tension: 0.2 },
{ label: 'Unique Visitors', data: analyticsData.dashboardVisitSeries.map(d => ({x: d.bucket, y: d.unique_visitors})), borderColor: '#7c3aed', backgroundColor: 'rgba(124,58,237,0.08)', fill: false, tension: 0.2 }
]
},
options: { responsive: true, scales: { x: { type: 'time', time: { unit: 'hour' }, title: { display: true, text: 'Time' } }, y: { beginAtZero: true, title: { display: true, text: 'Count' } } } }
});
}
<!-- Admin: Delete Analytics -->
{% if is_config_admin %}
<h3 style="margin-top: 40px; margin-bottom: 15px; color: #f87171;">⚠ Analytics Management</h3>
<div style="background: var(--bg-page); padding: 20px; border-radius: 8px; border: 1px solid #f87171;">
<p style="margin: 0 0 15px 0; color: var(--color-muted);">These actions permanently delete analytics data from the database and cannot be undone.</p>
<div style="display: flex; gap: 15px; flex-wrap: wrap;">
<button onclick="confirmDeleteAnalytics('global')" class="btn" style="background: #e67e22; border: none; cursor: pointer;">
🗑 Reset Global Analytics
</button>
<button onclick="confirmDeleteAnalytics('all')" class="btn" style="background: #e74c3c; border: none; cursor: pointer;">
🗑 Reset All Analytics (Global + Users)
</button>
</div>
</div>
<script>
function confirmDeleteAnalytics(scope) {
const msg = scope === 'global'
? 'Delete all analytics for global (non-user) providers/models? This cannot be undone.'
: 'Delete ALL analytics including all user data? This cannot be undone.';
const msg = scope === 'global' ? 'Delete all analytics for global (non-user) providers/models? This cannot be undone.' : 'Delete ALL analytics including all user data? This cannot be undone.';
if (!confirm(msg)) return;
if (!confirm('Are you sure? This is irreversible.')) return;
fetch(`${BASE_PATH}/api/admin/analytics/delete-${scope}`, {method: 'POST'})
......@@ -1043,14 +718,4 @@ function confirmDeleteAnalytics(scope) {
.catch(e => alert('Error: ' + e));
}
</script>
{% endif %}
<div style="margin-top: 30px; display: flex; gap: 10px; flex-wrap: wrap;">
<a href="{{ url_for(request, '/dashboard') }}" class="btn btn-secondary" data-i18n="analytics_page.back">Back to Dashboard</a>
<a href="{{ url_for(request, '/dashboard/analytics/provider-quotas') }}" class="btn btn-secondary">Provider Quota Debug</a>
{% if is_admin %}
<a href="{{ url_for(request, '/dashboard/rate-limits') }}" class="btn">Rate Limits</a>
<a href="{{ url_for(request, '/dashboard/response-cache') }}" class="btn">Response Cache</a>
{% endif %}
</div>
{% endblock %}
tests/test_analytics_savings.py 0 → 100644
View file @ d1787cd5
from datetime import datetime, timedelta
import aisbf.analytics as analytics_module
from aisbf.analytics import Analytics
from aisbf.database import DatabaseManager
class _UsageDb(DatabaseManager):
def get_provider_usage(self, user_id, provider_id):
return {
"usage_data": {
"free_tier": {
"limit_type": "requests",
"limit": 1,
"period": "week",
"used": 200_000_000,
"source": "provider",
}
}
}
def _make_db(tmp_path):
db_path = tmp_path / "analytics-savings.db"
return _UsageDb({
"type": "sqlite",
"sqlite_path": str(db_path),
})
def _seed_token_usage(db: DatabaseManager, rows: list[dict]):
with db._get_connection() as conn:
cursor = conn.cursor()
for row in rows:
cursor.execute(
"""
INSERT INTO token_usage (
user_id, provider_id, model_name, tokens_used, prompt_tokens, completion_tokens,
actual_cost, success, latency_ms, error_type, token_id, rotation_id,
autoselect_id, analytics_kind, timestamp
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
row.get("user_id"),
row["provider_id"],
row["model_name"],
row["tokens_used"],
row.get("prompt_tokens"),
row.get("completion_tokens"),
row.get("actual_cost"),
row.get("success", 1),
row.get("latency_ms", 100),
row.get("error_type"),
row.get("token_id"),
row.get("rotation_id"),
row.get("autoselect_id"),
row.get("analytics_kind", "execution"),
db._format_dt_db(row["timestamp"]),
),
)
conn.commit()
class _CacheStub:
def get_stats(self):
return {"hits": 0}
def get_user_stats(self, user_id):
return {"hits": 0}
class _BatcherStub:
def get_stats(self):
return {"batches_formed": 0, "requests_batched": 0}
class _ConfigStub:
def get_provider(self, provider_id, warn=False):
return None
def test_savings_overview_caps_single_provider_subscription_savings_by_estimated_cost(tmp_path, monkeypatch):
db = _make_db(tmp_path)
now = datetime.now()
_seed_token_usage(db, [
{
"provider_id": "codex-free",
"model_name": "gpt-5-mini",
"tokens_used": 200_000_000,
"prompt_tokens": 100_000_000,
"completion_tokens": 100_000_000,
"timestamp": now - timedelta(hours=1),
}
])
monkeypatch.setattr("aisbf.analytics.get_response_cache", lambda: _CacheStub(), raising=False)
monkeypatch.setattr("aisbf.cache.get_response_cache", lambda: _CacheStub())
monkeypatch.setattr("aisbf.batching.get_request_batcher", lambda: _BatcherStub())
monkeypatch.setattr(analytics_module, "config", _ConfigStub(), raising=False)
analytics = Analytics(db)
overview = analytics.get_savings_overview(
from_datetime=now - timedelta(days=1),
to_datetime=now,
provider_filter="codex-free",
model_filter="gpt-5-mini",
)
assert overview is not None
assert overview["provider_equivalents"] == []
assert overview["total_cost_saved"] == 0
assert overview["direct_feature_savings"]["cost_saved"] == 0
assert overview["free_tier_equivalent_savings"]["cost_saved"] == 0
def test_savings_overview_caps_multi_provider_free_tier_equivalent_by_estimated_cost(tmp_path, monkeypatch):
db = _make_db(tmp_path)
now = datetime.now()
_seed_token_usage(db, [
{
"provider_id": "codex-free",
"model_name": "gpt-5-mini",
"tokens_used": 200_000_000,
"prompt_tokens": 100_000_000,
"completion_tokens": 100_000_000,
"timestamp": now - timedelta(hours=1),
}
])
monkeypatch.setattr("aisbf.analytics.get_response_cache", lambda: _CacheStub(), raising=False)
monkeypatch.setattr("aisbf.cache.get_response_cache", lambda: _CacheStub())
monkeypatch.setattr("aisbf.batching.get_request_batcher", lambda: _BatcherStub())
monkeypatch.setattr(analytics_module, "config", _ConfigStub(), raising=False)
analytics = Analytics(db)
overview = analytics.get_savings_overview(
from_datetime=now - timedelta(days=1),
to_datetime=now,
)
assert overview is not None
assert len(overview["provider_equivalents"]) == 1
provider_equivalent = overview["provider_equivalents"][0]
assert provider_equivalent["estimated_payg_cost"] > 0
assert provider_equivalent["covered_usage_amount"] == 1
assert provider_equivalent["coverage_ratio"] == 1
assert provider_equivalent["equivalent_saved_cost"] < provider_equivalent["estimated_payg_cost"]
assert provider_equivalent["equivalent_saved_cost"] <= provider_equivalent["premium_reference_monthly_cost"]
assert overview["total_cost_saved"] == provider_equivalent["equivalent_saved_cost"]
assert overview["free_tier_equivalent_savings"]["cost_saved"] == provider_equivalent["equivalent_saved_cost"]
assert overview["direct_feature_savings"]["cost_saved"] == 0
tests/test_dashboard_proxy_events.py 0 → 100644
View file @ d1787cd5
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from aisbf.handlers import RequestHandler
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
class RequestStateStub:
def __init__(self, user_id=17, token_id=23):
self.user_id = user_id
self.token_id = token_id
class RequestStub:
def __init__(self):
self.url = SimpleNamespace(path="/api/test-provider/chat/completions")
self.method = "POST"
self.session = {"username": "alice"}
self.state = RequestStateStub()
self.headers = {}
class ProviderHandlerStub:
def __init__(self, response):
self.response = response
self.success_calls = 0
self.failure_calls = 0
def is_rate_limited(self):
return False
async def apply_rate_limit(self):
return None
async def handle_request(self, **kwargs):
return self.response
def record_success(self):
self.success_calls += 1
def record_failure(self):
self.failure_calls += 1
class AnalyticsStub:
def __init__(self):
self.calls = []
def record_request(self, **kwargs):
self.calls.append(kwargs)
def estimate_cost(self, provider_id, total_tokens, prompt_tokens, completion_tokens):
return 0.0
class DbStub:
def __init__(self):
self.events = []
self.prompt_runs = []
self.prompt_findings = []
def record_dashboard_event(self, **kwargs):
self.events.append(kwargs)
return len(self.events)
def get_user_providers(self, user_id):
return []
def get_user_rotations(self, user_id):
return []
def get_user_autoselects(self, user_id):
return []
def record_prompt_analysis_run(self, **kwargs):
self.prompt_runs.append(kwargs)
return len(self.prompt_runs)
def record_prompt_analysis_findings(self, run_id, findings):
self.prompt_findings.append({"run_id": run_id, "findings": findings})
class MysqlCursorStub:
def __init__(self, lastrowid=0, fallback_id=41):
self.lastrowid = lastrowid
self.fallback_id = fallback_id
self.executed = []
self._fetchone = None
def execute(self, sql, params=None):
self.executed.append((sql, params))
if "SELECT LAST_INSERT_ID()" in sql:
self._fetchone = (self.fallback_id,)
else:
self._fetchone = None
def fetchone(self):
return self._fetchone
class MysqlConnStub:
def __init__(self, cursor):
self._cursor = cursor
self.commit_calls = 0
def cursor(self):
return self._cursor
def commit(self):
self.commit_calls += 1
class MysqlConnCtxStub:
def __init__(self, conn):
self.conn = conn
def __enter__(self):
return self.conn
def __exit__(self, exc_type, exc, tb):
return False
class CacheStub:
def __init__(self, cached_response=None):
self.cached_response = cached_response
self.get_calls = 0
self.set_calls = 0
def get(self, *args, **kwargs):
self.get_calls += 1
return self.cached_response
def set(self, *args, **kwargs):
self.set_calls += 1
class ConfigStub:
def __init__(self, aisbf_config=None):
self._aisbf_config = aisbf_config
def get_aisbf_config(self):
return self._aisbf_config
def get_condensation(self):
return None
def resolve_feature_enabled(self, feature_name, **kwargs):
if feature_name == "response_cache":
return True
if feature_name == "prompt_security":
return True
if feature_name == "context_lens":
return True
if feature_name == "block_high_risk_prompts":
return False
if feature_name == "context_condensation":
return False
return False
@pytest.mark.asyncio
async def test_handle_chat_completion_records_dashboard_proxy_event(monkeypatch):
request = RequestStub()
response = {
"choices": [{"message": {"content": "Hello back"}}],
"usage": {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17},
}
provider_handler = ProviderHandlerStub(response)
analytics = AnalyticsStub()
db = DbStub()
monkeypatch.setattr("aisbf.handlers.get_provider_handler", lambda provider_id, api_key, user_id=None: provider_handler)
monkeypatch.setattr("aisbf.handlers.get_analytics", lambda: analytics)
monkeypatch.setattr("aisbf.handlers.DatabaseRegistry.get_config_database", staticmethod(lambda: db))
monkeypatch.setattr("aisbf.handlers.get_response_cache", lambda *args, **kwargs: type("CacheStub", (), {"get": lambda self, *a, **k: None, "set": lambda self, *a, **k: None})())
monkeypatch.setattr("aisbf.handlers.count_messages_tokens", lambda messages, model: 12)
monkeypatch.setattr("aisbf.handlers.get_context_config_for_model", lambda **kwargs: {})
monkeypatch.setattr("aisbf.handlers.get_max_request_tokens_for_model", lambda **kwargs: None)
monkeypatch.setattr(RequestHandler, "_settle_market_result", lambda *args, **kwargs: None)
handler = RequestHandler(user_id=17)
handler.config = ConfigStub(aisbf_config=SimpleNamespace(response_cache=SimpleNamespace(model_dump=lambda: {})))
handler.user_providers = {
"test-provider": {
"type": "openai",
"endpoint": "https://example.test/v1",
"api_key_required": False,
}
}
result = await handler.handle_chat_completion(
request,
"test-provider",
{
"model": "test-model",
"messages": [{"role": "user", "content": "Hello"}],
"stream": False,
},
)
assert result == response
assert provider_handler.success_calls == 1
assert len(db.events) == 1
event = db.events[0]
assert event["event_type"] == "request_proxied"
assert event["path"] == "/api/test-provider/chat/completions"
assert event["user_id"] == 17
assert event["username"] == "alice"
assert event["provider_id"] == "test-provider"
assert event["status_code"] == 200
assert event["metadata"]["model_name"] == "test-model"
assert event["metadata"]["prompt_tokens"] == 12
assert event["metadata"]["completion_tokens"] == 5
assert event["metadata"]["total_tokens"] == 17
assert event["metadata"]["stream"] is False
assert db.prompt_runs
assert db.prompt_runs[0]["provider_id"] == "test-provider"
assert db.prompt_findings
def test_record_dashboard_proxy_event_preserves_streaming_metadata(monkeypatch):
request = RequestStub()
db = DbStub()
monkeypatch.setattr("aisbf.handlers.DatabaseRegistry.get_config_database", staticmethod(lambda: db))
handler = RequestHandler()
handler.config = ConfigStub()
handler._record_dashboard_proxy_event(
request,
"test-provider",
"stream-model",
True,
18.5,
metadata={
"prompt_tokens": 14,
"completion_tokens": 9,
"total_tokens": 23,
"stream": True,
"rotation_id": "rotation-a",
},
)
assert len(db.events) == 1
event = db.events[0]
assert event["event_type"] == "request_proxied"
assert event["provider_id"] == "test-provider"
assert event["rotation_id"] == "rotation-a"
assert event["status_code"] == 200
assert event["metadata"]["model_name"] == "stream-model"
assert event["metadata"]["prompt_tokens"] == 14
assert event["metadata"]["completion_tokens"] == 9
assert event["metadata"]["total_tokens"] == 23
assert event["metadata"]["stream"] is True
@pytest.mark.asyncio
async def test_handle_chat_completion_records_autoselect_id_on_proxy_event(monkeypatch):
request = RequestStub()
response = {
"choices": [{"message": {"content": "Selected response"}}],
"usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12},
}
provider_handler = ProviderHandlerStub(response)
analytics = AnalyticsStub()
db = DbStub()
monkeypatch.setattr("aisbf.handlers.get_provider_handler", lambda provider_id, api_key, user_id=None: provider_handler)
monkeypatch.setattr("aisbf.handlers.get_analytics", lambda: analytics)
monkeypatch.setattr("aisbf.handlers.DatabaseRegistry.get_config_database", staticmethod(lambda: db))
monkeypatch.setattr("aisbf.handlers.get_response_cache", lambda *args, **kwargs: type("CacheStub", (), {"get": lambda self, *a, **k: None, "set": lambda self, *a, **k: None})())
monkeypatch.setattr("aisbf.handlers.count_messages_tokens", lambda messages, model: 8)
monkeypatch.setattr("aisbf.handlers.get_context_config_for_model", lambda **kwargs: {})
monkeypatch.setattr("aisbf.handlers.get_max_request_tokens_for_model", lambda **kwargs: None)
monkeypatch.setattr(RequestHandler, "_settle_market_result", lambda *args, **kwargs: None)
handler = RequestHandler(user_id=17)
handler.config = ConfigStub(aisbf_config=SimpleNamespace(response_cache=SimpleNamespace(model_dump=lambda: {})))
handler.user_providers = {
"test-provider": {
"type": "openai",
"endpoint": "https://example.test/v1",
"api_key_required": False,
}
}
await handler.handle_chat_completion(
request,
"test-provider",
{
"model": "test-model",
"messages": [{"role": "user", "content": "Hello"}],
"stream": False,
"_autoselect_id": "auto-main",
},
)
assert len(db.events) == 1
event = db.events[0]
assert event["autoselect_id"] == "auto-main"
assert event["metadata"]["autoselect_id"] == "auto-main"
def test_request_handler_resolves_provider_cache_override():
handler = RequestHandler(user_id=17)
class ConfigWithProviderOverride(ConfigStub):
def resolve_feature_enabled(self, feature_name, **kwargs):
if feature_name == "response_cache":
provider_config = kwargs.get("provider_config") or {}
return provider_config.get("enable_response_cache") is True
return super().resolve_feature_enabled(feature_name, **kwargs)
handler.config = ConfigWithProviderOverride(aisbf_config=SimpleNamespace(response_cache=SimpleNamespace(enabled=True, model_dump=lambda: {})))
assert handler._should_cache_response(provider_config={"enable_response_cache": False}) is False
assert handler._should_cache_response(provider_config={"enable_response_cache": True}) is True
def test_record_prompt_analysis_run_uses_mysql_last_insert_id_fallback():
from aisbf.database import DatabaseManager
manager = DatabaseManager.__new__(DatabaseManager)
manager.db_type = "mysql"
cursor = MysqlCursorStub(lastrowid=0, fallback_id=77)
conn = MysqlConnStub(cursor)
manager._get_connection = lambda: MysqlConnCtxStub(conn)
run_id = manager.record_prompt_analysis_run(
user_id=5,
token_id=9,
provider_id="provider-a",
model_name="model-a",
summary_json={"composition": {"has_tools": True}},
)
assert run_id == 77
assert any("SELECT LAST_INSERT_ID()" in sql for sql, _ in cursor.executed)
insert_sql, insert_params = cursor.executed[0]
assert "INSERT INTO prompt_analysis_runs" in insert_sql
assert '"has_tools": true' in insert_params[-1]
@pytest.mark.asyncio
async def test_handle_chat_completion_uses_cache_when_provider_override_enables_it(monkeypatch):
request = RequestStub()
cached_response = {
"choices": [{"message": {"content": "Cached"}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
provider_handler = ProviderHandlerStub({"choices": [{"message": {"content": "Live"}}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}})
analytics = AnalyticsStub()
db = DbStub()
cache = CacheStub(cached_response=cached_response)
monkeypatch.setattr("aisbf.handlers.get_provider_handler", lambda provider_id, api_key, user_id=None: provider_handler)
monkeypatch.setattr("aisbf.handlers.get_analytics", lambda: analytics)
monkeypatch.setattr("aisbf.handlers.DatabaseRegistry.get_config_database", staticmethod(lambda: db))
monkeypatch.setattr("aisbf.handlers.get_response_cache", lambda *args, **kwargs: cache)
monkeypatch.setattr("aisbf.handlers.count_messages_tokens", lambda messages, model: 12)
monkeypatch.setattr("aisbf.handlers.get_context_config_for_model", lambda **kwargs: {})
monkeypatch.setattr("aisbf.handlers.get_max_request_tokens_for_model", lambda **kwargs: None)
monkeypatch.setattr(RequestHandler, "_settle_market_result", lambda *args, **kwargs: None)
handler = RequestHandler(user_id=17)
handler.config = ConfigStub(aisbf_config=SimpleNamespace(response_cache=SimpleNamespace(enabled=True, model_dump=lambda: {})))
handler.user_providers = {
"test-provider": {
"type": "openai",
"endpoint": "https://example.test/v1",
"api_key_required": False,
"enable_response_cache": True,
}
}
result = await handler.handle_chat_completion(
request,
"test-provider",
{
"model": "test-model",
"messages": [{"role": "user", "content": "Hello"}],
"stream": False,
},
)
assert result == cached_response
assert cache.get_calls == 1
assert cache.set_calls == 0
assert provider_handler.success_calls == 0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment