Implement base64 encoding for chatbot responses to avoid formatting issues

- Change prompt instruction to request base64 encoded responses instead of escaped text
- Add base64 import and decoding logic in forward_to_chatbot function
- Update JSON validation to verify base64 format using regex and atob() validation
- Add comprehensive error handling for base64 decoding with fallback to raw response
- Enhanced logging to track base64 decoding success/failure
- Prevent issues with special characters, HTML entities, and complex formatting

This approach eliminates problems with HTML encoding, escape sequences,
and special characters by having the chatbot encode the entire response
in base64 format, which we then decode on the proxy side.

olproxy.py

@@ -24,6 +24,7 @@ import asyncio
 import datetime  # Import datetime module
 import uuid
 import re
+import base64
 from playwright.async_api import async_playwright, BrowserContext
 from aiohttp import web
 import logging
@@ -369,8 +370,8 @@ async def forward_to_chatbot(chatbot_name, config, prompt):
     # Generate unique ID for this request using UUID for better uniqueness
     request_id = str(uuid.uuid4()).replace('-', '')[:16]  # 16-character unique ID
     
-    # Create JSON-based prompt with unique ID
-    json_instruction = f'Format all responses as a raw JSON object with a "response" key containing the answer as a string, exactly as it would appear unfiltered from an API, including all content (code, explanations, instructions) with escaped newlines and special characters as well as command and special formatting from the API, use tools and agents as declared in the question when needed, include any formatting, xml, or any markup language as requested in the reponse key as well, and a second "id" key containing a unique id "{request_id}, nothing should be outside of the json, the whole reponse include in it. \n\n'
+    # Create JSON-based prompt with unique ID and base64 encoding
+    json_instruction = f'Format all responses as a raw JSON object with a "response" key containing the answer encoded in base64 format, and a second "id" key containing the unique id "{request_id}". The response should contain the complete answer exactly as it would appear unfiltered from an API, including all content (code, explanations, instructions, formatting, XML, markup language, etc.). Encode the entire response content in base64 before placing it in the "response" key. Nothing should be outside of the JSON object. \n\n'
     
     modified_prompt = f"{json_instruction}\n\n{prompt}"
     logging.info(f"Request ID: {request_id}, Modified prompt: {modified_prompt}")
@@ -413,6 +414,18 @@ async def forward_to_chatbot(chatbot_name, config, prompt):
         logging.info(f"Progressive detection failed, using latest response fallback for {chatbot_name} (Request ID: {request_id})")
         response_text = await detect_latest_response(page, container_selector, prompt, modified_prompt, request_id, chatbot_name)
 
+    # Decode base64 response if we got a valid response
+    if response_text and not response_text.startswith("Error:"):
+        try:
+            # Try to decode the base64 response
+            decoded_response = base64.b64decode(response_text).decode('utf-8')
+            logging.info(f"Successfully decoded base64 response from {chatbot_name} (Request ID: {request_id}): {decoded_response[:200]}...")
+            return decoded_response.strip()
+        except Exception as e:
+            logging.warning(f"Failed to decode base64 response from {chatbot_name} (Request ID: {request_id}): {str(e)}")
+            logging.info(f"Returning raw response: {response_text[:200]}...")
+            return response_text.strip()
+    
     logging.info(f"Final response from {chatbot_name} (Request ID: {request_id}): {response_text[:200] if response_text else 'None'}...")
     return (response_text or "Error: No response detected").strip()
 
@@ -638,11 +651,27 @@ async def detect_json_response_with_id(page, container_selector, request_id, pro
                                                    typeof jsonObj.response === 'string' &&
                                                    jsonObj.response.trim().length > 0;
                             
-                            if (hasValidId && hasValidResponse) {
-                                console.log(`✓ Complete valid JSON found - ID: ${jsonObj.id}, Response length: ${jsonObj.response.length}`);
+                            // Additional validation: check if response looks like base64
+                            let isValidBase64 = false;
+                            if (hasValidResponse) {
+                                try {
+                                    // Basic base64 validation - should be valid base64 string
+                                    const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
+                                    isValidBase64 = base64Regex.test(jsonObj.response.trim());
+                                    if (isValidBase64) {
+                                        // Try to decode to verify it's valid base64
+                                        atob(jsonObj.response.trim());
+                                    }
+                                } catch (e) {
+                                    isValidBase64 = false;
+                                }
+                            }
+                            
+                            if (hasValidId && hasValidResponse && isValidBase64) {
+                                console.log(`✓ Complete valid JSON found - ID: ${jsonObj.id}, Base64 Response length: ${jsonObj.response.length}`);
                                 return true;
                             } else {
-                                console.log(`✗ Incomplete JSON - ID valid: ${hasValidId}, Response valid: ${hasValidResponse}`);
+                                console.log(`✗ Incomplete JSON - ID valid: ${hasValidId}, Response valid: ${hasValidResponse}, Base64 valid: ${isValidBase64}`);
                                 return false;
                             }
                         } catch (e) {