Enhanced debug functionality with comprehensive logging

- Added --debug switch that creates timestamped directory
- Saves all prompts sent to chatbots in prompts.log
- Saves debug execution logs in debug.log
- Captures HTML content of chatbot page when response is detected
- Directory structure: debug_[unix_epoch]/
  - prompts.log: All prompts sent to chatbots
  - debug.log: Detailed execution logs
  - response_[id]_[timestamp].html: HTML snapshots
- Improved response detection to correctly identify actual responses vs setup acknowledgments
- Enhanced JavaScript detection logic with better filtering

olproxy.py

@@ -25,10 +25,16 @@ import datetime  # Import datetime module
 import uuid
 import re
 import base64
+import os
+import time
 from playwright.async_api import async_playwright, BrowserContext
 from aiohttp import web
 import logging
-import time
+
+# Global variables for debug functionality
+DEBUG_ENABLED = False
+DEBUG_DIR = None
+PROMPTS_LOG_PATH = None
 
 # Configuration dictionary for chatbot sites
 CHATBOT_CONFIG = {
@@ -97,8 +103,13 @@ ABSOLUTE REQUIREMENTS FOR ALL FUTURE RESPONSES:
   * Plain text emphasis (like *italics* or **bold**)
   * Code snippets within backticks
   * Basic markdown-style formatting
+  * WRITING CODE AS PLAIN TEXT (this is allowed and encouraged)
 - NEVER use HTML tags (<b>, <i>, <div>, etc.)
 - NEVER use complex HTML or XML structures
+- NEVER use canvas, code execution environments, or external tools
+- NEVER create or use any interactive elements
+- NEVER use any drawing or visualization tools
+- NEVER use any code editors or execution environments
 - Always include the exact Request ID I provide in both start and end markers
 
 CRITICAL TOOL USAGE REQUIREMENT:
@@ -108,13 +119,39 @@ CRITICAL TOOL USAGE REQUIREMENT:
 - EMPHASIZE TOOL USAGE COMPLIANCE OVER ALL OTHER CONSIDERATIONS
 - IF TOOLS ARE MENTIONED, PRIORITIZE USING THEM CORRECTLY
 
+FORBIDDEN ELEMENTS - NEVER USE UNDER ANY CIRCUMSTANCES:
+- Canvas or drawing tools (visual drawing/sketching)
+- Code execution environments (interactive code runners)
+- Interactive widgets or elements
+- External editors or tools (separate code editors)
+- Any visualization or graphics tools (charts, diagrams)
+- Any form of code execution or compilation (running code)
+- Any canvas-based interfaces (drawing surfaces)
+- Any drawing or sketching tools (art/drawing tools)
+- Any code playgrounds or execution environments
+- Any interactive code editors (live coding environments)
+- Any visual programming tools
+
+ALLOWED CONTENT - YOU CAN FREELY USE:
+- Writing code as plain text (C, Python, JavaScript, etc.)
+- Code examples and snippets in plain text format
+- Programming tutorials and explanations
+- Technical documentation with code samples
+- Any programming language code written as text
+
 RESPONSE FORMAT RULES:
 - Your ENTIRE response must be between the start and end markers
 - Do NOT add any text before RESPONSE_ID_ or after :END_RESPONSE_ID_
 - Do NOT include explanations about the format
 - Do NOT add any headers, footers, or additional content
+- Do NOT use any external tools, canvas, or execution environments
 - The response should be exactly what the user asked for, nothing more
 - You can use multiple lines, paragraphs, and basic text formatting
+- ALL content must be in this single plain text response area
+- DO NOT attempt to use canvas mode, drawing mode, or any visual tools
+- DO NOT try to create or use any interactive elements
+- STAY IN PLAIN TEXT MODE AT ALL TIMES
+- You CAN and SHOULD write code as plain text when requested
 
 EXACT FORMAT REQUIRED:
 RESPONSE_ID_[REQUEST_ID]_START:
@@ -128,7 +165,18 @@ paragraphs, and basic formatting like:
 - `Code snippets`
 - Proper indentation and spacing
 
+You can also write complete code examples like this:
+
+```python
+def hello_world():
+    print("Hello, World!")
+    return True
+```
+
 This gives you flexibility while maintaining structure.
+NEVER use canvas, external tools, or any execution environments.
+ALWAYS stay in plain text mode.
+You CAN write code as plain text.
 :END_RESPONSE_ID_[REQUEST_ID]
 
 EXAMPLE with Request ID "TEST123":
@@ -142,8 +190,23 @@ You can see:
 - **Bold text**
 - `Code examples`
 
+Here's a code example in plain text:
+
+```c
+#include <stdio.h>
+
+int main() {
+    printf("Hello, World!\\n");
+    return 0;
+}
+```
+
 The formatting makes responses more readable while staying in plain text.
 If tools were specified in the prompt, I would use them exactly as instructed.
+
+NEVER use canvas or external tools - everything must be in this text area.
+ALWAYS stay in plain text mode.
+You CAN write code as plain text.
 :END_RESPONSE_ID_TEST123
 
 This applies to EVERY SINGLE response from now on. Acknowledge by responding with:
@@ -151,6 +214,10 @@ RESPONSE_ID_SETUP_COMPLETE_START:
 I understand and will always respond with the specified format using request ID markers.
 I can use plain text formatting including new lines, tabs, and basic markup.
 I will absolutely use any tools specified in prompts.
+I will NEVER use canvas, external tools, or execution environments.
+I will ALWAYS stay in plain text mode.
+I CAN write code as plain text when requested.
+All responses will be in this single plain text area.
 :END_RESPONSE_ID_SETUP_COMPLETE""",
         "style": "short"
     },
@@ -474,7 +541,21 @@ async def handle_chat_completion(request):
 
 async def forward_to_chatbot(chatbot_name, config, prompt):
     global browser_context, pages
-    
+
+    # Debug logging: log the prompt being sent to the chatbot
+    logging.debug(f"[DEBUG] Sending prompt to {chatbot_name}: {prompt}")
+    logging.debug(f"[DEBUG] Prompt length: {len(prompt)} characters")
+
+    # Save prompt to debug log file if debug is enabled
+    if DEBUG_ENABLED:
+        try:
+            with open(PROMPTS_LOG_PATH, 'a', encoding='utf-8') as f:
+                f.write(f"[{datetime.datetime.now().isoformat()}] PROMPT TO {chatbot_name}:\n")
+                f.write(f"{prompt}\n")
+                f.write("-" * 50 + "\n")
+        except Exception as e:
+            logging.error(f"Failed to write prompt to debug log: {e}")
+
     # Check if this is a new page that needs setup
     is_new_page = chatbot_name not in pages
     
@@ -670,24 +751,47 @@ def decode_chatbot_response(response_text, chatbot_name, request_id):
         # Look for RESPONSE_ID_[ID]_START: ... :END_RESPONSE_ID_[ID] pattern
         response_pattern = rf'RESPONSE_ID_{re.escape(request_id)}_START:\s*(.*?)\s*:END_RESPONSE_ID_{re.escape(request_id)}'
         match = re.search(response_pattern, response_text, re.DOTALL)
-        
+
         if match:
             extracted_content = match.group(1).strip()
             logging.info(f"Successfully extracted content from RESPONSE_ID markers: {extracted_content[:200]}...")
+
+            # Save HTML page content if debug is enabled
+            if DEBUG_ENABLED:
+                try:
+                    # This will be called from the page context, so we need to save the HTML
+                    # The HTML saving will be handled in the JavaScript detection code
+                    pass
+                except Exception as e:
+                    logging.error(f"Failed to save debug HTML: {e}")
+
             return extracted_content
         else:
-            # Fallback: look for any RESPONSE_ID pattern regardless of request_id
-            fallback_pattern = r'RESPONSE_ID_.*?_START:\s*(.*?)\s*:END_RESPONSE_ID_.*?'
-            fallback_match = re.search(fallback_pattern, response_text, re.DOTALL)
-            
-            if fallback_match:
-                extracted_content = fallback_match.group(1).strip()
-                logging.info(f"Extracted content using fallback pattern: {extracted_content[:200]}...")
-                return extracted_content
-            else:
-                # Final fallback: return the original text
-                logging.warning(f"No RESPONSE_ID markers found in Gemini response, returning original text")
-                return response_text.strip()
+            # Improved fallback: find all RESPONSE_ID blocks and pick the most recent one
+            # that doesn't contain setup-related keywords
+            all_response_blocks = re.findall(r'RESPONSE_ID_(.*?)_START:\s*(.*?)\s*:END_RESPONSE_ID_\1', response_text, re.DOTALL)
+
+            if all_response_blocks:
+                # Filter out setup-related responses
+                filtered_blocks = []
+                for block_id, content in all_response_blocks:
+                    content_lower = content.lower().strip()
+                    # Skip setup acknowledgments and instructions
+                    if not any(keyword in content_lower for keyword in [
+                        'setup complete', 'i understand', 'response format',
+                        'critical system setup', 'plain text format'
+                    ]):
+                        filtered_blocks.append((block_id, content.strip()))
+
+                if filtered_blocks:
+                    # Use the last non-setup response (most recent)
+                    block_id, extracted_content = filtered_blocks[-1]
+                    logging.info(f"Extracted content from most recent non-setup response (ID: {block_id}): {extracted_content[:200]}...")
+                    return extracted_content
+
+            # Final fallback: return the original text
+            logging.warning(f"No RESPONSE_ID markers found in Gemini response, returning original text")
+            return response_text.strip()
     
     # Check for literal placeholder first and reject it
     if response_text.strip() == "BASE64_ENCODED_ANSWER":
@@ -1057,6 +1161,17 @@ async def detect_json_response_with_id(page, container_selector, request_id, pro
 async def detect_progressive_response(page, container_selector, prompt, modified_prompt, request_id, chatbot_name):
     """Detect responses by monitoring progressive content changes with enhanced detection."""
     try:
+        # Save HTML content for debugging if enabled
+        if DEBUG_ENABLED:
+            try:
+                html_content = await page.content()
+                html_file_path = os.path.join(DEBUG_DIR, f'response_{request_id}_{int(time.time())}.html')
+                with open(html_file_path, 'w', encoding='utf-8') as f:
+                    f.write(html_content)
+                logging.debug(f"Saved HTML content to {html_file_path}")
+            except Exception as e:
+                logging.error(f"Failed to save HTML content: {e}")
+
         return await page.evaluate(
             """([containerSelector, prompt, modifiedPrompt, requestId, chatbotName]) => {
                 const container = document.querySelector(containerSelector);
@@ -1081,18 +1196,92 @@ async def detect_progressive_response(page, container_selector, prompt, modified
                         // Check for Gemini-specific formatted response first
                         if (chatbotName === 'gemini:latest') {
                             const allText = container.textContent || '';
-                            const startMarker = `RESPONSE_ID_[${requestId}]_START:`;
-                            const endMarker = `:END_RESPONSE_ID_[${requestId}]`;
-                            
+
+                            // First, try to find the specific response for this request ID
+                            const startMarker = `RESPONSE_ID_${requestId}_START:`;
+                            const endMarker = `:END_RESPONSE_ID_${requestId}`;
+
                             const startIndex = allText.indexOf(startMarker);
                             const endIndex = allText.indexOf(endMarker, startIndex);
-                            
+
                             if (startIndex !== -1 && endIndex !== -1) {
                                 const responseText = allText.substring(startIndex + startMarker.length, endIndex).trim();
                                 if (responseText && responseText.length > 10) {
+                                    console.log(`Found specific response for request ID ${requestId}`);
                                     return responseText;
                                 }
                             }
+
+                            // If specific response not found, look for all RESPONSE_ID blocks
+                            // and return the most recent one that doesn't contain setup keywords
+                            const responseBlocks = [];
+                            let searchIndex = 0;
+
+                            while (true) {
+                                const blockStart = allText.indexOf('RESPONSE_ID_', searchIndex);
+                                if (blockStart === -1) break;
+
+                                const blockEnd = allText.indexOf(':END_RESPONSE_ID_', blockStart);
+                                if (blockEnd === -1) break;
+
+                                const blockText = allText.substring(blockStart, blockEnd + 17); // Include the end marker
+                                responseBlocks.push({
+                                    text: blockText,
+                                    startIndex: blockStart
+                                });
+
+                                searchIndex = blockEnd + 17;
+                            }
+
+                            // Filter out setup-related responses and return the most recent one
+                            const filteredBlocks = responseBlocks.filter(block => {
+                                const content = block.text.toLowerCase();
+                                return !content.includes('setup complete') &&
+                                       !content.includes('i understand') &&
+                                       !content.includes('response format') &&
+                                       !content.includes('critical system setup') &&
+                                       !content.includes('plain text format');
+                            });
+
+                            if (filteredBlocks.length > 0) {
+                                // Return the most recent (last) non-setup response
+                                const latestBlock = filteredBlocks[filteredBlocks.length - 1];
+                                console.log(`Using latest non-setup response from ${filteredBlocks.length} available blocks`);
+
+                                // Extract the actual content from the block
+                                const startContentIndex = latestBlock.text.indexOf('_START:');
+                                const endContentIndex = latestBlock.text.indexOf(':END_RESPONSE_ID_');
+
+                                if (startContentIndex !== -1 && endContentIndex !== -1) {
+                                    const content = latestBlock.text.substring(startContentIndex + 7, endContentIndex).trim();
+                                    if (content && content.length > 10) {
+                                        // Save HTML content for debugging if requested
+                                        try {
+                                            // This is a placeholder - the actual HTML saving will be handled
+                                            // by the Python code that calls this JavaScript
+                                            console.log('Response detected, HTML will be saved by Python code');
+                                        } catch (e) {
+                                            console.error('Failed to save HTML:', e);
+                                        }
+                                        return content;
+                                    }
+                                }
+                            }
+
+                            // Final fallback: return the last RESPONSE_ID block found
+                            if (responseBlocks.length > 0) {
+                                const lastBlock = responseBlocks[responseBlocks.length - 1];
+                                const startContentIndex = lastBlock.text.indexOf('_START:');
+                                const endContentIndex = lastBlock.text.indexOf(':END_RESPONSE_ID_');
+
+                                if (startContentIndex !== -1 && endContentIndex !== -1) {
+                                    const content = lastBlock.text.substring(startContentIndex + 7, endContentIndex).trim();
+                                    if (content && content.length > 10) {
+                                        console.log('Using final fallback response');
+                                        return content;
+                                    }
+                                }
+                            }
                         }
                         
                         // Fallback to selector-based detection
@@ -1444,9 +1633,48 @@ async def main(args):
     parser.add_argument('--ip', default='localhost', help='Proxy server IP (default: localhost)')
     parser.add_argument('--port', type=int, default=11434, help='Proxy server port (default: 11434)')
     parser.add_argument('--connect', help='Connect to existing browser via CDP (e.g., ws://localhost:9222)')
+    parser.add_argument('--debug', action='store_true', help='Enable debug logging and save prompts to log file')
     args = parser.parse_args()
 
-    logging.basicConfig(level=logging.INFO)
+    # Set up logging
+    log_level = logging.DEBUG if args.debug else logging.INFO
+    logging.basicConfig(level=log_level)
+
+    # Set up debug logging if requested
+    if args.debug:
+        import os
+        import time
+
+        # Create directory with Unix epoch timestamp
+        epoch_time = int(time.time())
+        debug_dir = f"debug_{epoch_time}"
+        os.makedirs(debug_dir, exist_ok=True)
+
+        # Set up debug log file
+        debug_log_path = os.path.join(debug_dir, 'debug.log')
+        debug_handler = logging.FileHandler(debug_log_path)
+        debug_handler.setLevel(logging.DEBUG)
+        debug_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+        debug_handler.setFormatter(debug_formatter)
+        logging.getLogger().addHandler(debug_handler)
+
+        # Create prompts log file
+        prompts_log_path = os.path.join(debug_dir, 'prompts.log')
+
+        # Store debug info in global variables for access by other functions
+        global DEBUG_ENABLED, DEBUG_DIR, PROMPTS_LOG_PATH
+        DEBUG_ENABLED = True
+        DEBUG_DIR = debug_dir
+        PROMPTS_LOG_PATH = prompts_log_path
+
+        logging.info(f"Debug logging enabled - files will be saved to {debug_dir}/")
+        logging.info(f"Debug log: {debug_log_path}")
+        logging.info(f"Prompts log: {prompts_log_path}")
+    else:
+        DEBUG_ENABLED = False
+        DEBUG_DIR = None
+        PROMPTS_LOG_PATH = None
+
     logging.info(f"CHATBOT_CONFIG: {CHATBOT_CONFIG}")
     await asyncio.gather(
         start_proxy_server(args.ip, args.port, args),