Enhanced debug logging with request ID tracking and response logging

- Added request ID logging to prompts.log for proper correlation
- Log decoded responses in prompts.log to show what gets sent back via API
- Improved HTML debug output with body-only extraction and response markers
- Better request ID consistency throughout the system
- Enhanced debug information for troubleshooting response detection

olproxy.py

@@ -642,6 +642,14 @@ async def forward_to_chatbot(chatbot_name, config, prompt):
     
     logging.info(f"Request ID: {request_id}, User prompt: {prompt}")
 
+    # Log request ID to debug file if debug is enabled
+    if DEBUG_ENABLED:
+        try:
+            with open(PROMPTS_LOG_PATH, 'a', encoding='utf-8') as f:
+                f.write(f"[{datetime.datetime.now().isoformat()}] REQUEST_ID: {request_id}\n")
+        except Exception as e:
+            logging.error(f"Failed to write request ID to debug log: {e}")
+
     try:
         # Wait a moment to ensure previous interaction is complete
         await asyncio.sleep(1.0)
@@ -732,7 +740,17 @@ async def forward_to_chatbot(chatbot_name, config, prompt):
         
         # Log the final decoded response
         logging.info(f"Final decoded response from {chatbot_name} (Request ID: {request_id}): {decoded_response[:200] if decoded_response else 'None'}...")
-        
+
+        # Log the decoded response to debug file if debug is enabled
+        if DEBUG_ENABLED:
+            try:
+                with open(PROMPTS_LOG_PATH, 'a', encoding='utf-8') as f:
+                    f.write(f"[{datetime.datetime.now().isoformat()}] DECODED_RESPONSE:\n")
+                    f.write(f"{decoded_response if decoded_response else 'None'}\n")
+                    f.write("-" * 50 + "\n")
+            except Exception as e:
+                logging.error(f"Failed to write decoded response to debug log: {e}")
+
         return decoded_response.strip()
     
     logging.info(f"Final response from {chatbot_name} (Request ID: {request_id}): {response_text[:200] if response_text else 'None'}...")
@@ -1167,23 +1185,79 @@ async def detect_progressive_response(page, container_selector, prompt, modified
                 html_content = await page.content()
                 html_file_path = os.path.join(DEBUG_DIR, f'response_{request_id}_{int(time.time())}.html')
 
-                # Try to format HTML with indentation for readability
+                # Extract body content and add markers around detected reply area
                 try:
                     from bs4 import BeautifulSoup
                     soup = BeautifulSoup(html_content, 'html.parser')
-                    formatted_html = soup.prettify()
+
+                    # Find body tag
+                    body = soup.find('body')
+                    if body:
+                        # Get the text content of the body
+                        body_content = str(body)
+
+                        # Try to find the detected response area and add markers
+                        all_text = soup.get_text()
+                        start_marker = f'RESPONSE_ID_{request_id}_START:'
+                        end_marker = f':END_RESPONSE_ID_{request_id}'
+
+                        if start_marker in all_text and end_marker in all_text:
+                            # Add visual markers around the response area
+                            start_index = body_content.find(start_marker)
+                            end_index = body_content.find(end_marker) + len(end_marker)
+
+                            if start_index != -1 and end_index != -1:
+                                # Add markers with empty lines
+                                marked_content = (
+                                    body_content[:start_index] +
+                                    '\n\n' + '='*50 + ' START OF DETECTED RESPONSE ' + '='*50 + '\n\n' +
+                                    body_content[start_index:end_index] +
+                                    '\n\n' + '='*50 + ' END OF DETECTED RESPONSE ' + '='*50 + '\n\n' +
+                                    body_content[end_index:]
+                                )
+                                body_content = marked_content
+
+                        # Format the body content
+                        formatted_html = f"""<!DOCTYPE html>
+<html>
+<head>
+    <title>Debug HTML - Request ID: {request_id}</title>
+    <style>
+        body {{ font-family: monospace; white-space: pre-wrap; }}
+        .marker {{ background-color: #ffff00; font-weight: bold; }}
+    </style>
+</head>
+<body>
+{body_content}
+</body>
+</html>"""
+                    else:
+                        # Fallback if no body tag found
+                        formatted_html = f"""<!DOCTYPE html>
+<html>
+<head>
+    <title>Debug HTML - Request ID: {request_id}</title>
+</head>
+<body>
+{html_content}
+</body>
+</html>"""
+
                 except ImportError:
-                    # If BeautifulSoup is not available, fall back to basic formatting
-                    import re
-                    formatted_html = re.sub(r'>\s*<', '>\n<', html_content)
-                    formatted_html = re.sub(r'(<[^/][^>]*>)', r'\n\1', formatted_html)
-                    formatted_html = re.sub(r'(</[^>]+>)', r'\1\n', formatted_html)
-                    # Clean up excessive newlines
-                    formatted_html = re.sub(r'\n{3,}', '\n\n', formatted_html).strip()
+                    # If BeautifulSoup is not available, create basic HTML structure
+                    formatted_html = f"""<!DOCTYPE html>
+<html>
+<head>
+    <title>Debug HTML - Request ID: {request_id}</title>
+</head>
+<body>
+{html_content}
+</body>
+</html>"""
 
                 with open(html_file_path, 'w', encoding='utf-8') as f:
                     f.write(formatted_html)
-                logging.debug(f"Saved formatted HTML content to {html_file_path}")
+                logging.debug(f"Saved cleaned HTML content to {html_file_path}")
             except Exception as e:
                 logging.error(f"Failed to save HTML content: {e}")