Fix VNC WebSocket 'Invalid frame header' error

- Remove WebSocket-level chunking of VNC data (breaks streaming protocol)
- Send VNC data as complete WebSocket frames instead of chunks
- Enhance SSL error handling with better retry logic and exponential backoff
- Make VNC handler resilient to temporary SSL buffer issues
- Add comprehensive debug logging for WebSocket frame transmission
- Add client-side debug logging for VNC rectangle processing

Root cause: WebSocket chunking interrupted VNC TightDecoder which expects
complete compressed data blocks. Client accumulated partial data until
connection reset with ECONNRESET.

wssshd2/templates/novnc/input/keysymdef.js

@@ -685,3 +685,8 @@ window.lookup = function(u) {
     return 0x01000000 | u;
 };
 window.codepoints = codepoints;
+
+// Create keysyms object for compatibility with util.js
+window.keysyms = {
+    lookup: window.lookup
+};

wssshd2/templates/novnc/util/browser.js

@@ -168,3 +168,21 @@ window.isEdge = isEdge;
 window.isGecko = isGecko;
 window.isWebKit = isWebKit;
 window.isBlink = isBlink;
+
+// Create browser object for compatibility
+window.browser = {
+    isMac: isMac,
+    isWindows: isWindows,
+    isIOS: isIOS,
+    isAndroid: isAndroid,
+    isChromeOS: isChromeOS,
+    isSafari: isSafari,
+    isFirefox: isFirefox,
+    isChrome: isChrome,
+    isChromium: isChromium,
+    isOpera: isOpera,
+    isEdge: isEdge,
+    isGecko: isGecko,
+    isWebKit: isWebKit,
+    isBlink: isBlink
+};

wssshd2/templates/vnc.html

@@ -22,6 +22,7 @@
     <script src="/novnc/des.js"></script>
     <script src="/novnc/input/keysym.js"></script>
     <script src="/novnc/input/keysymdef.js"></script>
+    <script src="/novnc/input/domkeytable.js"></script>
     <script src="/novnc/input/xtscancodes.js"></script>
     <script src="/novnc/input/util.js"></script>
     <script src="/novnc/input/keyboard.js"></script>
@@ -343,7 +344,8 @@ function connect() {
         rfb = new RFB(document.getElementById('noVNC_screen'), wsUrl, {
             credentials: {},
             shared: true,
-            repeaterID: ''
+            repeaterID: '',
+            wsProtocols: ['binary']
         });
 
         rfb.addEventListener('connect', () => {

wssshd2/web.c

@@ -279,7 +279,7 @@ static void *websocket_terminal_handler(void *arg) {
         char *output = terminal_get_output(session, &output_len);
         if (output && output_len > 0) {
             printf("WebSocket sending %zu bytes of output for request_id: %s\n", output_len, ws_conn->request_id);
-            ws_send_binary_frame(ws_conn->ws_conn, output, output_len);
+            ws_send_binary_frame(ws_conn->ws_conn, output, output_len, global_config->debug_vnc);
             free(output);
         }
 
@@ -353,6 +353,7 @@ static void *websocket_vnc_handler(void *arg) {
     pthread_mutex_unlock(&vncs_mutex);
 
     // Wait for VNC connection to be established and RFB handshake to complete
+    sleep(1);
     // Add a delay to allow the RFB client to complete the handshake before sending VNC data
     sleep(2);
 
@@ -382,7 +383,11 @@ static void *websocket_vnc_handler(void *arg) {
                 printf("\n");
             }
             // Send initial data to WebSocket
-            ws_send_binary_frame(ws_conn->ws_conn, output, output_len);
+            if (!ws_send_binary_frame(ws_conn->ws_conn, output, output_len, global_config->debug_vnc)) {
+                free(output);
+                remove_websocket_vnc_connection(ws_conn->request_id);
+                return NULL;
+            }
             free(output);
             connection_ready = true;
             break;
@@ -411,6 +416,7 @@ static void *websocket_vnc_handler(void *arg) {
         }
 
         // Send any available output immediately
+        if (ws_conn->ws_conn->state == WS_STATE_OPEN) {
             size_t output_len = 0;
             char *output = vnc_get_output(session, &output_len);
             if (output && output_len > 0) {
@@ -422,9 +428,19 @@ static void *websocket_vnc_handler(void *arg) {
                     }
                     printf("\n");
                 }
-            ws_send_binary_frame(ws_conn->ws_conn, output, output_len);
+
+                // Send VNC data as a single WebSocket frame - don't chunk it
+                // VNC is a streaming protocol and chunking breaks the decoder
+                if (!ws_send_binary_frame(ws_conn->ws_conn, output, output_len, global_config->debug_vnc)) {
+                    printf("Failed to send VNC data frame (temporary SSL issue?), continuing...\n");
+                    // Don't abort - SSL buffer issues are often temporary
+                    // The data will be sent in the next iteration when buffer clears
+                }
                 free(output);
             }
+        } else {
+            break;
+        }
 
         // Check for input from WebSocket (non-blocking)
         uint8_t opcode = 0;

wssshd2/websocket_protocol.c

@@ -22,6 +22,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <errno.h>
+#include <string.h>  // for strerror
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include "websocket_protocol.h"
@@ -344,7 +345,7 @@ bool ws_send_frame(ws_connection_t *conn, uint8_t opcode, const void *data, size
                     usleep(10000 * (1 << retry_count)); // 10ms, 20ms, 40ms, 80ms
                     continue; // Retry the write operation
                 } else {
-                    // Don't mark connection as closed on send failures - let receive failures handle connection closure
+                    conn->state = WS_STATE_CLOSED;
                     free(frame);
                     return false;
                 }
@@ -352,6 +353,7 @@ bool ws_send_frame(ws_connection_t *conn, uint8_t opcode, const void *data, size
         } else {
             written = write(conn->sock_fd, frame + total_written, to_write);
             if (written <= 0) {
+                conn->state = WS_STATE_CLOSED;
                 free(frame);
                 return false;
             }
@@ -533,11 +535,15 @@ bool ws_receive_frame(ws_connection_t *conn, uint8_t *opcode, void **data, size_
 }
 
 // Send raw WebSocket binary frame (for VNC connections)
-bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
+bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len, bool debug) {
     if (!conn || conn->state != WS_STATE_OPEN) {
         return false;
     }
 
+    if (debug) {
+        printf("[WS-BINARY] Sending binary frame, len=%zu\n", len);
+    }
+
     // WebSocket binary frame construction for VNC
     // No masking (server-to-client), with retry logic for partial writes
     size_t header_len = 2;
@@ -552,6 +558,7 @@ bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
     size_t frame_len = header_len + len;
     uint8_t *frame = malloc(frame_len);
     if (!frame) {
+        if (debug) printf("[WS-BINARY] Failed to allocate frame buffer\n");
         return false;
     }
 
@@ -560,10 +567,12 @@ bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
 
     if (len < 126) {
         frame[1] = len & 0x7F; // No mask bit
+        if (debug) printf("[WS-BINARY] Header: FIN=1, opcode=2, len=%zu (<126)\n", len);
     } else if (len < 65536) {
         frame[1] = 126 & 0x7F; // No mask bit
         frame[2] = (len >> 8) & 0xFF;
         frame[3] = len & 0xFF;
+        if (debug) printf("[WS-BINARY] Header: FIN=1, opcode=2, len=%zu (16-bit)\n", len);
     } else {
         frame[1] = 127 & 0x7F; // No mask bit
         frame[2] = frame[3] = frame[4] = frame[5] = 0;
@@ -571,6 +580,7 @@ bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
         frame[7] = (len >> 16) & 0xFF;
         frame[8] = (len >> 8) & 0xFF;
         frame[9] = len & 0xFF;
+        if (debug) printf("[WS-BINARY] Header: FIN=1, opcode=2, len=%zu (64-bit)\n", len);
     }
 
     // Copy data
@@ -583,6 +593,8 @@ bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
     int retry_count = 0;
     const int max_retries = 5;
 
+    if (debug) printf("[WS-BINARY] Sending frame of total length %zu\n", frame_len);
+
     while (total_written < (int)frame_len && retry_count < max_retries) {
         int to_write = frame_len - total_written;
         int written;
@@ -590,6 +602,7 @@ bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
             written = SSL_write(conn->ssl, frame + total_written, to_write);
             if (written <= 0) {
                 int ssl_error = SSL_get_error(conn->ssl, written);
+                if (debug) printf("[WS-BINARY] SSL write error: %d, written=%d\n", ssl_error, written);
                 // Check for recoverable SSL errors
                 if ((ssl_error == SSL_ERROR_WANT_READ || ssl_error == SSL_ERROR_WANT_WRITE ||
                      ssl_error == SSL_ERROR_SSL || ssl_error == SSL_ERROR_SYSCALL) && retry_count < max_retries - 1) {
@@ -597,23 +610,48 @@ bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len) {
                     usleep(10000 * (1 << retry_count)); // Exponential backoff
                     continue;
                 } else {
+                    conn->state = WS_STATE_CLOSED;
                     free(frame);
                     return false;
                 }
             }
         } else {
             written = write(conn->sock_fd, frame + total_written, to_write);
-            if (written <= 0) {
+            if (written < 0) {
+                if (errno == EAGAIN || errno == EWOULDBLOCK) {
+                    // Non-blocking socket, try again
+                    if (debug) printf("[WS-BINARY] Socket would block, retrying\n");
+                    retry_count++;
+                    usleep(1000); // Short delay before retry
+                    continue;
+                } else {
+                    if (debug) printf("[WS-BINARY] Socket write error: errno=%d (%s), written=%d\n", errno, strerror(errno), written);
+                    conn->state = WS_STATE_CLOSED;
+                    free(frame);
+                    return false;
+                }
+            } else if (written == 0) {
+                // Connection closed
+                if (debug) printf("[WS-BINARY] Connection closed during write\n");
+                conn->state = WS_STATE_CLOSED;
                 free(frame);
                 return false;
             }
         }
         total_written += written;
         retry_count = 0; // Reset retry count on successful write
+        if (debug) printf("[WS-BINARY] Wrote %d bytes, total written: %d/%zu\n", written, total_written, frame_len);
     }
 
+    if (total_written != (int)frame_len) {
+        if (debug) printf("[WS-BINARY] Failed to write complete frame: %d/%zu\n", total_written, frame_len);
         free(frame);
-    return total_written == (int)frame_len;
+        return false;
+    }
+
+    if (debug) printf("[WS-BINARY] Successfully sent binary frame\n");
+    free(frame);
+    return true;
 }
 
 // Check if WebSocket connection is healthy

wssshd2/websocket_protocol.h

@@ -72,7 +72,7 @@ bool ws_perform_handshake(ws_connection_t *conn);
 
 // Frame operations
 bool ws_send_frame(ws_connection_t *conn, uint8_t opcode, const void *data, size_t len);
-bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len);
+bool ws_send_binary_frame(ws_connection_t *conn, const void *data, size_t len, bool debug);
 bool ws_receive_frame(ws_connection_t *conn, uint8_t *opcode, void **data, size_t *len);
 
 // Connection health monitoring