Add sha1.*. Remove UTF-8 encode. Protocol handling.

Add common/sha1.h and common/sha1.c so that we have the SHA routines
even if openssl is not available. From the IETF SHA RFC example code.

Remove the UTF-8 encoding hack. This was really just an experiment.

If the protocol passed in the handshake has "binary" then don't base64
encode for the HyBi protocol. This will allow noVNC to request the
binary data be passed raw and not base64 encoded. Unfortunately, the
client doesn't speak first in VNC protocol (bad original design). If
it did then we could determine whether to base64 encode or not based
on the first HyBi frame from the client and whether the binary bit is
set or not. Oh well.

Misc Cleanup:

- Always free response and buf in handshake routine.

- Remove some unused variables.

common/sha1.h

+/*
+ * Copyright (C) The Internet Society (2001).  All Rights Reserved.
+ *
+ * This document and translations of it may be copied and furnished to
+ * others, and derivative works that comment on or otherwise explain it
+ * or assist in its implementation may be prepared, copied, published
+ * and distributed, in whole or in part, without restriction of any
+ * kind, provided that the above copyright notice and this paragraph are
+ * included on all such copies and derivative works.  However, this
+ * document itself may not be modified in any way, such as by removing
+ * the copyright notice or references to the Internet Society or other
+ * Internet organizations, except as needed for the purpose of
+ * developing Internet standards in which case the procedures for
+ * copyrights defined in the Internet Standards process must be
+ * followed, or as required to translate it into languages other than
+ * English.
+ *
+ * The limited permissions granted above are perpetual and will not be
+ * revoked by the Internet Society or its successors or assigns.
+ *
+ * This document and the information contained herein is provided on an
+ * "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING
+ * TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION
+ * HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/*
+ *  sha1.h
+ *
+ *  Description:
+ *      This is the header file for code which implements the Secure
+ *      Hashing Algorithm 1 as defined in FIPS PUB 180-1 published
+ *      April 17, 1995.
+ *
+ *      Many of the variable names in this code, especially the
+ *      single character names, were used because those were the names
+ *      used in the publication.
+ *
+ *      Please read the file sha1.c for more information.
+ *
+ */
+
+
+#ifndef _SHA1_H_
+#define _SHA1_H_
+
+#include <stdint.h>
+/*
+ * If you do not have the ISO standard stdint.h header file, then you
+ * must typdef the following:
+ *    name              meaning
+ *  uint32_t         unsigned 32 bit integer
+ *  uint8_t          unsigned 8 bit integer (i.e., unsigned char)
+ *  int_least16_t    integer of >= 16 bits
+ *
+ */
+
+#ifndef _SHA_enum_
+#define _SHA_enum_
+enum
+{
+    shaSuccess = 0,
+    shaNull,            /* Null pointer parameter */
+    shaInputTooLong,    /* input data too long */
+    shaStateError       /* called Input after Result */
+};
+#endif
+#define SHA1HashSize 20
+
+/*
+ *  This structure will hold context information for the SHA-1
+ *  hashing operation
+ */
+typedef struct SHA1Context
+{
+    uint32_t Intermediate_Hash[SHA1HashSize/4]; /* Message Digest  */
+
+    uint32_t Length_Low;            /* Message length in bits      */
+    uint32_t Length_High;           /* Message length in bits      */
+
+                               /* Index into message block array   */
+    int_least16_t Message_Block_Index;
+    uint8_t Message_Block[64];      /* 512-bit message blocks      */
+
+    int Computed;               /* Is the digest computed?         */
+    int Corrupted;             /* Is the message digest corrupted? */
+} SHA1Context;
+
+/*
+ *  Function Prototypes
+ */
+int SHA1Reset(  SHA1Context *);
+int SHA1Input(  SHA1Context *,
+                const uint8_t *,
+                unsigned int);
+int SHA1Result( SHA1Context *,
+                uint8_t Message_Digest[SHA1HashSize]);
+
+#endif

libvncserver/Makefile.am

@@ -24,7 +24,7 @@ WEBSOCKETSSSLSRCS = rfbssl_none.c
 #endif
 endif
 
-WEBSOCKETSSRCS = websockets.c ../common/md5.c $(WEBSOCKETSSSLSRCS)
+WEBSOCKETSSRCS = websockets.c ../common/md5.c ../common/sha1.c $(WEBSOCKETSSSLSRCS)
 endif
 
 includedir=$(prefix)/include/rfb

libvncserver/websockets.c

@@ -31,8 +31,10 @@
 /* errno */
 #include <errno.h>
 
-#include <md5.h>
 #include <byteswap.h>
+#include <string.h>
+#include "md5.h"
+#include "sha1.h"
 #include "rfbconfig.h"
 #include "rfbssl.h"
 
@@ -58,10 +60,12 @@ enum {
   WEBSOCKETS_VERSION_HYBI
 };
 
+#if 0
 #include <sys/syscall.h>
 static int gettid() {
     return (int)syscall(SYS_gettid);
 }
+#endif
 
 typedef int (*wsEncodeFunc)(rfbClientPtr cl, const char *src, int len, char **dst);
 typedef int (*wsDecodeFunc)(rfbClientPtr cl, char *dst, int len);
@@ -162,23 +166,36 @@ min (int a, int b) {
     return a < b ? a : b;
 }
 
-#ifdef LIBVNCSERVER_WITH_CLIENT_GCRYPT
-#else
-#include <openssl/sha.h>
 
-static void webSocketsGenSha1Key(char *target, int size, char *key)
+void
+webSocketsGenSha1Key(char * target, int size, char *key)
 {
-    SHA_CTX c;
-    unsigned char tmp[SHA_DIGEST_LENGTH];
-
-    SHA1_Init(&c);
-    SHA1_Update(&c, key, strlen(key));
-    SHA1_Update(&c, GUID, sizeof(GUID) - 1);
-    SHA1_Final(tmp, &c);
-    if (-1 == __b64_ntop(tmp, SHA_DIGEST_LENGTH, target, size))
-	rfbErr("b64_ntop failed\n");
+    int len;
+    SHA1Context sha;
+    uint8_t digest[SHA1HashSize];
+    
+    if (size < B64LEN(SHA1HashSize) + 1) {
+        rfbErr("webSocketsGenSha1Key: not enough space in target\n");
+        target[0] = '\0';
+        return;
+    }
+
+    SHA1Reset(&sha);
+    SHA1Input(&sha, (unsigned char *)key, strlen(key));
+    SHA1Input(&sha, (unsigned char *)GUID, strlen(GUID));
+    SHA1Result(&sha, digest);
+
+    len = __b64_ntop((unsigned char *)digest, SHA1HashSize, target, size);
+    if (len < size - 1) {
+        rfbErr("webSocketsGenSha1Key: b64_ntop failed\n");
+        target[0] = '\0';
+        return;
+    }
+
+    target[len] = '\0';
+    return;
 }
-#endif
+
 
 /*
  * rfbWebSocketsHandshake is called to handle new WebSockets connections
@@ -237,7 +254,7 @@ static rfbBool
 webSocketsHandshake(rfbClientPtr cl, char *scheme)
 {
     char *buf, *response, *line;
-    int n, linestart = 0, len = 0, llen, base64 = 0;
+    int n, linestart = 0, len = 0, llen, base64 = TRUE;
     char prefix[5], trailer[17];
     char *path = NULL, *host = NULL, *origin = NULL, *protocol = NULL;
     char *key1 = NULL, *key2 = NULL, *key3 = NULL;
@@ -268,6 +285,8 @@ webSocketsHandshake(rfbClientPtr cl, char *scheme)
                 rfbLog("webSocketsHandshake: client gone\n");
             else
                 rfbLogPerror("webSocketsHandshake: read");
+            free(response);
+            free(buf);
             return FALSE;
         }
 
@@ -285,6 +304,8 @@ webSocketsHandshake(rfbClientPtr cl, char *scheme)
                             rfbLog("webSocketsHandshake: client gone\n");
                         else
                             rfbLogPerror("webSocketsHandshake: read");
+                        free(response);
+                        free(buf);
                         return FALSE;
                     }
                     rfbLog("Got key3\n");
@@ -298,7 +319,6 @@ webSocketsHandshake(rfbClientPtr cl, char *scheme)
                 /* 16 = 4 ("GET ") + 1 ("/.*") + 11 (" HTTP/1.1\r\n") */
                 path = line+4;
                 buf[len-11] = '\0'; /* Trim trailing " HTTP/1.1\r\n" */
-                base64 = TRUE;
                 cl->wspath = strdup(path);
                 /* rfbLog("Got path: %s\n", path); */
             } else if ((strncasecmp("host: ", line, min(llen,6))) == 0) {
@@ -345,14 +365,25 @@ webSocketsHandshake(rfbClientPtr cl, char *scheme)
         return FALSE;
     }
 
-    /*
-    if ((!protocol) || (!strcasestr(protocol, "base64"))) {
-        rfbErr("webSocketsHandshake: base64 subprotocol not supported by client\n");
+    if ((protocol) && (strstr(protocol, "binary"))) {
+        if (! sec_ws_version) {
+            rfbErr("webSocketsHandshake: 'binary' protocol not supported with Hixie\n");
             free(response);
             free(buf);
             return FALSE;
         }
-    */
+        rfbLog("  - webSocketsHandshake: using binary/raw encoding\n");
+        base64 = FALSE;
+        protocol = "binary";
+    } else {
+        rfbLog("  - webSocketsHandshake: using base64 encoding\n");
+        base64 = TRUE;
+        if ((protocol) && (strstr(protocol, "base64"))) {
+            protocol = "base64";
+        } else {
+            protocol = "";
+        }
+    }
 
     /*
      * Generate the WebSockets server response based on the the headers sent
@@ -360,7 +391,7 @@ webSocketsHandshake(rfbClientPtr cl, char *scheme)
      */
 
     if (sec_ws_version) {
-	char accept[SHA_DIGEST_LENGTH * 3];
+	char accept[B64LEN(SHA1HashSize) + 1];
 	rfbLog("  - WebSockets client version hybi-%02d\n", sec_ws_version);
 	webSocketsGenSha1Key(accept, sizeof(accept), sec_ws_key);
 	len = snprintf(response, WEBSOCKETS_MAX_HANDSHAKE_LEN,
@@ -457,38 +488,16 @@ webSocketsGenMd5(char * target, char *key1, char *key2, char *key3)
 static int
 webSocketsEncodeHixie(rfbClientPtr cl, const char *src, int len, char **dst)
 {
-    int i, sz = 0;
-    unsigned char chr;
+    int sz = 0;
     ws_ctx_t *wsctx = (ws_ctx_t *)cl->wsctx;
 
     wsctx->encodeBuf[sz++] = '\x00';
-    if (wsctx->base64) {
     len = __b64_ntop((unsigned char *)src, len, wsctx->encodeBuf+sz, sizeof(wsctx->encodeBuf) - (sz + 1));
     if (len < 0) {
         return len;
     }
     sz += len;
-    } else {
-        for (i=0; i < len; i++) {
-            chr = src[i];
-            if (chr < 128) {
-                if (chr == 0x00) {
-                    wsctx->encodeBuf[sz++] = '\xc4';
-                    wsctx->encodeBuf[sz++] = '\x80';
-                } else {
-                    wsctx->encodeBuf[sz++] = chr;
-                }
-            } else {
-                if (chr < 192) {
-                    wsctx->encodeBuf[sz++] = '\xc2';
-                    wsctx->encodeBuf[sz++] = chr;
-                } else {
-                    wsctx->encodeBuf[sz++] = '\xc3';
-                    wsctx->encodeBuf[sz++] = chr - 64;
-                }
-            }
-        }
-    }
+
     wsctx->encodeBuf[sz++] = '\xff';
     *dst = wsctx->encodeBuf;
     return sz;
@@ -524,9 +533,8 @@ ws_peek(rfbClientPtr cl, char *buf, int len)
 static int
 webSocketsDecodeHixie(rfbClientPtr cl, char *dst, int len)
 {
-    int retlen = 0, n, i, avail, modlen, needlen, actual;
+    int retlen = 0, n, i, avail, modlen, needlen;
     char *buf, *end = NULL;
-    unsigned char chr, chr2;
     ws_ctx_t *wsctx = (ws_ctx_t *)cl->wsctx;
 
     buf = wsctx->decodeBuf;
@@ -539,7 +547,6 @@ webSocketsDecodeHixie(rfbClientPtr cl, char *dst, int len)
     }
 
 
-    if (wsctx->base64) {
     /* Base64 encoded WebSockets stream */
 
     if (buf[0] == '\xff') {
@@ -610,63 +617,6 @@ webSocketsDecodeHixie(rfbClientPtr cl, char *dst, int len)
         /* rfbLog("Saving carryover %d\n", dst[retlen + i]); */
         wsctx->carryBuf[i] = dst[retlen + i];
     }
-    } else {
-        /* UTF-8 encoded WebSockets stream */
-
-        actual = 0;
-        for (needlen = 0; needlen < n && actual < len; needlen++) {
-            chr = buf[needlen];
-            if ((chr > 0) && (chr < 128)) {
-                actual++;
-            } else if ((chr > 127) && (chr < 255)) {
-                if (needlen + 1 >= n) {
-                    break;
-                }
-                needlen++;
-                actual++;
-            }
-        }
-
-        if (actual < len) {
-            errno = EAGAIN;
-            return -1;
-        }
-
-        /* Consume what we need */
-        if ((n = ws_read(cl, buf, needlen)) < needlen) {
-            return n;
-        }
-
-        while (retlen < len) {
-            chr = buf[0];
-            buf += 1;
-            if (chr == 0) {
-                /* Begin frame marker, just skip it */
-            } else if (chr == 255) {
-                /* Begin frame marker, just skip it */
-	    } else if (chr < 128) {
-                dst[retlen++] = chr;
-            } else {
-                chr2 = buf[0];
-                buf += 1;
-                switch (chr) {
-                case (unsigned char) '\xc2':
-                    dst[retlen++] = chr2;
-                    break;
-                case (unsigned char) '\xc3':
-                    dst[retlen++] = chr2 + 64;
-                    break;
-                case (unsigned char) '\xc4':
-                    dst[retlen++] = 0;
-                    break;
-                default:
-                    rfbErr("Invalid UTF-8 encoding\n");
-                    errno = EIO;
-                    return -1;
-                }
-            }
-        }
-    }
 
     /* rfbLog("<< webSocketsDecode, retlen: %d\n", retlen); */
     return retlen;
@@ -675,7 +625,7 @@ webSocketsDecodeHixie(rfbClientPtr cl, char *dst, int len)
 static int
 webSocketsDecodeHybi(rfbClientPtr cl, char *dst, int len)
 {
-    char *buf, *payload, *rbuf;
+    char *buf, *payload;
     int ret = -1, result = -1;
     int total = 0;
     ws_mask_t mask;
@@ -683,7 +633,7 @@ webSocketsDecodeHybi(rfbClientPtr cl, char *dst, int len)
     int i, j;
     unsigned char opcode;
     ws_ctx_t *wsctx = (ws_ctx_t *)cl->wsctx;
-    int flength, fin, fhlen, blen;
+    int flength, fin, fhlen;
 
     // rfbLog(" <== %s[%d]: %d cl: %p, wsctx: %p-%p (%d)\n", __func__, gettid(), len, cl, wsctx, (char *)wsctx + sizeof(ws_ctx_t), sizeof(ws_ctx_t));