websockets: Add encryption support

[jes: moved out GnuTLS and OpenSSL support, added a dummy support, to
separate changes better, and to keep things compiling]
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

libvncserver/Makefile.am

@@ -13,7 +13,7 @@ TIGHTVNCFILETRANSFERSRCS = tightvnc-filetransfer/rfbtightserver.c \
 endif
 
 if WITH_WEBSOCKETS
-WEBSOCKETSSRCS = websockets.c md5.c
+WEBSOCKETSSRCS = websockets.c md5.c rfbssl_none.c
 endif
 
 includedir=$(prefix)/include/rfb

libvncserver/rfbserver.c

@@ -73,6 +73,10 @@
 /* strftime() */
 #include <time.h>
 
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+#include "rfbssl.h"
+#endif
+
 #ifdef __MINGW32__
 static int compat_mkdir(const char *path, int mode)
 {
@@ -360,7 +364,6 @@ rfbNewTCPOrUDPClient(rfbScreenInfoPtr rfbScreen,
 
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
       cl->webSockets       = FALSE;
-      cl->webSocketsSSL    = FALSE;
       cl->webSocketsBase64 = FALSE;
       cl->dblen= 0;
       cl->carrylen = 0;
@@ -1841,26 +1844,62 @@ rfbProcessClientNormalMessage(rfbClientPtr cl)
 
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
     if (cl->webSockets) {
+	if (cl->sslctx)
+	    n = rfbssl_peek(cl, encBuf, 4);
+	else
 	    n = recv(cl->sock, encBuf, 4, MSG_PEEK);
+
+	if (n <= 0) {
+	    if (n != 0)
+		rfbLogPerror("rfbProcessClientNormalMessage: peek");
+	    rfbCloseClient(cl);
+	    return;
+	}
+
 	if (cl->webSocketsBase64) {
             /* With Base64 encoding we need at least 4 bytes */
             if ((n > 0) && (n < 4)) {
                 if (encBuf[0] == '\xff') {
+		    int doclose = 0;
                     /* Make sure we don't miss a client disconnect on an end frame
-                    * marker */
-                    n = read(cl->sock, encBuf, 1);
+                     * marker. Because we use a peek buffer in some cases it is not
+		     * applicable to wait for more data per select(). */
+		    switch (n) {
+			case 3:
+			    if (encBuf[1] == '\xff' && encBuf[2] == '\x00')
+				doclose = 1;
+			    break;
+			case 2:
+			    if (encBuf[1] == '\x00')
+				doclose = 1;
+			    break;
+			default:
+			    ;
+		    }
+
+		    if (cl->sslctx)
+			n = rfbssl_read(cl, encBuf, n);
+		    else
+			n = read(cl->sock, encBuf, n);
+
+		    if (doclose) {
+			rfbErr("rfbProcessClientNormalMessage: websocket close frame received\n");
+			rfbCloseClient(cl);
 		    }
 		    return;
                 }
+            }
         } else {
             /* With UTF-8 encoding we need at least 3 bytes (framing + 1) */
             if ((n == 1) || (n == 2)) {
                 if (encBuf[0] == '\xff') {
                     /* Make sure we don't miss a client disconnect on an end frame
                     * marker */
+		    if (cl->sslctx)
+			    n = rfbssl_read(cl, encBuf, 1);
+		    else
 			    n = read(cl->sock, encBuf, 1);
                 }
-                return;
             }
         }
     }

libvncserver/rfbssl.h

+#ifndef _VNCSSL_H
+#define _VNCSSL_H 1
+
+#include "rfb/rfb.h"
+#include "rfb/rfbconfig.h"
+
+int rfbssl_init(rfbClientPtr cl);
+int rfbssl_pending(rfbClientPtr cl);
+int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize);
+int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize);
+int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize);
+void rfbssl_destroy(rfbClientPtr cl);
+
+
+#endif /* _VNCSSL_H */

libvncserver/rfbssl_none.c

+/*
+ * rfbssl_none.c - Secure socket functions (fallback to failing)
+ */
+
+/*
+ *  Copyright (C) 2011 Johannes Schindelin
+ *
+ *  This is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This software is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this software; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ */
+
+#include "rfbssl.h"
+
+struct rfbssl_ctx *rfbssl_init_global(char *key, char *cert)
+{
+    return NULL;
+}
+
+int rfbssl_init(rfbClientPtr cl)
+{
+    return -1;
+}
+
+int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
+{
+    return -1;
+}
+
+int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
+{
+    return -1;
+}
+
+int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
+{
+    return -1;
+}
+
+int rfbssl_pending(rfbClientPtr cl)
+{
+    return -1;
+}
+
+void rfbssl_destroy(rfbClientPtr cl)
+{
+}

libvncserver/sockets.c

@@ -62,6 +62,10 @@
 #include <unistd.h>
 #endif
 
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+#include "rfbssl.h"
+#endif
+
 #if defined(__linux__) && defined(NEED_TIMEVAL)
 struct timeval 
 {
@@ -392,6 +396,10 @@ rfbCloseClient(rfbClientPtr cl)
 	  while(cl->screen->maxFd>0
 		&& !FD_ISSET(cl->screen->maxFd,&(cl->screen->allFds)))
 	    cl->screen->maxFd--;
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+	if (cl->sslctx)
+	    rfbssl_destroy(cl);
+#endif
 #ifndef __MINGW32__
 	shutdown(cl->sock,SHUT_RDWR);
 #endif
@@ -460,6 +468,8 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
         if (cl->webSockets) {
             n = webSocketsDecode(cl, buf, len);
+        } else if (cl->sslctx) {
+	    n = rfbssl_read(cl, buf, len);
 	} else {
             n = read(sock, buf, len);
         }
@@ -490,6 +500,12 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
                 return n;
             }
 
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+	    if (cl->sslctx) {
+		if (rfbssl_pending(cl))
+		    continue;
+	    }
+#endif
             FD_ZERO(&fds);
             FD_SET(sock, &fds);
             tv.tv_sec = timeout / 1000;
@@ -500,6 +516,7 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
                 return n;
             }
             if (n == 0) {
+                rfbErr("ReadExact: select timeout\n");
                 errno = ETIMEDOUT;
                 return -1;
             }
@@ -540,6 +557,11 @@ rfbPeekExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
     struct timeval tv;
 
     while (len > 0) {
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+	if (cl->sslctx)
+	    n = rfbssl_peek(cl, buf, len);
+	else
+#endif
 	    n = recv(sock, buf, len, MSG_PEEK);
 
         if (n == len) {
@@ -564,13 +586,19 @@ rfbPeekExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
                 return n;
             }
 
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+	    if (cl->sslctx) {
+		if (rfbssl_pending(cl))
+		    continue;
+	    }
+#endif
             FD_ZERO(&fds);
             FD_SET(sock, &fds);
             tv.tv_sec = timeout / 1000;
             tv.tv_usec = (timeout % 1000) * 1000;
             n = select(sock+1, &fds, NULL, &fds, &tv);
             if (n < 0) {
-                rfbLogPerror("ReadExact: select");
+                rfbLogPerror("PeekExact: select");
                 return n;
             }
             if (n == 0) {
@@ -581,7 +609,7 @@ rfbPeekExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
     }
 #undef DEBUG_READ_EXACT
 #ifdef DEBUG_READ_EXACT
-    rfbLog("ReadExact %d bytes\n",len);
+    rfbLog("PeekExact %d bytes\n",len);
     for(n=0;n<len;n++)
 	    fprintf(stderr,"%02x ",(unsigned char)buf[n]);
     fprintf(stderr,"\n");
@@ -628,6 +656,11 @@ rfbWriteExact(rfbClientPtr cl,
 
     LOCK(cl->outputMutex);
     while (len > 0) {
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+        if (cl->sslctx)
+	    n = rfbssl_write(cl, buf, len);
+	else
+#endif
 	    n = write(sock, buf, len);
 
         if (n > 0) {

libvncserver/websockets.c

@@ -32,6 +32,7 @@
 #include <errno.h>
 
 #include <md5.h>
+#include "rfbssl.h"
 
 #define FLASH_POLICY_RESPONSE "<cross-domain-policy><allow-access-from domain=\"*\" to-ports=\"*\" /></cross-domain-policy>\n"
 #define SZ_FLASH_POLICY_RESPONSE 93
@@ -91,15 +92,15 @@ webSocketsCheck (rfbClientPtr cl)
             rfbErr("webSocketsHandshake: failed sending Flash policy response");
         }
         return FALSE;
-    } else if (strncmp(bbuf, "\x16", 1) == 0) {
-        cl->webSocketsSSL = TRUE;
+    } else if (strncmp(bbuf, "\x16", 1) == 0 || strncmp(bbuf, "\x80", 1) == 0) {
         rfbLog("Got TLS/SSL WebSockets connection\n");
-        scheme = "wss";
-        /* TODO */
-        /* bbuf = ... */
+        if (-1 == rfbssl_init(cl)) {
+	  rfbErr("webSocketsHandshake: rfbssl_init failed\n");
 	  return FALSE;
+	}
+	ret = rfbPeekExactTimeout(cl, bbuf, 4, WEBSOCKETS_CLIENT_CONNECT_WAIT_MS);
+        scheme = "wss";
     } else {
-        cl->webSocketsSSL = FALSE;
         scheme = "ws";
     }
 
@@ -334,6 +335,30 @@ webSocketsEncode(rfbClientPtr cl, const char *src, int len)
     return sz;
 }
 
+static int
+ws_read(rfbClientPtr cl, char *buf, int len)
+{
+    int n;
+    if (cl->sslctx) {
+	n = rfbssl_read(cl, buf, len);
+    } else {
+	n = read(cl->sock, buf, len);
+    }
+    return n;
+}
+
+static int
+ws_peek(rfbClientPtr cl, char *buf, int len)
+{
+    int n;
+    if (cl->sslctx) {
+	n = rfbssl_peek(cl, buf, len);
+    } else {
+	n = recv(cl->sock, buf, len, MSG_PEEK);
+    }
+    return n;
+}
+
 int
 webSocketsDecode(rfbClientPtr cl, char *dst, int len)
 {
@@ -343,10 +368,10 @@ webSocketsDecode(rfbClientPtr cl, char *dst, int len)
 
     buf = cl->decodeBuf;
 
-    n = recv(cl->sock, buf, len*2+2, MSG_PEEK);
+    n = ws_peek(cl, buf, len*2+2);
 
     if (n <= 0) {
-        rfbLog("recv of %d\n", n);
+        rfbErr("%s: recv of %d\n", __func__, n);
         return n;
     }
 
@@ -355,7 +380,7 @@ webSocketsDecode(rfbClientPtr cl, char *dst, int len)
         /* Base64 encoded WebSockets stream */
 
         if (buf[0] == '\xff') {
-            i = read(cl->sock, buf, 1); /* Consume marker */
+            i = ws_read(cl, buf, 1); /* Consume marker */
             buf++;
             n--;
         }
@@ -364,7 +389,7 @@ webSocketsDecode(rfbClientPtr cl, char *dst, int len)
             return -1;
         }
         if (buf[0] == '\x00') {
-            i = read(cl->sock, buf, 1); /* Consume marker */
+            i = ws_read(cl, buf, 1); /* Consume marker */
             buf++;
             n--;
         }
@@ -414,7 +439,7 @@ webSocketsDecode(rfbClientPtr cl, char *dst, int len)
         retlen += n;
 
         /* Consume the data from socket */
-        i = read(cl->sock, buf, needlen);
+        i = ws_read(cl, buf, needlen);
 
         cl->carrylen = n - len;
         retlen -= cl->carrylen;
@@ -445,7 +470,7 @@ webSocketsDecode(rfbClientPtr cl, char *dst, int len)
         }
 
         /* Consume what we need */
-        if ((n = read(cl->sock, buf, needlen)) < needlen) {
+        if ((n = ws_read(cl, buf, needlen)) < needlen) {
             return n;
         }
 

rfb/rfb.h

@@ -368,6 +368,10 @@ typedef struct _rfbScreenInfo
     rfbDisplayFinishedHookPtr displayFinishedHook;
     /** xvpHook is called to handle an xvp client message */
     rfbXvpHookPtr xvpHook;
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+    char *sslkeyfile;
+    char *sslcertfile;
+#endif
 } rfbScreenInfo, *rfbScreenInfoPtr;
 
 
@@ -414,6 +418,8 @@ typedef struct _rfbStatList {
     struct _rfbStatList *Next;
 } rfbStatList;
 
+typedef struct _rfbSslCtx rfbSslCtx;
+
 typedef struct _rfbClientRec {
 
     /** back pointer to the screen */
@@ -637,8 +643,9 @@ typedef struct _rfbClientRec {
     rfbBool webSocketsSSL;
     rfbBool webSocketsBase64;
 
-    char encodeBuf[UPDATE_BUF_SIZE*2 + 2]; /* UTF-8 could double it + framing */
+    rfbSslCtx *sslctx;
 
+    char encodeBuf[UPDATE_BUF_SIZE*2 + 2]; /* UTF-8 could double it + framing */
     char decodeBuf[8192];                  /* TODO: what makes sense? */
     int dblen;
     char carryBuf[3];                      /* For base64 carry-over */