Add support for different crypto implementations

libvncserver/Makefile.am

@@ -15,16 +15,18 @@ endif
 if WITH_WEBSOCKETS
 
 if HAVE_LIBSSL
-WEBSOCKETSSSLSRCS = rfbssl_openssl.c
+WEBSOCKETSSSLSRCS = rfbssl_openssl.c rfbcrypto_openssl.c
+WEBSOCKETSSSLLIBS = @SSL_LIBS@ @CRYPT_LIBS@
 else
 if HAVE_GNUTLS
-WEBSOCKETSSSLSRCS = rfbssl_gnutls.c
+WEBSOCKETSSSLSRCS = rfbssl_gnutls.c rfbcrypto_gnutls.c
+WEBSOCKETSSSLLIBS = @GNUTLS_LIBS@
 else
-WEBSOCKETSSSLSRCS = rfbssl_none.c
+WEBSOCKETSSSLSRCS = rfbssl_none.c rfbcrypto_included.c ../common/md5.c ../common/sha1.c
 endif
 endif
 
-WEBSOCKETSSRCS = websockets.c ../common/md5.c ../common/sha1.c $(WEBSOCKETSSSLSRCS)
+WEBSOCKETSSRCS = websockets.c $(WEBSOCKETSSSLSRCS)
 endif
 
 includedir=$(prefix)/include/rfb
@@ -59,7 +61,7 @@ LIB_SRCS = main.c rfbserver.c rfbregion.c auth.c sockets.c $(WEBSOCKETSSRCS) \
 	$(ZLIBSRCS) $(TIGHTSRCS) $(TIGHTVNCFILETRANSFERSRCS)
 
 libvncserver_la_SOURCES=$(LIB_SRCS)
-libvncserver_la_LIBADD=@SSL_LIBS@ @CRYPT_LIBS@
+libvncserver_la_LIBADD=$(WEBSOCKETSSSLLIBS)
 
 lib_LTLIBRARIES=libvncserver.la
 

libvncserver/rfbcrypto.h

+#ifndef _RFB_CRYPTO_H
+#define _RFB_CRYPTO_H 1
+
+#include <sys/uio.h>
+
+#define SHA1_HASH_SIZE 20
+#define MD5_HASH_SIZE 16
+
+void digestmd5(const struct iovec *iov, int iovcnt, void *dest);
+void digestsha1(const struct iovec *iov, int iovcnt, void *dest);
+
+#endif

libvncserver/rfbcrypto_gnutls.c

+/*
+ * rfbcrypto_gnutls.c - Crypto wrapper (gnutls version)
+ */
+
+/*
+ *  Copyright (C) 2011 Gernot Tenchio
+ *
+ *  This is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This software is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this software; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ */
+
+#include <string.h>
+#include <gcrypt.h>
+#include "rfbcrypto.h"
+
+void digestmd5(const struct iovec *iov, int iovcnt, void *dest)
+{
+    gcry_md_hd_t c;
+    int i;
+
+    gcry_md_open(&c, GCRY_MD_MD5, 0);
+    for (i = 0; i < iovcnt; i++)
+	gcry_md_write(c, iov[i].iov_base, iov[i].iov_len);
+    gcry_md_final(c);
+    memcpy(dest, gcry_md_read(c, 0), gcry_md_get_algo_dlen(GCRY_MD_MD5));
+}
+
+void digestsha1(const struct iovec *iov, int iovcnt, void *dest)
+{
+    gcry_md_hd_t c;
+    int i;
+
+    gcry_md_open(&c, GCRY_MD_SHA1, 0);
+    for (i = 0; i < iovcnt; i++)
+	gcry_md_write(c, iov[i].iov_base, iov[i].iov_len);
+    gcry_md_final(c);
+    memcpy(dest, gcry_md_read(c, 0), gcry_md_get_algo_dlen(GCRY_MD_SHA1));
+}

libvncserver/rfbcrypto_included.c

+/*
+ * rfbcrypto_included.c - Crypto wrapper (included version)
+ */
+
+/*
+ *  Copyright (C) 2011 Gernot Tenchio
+ *
+ *  This is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This software is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this software; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ */
+
+#include <string.h>
+#include "md5.h"
+#include "sha1.h"
+#include "rfbcrypto.h"
+
+void digestmd5(const struct iovec *iov, int iovcnt, void *dest)
+{
+    md5_context c;
+    int i;
+
+    __md5_init_ctx(&c);
+    for (i = 0; i < iovcnt; i++)
+	__md5_process_bytes(&c, iov[i].iov_base, iov[i].iov_len);
+    __md5_finish_ctx(&c, dest);
+}
+
+void digestsha1(const struct iovec *iov, int iovcnt, void *dest)
+{
+    SHA1Context c;
+    int i;
+    
+    SHA1Reset(&c);
+    for (i = 0; i < iovcnt; i++)
+	SHA1Input(&c, iov[i].iov_base, iov[i].iov_len);
+    SHA1Result(&c, dest);
+}

libvncserver/rfbcrypto_openssl.c

+/*
+ * rfbcrypto_openssl.c - Crypto wrapper (openssl version)
+ */
+
+/*
+ *  Copyright (C) 2011 Gernot Tenchio
+ *
+ *  This is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This software is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this software; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ */
+
+#include <string.h>
+#include <openssl/sha.h>
+#include <openssl/md5.h>
+#include "rfbcrypto.h"
+
+void digestmd5(const struct iovec *iov, int iovcnt, void *dest)
+{
+    MD5_CTX c;
+    int i;
+    
+    MD5_Init(&c);
+    for (i = 0; i < iovcnt; i++)
+	MD5_Update(&c, iov[i].iov_base, iov[i].iov_len);
+    MD5_Final(dest, &c);
+}
+
+void digestsha1(const struct iovec *iov, int iovcnt, void *dest)
+{
+    SHA_CTX c;
+    int i;
+    
+    SHA1_Init(&c);
+    for (i = 0; i < iovcnt; i++)
+	SHA1_Update(&c, iov[i].iov_base, iov[i].iov_len);
+    SHA1_Final(dest, &c);
+}

libvncserver/rfbcrypto_polarssl.c

+#include <string.h>
+#include <polarssl/md5.h>
+#include <polarssl/sha1.h>
+#include "rfbcrypto.h"
+
+void digestmd5(const struct iovec *iov, int iovcnt, void *dest)
+{
+    md5_context c;
+    int i;
+    
+    md5_starts(&c);
+    for (i = 0; i < iovcnt; i++)
+	md5_update(&c, iov[i].iov_base, iov[i].iov_len);
+    md5_finish(dest, &c);
+}
+
+void digestsha1(const struct iovec *iov, int iovcnt, void *dest)
+{
+    sha1_context c;
+    int i;
+    
+    sha1_starts(&c);
+    for (i = 0; i < iovcnt; i++)
+	sha1_update(&c, iov[i].iov_base, iov[i].iov_len);
+    sha1_finish(dest, &c);
+}

libvncserver/websockets.c

@@ -33,10 +33,9 @@
 
 #include <byteswap.h>
 #include <string.h>
-#include "md5.h"
-#include "sha1.h"
 #include "rfbconfig.h"
 #include "rfbssl.h"
+#include "rfbcrypto.h"
 
 #if defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && __BYTE_ORDER == __BIG_ENDIAN
 #define WS_NTOH64(n) (n)
@@ -165,36 +164,20 @@ min (int a, int b) {
     return a < b ? a : b;
 }
 
-void
-webSocketsGenSha1Key(char * target, int size, char *key)
+static void webSocketsGenSha1Key(char *target, int size, char *key)
 {
-    int len;
-    SHA1Context sha;
-    uint8_t digest[SHA1HashSize];
-
-    if (size < B64LEN(SHA1HashSize) + 1) {
-        rfbErr("webSocketsGenSha1Key: not enough space in target\n");
-        target[0] = '\0';
-        return;
-    }
-
-    SHA1Reset(&sha);
-    SHA1Input(&sha, (unsigned char *)key, strlen(key));
-    SHA1Input(&sha, (unsigned char *)GUID, strlen(GUID));
-    SHA1Result(&sha, digest);
-
-    len = __b64_ntop((unsigned char *)digest, SHA1HashSize, target, size);
-    if (len < size - 1) {
-        rfbErr("webSocketsGenSha1Key: b64_ntop failed\n");
-        target[0] = '\0';
-        return;
-    }
-
-    target[len] = '\0';
-    return;
+    struct iovec iov[2];
+    unsigned char hash[20];
+
+    iov[0].iov_base = key;
+    iov[0].iov_len = strlen(key);
+    iov[1].iov_base = GUID;
+    iov[1].iov_len = sizeof(GUID) - 1;
+    digestsha1(iov, 2, hash);
+    if (-1 == __b64_ntop(hash, sizeof(hash), target, size))
+	rfbErr("b64_ntop failed\n");
 }
 
-
 /*
  * rfbWebSocketsHandshake is called to handle new WebSockets connections
  */
@@ -389,7 +372,7 @@ webSocketsHandshake(rfbClientPtr cl, char *scheme)
      */
 
     if (sec_ws_version) {
-	char accept[B64LEN(SHA1HashSize) + 1];
+	char accept[B64LEN(SHA1_HASH_SIZE) + 1];
 	rfbLog("  - WebSockets client version hybi-%02d\n", sec_ws_version);
 	webSocketsGenSha1Key(accept, sizeof(accept), sec_ws_key);
 	len = snprintf(response, WEBSOCKETS_MAX_HANDSHAKE_LEN,
@@ -443,6 +426,8 @@ webSocketsGenMd5(char * target, char *key1, char *key2, char *key3)
     unsigned int i, spaces1 = 0, spaces2 = 0;
     unsigned long num1 = 0, num2 = 0;
     unsigned char buf[17];
+    struct iovec iov[1];
+
     for (i=0; i < strlen(key1); i++) {
         if (key1[i] == ' ') {
             spaces1 += 1;
@@ -477,7 +462,9 @@ webSocketsGenMd5(char * target, char *key1, char *key2, char *key3)
     strncpy((char *)buf+8, key3, 8);
     buf[16] = '\0';
 
-    md5_buffer((char *)buf, 16, target);
+    iov[0].iov_base = buf;
+    iov[0].iov_len = 16;
+    digestmd5(iov, 1, target);
     target[16] = '\0';
 
     return;