Commit a2a6e256 authored by Gernot Tenchio's avatar Gernot Tenchio Committed by Johannes Schindelin

websockets: add GnuTLS and OpenSSL support

For now, only OpenSSL support is activated through configure, since GnuTLS
is only used in LibVNCClient.

[jes: separated this out from the commit adding encryption support, added
autoconf support.]
Signed-off-by: 's avatarJohannes Schindelin <johannes.schindelin@gmx.de>
parent 4aa35863
......@@ -54,6 +54,99 @@ AM_CONDITIONAL(HAVE_MP3LAME, test "$HAVE_MP3LAME" = "true")
# before it seemed to be inside the with_jpeg conditional.
AC_CHECK_HEADER(thenonexistentheader.h, HAVE_THENONEXISTENTHEADER_H="true")
# set some ld -R nonsense
#
uname_s=`(uname -s) 2>/dev/null`
ld_minus_R="yes"
if test "x$uname_s" = "xHP-UX"; then
ld_minus_R="no"
elif test "x$uname_s" = "xOSF1"; then
ld_minus_R="no"
elif test "x$uname_s" = "xDarwin"; then
ld_minus_R="no"
fi
# Check for OpenSSL
AH_TEMPLATE(HAVE_LIBCRYPT, [libcrypt library present])
AC_ARG_WITH(crypt,
[ --without-crypt disable support for libcrypt],,)
if test "x$with_crypt" != "xno"; then
AC_CHECK_FUNCS([crypt], HAVE_LIBC_CRYPT="true")
if test -z "$HAVE_LIBC_CRYPT"; then
AC_CHECK_LIB(crypt, crypt,
CRYPT_LIBS="-lcrypt"
[AC_DEFINE(HAVE_LIBCRYPT)], ,)
fi
fi
AC_SUBST(CRYPT_LIBS)
# some OS's need both -lssl and -lcrypto on link line:
AH_TEMPLATE(HAVE_LIBCRYPTO, [openssl libcrypto library present])
AC_ARG_WITH(crypto,
[ --without-crypto disable support for openssl libcrypto],,)
AH_TEMPLATE(HAVE_LIBSSL, [openssl libssl library present])
AC_ARG_WITH(ssl,
[ --without-ssl disable support for openssl libssl]
[ --with-ssl=DIR use openssl include/library files in DIR],,)
if test "x$with_crypto" != "xno" -a "x$with_ssl" != "xno"; then
if test ! -z "$with_ssl" -a "x$with_ssl" != "xyes"; then
saved_CPPFLAGS="$CPPFLAGS"
saved_LDFLAGS="$LDFLAGS"
CPPFLAGS="$CPPFLAGS -I$with_ssl/include"
LDFLAGS="$LDFLAGS -L$with_ssl/lib"
if test "x$ld_minus_R" = "xno"; then
:
elif test "x$GCC" = "xyes"; then
LDFLAGS="$LDFLAGS -Xlinker -R$with_ssl/lib"
else
LDFLAGS="$LDFLAGS -R$with_ssl/lib"
fi
fi
AC_CHECK_LIB(crypto, RAND_file_name,
[AC_DEFINE(HAVE_LIBCRYPTO) HAVE_LIBCRYPTO="true"], ,)
if test ! -z "$with_ssl" -a "x$with_ssl" != "xyes"; then
if test "x$HAVE_LIBCRYPTO" != "xtrue"; then
CPPFLAGS="$saved_CPPFLAGS"
LDFLAGS="$saved_LDFLAGS"
fi
fi
fi
AH_TEMPLATE(HAVE_X509_PRINT_EX_FP, [open ssl X509_print_ex_fp available])
if test "x$with_ssl" != "xno"; then
if test "x$HAVE_LIBCRYPTO" = "xtrue"; then
AC_CHECK_LIB(ssl, SSL_library_init,
SSL_LIBS="-lssl -lcrypto"
[AC_DEFINE(HAVE_LIBSSL) HAVE_LIBSSL="true"], ,
-lcrypto)
else
AC_CHECK_LIB(ssl, SSL_library_init,
SSL_LIBS="-lssl"
[AC_DEFINE(HAVE_LIBSSL) HAVE_LIBSSL="true"], ,)
fi
fi
AC_SUBST(SSL_LIBS)
if test "x$HAVE_LIBSSL" != "xtrue" -a "x$with_ssl" != "xno"; then
AC_MSG_WARN([
==========================================================================
*** The openssl encryption library libssl.so was not found. ***
An x11vnc built this way will not support SSL encryption. To enable
SSL install the necessary development packages (perhaps it is named
something like libssl-dev) and run configure again.
==========================================================================
])
sleep 5
elif test "x$with_ssl" != "xno"; then
AC_CHECK_LIB(ssl, X509_print_ex_fp,
[AC_DEFINE(HAVE_X509_PRINT_EX_FP) HAVE_X509_PRINT_EX_FP="true"], , $SSL_LIBS
)
fi
AM_CONDITIONAL(HAVE_LIBSSL, test "x$with_crypto" != "xno" -a "x$with_ssl" != "xno")
# Checks for X libraries
HAVE_X11="false"
AC_PATH_XTRA
......@@ -296,98 +389,6 @@ configure again.
sleep 5
fi
# set some ld -R nonsense
#
uname_s=`(uname -s) 2>/dev/null`
ld_minus_R="yes"
if test "x$uname_s" = "xHP-UX"; then
ld_minus_R="no"
elif test "x$uname_s" = "xOSF1"; then
ld_minus_R="no"
elif test "x$uname_s" = "xDarwin"; then
ld_minus_R="no"
fi
AH_TEMPLATE(HAVE_LIBCRYPT, [libcrypt library present])
AC_ARG_WITH(crypt,
[ --without-crypt disable support for libcrypt],,)
if test "x$with_crypt" != "xno"; then
AC_CHECK_FUNCS([crypt], HAVE_LIBC_CRYPT="true")
if test -z "$HAVE_LIBC_CRYPT"; then
AC_CHECK_LIB(crypt, crypt,
CRYPT_LIBS="-lcrypt"
[AC_DEFINE(HAVE_LIBCRYPT)], ,)
fi
fi
AC_SUBST(CRYPT_LIBS)
# some OS's need both -lssl and -lcrypto on link line:
AH_TEMPLATE(HAVE_LIBCRYPTO, [openssl libcrypto library present])
AC_ARG_WITH(crypto,
[ --without-crypto disable support for openssl libcrypto],,)
AH_TEMPLATE(HAVE_LIBSSL, [openssl libssl library present])
AC_ARG_WITH(ssl,
[ --without-ssl disable support for openssl libssl]
[ --with-ssl=DIR use openssl include/library files in DIR],,)
if test "x$with_crypto" != "xno" -a "x$with_ssl" != "xno"; then
if test ! -z "$with_ssl" -a "x$with_ssl" != "xyes"; then
saved_CPPFLAGS="$CPPFLAGS"
saved_LDFLAGS="$LDFLAGS"
CPPFLAGS="$CPPFLAGS -I$with_ssl/include"
LDFLAGS="$LDFLAGS -L$with_ssl/lib"
if test "x$ld_minus_R" = "xno"; then
:
elif test "x$GCC" = "xyes"; then
LDFLAGS="$LDFLAGS -Xlinker -R$with_ssl/lib"
else
LDFLAGS="$LDFLAGS -R$with_ssl/lib"
fi
fi
AC_CHECK_LIB(crypto, RAND_file_name,
[AC_DEFINE(HAVE_LIBCRYPTO) HAVE_LIBCRYPTO="true"], ,)
if test ! -z "$with_ssl" -a "x$with_ssl" != "xyes"; then
if test "x$HAVE_LIBCRYPTO" != "xtrue"; then
CPPFLAGS="$saved_CPPFLAGS"
LDFLAGS="$saved_LDFLAGS"
fi
fi
fi
AH_TEMPLATE(HAVE_X509_PRINT_EX_FP, [open ssl X509_print_ex_fp available])
if test "x$with_ssl" != "xno"; then
if test "x$HAVE_LIBCRYPTO" = "xtrue"; then
AC_CHECK_LIB(ssl, SSL_library_init,
SSL_LIBS="-lssl -lcrypto"
[AC_DEFINE(HAVE_LIBSSL) HAVE_LIBSSL="true"], ,
-lcrypto)
else
AC_CHECK_LIB(ssl, SSL_library_init,
SSL_LIBS="-lssl"
[AC_DEFINE(HAVE_LIBSSL) HAVE_LIBSSL="true"], ,)
fi
fi
AC_SUBST(SSL_LIBS)
if test "x$HAVE_LIBSSL" != "xtrue" -a "x$with_ssl" != "xno"; then
AC_MSG_WARN([
==========================================================================
*** The openssl encryption library libssl.so was not found. ***
An x11vnc built this way will not support SSL encryption. To enable
SSL install the necessary development packages (perhaps it is named
something like libssl-dev) and run configure again.
==========================================================================
])
sleep 5
elif test "x$with_ssl" != "xno"; then
AC_CHECK_LIB(ssl, X509_print_ex_fp,
[AC_DEFINE(HAVE_X509_PRINT_EX_FP) HAVE_X509_PRINT_EX_FP="true"], , $SSL_LIBS
)
fi
if test "x$with_v4l" != "xno"; then
AC_CHECK_HEADER(linux/videodev.h,
[AC_DEFINE(HAVE_LINUX_VIDEODEV_H)],,)
......@@ -720,7 +721,7 @@ if test "x$HAVE_B64" != "xtrue"; then
with_websockets=""
fi
if test "x$with_websockets" = "xyes"; then
LIBS="$LIBS -lresolv"
LIBS="$LIBS -lresolv $SSL_LIBS"
AC_DEFINE(WITH_WEBSOCKETS)
fi
AM_CONDITIONAL(WITH_WEBSOCKETS, test "$with_websockets" = "yes")
......
......@@ -13,7 +13,18 @@ TIGHTVNCFILETRANSFERSRCS = tightvnc-filetransfer/rfbtightserver.c \
endif
if WITH_WEBSOCKETS
WEBSOCKETSSRCS = websockets.c md5.c rfbssl_none.c
if HAVE_LIBSSL
WEBSOCKETSSSLSRCS = rfbssl_openssl.c
else
#if HAVE_GNUTLS
#WEBSOCKETSSSLSRCS = rfbssl_gnutls.c
#else
WEBSOCKETSSSLSRCS = rfbssl_none.c
#endif
endif
WEBSOCKETSSRCS = websockets.c md5.c $(WEBSOCKETSSSLSRCS)
endif
includedir=$(prefix)/include/rfb
......
/*
* rfbssl_gnutls.c - Secure socket funtions (gnutls version)
*/
/*
* Copyright (C) 2011 Gernot Tenchio
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this software; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
* USA.
*/
#include "rfbssl.h"
#include <gnutls/gnutls.h>
#include <errno.h>
struct rfbssl_ctx {
char peekbuf[2048];
int peeklen;
int peekstart;
gnutls_session_t session;
gnutls_certificate_credentials_t x509_cred;
gnutls_dh_params_t dh_params;
#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
gnutls_rsa_params_t rsa_params;
#endif
};
void rfbssl_log_func(int level, const char *msg)
{
rfbErr("SSL: %s", msg);
}
static void rfbssl_error(const char *msg, int e)
{
rfbErr("%s: %s (%ld)\n", msg, gnutls_strerror(e), e);
}
static int rfbssl_init_session(struct rfbssl_ctx *ctx, int fd)
{
gnutls_session_t session;
int ret;
if (!GNUTLS_E_SUCCESS == (ret = gnutls_init(&session, GNUTLS_SERVER))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_priority_set_direct(session, "EXPORT", NULL))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, ctx->x509_cred))) {
/* */
} else {
gnutls_session_enable_compatibility_mode(session);
gnutls_transport_set_ptr(session, (gnutls_transport_ptr_t)fd);
ctx->session = session;
}
return ret;
}
static int generate_dh_params(struct rfbssl_ctx *ctx)
{
int ret;
if (GNUTLS_E_SUCCESS == (ret = gnutls_dh_params_init(&ctx->dh_params)))
ret = gnutls_dh_params_generate2(ctx->dh_params, 1024);
return ret;
}
#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
static int generate_rsa_params(struct rfbssl_ctx *ctx)
{
int ret;
if (GNUTLS_E_SUCCESS == (ret = gnutls_rsa_params_init(&ctx->rsa_params)))
ret = gnutls_rsa_params_generate2(ctx->rsa_params, 512);
return ret;
}
#endif
struct rfbssl_ctx *rfbssl_init_global(char *key, char *cert)
{
int ret = GNUTLS_E_SUCCESS;
struct rfbssl_ctx *ctx = NULL;
if (NULL == (ctx = malloc(sizeof(struct rfbssl_ctx)))) {
ret = GNUTLS_E_MEMORY_ERROR;
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_global_init())) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_allocate_credentials(&ctx->x509_cred))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_set_x509_trust_file(ctx->x509_cred, cert, GNUTLS_X509_FMT_PEM))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_set_x509_key_file(ctx->x509_cred, cert, key, GNUTLS_X509_FMT_PEM))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = generate_dh_params(ctx))) {
/* */
#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
} else if (!GNUTLS_E_SUCCESS == (ret = generate_rsa_params(ctx))) {
/* */
#endif
} else {
gnutls_global_set_log_function(rfbssl_log_func);
gnutls_global_set_log_level(1);
gnutls_certificate_set_dh_params(ctx->x509_cred, ctx->dh_params);
return ctx;
}
free(ctx);
return NULL;
}
int rfbssl_init(rfbClientPtr cl)
{
int ret = -1;
struct rfbssl_ctx *ctx;
char *keyfile;
if (!(keyfile = cl->screen->sslkeyfile))
keyfile = cl->screen->sslcertfile;
if (NULL == (ctx = rfbssl_init_global(keyfile, cl->screen->sslcertfile))) {
/* */
} else if (GNUTLS_E_SUCCESS != (ret = rfbssl_init_session(ctx, cl->sock))) {
/* */
} else {
while (GNUTLS_E_SUCCESS != (ret = gnutls_handshake(ctx->session))) {
if (ret == GNUTLS_E_AGAIN)
continue;
break;
}
}
if (ret != GNUTLS_E_SUCCESS) {
rfbssl_error(__func__, ret);
} else {
cl->sslctx = (rfbSslCtx *)ctx;
rfbLog("%s protocol initialized\n", gnutls_protocol_get_name(gnutls_protocol_get_version(ctx->session)));
}
return ret;
}
static int rfbssl_do_read(rfbClientPtr cl, char *buf, int bufsize)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
int ret;
while ((ret = gnutls_record_recv(ctx->session, buf, bufsize)) < 0) {
if (ret == GNUTLS_E_AGAIN) {
/* continue */
} else if (ret == GNUTLS_E_INTERRUPTED) {
/* continue */
} else {
break;
}
}
if (ret < 0) {
rfbssl_error(__func__, ret);
errno = EIO;
ret = -1;
}
return ret < 0 ? -1 : ret;
}
int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
int ret;
while ((ret = gnutls_record_send(ctx->session, buf, bufsize)) < 0) {
if (ret == GNUTLS_E_AGAIN) {
/* continue */
} else if (ret == GNUTLS_E_INTERRUPTED) {
/* continue */
} else {
break;
}
}
if (ret < 0)
rfbssl_error(__func__, ret);
return ret;
}
int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
{
int ret = -1;
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
if (ctx->peekstart) {
int spaceleft = sizeof(ctx->peekbuf) - ctx->peeklen - ctx->peekstart;
if (spaceleft < bufsize) {
memmove(ctx->peekbuf, ctx->peekbuf + ctx->peekstart, ctx->peeklen);
ctx->peekstart = 0;
}
}
/* If we have any peek data, simply return that. */
if (ctx->peeklen) {
if (bufsize > ctx->peeklen) {
/* more than we have, so we are trying to read the remaining
* bytes
**/
int required = bufsize - ctx->peeklen;
int total = ctx->peekstart + ctx->peeklen;
int n, avail = sizeof(ctx->peekbuf) - total;
if (required > avail)
required = avail;
if (!required) {
rfbErr("%s: no space left\n", __func__);
} else if ((n = rfbssl_do_read(cl, ctx->peekbuf + total, required)) < 0) {
rfbErr("%s: read error\n", __func__);
return n;
} else {
ctx->peeklen += n;
}
ret = ctx->peeklen;
} else {
/* simply return what we have */
ret = bufsize;
}
} else {
ret = bufsize;
if (ret > sizeof(ctx->peekbuf))
ret = sizeof(ctx->peekbuf);
if ((ret = rfbssl_do_read(cl, ctx->peekbuf, ret)) > 0)
ctx->peeklen = ret;
}
if (ret >= 0) {
memcpy(buf, ctx->peekbuf + ctx->peekstart, ret);
}
return ret;
}
int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
{
int ret;
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
if (ctx->peeklen) {
/* If we have any peek data, simply return that. */
ret = bufsize < ctx->peeklen ? bufsize : ctx->peeklen;
memcpy (buf, ctx->peekbuf + ctx->peekstart, ret);
ctx->peeklen -= ret;
if (ctx->peeklen != 0)
ctx->peekstart += ret;
else
ctx->peekstart = 0;
} else {
ret = rfbssl_do_read(cl, buf, bufsize);
}
return ret;
}
int rfbssl_pending(rfbClientPtr cl)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
int ret = ctx->peeklen;
if (ret <= 0)
ret = gnutls_record_check_pending(ctx->session);
return ret;
}
void rfbssl_destroy(rfbClientPtr cl)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
gnutls_bye(ctx->session, GNUTLS_SHUT_WR);
gnutls_deinit(ctx->session);
gnutls_certificate_free_credentials(ctx->x509_cred);
}
/*
* rfbssl_openssl.c - Secure socket funtions (openssl version)
*/
/*
* Copyright (C) 2011 Gernot Tenchio
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this software; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
* USA.
*/
#include "rfbssl.h"
#include <openssl/ssl.h>
#include <openssl/err.h>
struct rfbssl_ctx {
SSL_CTX *ssl_ctx;
SSL *ssl;
};
static void rfbssl_error(void)
{
char buf[1024];
unsigned long e = ERR_get_error();
rfbErr("%s (%ld)\n", ERR_error_string(e, buf), e);
}
int rfbssl_init(rfbClientPtr cl)
{
char *keyfile;
int r, ret = -1;
struct rfbssl_ctx *ctx;
SSL_library_init();
SSL_load_error_strings();
if (cl->screen->sslkeyfile && *cl->screen->sslkeyfile) {
keyfile = cl->screen->sslkeyfile;
} else {
keyfile = cl->screen->sslcertfile;
}
if (NULL == (ctx = malloc(sizeof(struct rfbssl_ctx)))) {
rfbErr("OOM\n");
} else if (!cl->screen->sslcertfile || !cl->screen->sslcertfile[0]) {
rfbErr("SSL connection but no cert specified\n");
} else if (NULL == (ctx->ssl_ctx = SSL_CTX_new(TLSv1_server_method()))) {
rfbssl_error();
} else if (SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, keyfile, SSL_FILETYPE_PEM) <= 0) {
rfbErr("Unable to load private key file %s\n", keyfile);
} else if (SSL_CTX_use_certificate_file(ctx->ssl_ctx, cl->screen->sslcertfile, SSL_FILETYPE_PEM) <= 0) {
rfbErr("Unable to load certificate file %s\n", cl->screen->sslcertfile);
} else if (NULL == (ctx->ssl = SSL_new(ctx->ssl_ctx))) {
rfbErr("SSL_new failed\n");
rfbssl_error();
} else if (!(SSL_set_fd(ctx->ssl, cl->sock))) {
rfbErr("SSL_set_fd failed\n");
rfbssl_error();
} else {
while ((r = SSL_accept(ctx->ssl)) < 0) {
if (SSL_get_error(ctx->ssl, r) != SSL_ERROR_WANT_READ)
break;
}
if (r < 0) {
rfbErr("SSL_accept failed %d\n", SSL_get_error(ctx->ssl, r));
} else {
cl->sslctx = (rfbSslCtx *)ctx;
ret = 0;
}
}
return ret;
}
int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
{
int ret;
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
while ((ret = SSL_write(ctx->ssl, buf, bufsize)) <= 0) {
if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_WRITE)
break;
}
return ret;
}
int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
{
int ret;
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
while ((ret = SSL_peek(ctx->ssl, buf, bufsize)) <= 0) {
if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_READ)
break;
}
return ret;
}
int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
{
int ret;
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
while ((ret = SSL_read(ctx->ssl, buf, bufsize)) <= 0) {
if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_READ)
break;
}
return ret;
}
int rfbssl_pending(rfbClientPtr cl)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
return SSL_pending(ctx->ssl);
}
void rfbssl_destroy(rfbClientPtr cl)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
if (ctx->ssl)
SSL_free(ctx->ssl);
if (ctx->ssl_ctx)
SSL_CTX_free(ctx->ssl_ctx);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment