support VNC protocol version 3.7

ChangeLog

+2005-09-26  Rohit Kumar <rokumar@novell.com>
+	* libvncserver/{auth,main,rfbserver}.c, rfb/{rfb,rfbproto}.h:
+	  support VNC protocol version 3.7. This allows to add security
+	  types.
+
 2005-08-21  Alberto Lusiani <alusiani@gmail.com>
 	* libvncserver.spec.in: split rpm into libvncserver, -devel and x11vnc
 

libvncserver/auth.c

@@ -6,6 +6,7 @@
  */
 
 /*
+ *  Copyright (C) 2005 Rohit Kumar, Johannes E. Schindelin
  *  OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
  *  Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.  
  *  All Rights Reserved.
@@ -29,38 +30,216 @@
 #include <rfb/rfb.h>
 
 /*
- * rfbAuthNewClient is called when we reach the point of authenticating
- * a new client.  If authentication isn't being used then we simply send
- * rfbNoAuth.  Otherwise we send rfbVncAuth plus the challenge.
+ * Handle security types
  */
 
 void
-rfbAuthNewClient(rfbClientPtr cl)
+rfbRegisterSecurityHandler(rfbScreenInfoPtr server, rfbSecurityHandler* handler)
 {
-    char buf[4 + CHALLENGESIZE];
-    int len;
+	rfbSecurityHandler* last = handler;
+
+	while(last->next)
+		last = last->next;
+
+	last->next = server->securityHandlers;
+	server->securityHandlers = handler;
+}
 
-    cl->state = RFB_AUTHENTICATION;
 
-    if (cl->screen->authPasswdData && !cl->reverseConnection) {
-        *(uint32_t *)buf = Swap32IfLE(rfbVncAuth);
+/*
+ * Send the authentication challenge.
+ */
+
+static void
+rfbVncAuthSendChallenge(rfbClientPtr cl)
+{
+	
+    /* 4 byte header is alreay sent. Which is rfbSecTypeVncAuth 
+       (same as rfbVncAuth). Just send the challenge. */
     rfbRandomBytes(cl->authChallenge);
-        memcpy(&buf[4], (char *)cl->authChallenge, CHALLENGESIZE);
-        len = 4 + CHALLENGESIZE;
-    } else {
-        *(uint32_t *)buf = Swap32IfLE(rfbNoAuth);
-        len = 4;
+    if (rfbWriteExact(cl, (char *)cl->authChallenge, CHALLENGESIZE) < 0) {
+        rfbLogPerror("rfbAuthNewClient: write");
+        rfbCloseClient(cl);
+        return;
+    }
+    
+    /* Dispatch client input to rfbVncAuthProcessResponse. */
+    cl->state = RFB_AUTHENTICATION;
+}
+
+
+/*
+ * Advertise the supported security types (protocol 3.7). Here before sending 
+ * the list of security types to the client one more security type is added 
+ * to the list if primaryType is not set to rfbSecTypeInvalid. This security
+ * type is the standard vnc security type which does the vnc authentication
+ * or it will be security type for no authentication.
+ * Different security types will be added by applications using this library.
+ */
+
+static void
+rfbSendSecurityTypeList(rfbClientPtr cl, int primaryType)
+{
+    /* The size of the message is the count of security types +1,
+     * since the first byte is the number of types. */
+    int size = 1;
+    rfbSecurityHandler* handler;
+#define MAX_SECURITY_TYPES 255
+    uint8_t buffer[MAX_SECURITY_TYPES+1];
+
+    /* Fill in the list of security types in the client structure. */
+    if (primaryType != rfbSecTypeInvalid) {
+	rfbSecurityHandler* handler = calloc(sizeof(rfbSecurityHandler),1);
+	handler->type = primaryType;
+	handler->handler = rfbVncAuthSendChallenge;
+	handler->next = NULL;
+	rfbRegisterSecurityHandler(cl->screen, handler);
+    }
+
+    for (handler = cl->screen->securityHandlers;
+	    handler && size<MAX_SECURITY_TYPES; handler = handler->next) {
+	buffer[size] = handler->type;
+	size++;
+    }
+    buffer[0] = (unsigned char)size-1;
+
+    /* Send the list. */
+    if (rfbWriteExact(cl, (char *)buffer, size) < 0) {
+	rfbLogPerror("rfbSendSecurityTypeList: write");
+	rfbCloseClient(cl);
+	return;
+    }
+
+    /*
+      * if count is 0, we need to send the reason and close the connection.
+      */
+    if(size <= 1) {
+	/* This means total count is Zero and so reason msg should be sent */
+	/* The execution should never reach here */
+	char* reason = "No authentication mode is registered!";
+
+	rfbClientConnFailed(cl, reason);
+	return;
+    }
+
+    /* Dispatch client input to rfbProcessClientSecurityType. */
+    cl->state = RFB_SECURITY_TYPE;
+}
+
+
+
+
+/*
+ * Tell the client what security type will be used (protocol 3.3).
+ */
+static void
+rfbSendSecurityType(rfbClientPtr cl, int32_t securityType)
+{
+    uint32_t value32;
+
+    /* Send the value. */
+    value32 = Swap32IfLE(securityType);
+    if (rfbWriteExact(cl, (char *)&value32, 4) < 0) {
+	rfbLogPerror("rfbSendSecurityType: write");
+	rfbCloseClient(cl);
+	return;
+    }
+
+    /* Decide what to do next. */
+    switch (securityType) {
+    case rfbSecTypeNone:
+	/* Dispatch client input to rfbProcessClientInitMessage. */
 	cl->state = RFB_INITIALISATION;
+	break;
+    case rfbSecTypeVncAuth:
+	/* Begin the standard VNC authentication procedure. */
+	rfbVncAuthSendChallenge(cl);
+	break;
+    default:
+	/* Impossible case (hopefully). */
+	rfbLogPerror("rfbSendSecurityType: assertion failed");
+	rfbCloseClient(cl);
     }
+}
 
-    if (rfbWriteExact(cl, buf, len) < 0) {
-        rfbLogPerror("rfbAuthNewClient: write");
+
+
+/*
+ * rfbAuthNewClient is called right after negotiating the protocol
+ * version. Depending on the protocol version, we send either a code
+ * for authentication scheme to be used (protocol 3.3), or a list of
+ * possible "security types" (protocol 3.7).
+ */
+
+void
+rfbAuthNewClient(rfbClientPtr cl)
+{
+    int32_t securityType = rfbSecTypeInvalid;
+
+    if (!cl->screen->authPasswdData || cl->reverseConnection) {
+	// chk if this condition is valid or not.
+	securityType = rfbSecTypeNone;
+    } else if (cl->screen->authPasswdData) {
+ 	    securityType = rfbSecTypeVncAuth;
+    }
+
+    if (cl->protocolMinorVersion < 7) {
+	/* Make sure we use only RFB 3.3 compatible security types. */
+	if (securityType == rfbSecTypeInvalid) {
+	    rfbLog("VNC authentication disabled - RFB 3.3 client rejected\n");
+	    rfbClientConnFailed(cl, "Your viewer cannot handle required "
+				"authentication methods");
+	    return;
+	}
+	rfbSendSecurityType(cl, securityType);
+    } else {
+	/* Here it's ok when securityType is set to rfbSecTypeInvalid. */
+	rfbSendSecurityTypeList(cl, securityType);
+    }
+}
+
+/*
+ * Read the security type chosen by the client (protocol 3.7).
+ */
+
+void
+rfbProcessClientSecurityType(rfbClientPtr cl)
+{
+    int n, i;
+    uint8_t chosenType;
+    rfbSecurityHandler* handler;
+
+    /* Read the security type. */
+    n = rfbReadExact(cl, (char *)&chosenType, 1);
+    if (n <= 0) {
+	if (n == 0)
+	    rfbLog("rfbProcessClientSecurityType: client gone\n");
+	else
+	    rfbLogPerror("rfbProcessClientSecurityType: read");
 	rfbCloseClient(cl);
 	return;
     }
+
+    if(chosenType == rfbSecTypeNone) {
+       cl->state = RFB_INITIALISATION;
+	return;
+    }
+	
+
+    /* Make sure it was present in the list sent by the server. */
+    for (handler = cl->screen->securityHandlers; handler;
+	    handler = handler->next)
+	if (chosenType == handler->type) {
+	    handler->handler(cl);
+	    return;
+	}
+
+    rfbLog("rfbProcessClientSecurityType: wrong security type requested\n");
+    rfbCloseClient(cl);
 }
 
 
+
 /*
  * rfbAuthProcessClientMessage is called when the client sends its
  * authentication response.

libvncserver/main.c

@@ -668,6 +668,7 @@ rfbScreenInfoPtr rfbGetScreen(int* argc,char** argv,
    screen->newClientHook = rfbDefaultNewClientHook;
    screen->displayHook = NULL;
    screen->processCustomClientMessage = rfbDefaultProcessCustomClientMessage;
+   screen->securityHandlers = NULL;
 
    /* initialize client list and iterator mutex */
    rfbClientListInit(screen);

libvncserver/rfbserver.c

@@ -487,6 +487,9 @@ rfbProcessClientMessage(rfbClientPtr cl)
     case RFB_PROTOCOL_VERSION:
         rfbProcessClientProtocolVersion(cl);
         return;
+    case RFB_SECURITY_TYPE:
+        rfbProcessClientSecurityType(cl);
+        return;
     case RFB_AUTHENTICATION:
         rfbAuthProcessClientMessage(cl);
         return;
@@ -545,9 +548,17 @@ rfbProcessClientProtocolVersion(rfbClientPtr cl)
         return;
     }
 
-    if (minor_ != rfbProtocolMinorVersion) {
-        /* Minor version mismatch - warn but try to continue */
-        rfbLog("Ignoring minor version mismatch\n");
+    // Chk for the minor version use either of the two standard version of RFB
+    cl->protocolMinorVersion = minor_;
+    if (minor_ > rfbProtocolMinorVersion) {
+       cl->protocolMinorVersion = rfbProtocolMinorVersion;
+    } else if (minor_ < rfbProtocolMinorVersion) {
+       cl->protocolMinorVersion = rfbProtocolFallbackMinorVersion;
+    }
+    if (minor_ != rfbProtocolMinorVersion &&
+       minor_ != rfbProtocolFallbackMinorVersion) {
+       rfbLog("Non-standard protocol version %d.%d, using %d.%d instead\n",
+              major_, minor_, rfbProtocolMajorVersion, cl->protocolMinorVersion);
     }
 
     rfbAuthNewClient(cl);

rfb/rfb.h

@@ -6,6 +6,8 @@
  */
 
 /*
+ *  Copyright (C) 2005 Rohit Kumar <rokumar@novell.com>,
+ *                     Johannes E. Schindelin <johannes.schindelin@gmx.de>
  *  Copyright (C) 2002 RealVNC Ltd.
  *  OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
  *  Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.  
@@ -142,6 +144,17 @@ typedef struct {
   } data; /* there have to be count*3 entries */
 } rfbColourMap;
 
+/*
+ *  Security handling (RFB protocol version 3.7
+ */
+
+typedef struct _rfbSecurity {
+	uint8_t type;
+	void (*handler)(struct _rfbClientRec* cl);
+	struct _rfbSecurity* next;
+} rfbSecurityHandler;
+
+
 /*
  * Per-screen (framebuffer) structure.  There can be as many as you wish,
  * each serving different clients. However, you have to call
@@ -265,6 +278,8 @@ typedef struct _rfbScreenInfo
     rfbProcessCustomClientMessageProcPtr processCustomClientMessage;
 
     in_addr_t listenInterface;
+
+    rfbSecurityHandler* securityHandlers;
 } rfbScreenInfo, *rfbScreenInfoPtr;
 
 
@@ -307,12 +322,16 @@ typedef struct _rfbClientRec {
     SOCKET sock;
     char *host;
 
+    /* RFB protocol minor version number */
+    int protocolMinorVersion;
+
 #ifdef LIBVNCSERVER_HAVE_LIBPTHREAD
     pthread_t client_thread;
 #endif
                                 /* Possible client states: */
     enum {
         RFB_PROTOCOL_VERSION,   /* establishing protocol version */
+	RFB_SECURITY_TYPE,      /* negotiating security (RFB v.3.7) */
         RFB_AUTHENTICATION,     /* authenticating */
         RFB_INITIALISATION,     /* sending initialisation messages */
         RFB_NORMAL              /* normal protocol messages */
@@ -582,7 +601,8 @@ extern void rfbHttpCheckFds(rfbScreenInfoPtr rfbScreen);
 
 extern void rfbAuthNewClient(rfbClientPtr cl);
 extern void rfbAuthProcessClientMessage(rfbClientPtr cl);
-
+extern void rfbRegisterSecurityHandler(rfbScreenInfoPtr server,
+	rfbSecurityHandler* handler);
 
 /* rre.c */
 

rfb/rfbproto.h

@@ -2,6 +2,7 @@
 #define RFBPROTO_H
 
 /*
+ *  Copyright (C) 2005 Rohit Kumar, Johannes E. Schindelin
  *  Copyright (C) 2000-2002 Constantin Kaplinsky.  All Rights Reserved.
  *  Copyright (C) 2000 Tridia Corporation.  All Rights Reserved.
  *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
@@ -217,12 +218,30 @@ typedef struct {
 
 #define rfbProtocolVersionFormat "RFB %03d.%03d\n"
 #define rfbProtocolMajorVersion 3
-#define rfbProtocolMinorVersion 3
+#define rfbProtocolMinorVersion 7
+#define rfbProtocolFallbackMinorVersion 3
 
 typedef char rfbProtocolVersionMsg[13];	/* allow extra byte for null */
 
 #define sz_rfbProtocolVersionMsg 12
 
+/*
+ * Negotiation of the security type (protocol version 3.7)
+ *
+ * Once the protocol version has been decided, the server either sends a list
+ * of supported security types, or informs the client about an error (when the
+ * number of security types is 0).  Security type rfbSecTypeTight is used to
+ * enable TightVNC-specific protocol extensions.  The value rfbSecTypeVncAuth
+ * stands for classic VNC authentication.
+ *
+ * The client selects a particular security type from the list provided by the
+ * server.
+ */
+
+#define rfbSecTypeInvalid 0
+#define rfbSecTypeNone 1
+#define rfbSecTypeVncAuth 2
+
 
 /*-----------------------------------------------------------------------------
  * Authentication