Add sha1.*. Remove UTF-8 encode. Protocol handling.

Add common/sha1.h and common/sha1.c so that we have the SHA routines
even if openssl is not available. From the IETF SHA RFC example code.

Remove the UTF-8 encoding hack. This was really just an experiment.

If the protocol passed in the handshake has "binary" then don't base64
encode for the HyBi protocol. This will allow noVNC to request the
binary data be passed raw and not base64 encoded. Unfortunately, the
client doesn't speak first in VNC protocol (bad original design). If
it did then we could determine whether to base64 encode or not based
on the first HyBi frame from the client and whether the binary bit is
set or not. Oh well.

Misc Cleanup:

- Always free response and buf in handshake routine.

- Remove some unused variables.

common/sha1.h

+/*
+ * Copyright (C) The Internet Society (2001).  All Rights Reserved.
+ *
+ * This document and translations of it may be copied and furnished to
+ * others, and derivative works that comment on or otherwise explain it
+ * or assist in its implementation may be prepared, copied, published
+ * and distributed, in whole or in part, without restriction of any
+ * kind, provided that the above copyright notice and this paragraph are
+ * included on all such copies and derivative works.  However, this
+ * document itself may not be modified in any way, such as by removing
+ * the copyright notice or references to the Internet Society or other
+ * Internet organizations, except as needed for the purpose of
+ * developing Internet standards in which case the procedures for
+ * copyrights defined in the Internet Standards process must be
+ * followed, or as required to translate it into languages other than
+ * English.
+ *
+ * The limited permissions granted above are perpetual and will not be
+ * revoked by the Internet Society or its successors or assigns.
+ *
+ * This document and the information contained herein is provided on an
+ * "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING
+ * TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION
+ * HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/*
+ *  sha1.h
+ *
+ *  Description:
+ *      This is the header file for code which implements the Secure
+ *      Hashing Algorithm 1 as defined in FIPS PUB 180-1 published
+ *      April 17, 1995.
+ *
+ *      Many of the variable names in this code, especially the
+ *      single character names, were used because those were the names
+ *      used in the publication.
+ *
+ *      Please read the file sha1.c for more information.
+ *
+ */
+
+
+#ifndef _SHA1_H_
+#define _SHA1_H_
+
+#include <stdint.h>
+/*
+ * If you do not have the ISO standard stdint.h header file, then you
+ * must typdef the following:
+ *    name              meaning
+ *  uint32_t         unsigned 32 bit integer
+ *  uint8_t          unsigned 8 bit integer (i.e., unsigned char)
+ *  int_least16_t    integer of >= 16 bits
+ *
+ */
+
+#ifndef _SHA_enum_
+#define _SHA_enum_
+enum
+{
+    shaSuccess = 0,
+    shaNull,            /* Null pointer parameter */
+    shaInputTooLong,    /* input data too long */
+    shaStateError       /* called Input after Result */
+};
+#endif
+#define SHA1HashSize 20
+
+/*
+ *  This structure will hold context information for the SHA-1
+ *  hashing operation
+ */
+typedef struct SHA1Context
+{
+    uint32_t Intermediate_Hash[SHA1HashSize/4]; /* Message Digest  */
+
+    uint32_t Length_Low;            /* Message length in bits      */
+    uint32_t Length_High;           /* Message length in bits      */
+
+                               /* Index into message block array   */
+    int_least16_t Message_Block_Index;
+    uint8_t Message_Block[64];      /* 512-bit message blocks      */
+
+    int Computed;               /* Is the digest computed?         */
+    int Corrupted;             /* Is the message digest corrupted? */
+} SHA1Context;
+
+/*
+ *  Function Prototypes
+ */
+int SHA1Reset(  SHA1Context *);
+int SHA1Input(  SHA1Context *,
+                const uint8_t *,
+                unsigned int);
+int SHA1Result( SHA1Context *,
+                uint8_t Message_Digest[SHA1HashSize]);
+
+#endif

libvncserver/Makefile.am

@@ -24,7 +24,7 @@ WEBSOCKETSSSLSRCS = rfbssl_none.c
 #endif
 endif
 
-WEBSOCKETSSRCS = websockets.c ../common/md5.c $(WEBSOCKETSSSLSRCS)
+WEBSOCKETSSRCS = websockets.c ../common/md5.c ../common/sha1.c $(WEBSOCKETSSSLSRCS)
 endif
 
 includedir=$(prefix)/include/rfb