LibVNCClient: Add H.264 encoding for framebuffer updates

This patch implements support in LibVNCClient for framebuffer updates
encoded as H.264 frames. Hardware accelerated decoding is performed
using VA API.

This is experimental support to let the community explore the possibilities
offered by the potential bandwidth and latency reductions that H.264 encoding
allows. This may be particularly useful for use cases such as online gaming,
hosted desktops, hosted set top boxes...

This patch only provides the client side support and is meant to be used
with corresponding server-side support, as provided by an upcoming patch for
qemu ui/vnc module (to view the display of a virtual machine executing under
QEMU).

With this H.264-based encoding, if multiple framebuffer update messages
are generated for a single server framebuffer modification, the H.264
frame data is sent only with the first update message. Subsequent update
framebuffer messages will contain only the coordinates and size of the
additional updated regions.

Instructions/Requirements:
* The patch should be applied on top of the previous patch I submitted with
minor enhancements to the gtkvncviewer application:
http://sourceforge.net/mailarchive/message.php?msg_id=30323804
* Currently only works with libva 1.0: use branch "v1.0-branch" for libva and
intel-driver. Those can be built as follows:
   cd libva
   git checkout v1.0-branch
   ./autogen.sh
   make
   sudo make install
   cd ..
   git clone git://anongit.freedesktop.org/vaapi/intel-driver
   cd intel-driver
   git checkout v1.0-branch
   ./autogen.sh
   make
   sudo make install
Signed-off-by: David Verbeiren <david.verbeiren@intel.com>

client_examples/gtkvncviewer.c

@@ -23,6 +23,10 @@
 #include <gdk/gdkkeysyms.h>
 #include <rfb/rfbclient.h>
 
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+#include <gdk/gdkx.h>
+#endif
+
 static rfbClient *cl;
 static gchar *server_cut_text = NULL;
 static gboolean framebuffer_allocated = FALSE;
@@ -57,6 +61,14 @@ static gboolean expose_event (GtkWidget      *widget,
 		cl->format.greenMax = (1 << image->visual->green_prec) - 1;
 		cl->format.blueMax  = (1 << image->visual->blue_prec) - 1;
 
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+		/* Allow libvncclient to use a more efficient way
+		 * of putting the framebuffer on the screen when
+		 * using the H.264 format.
+		 */
+		cl->outputWindow = GDK_WINDOW_XID(widget->window);
+#endif
+
 		SetFormatAndEncodings (cl);
 
 		framebuffer_allocated = TRUE;
@@ -67,12 +79,14 @@ static gboolean expose_event (GtkWidget      *widget,
 		gdk_cursor_unref( cur );
 	}
 
+#ifndef LIBVNCSERVER_CONFIG_LIBVA
 	gdk_draw_image (GDK_DRAWABLE (widget->window),
 	                widget->style->fg_gc[gtk_widget_get_state(widget)],
 	                image,
 	                event->area.x, event->area.y,
 	                event->area.x, event->area.y,
 	                event->area.width, event->area.height);
+#endif
 
 	return FALSE;
 }
@@ -462,10 +476,12 @@ static void update (rfbClient *cl, int x, int y, int w, int h) {
 		dialog_connecting = NULL;
 	}
 
+#ifndef LIBVNCSERVER_CONFIG_LIBVA
 	GtkWidget *drawing_area = rfbClientGetClientData (cl, gtk_init);
 
 	if (drawing_area != NULL)
 		gtk_widget_queue_draw_area (drawing_area, x, y, w, h);
+#endif
 }
 
 static void kbd_leds (rfbClient *cl, int value, int pad) {

configure.ac

@@ -151,6 +151,20 @@ HAVE_X11="false"
 AC_PATH_XTRA
 AH_TEMPLATE(HAVE_X11, [X11 build environment present])
 
+# See if we want libva support
+# TODO: check if library actually exists
+AH_TEMPLATE(CONFIG_LIBVA, [Build libva support])
+AC_ARG_WITH(libva,
+[  --with-libva                   build libva support],,)
+if test "x$with_libva" != "xno"; then
+    AC_CHECK_LIB(va, vaInitialize,
+        VA_LIBS="-lva -lva-x11"
+        [AC_DEFINE(CONFIG_LIBVA) CONFIG_LIBVA="true"], ,)
+fi
+AC_SUBST(VA_LIBS)
+AM_CONDITIONAL(CONFIG_LIBVA, test ! -z "$VA_LIBS")
+
+
 # See if we are to build x11vnc:
 AH_TEMPLATE(HAVE_SYSTEM_LIBVNCSERVER, [Use the system libvncserver build environment for x11vnc.])
 AC_ARG_WITH(system-libvncserver,

libvncclient/Makefile.am

@@ -14,7 +14,7 @@ endif
 
 
 libvncclient_la_SOURCES=cursor.c listen.c rfbproto.c sockets.c vncviewer.c ../common/minilzo.c $(TLSSRCS)
-libvncclient_la_LIBADD=$(TLSLIBS)
+libvncclient_la_LIBADD=$(TLSLIBS) $(VA_LIBS)
 
 noinst_HEADERS=../common/lzodefs.h ../common/lzoconf.h ../common/minilzo.h tls.h
 

libvncclient/rfbproto.c

@@ -158,6 +158,10 @@ static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_
 static void CopyRectangle(rfbClient* client, uint8_t* buffer, int x, int y, int w, int h) {
   int j;
 
+  if (client->frameBuffer == NULL) {
+      return;
+  }
+
 #define COPY_RECT(BPP) \
   { \
     int rs = w * BPP / 8, rs2 = client->width * BPP / 8; \
@@ -260,6 +264,9 @@ static rfbBool HandleZRLE24Up(rfbClient* client, int rx, int ry, int rw, int rh)
 static rfbBool HandleZRLE24Down(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleZRLE32(rfbClient* client, int rx, int ry, int rw, int rh);
 #endif
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+static rfbBool HandleH264 (rfbClient* client, int rx, int ry, int rw, int rh);
+#endif
 
 /*
  * Server Capability Functions
@@ -1344,6 +1351,10 @@ SetFormatAndEncodings(rfbClient* client)
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCoRRE);
       } else if (strncasecmp(encStr,"rre",encStrLen) == 0) {
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRRE);
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+      } else if (strncasecmp(encStr,"h264",encStrLen) == 0) {
+	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingH264);
+#endif
       } else {
 	rfbClientLog("Unknown encoding '%.*s'\n",encStrLen,encStr);
       }
@@ -1412,6 +1423,10 @@ SetFormatAndEncodings(rfbClient* client)
       encs[se->nEncodings++] = rfbClientSwap32IfLE(client->appData.qualityLevel +
 					  rfbEncodingQualityLevel0);
     }
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingH264);
+    rfbClientLog("h264 encoding added\n");
+#endif
   }
 
 
@@ -2127,6 +2142,14 @@ HandleRFBServerMessage(rfbClient* client)
 	break;
      }
 
+#endif
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+      case rfbEncodingH264:
+      {
+	if (!HandleH264(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+	  return FALSE;
+	break;
+      }
 #endif
 
       default:
@@ -2361,6 +2384,7 @@ HandleRFBServerMessage(rfbClient* client)
 #define UNCOMP -8
 #include "zrle.c"
 #undef BPP
+#include "h264.c"
 
 
 /*

libvncclient/vncviewer.c

@@ -91,7 +91,11 @@ static rfbBool MallocFrameBuffer(rfbClient* client) {
 static void initAppData(AppData* data) {
 	data->shareDesktop=TRUE;
 	data->viewOnly=FALSE;
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+	data->encodingsString="h264 tight zrle ultra copyrect hextile zlib corre rre raw";
+#else
 	data->encodingsString="tight zrle ultra copyrect hextile zlib corre rre raw";
+#endif
 	data->useBGR233=FALSE;
 	data->nColours=0;
 	data->forceOwnCmap=FALSE;
@@ -129,6 +133,9 @@ rfbClient* rfbGetClient(int bitsPerSample,int samplesPerPixel,
   /* default: use complete frame buffer */ 
   client->updateRect.x = -1;
  
+  client->frameBuffer = NULL;
+  client->outputWindow = 0;
+ 
   client->format.bitsPerPixel = bytesPerPixel*8;
   client->format.depth = bitsPerSample*samplesPerPixel;
   client->appData.requestedDepth=client->format.depth;

rfb/rfbclient.h

@@ -175,6 +175,7 @@ typedef void (*GotCopyRectProc)(struct _rfbClient* client, int src_x, int src_y,
 
 typedef struct _rfbClient {
 	uint8_t* frameBuffer;
+	unsigned long outputWindow; /* Output Window ID. When set, client application enables libvncclient to perform direct rendering in its window */
 	int width, height;
 
 	int endianTest;

rfb/rfbproto.h

@@ -514,6 +514,9 @@ typedef struct {
 #define rfbEncodingSupportedEncodings 0xFFFE0002
 #define rfbEncodingServerIdentity     0xFFFE0003
 
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+#define rfbEncodingH264               0x48323634
+#endif
 
 /*****************************************************************************
  *
@@ -868,6 +871,21 @@ typedef struct {
 
 #endif
 
+/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * h264 - h264 encoding.  We have an rfbH264Header structure
+ * giving the number of bytes following.  Finally the data follows is
+ * h264 encoded frame.
+ */
+
+typedef struct {
+    uint32_t nBytes;
+	uint32_t slice_type;
+	uint32_t width;
+	uint32_t height;
+} rfbH264Header;
+
+#define sz_rfbH264Header 16
+
 /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  * XCursor encoding. This is a special encoding used to transmit X-style
  * cursor shapes from server to clients. Note that for this encoding,