add cursor alphablending to rfb.h cursor.c, x11vnc -alphablend -snapfb etc..

ChangeLog

+2005-01-15  Karl Runge <runge@karlrunge.com>
+	* rfb/rfb.h: add alphaSource and alphaPreMultiplied to rfbCursor. 
+	* libvncserver/cursor.c: do cursor alpha blending in rfbDrawCursor()
+	  for non-cursorshapeupdates clients.
+	* x11vnc: -alphablend, cursors fixes, -snapfb, more tweaks and bug
+	  fixes.
+
 2004-12-27  Karl Runge <runge@karlrunge.com>
 	* x11vnc: improve alpha channel handling for XFIXES cursors. 
 	* add more parameters to remote control.

libvncserver/cursor.c

@@ -310,6 +310,8 @@ void rfbFreeCursor(rfbCursorPtr cursor)
    if(cursor) {
        if(cursor->cleanupRichSource && cursor->richSource)
 	   free(cursor->richSource);
+       if(cursor->cleanupRichSource && cursor->alphaSource)
+	   free(cursor->alphaSource);
        if(cursor->cleanupSource && cursor->source)
 	   free(cursor->source);
        if(cursor->cleanupMask && cursor->mask)
@@ -320,6 +322,7 @@ void rfbFreeCursor(rfbCursorPtr cursor)
 	   cursor->cleanup=cursor->cleanupSource=cursor->cleanupMask
 	       =cursor->cleanupRichSource=FALSE;
 	   cursor->source=cursor->mask=cursor->richSource=0;
+	   cursor->alphaSource=0;
        }
    }
    
@@ -489,12 +492,84 @@ void rfbDrawCursor(rfbScreenInfoPtr s)
    if(!c->richSource)
      rfbMakeRichCursorFromXCursor(s,c);
    
-   /* now the cursor has to be drawn */
-   for(j=0;j<y2;j++)
-     for(i=0;i<x2;i++)
-       if((c->mask[(j+j1)*w+(i+i1)/8]<<((i+i1)&7))&0x80)
-	 memcpy(s->frameBuffer+(j+y1)*rowstride+(i+x1)*bpp,
-		c->richSource+(j+j1)*c->width*bpp+(i+i1)*bpp,bpp);
+   if (c->alphaSource) {
+	int rmax, rshift;
+	int gmax, gshift;
+	int bmax, bshift;
+	int amax = 255;	/* alphaSource is always 8bits of info per pixel */
+	unsigned long rmask, gmask, bmask;
+
+	rmax   = s->serverFormat.redMax;
+	gmax   = s->serverFormat.greenMax;
+	bmax   = s->serverFormat.blueMax;
+	rshift = s->serverFormat.redShift;
+	gshift = s->serverFormat.greenShift;
+	bshift = s->serverFormat.blueShift;
+
+	rmask = (rmax << rshift);
+	gmask = (gmax << gshift);
+	bmask = (bmax << bshift);
+
+	for(j=0;j<y2;j++) {
+		for(i=0;i<x2;i++) {
+			/*
+			 * we loop over the whole cursor ignoring c->mask[],
+			 * using the extracted alpha value instead.
+			 */
+			char *dest, *src, *aptr;
+			unsigned long val, *dv, *sv;
+			int rdst, gdst, bdst;		/* fb RGB */
+			int asrc, rsrc, gsrc, bsrc;	/* rich source ARGB */
+
+			dest = s->frameBuffer + (j+y1)*rowstride + (i+x1)*bpp;
+			src  = c->richSource  + (j+j1)*c->width*bpp + (i+i1)*bpp;
+			aptr = c->alphaSource + (j+j1)*c->width + (i+i1);
+
+			dv = (unsigned long *)dest;
+			sv = (unsigned long *)src;
+
+			asrc = *((unsigned char *)aptr);
+
+			if (!asrc) {
+				continue;
+			}
+
+			/* extract dest and src RGB */
+			rdst = (*dv & rmask) >> rshift;	/* fb */
+			gdst = (*dv & gmask) >> gshift;
+			bdst = (*dv & bmask) >> bshift;
+
+			rsrc = (*sv & rmask) >> rshift;	/* richcursor */
+			gsrc = (*sv & gmask) >> gshift;
+			bsrc = (*sv & bmask) >> bshift;
+
+			/* blend in fb data. */
+			if (! c->alphaPreMultiplied) {
+				rsrc = (asrc * rsrc)/amax;
+				gsrc = (asrc * gsrc)/amax;
+				bsrc = (asrc * bsrc)/amax;
+			}
+			rdst = rsrc + ((amax - asrc) * rdst)/amax;
+			gdst = gsrc + ((amax - asrc) * gdst)/amax;
+			bdst = bsrc + ((amax - asrc) * bdst)/amax;
+
+			val = 0;
+			val |= (rdst << rshift);
+			val |= (gdst << gshift);
+			val |= (bdst << bshift);
+
+			/* insert the cooked pixel into the fb */
+			memcpy(dest, &val, bpp);
+		}
+	}
+   } else {
+      /* now the cursor has to be drawn */
+      for(j=0;j<y2;j++)
+        for(i=0;i<x2;i++)
+          if((c->mask[(j+j1)*w+(i+i1)/8]<<((i+i1)&7))&0x80)
+   	 memcpy(s->frameBuffer+(j+y1)*rowstride+(i+x1)*bpp,
+   		c->richSource+(j+j1)*c->width*bpp+(i+i1)*bpp,bpp);
+   }
 
    if(wasChanged)
      rfbMarkRectAsModified(s,x1,y1,x1+x2,y1+y2);

rfb/rfb.h

@@ -664,6 +664,8 @@ typedef struct rfbCursor {
     unsigned short foreRed, foreGreen, foreBlue; /* device-independent colour */
     unsigned short backRed, backGreen, backBlue; /* device-independent colour */
     unsigned char *richSource; /* source bytes for a rich cursor */
+    unsigned char *alphaSource; /* source for alpha blending info */
+    rfbBool alphaPreMultiplied; /* if richSource already has alpha applied */
 } rfbCursor, *rfbCursorPtr;
 extern unsigned char rfbReverseByte[0x100];
 

x11vnc/ChangeLog

+2005-01-15  Karl Runge <runge@karlrunge.com>
+	* adjust alpha blending parameters, -alphablend, handle 24bpp.
+	* add -snapfb snapshot fb, not clear how useful it is..
+	* more functions etc for -pointer_mode 4, still not finished.
+	* scan_for_updates() "count only" mode.
+	* increase max shm size on Linux.
+	* -oa -logappend, -pm, -speeds 
+	* fix bugs in -allow, -R connect, screen == NULL
+
 2004-12-27  Karl Runge <runge@karlrunge.com>
 	* allow -DLIBVNCSERVER_HAVE_FOO=0 override everything
 	* get_xfixes_cursor() try to more carefully convert alpha channel