x11vnc: support colormaps for depths other than 8. xinerama warppointer only...

x11vnc: support colormaps for depths other than 8.  xinerama warppointer only if more than one subscreen.

x11vnc/8to24.c

@@ -59,10 +59,11 @@ enum mark_8bpp_modes {
 	MARK_8BPP_TOP
 };
 
+
 #define NCOLOR 256
 
 static Colormap root_cmap = 0;
-static unsigned int root_rgb[NCOLOR];
+static unsigned int *root_rgb = NULL;
 
 static void set_root_cmap(void) {
 #if NO_X11
@@ -72,11 +73,25 @@ static void set_root_cmap(void) {
 	static time_t last_set = 0;
 	time_t now = time(NULL);
 	XWindowAttributes attr;
-	static XColor color[NCOLOR];
+	static XColor *color = NULL;
 	int redo = 0;
+	int ncolor = 0;
 
 	RAWFB_RET_VOID
 
+	if (depth > 8) {
+		ncolor = 1 << depth;
+	} else {
+		ncolor = NCOLOR;
+	}
+
+	if (!root_rgb) {
+		root_rgb = (unsigned int *) malloc(ncolor * sizeof(unsigned int));
+	}
+	if (!color) {
+		color = (XColor *) malloc(ncolor * sizeof(XColor));
+	}
+
 	if (now > last_set + 10) {
 		redo = 1;
 	}
@@ -87,7 +102,11 @@ static void set_root_cmap(void) {
 			return;
 		}
 		if (attr.colormap) {
-			int i, ncells = NCOLOR;
+			int i, ncells = ncolor;
+
+			if (depth < 8) {
+				ncells = CellsOfScreen(ScreenOfDisplay(dpy, scr));
+			}
 			for (i=0; i < ncells; i++) {
 				color[i].pixel = i;
 				color[i].pad = 0;
@@ -236,7 +255,11 @@ static void set_poll_fb(void) {
 		return;		/* this saves a bit of RAM */
 	}
 	pfb(4, &poll24_fb, &poll24_fb_w, &poll24_fb_h);
+	if (depth > 8) {
+		pfb(2, &poll8_fb,  &poll8_fb_w,  &poll8_fb_h);	/* 2X for rare 16bpp colormap case */
+	} else {
 		pfb(1, &poll8_fb,  &poll8_fb_w,  &poll8_fb_h);
+	}
 }
 
 int MV_glob = 0;
@@ -428,7 +451,7 @@ if (db24 > 2) fprintf(stderr, " check_for_multivis: %.4f\n", now - last_call);
 		if (check_depth(win, win, doall)) {
 			/*
 			 * returns 1 if no need to recurse down e.g. It
-			 * is 8bpp and we assume all lower one are too.
+			 * is 8bpp and we assume all lower ones are too.
 			 */
 			continue;
 		}
@@ -529,7 +552,7 @@ if (0) fprintf(stderr, "MV_count: %d hit: %d %.4f  %10.2f\n", MV_count, MV_hit,
 		mark_8bpp(MARK_8BPP_ALL);
 		last_poll = now;
 
-	} else if (depth == 8 && multivis_24count) {
+	} else if (depth <= 16 && multivis_24count) {
 		static double last_check = 0.0;
 		if (now > last_check + 0.4) {
 			last_check = now;
@@ -656,8 +679,7 @@ static int check_depth_win(Window win, Window top, XWindowAttributes *attr) {
 	if (attr->depth > 0) {
 		if (depth == 24 && attr->depth != 24) {
 			store_it = 1;
-		} else if (depth == 8 && root_cmap && attr->colormap !=
-		    root_cmap) {
+		} else if (depth <= 16 && root_cmap && attr->colormap != root_cmap) {
 			store_it = 1;
 		}
 	}
@@ -759,6 +781,7 @@ if (db24 > 1) fprintf(stderr, "          ------------ 0x%lx i=%d\n", windows_8bp
 	return 0;
 }
 
+/* polling line XImage */
 static XImage *p_xi(XImage *xi, Visual *visual, int win_depth, int *w) {
 	RAWFB_RET(NULL)
 
@@ -771,8 +794,12 @@ static XImage *p_xi(XImage *xi, Visual *visual, int win_depth, int *w) {
 		if (xi) {
 			XDestroyImage(xi);
 		}
-		if (win_depth == 8) {
+		if (win_depth != 24) {
+			if (win_depth > 8) {
+				d = (char *) malloc(dpy_x * 2);
+			} else {
 				d = (char *) malloc(dpy_x * 1);
+			}
 		} else {
 			d = (char *) malloc(dpy_x * 4);
 		}
@@ -837,14 +864,18 @@ static int poll_line(int x1, int x2, int y1, int n, sraRegionPtr mod) {
 		last_win = win;
 	}
 
-	if (attr.depth != 8 && attr.depth != 24) {
+	if (attr.depth > 16 && attr.depth != 24) {
 		X_UNLOCK;
 		return 1;
-	} else if (attr.depth == 8) {
-		xi = xi8 = p_xi(xi8, attr.visual, 8, &xi8_w);
+	} else if (attr.depth <= 16) {
+		xi = xi8 = p_xi(xi8, attr.visual, attr.depth, &xi8_w);
 
 		poll_fb = poll8_fb;
+		if (attr.depth > 8) {
+			fac = 2;
+		} else {
 			fac = 1;
+		}
 		n_off = poll8_fb_w * y1 + x1;
 	} else {
 		xi = xi24 = p_xi(xi24, attr.visual, 24, &xi24_w);
@@ -1280,7 +1311,7 @@ if (db24 > 1) fprintf(stderr, "subtract:  0x%lx %d -- %d %d %d %d\n", swin, k, s
 					break;
 				}
 			}
-			if (! seen && attr.depth == 8) {
+			if (!seen && attr.depth <= 16) {
 				/* store only new ones: */
 				cmaps[ncmaps++] = attr.colormap;
 			}
@@ -1292,10 +1323,11 @@ if (db24 > 1) fprintf(stderr, "subtract:  0x%lx %d -- %d %d %d %d\n", swin, k, s
 	return mapcount;
 }
 
-static XColor color[CMAPMAX][NCOLOR];
-static unsigned int rgb[CMAPMAX][NCOLOR];
+static XColor *color[CMAPMAX];
+static unsigned int *rgb[CMAPMAX];
 static int cmap_failed[CMAPMAX];
-int histo[256];
+static int color_init = 0;
+int histo[65536];
 
 static int get_cmap(int j, Colormap cmap) {
 #if NO_X11
@@ -1303,27 +1335,38 @@ static int get_cmap(int j, Colormap cmap) {
 	if (!j || !cmap) {}
 	return 0;
 #else
-	int i, ncells;
+	int i, ncells, ncolor;
 	XErrorHandler old_handler = NULL;
 
 	RAWFB_RET(0)
 
-	if (0) {
+	if (depth > 8) {
+		ncolor = 1 << depth;
+	} else {
+		ncolor = NCOLOR;
+	}
+	if (!color_init) {
+		int cm;
+		for (cm = 0; cm < CMAPMAX; cm++) {
+			color[cm] = (XColor *) malloc(ncolor * sizeof(XColor));
+			rgb[cm] = (unsigned int *) malloc(ncolor * sizeof(unsigned int));
+		}
+		color_init = 1;
+	}
+
+	if (depth <= 16) {
 		/* not working properly for depth 24... */
 		X_LOCK;
 		ncells = CellsOfScreen(ScreenOfDisplay(dpy, scr));
 		X_UNLOCK;
-	} else {
-		ncells = NCOLOR;
 	}
 if (db24 > 1) fprintf(stderr, "get_cmap: %d 0x%x\n", j, (unsigned int) cmap);
 
-	/* ncells should "always" be 256. */
-	if (ncells > NCOLOR) {
-		ncells = NCOLOR;
-	} else if (ncells == 8) {
-		/* hmmm. see set_colormap() */
-		ncells = NCOLOR;
+	if (ncells > ncolor) {
+		ncells = ncolor;
+	} else if (ncells == 8 && depth != 3) {
+		/* XXX. see set_colormap() */
+		ncells = 1 << depth;
 	}
 
 	/* initialize XColor array: */
@@ -1404,7 +1447,7 @@ if (db24 > 1) fprintf(stderr, "ncmaps: %d\n", ncmaps);
 		}
 	}
 
-	if (windows_8bpp[n].depth == 8) {	/* 24 won't have a cmap */
+	if (windows_8bpp[n].depth != 24) {	/* 24 won't have a cmap */
 		if (failed || cm == -1) {
 			return;
 		}
@@ -1448,17 +1491,18 @@ static XImage *cmap_xi(XImage *xi, Window win, int win_depth) {
 	if (! dpy || ! valid_window(win, &attr, 1)) {
 		return (XImage *) NULL;
 	}
-	if (attr.depth != win_depth) {
-		return (XImage *) NULL;
-	} else if (win_depth == 8) {
-		d = (char *) malloc(dpy_x * dpy_y * 1);
-	} else if (win_depth == 24) {
+	if (win_depth == 24) {
 		d = (char *) malloc(dpy_x * dpy_y * 4);
+	} else if (win_depth <= 16) {
+		if (win_depth > 8) {
+			d = (char *) malloc(dpy_x * dpy_y * 2);
+		} else {
+			d = (char *) malloc(dpy_x * dpy_y * 1);
+		}
 	} else {
 		return (XImage *) NULL;
 	}
-	return XCreateImage(dpy, attr.visual, win_depth, ZPixmap, 0, d, dpy_x,
-	    dpy_y, 8, 0);
+	return XCreateImage(dpy, attr.visual, win_depth, ZPixmap, 0, d, dpy_x, dpy_y, 8, 0);
 #endif	/* NO_X11 */
 }
 
@@ -1472,6 +1516,7 @@ static void transform_rect(sraRect rect, Window win, int win_depth, int cm) {
 
 	char *src, *dst, *poll;
 	unsigned int *ui;
+	unsigned short *us;
 	unsigned char *uc;
 	int ps, pixelsize = bpp/8;
 	int poll_Bpl;
@@ -1495,7 +1540,7 @@ if (db24 > 1) fprintf(stderr, "transform %4d %4d %4d %4d cm: %d\n", rect.x1, rec
 	h = rect.y2 - rect.y1;
 	w = rect.x2 - rect.x1;
 
-	if (depth == 8) {
+	if (depth != 24) {
 		/* need to fetch depth 24 data. */
 		do_getimage = 1;
 	}
@@ -1524,21 +1569,32 @@ if (db24 > 1) fprintf(stderr, "transform %4d %4d %4d %4d cm: %d\n", rect.x1, rec
 		X_LOCK;
 #define GETSUBIMAGE
 #ifdef GETSUBIMAGE
-		if (win_depth == 8) {
-			if (xi_8 == NULL || xi_8->width != dpy_x ||
-			    xi_8->height != dpy_y) {
-				xi_8 = cmap_xi(xi_8, win, 8);
-			}
-			xi = xi_8;
-		} else if (win_depth == 24) {
+		if (win_depth == 24) {
 			if (xi_24 == NULL || xi_24->width != dpy_x ||
 			    xi_24->height != dpy_y) {
 				xi_24 = cmap_xi(xi_24, win, 24);
 			}
 			xi = xi_24;
+		} else if (win_depth <= 16) {
+			if (xi_8 == NULL || xi_8->width != dpy_x ||
+			    xi_8->height != dpy_y) {
+				if (win_depth > 8) {
+					/* XXX */
+					xi_8 = cmap_xi(xi_8, win, 16);
+				} else {
+					xi_8 = cmap_xi(xi_8, win, 8);
+				}
+			}
+			xi = xi_8;
 		}
 #endif
 
+		if (xi == NULL) {
+			rfbLog("transform_rect: xi is NULL\n");
+			X_UNLOCK;
+			clean_up_exit(1);
+		}
+
 		old_handler = XSetErrorHandler(trap_xerror);
 		trapped_xerror = 0;
 
@@ -1578,7 +1634,7 @@ if (db24 > 1) fprintf(stderr, "xi: 0x%p  %d %d %d %d -- %d %d\n", (void *)xi, xo
 		}
 		trapped_xerror = 0;
 
-		if (xi->depth != 8 && xi->depth != 24) {
+		if (xi->depth > 16 && xi->depth != 24) {
 #ifndef GETSUBIMAGE
 			X_LOCK;
 			XDestroyImage(xi);
@@ -1590,64 +1646,76 @@ if (db24) fprintf(stderr, "xi: wrong depth: %d\n", xi->depth);
 
 		set_poll_fb();
 
-		if (xi->depth == 8) {
-			int ps1, ps2, fac;
-
-			if (depth == 8) {
-				ps1 = 1;
-				ps2 = 4;
+		if (xi->depth == 24) {
+			/* line by line ... */
+			int ps1 = 4, fac;
+			if (depth <= 8) {
 				fac = 4;
+			} else if (depth <= 16) {
+				fac = 2;
 			} else {
-				ps1 = 1;
-				ps2 = pixelsize;
-				fac = 1;
+				fac = 1;	/* will not happen 24 on 24 */
 			}
 
 			src = xi->data;
 			dst = cmap8to24_fb + fac * n_off;
 
-			poll = poll8_fb + poll8_fb_w * rect.y1 + rect.x1;
-			poll_Bpl = poll8_fb_w * 1;
+			poll = poll24_fb + (poll24_fb_w * rect.y1 + rect.x1) * 4;
+			poll_Bpl = poll24_fb_w * 4;
 
-			/* line by line ... */
 			for (line = 0; line < h; line++) {
-				/* pixel by pixel... */
-				for (j = 0; j < w; j++) {
-
-					uc = (unsigned char *) (src + ps1 * j);
-					ui = (unsigned int *)  (dst + ps2 * j);
-
-					idx = (int) (*uc);
-
-					*ui = rgb[cm][idx];
+				memcpy(dst,  src, w * ps1);
+				memcpy(poll, src, w * ps1);
 
-					*(poll + ps1 * j) = *uc;
-				}
 				src += xi->bytes_per_line;
 				dst += main_bytes_per_line * fac;
 				poll += poll_Bpl;
 			}
-		} else if (xi->depth == 24) {
-			/* line by line ... */
-			int ps1 = 4, fac;
-			if (depth == 8) {
+		} else if (xi->depth <= 16) {
+			int ps1, ps2, fac;
+
+			if (depth <= 8) {
+				ps1 = 1;
+				ps2 = 4;
+				fac = 4;
+			} else if (depth <= 16) {
+				ps1 = 2;
+				ps2 = 4;
 				fac = 4;
 			} else {
-				fac = 1;	/* will not happen */
+				/* should be 24 case */
+				ps1 = 1;
+				ps2 = pixelsize;
+				fac = 1;
 			}
 
 			src = xi->data;
-			dst = cmap8to24_fb + fac * n_off;
+			dst = cmap8to24_fb + (fac/ps1) * n_off;
 
-			poll = poll24_fb + (poll24_fb_w * rect.y1 + rect.x1)*4;
-			poll_Bpl = poll24_fb_w * 4;
+			poll = poll8_fb + poll8_fb_w * rect.y1 * ps1 + rect.x1 * ps1;
+			poll_Bpl = poll8_fb_w * ps1;
 
+			/* line by line ... */
 			for (line = 0; line < h; line++) {
-				memcpy(dst,  src, w * ps1);
-				memcpy(poll, src, w * ps1);
+				/* pixel by pixel... */
+				for (j = 0; j < w; j++) {
+					if (ps1 == 2) {
+						unsigned short *ptmp;
+						us    = (unsigned short *) (src + ps1 * j);
+						idx   = (int) (*us);
+						ptmp  = (unsigned short *) (poll + ps1 * j);
+						*ptmp = *us;
+					} else {
+						uc  = (unsigned char *) (src + ps1 * j);
+						idx = (int) (*uc);
+						*(poll + ps1 * j) = *uc;
+					}
+					ui = (unsigned int *) (dst + ps2 * j);
+					*ui = rgb[cm][idx];
 
+				}
 				src += xi->bytes_per_line;
-				dst += main_bytes_per_line * fac;
+				dst += main_bytes_per_line * (fac/ps1);
 				poll += poll_Bpl;
 			}
 		}
@@ -1661,11 +1729,12 @@ if (db24) fprintf(stderr, "xi: wrong depth: %d\n", xi->depth);
 	} else if (! do_getimage) {
 		int fac;
 
-		if (depth == 8) {
+		if (depth <= 16) {
 			/* cooked up depth 24 TrueColor  */
 			/* but currently disabled (high bits no useful?) */
 			ps = 4;
 			fac = 4;
+			/* XXX not correct for depth > 8, but do we ever come here in that case? */
 			src = cmap8to24_fb + 4 * n_off;
 		} else {
 			ps = pixelsize;
@@ -1697,6 +1766,7 @@ if (db24) fprintf(stderr, "xi: wrong depth: %d\n", xi->depth);
 void bpp8to24(int x1, int y1, int x2, int y2) {
 	char *src, *dst;
 	unsigned char *uc;
+	unsigned short *us;
 	unsigned int *ui;
 	int idx, pixelsize = bpp/8;
 	int line, k, i, j, h, w;
@@ -1741,18 +1811,34 @@ if (db24 > 1) fprintf(stderr, "bpp8to24 %d %d %d %d %.4f\n", x1, y1, x2, y2, dno
 	h = y2 - y1;
 	w = x2 - x1;
 
-	if (depth == 8) {
+	if (depth == 24) {
+		/* pixelsize = 4 */
+		n_off = main_bytes_per_line * y1 + pixelsize * x1;
+
+		src = main_fb      + n_off;
+		dst = cmap8to24_fb + n_off;
+
+		/* otherwise, the pixel data as is */
+		for (line = 0; line < h; line++) {
+			memcpy(dst, src, w * pixelsize);
+			src += main_bytes_per_line;
+			dst += main_bytes_per_line;
+		}
+	} else if (depth <= 16) {
 		/* need to cook up to depth 24 TrueColor  */
-		/* pixelsize = 1 */
+		int ps1 = 1, ps2 = 4;
+		if (depth > 8) {
+			ps1 = 2;
+		}
 
+		/* pixelsize = 1, 2 */
 		n_off = main_bytes_per_line * y1 + pixelsize * x1;
 
 		src = main_fb + n_off;
-		dst = cmap8to24_fb + 4 * n_off;
+		dst = cmap8to24_fb + (4/ps1) * n_off;
 
 		set_root_cmap();
 		if (root_cmap) {
-			int ps1 = 1, ps2 = 4;
 #if 0
 			unsigned int hi;
 #endif
@@ -1761,11 +1847,14 @@ if (db24 > 1) fprintf(stderr, "bpp8to24 %d %d %d %d %.4f\n", x1, y1, x2, y2, dno
 			for (line = 0; line < h; line++) {
 				/* pixel by pixel... */
 				for (j = 0; j < w; j++) {
-
+					if (ps1 == 2) {
+						us = (unsigned short *) (src + ps1 * j);
+						idx = (int) (*us);
+					} else {
 						uc = (unsigned char *)  (src + ps1 * j);
-					ui = (unsigned int *)  (dst + ps2 * j);
-
 						idx = (int) (*uc);
+					}
+					ui = (unsigned int *)  (dst + ps2 * j);
 
 if (0 && line % 100 == 0 && j % 32 == 0) fprintf(stderr, "%d %d %u  x1=%d y1=%d\n", line, j, root_rgb[idx], x1, y1);
 #if 0
@@ -1779,23 +1868,10 @@ if (0 && line % 100 == 0 && j % 32 == 0) fprintf(stderr, "%d %d %u  x1=%d y1=%d\
 if (db24 > 2) histo[idx]++;
 				}
 				src += main_bytes_per_line;
-				dst += main_bytes_per_line * 4;
+				dst += main_bytes_per_line * (4/ps1);
 			}
 		}
 		
-	} else if (depth == 24) {
-		/* pixelsize = 4 */
-		n_off = main_bytes_per_line * y1 + pixelsize * x1;
-
-		src = main_fb      + n_off;
-		dst = cmap8to24_fb + n_off;
-
-		/* otherwise, the pixel data as is */
-		for (line = 0; line < h; line++) {
-			memcpy(dst, src, w * pixelsize);
-			src += main_bytes_per_line;
-			dst += main_bytes_per_line;
-		}
 	}
 
 	if (last_map_count > MAX_8BPP_WINDOWS/4) {
@@ -1871,7 +1947,7 @@ if (db24 > 1) fprintf(stderr, "bpp8to24 w=%d h=%d m=%p c=%p r=%p ncmaps=%d\n", w
 	 * now go back and transform and 8bpp regions to TrueColor in
 	 * cmap8to24_fb.
 	 */
-	if (last_map_count && (ncmaps || depth == 8)) {
+	if (last_map_count && (ncmaps || depth <= 16)) {
 		int i, j;
 		int win[MAX_8BPP_WINDOWS];
 		int did[MAX_8BPP_WINDOWS];

x11vnc/ChangeLog

+2008-05-31  Karl Runge <runge@karlrunge.com>
+	* x11vnc: Improvements to nonstandard indexed color support, e.g.
+	  depths 1, 4, 12, etc. instead of only 8.  Only enable xinerama
+	  xwarppointer if there is more than 1 subscreen.
+
 2008-05-07  Karl Runge <runge@karlrunge.com>
 	* x11vnc: add UltraVNC repeater proxy support.  fix to setp gui
 	  mode. -threads is now strongly discouraged.  Read PORT= in url.

x11vnc/connections.c

@@ -2734,6 +2734,23 @@ void check_gui_inputs(void) {
 	}
 }
 
+static int turn_off_truecolor = 0;
+
+static void turn_off_truecolor_ad(rfbClientPtr client) {
+	if (turn_off_truecolor) {
+		rfbLog("turning off truecolor advertising.\n");
+		screen->serverFormat.trueColour = FALSE;
+		screen->displayHook = NULL;
+		screen->serverFormat.redShift   = 0;
+		screen->serverFormat.greenShift = 0;
+		screen->serverFormat.blueShift  = 0;
+		screen->serverFormat.redMax     = 0;
+		screen->serverFormat.greenMax   = 0;
+		screen->serverFormat.blueMax    = 0;
+		turn_off_truecolor = 0;
+	}
+}
+
 /*
  * libvncserver callback for when a new client connects
  */
@@ -2877,6 +2894,43 @@ enum rfbNewClientAction new_client(rfbClientPtr client) {
 		check_ncache(1, 0);
 	}
 
+	if (advertise_truecolor && indexed_color) {
+		int rs = 0, gs = 2, bs = 4;
+		int rm = 3, gm = 3, bm = 3;
+		if (bpp >= 24) {
+			rs = 0, gs = 8, bs = 16;
+			rm = 255, gm = 255, bm = 255;
+		} else if (bpp >= 16) {
+			rs = 0, gs = 5, bs = 10;
+			rm = 31, gm = 31, bm = 31;
+		}
+		rfbLog("advertising truecolor.\n");
+		if (getenv("ADVERT_BMSHIFT")) {
+			bm--;
+		}
+
+		client->format.trueColour = TRUE;
+		client->format.redShift   = rs;
+		client->format.greenShift = gs;
+		client->format.blueShift  = bs;
+		client->format.redMax     = rm;
+		client->format.greenMax   = gm;
+		client->format.blueMax    = bm;
+
+		rfbSetTranslateFunction(client);
+
+		screen->serverFormat.trueColour = TRUE;
+		screen->serverFormat.redShift   = rs;
+		screen->serverFormat.greenShift = gs;
+		screen->serverFormat.blueShift  = bs;
+		screen->serverFormat.redMax     = rm;
+		screen->serverFormat.greenMax   = gm;
+		screen->serverFormat.blueMax    = bm;
+		screen->displayHook = turn_off_truecolor_ad;
+
+		turn_off_truecolor = 1;
+	}
+
 	if (unixpw) {
 		unixpw_in_progress = 1;
 		unixpw_client = client;

x11vnc/cursor.c

@@ -976,8 +976,8 @@ static rfbCursorPtr pixels2curs(unsigned long *pixels, int w, int h,
 		first = 0;
 	}
 
-	if (cmap8to24 && cmap8to24_fb && depth == 8) {
-		if (Bpp == 1) {
+	if (cmap8to24 && cmap8to24_fb && depth <= 16) {
+		if (Bpp <= 2) {
 			Bpp = 4;
 		}
 	}

x11vnc/help.c

@@ -140,6 +140,11 @@ void print_help(int mode) {
 "                       a colormap histogram.  Example: -shiftcmap 240\n"
 "-notruecolor           For 8bpp displays, force indexed color (i.e. a colormap)\n"
 "                       even if it looks like 8bpp TrueColor (rare problem).\n"
+"-advertise_truecolor   If the X11 display is indexed color, lie to clients\n"
+"                       when they first connect by telling them it is truecolor.\n"
+"                       To workaround RealVNC: inPF has colourMap but not 8bpp\n"
+"                       Use '-advertise_truecolor reset' to reset client fb too.\n"
+"\n"
 "-visual n              This option probably does not do what you think.\n"
 "                       It simply *forces* the visual used for the framebuffer;\n"
 "                       this may be a bad thing... (e.g. messes up colors or\n"
@@ -147,8 +152,10 @@ void print_help(int mode) {
 "                       workarounds.  n may be a decimal number, or 0x hex.\n"
 "                       Run xdpyinfo(1) for the values.  One may also use\n"
 "                       \"TrueColor\", etc. see <X11/X.h> for a list.  If the\n"
-"                       string ends in \":m\" then for better or for worse the\n"
-"                       visual depth is forced to be m.\n"
+"                       string ends in \":m\" then for better or for worse\n"
+"                       the visual depth is forced to be m.  You may want to\n"
+"                       use -noshm when using this option (so XGetImage may\n"
+"                       automatically translate the pixel data).\n"
 "\n"
 "-overlay               Handle multiple depth visuals on one screen, e.g. 8+24\n"
 "                       and 24+8 overlay visuals (the 32 bits per pixel are\n"

x11vnc/misc/Xdummy

@@ -396,7 +396,8 @@ make_so() {
 		warn "$tmp still exists."
 		exit 1
 	fi
-	tail +$n1 $0 | head -$dn > $tmp
+	#tail +$n1 $0 | head -$dn > $tmp
+	tail -n +$n1 $0 | head -n $dn > $tmp
 
 	# compile it to Xdummy.so:
 	rm -f $SO

x11vnc/options.c

@@ -114,6 +114,8 @@ int ping_interval = 0;		/* -ping */
 int flash_cmap = 0;		/* follow installed colormaps */
 int shift_cmap = 0;		/* ncells < 256 and needs shift of pixel values */
 int force_indexed_color = 0;	/* whether to force indexed color for 8bpp */
+int advertise_truecolor = 0;
+int advertise_truecolor_reset = 0;
 int cmap8to24 = 0;		/* -8to24 */
 int xform24to32 = 0;		/* -24to32 */
 char *cmap8to24_str = NULL;

x11vnc/options.h

@@ -89,6 +89,8 @@ extern int ping_interval;
 extern int flash_cmap;
 extern int shift_cmap;
 extern int force_indexed_color;
+extern int advertise_truecolor;
+extern int advertise_truecolor_reset;
 extern int cmap8to24;
 extern char *cmap8to24_str;
 extern int xform24to32;

x11vnc/scan.c

@@ -1285,8 +1285,12 @@ void scale_and_mark_rect(int X1, int Y1, int X2, int Y2, int mark) {
 
 	if (cmap8to24 && cmap8to24_fb) {
 		src_fb = cmap8to24_fb;
-		if (scaling && depth == 8) {
+		if (scaling) {
+			if (depth <= 8) {
 				fac = 4;
+			} else if (depth <= 16) {
+				fac = 2;
+			}
 		}
 	}
 	dst_fb = rfb_fb;

x11vnc/screen.c

@@ -79,6 +79,7 @@ int rawfb_vnc_reflect = 0;
  */
 #define NCOLOR 256
 
+/* this is only for rawfb */
 void set_greyscale_colormap(void) {
 	int i;
 	if (! screen) {
@@ -139,6 +140,8 @@ if (0) fprintf(stderr, "set_hi240_colormap: %s\n", raw_fb_pixfmt);
 
 	rfbSetClientColourMaps(screen, 0, 256);
 }
+
+/* this is only for rawfb */
 void unset_colormap(void) {
 	if (! screen) {
 		return;
@@ -151,35 +154,55 @@ void unset_colormap(void) {
 if (0) fprintf(stderr, "unset_colormap: %s\n", raw_fb_pixfmt);
 }
 
+/* this is X11 case */
 void set_colormap(int reset) {
+
 #if NO_X11
 	if (!reset) {}
 	return;
 #else
 	static int init = 1;
-	static XColor color[NCOLOR], prev[NCOLOR];
+	static XColor *color = NULL, *prev = NULL;
+	static int ncolor = 0;
 	Colormap cmap;
 	Visual *vis;
 	int i, ncells, diffs = 0;
 
 	if (reset) {
 		init = 1;
+		ncolor = 0;
 		if (screen->colourMap.data.shorts) {
 			free(screen->colourMap.data.shorts);
 			screen->colourMap.data.shorts = NULL;
 		}
+		if (color) {
+			free(color);
+			color = NULL;
+		}
+		if (prev) {
+			free(prev);
+			prev = NULL;
+		}
 	}
-if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 
 	if (init) {
-		screen->colourMap.count = NCOLOR;
+		if (depth > 8) {
+			ncolor = 1 << depth;
+		} else {
+			ncolor = NCOLOR;
+		}
+		screen->colourMap.count = ncolor;
 		screen->serverFormat.trueColour = FALSE;
 		screen->colourMap.is16 = TRUE;
 		screen->colourMap.data.shorts = (unsigned short *)
-			malloc(3*sizeof(unsigned short) * NCOLOR);
+			malloc(3*sizeof(unsigned short) * ncolor);
+	}
+	if (color == NULL) {
+		color = (XColor *) calloc(ncolor * sizeof(XColor), 1);
+		prev  = (XColor *) calloc(ncolor * sizeof(XColor), 1);
 	}
 
-	for (i=0; i < NCOLOR; i++) {
+	for (i=0; i < ncolor; i++) {
 		prev[i].red   = color[i].red;
 		prev[i].green = color[i].green;
 		prev[i].blue  = color[i].blue;
@@ -203,15 +226,15 @@ if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 		}
 	}
 
-	if (ncells != NCOLOR) {
-		if (init && ! quiet) {
-			rfbLog("set_colormap: number of cells is %d "
-			    "instead of %d.\n", ncells, NCOLOR);
-		}
+	if (ncells != ncolor) {
 		if (! shift_cmap) {
 			screen->colourMap.count = ncells;
 		}
 	}
+	if (init && ! quiet) {
+		rfbLog("set_colormap: number of cells: %d, "
+		    "ncolor(%d) is %d.\n", ncells, depth, ncolor);
+	}
 
 	if (flash_cmap && ! init) {
 		XWindowAttributes attr;
@@ -233,9 +256,9 @@ if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 			}
 		}
 	}
-	if (ncells > NCOLOR && ! quiet) {
+	if (ncells > ncolor && ! quiet) {
 		rfbLog("set_colormap: big problem: ncells=%d > %d\n",
-		    ncells, NCOLOR);
+		    ncells, ncolor);
 	}
 
 	if (vis->class == TrueColor || vis->class == DirectColor) {
@@ -245,7 +268,7 @@ if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 		 * mentioned in xdpyinfo.  Looks OK... perhaps fortuitously.
 		 */
 		if (ncells == 8 && ! shift_cmap) {
-			ncells = NCOLOR;
+			ncells = ncolor;
 		}
 	}
 
@@ -271,7 +294,7 @@ if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 			diffs++;
 		}
 
-		if (shift_cmap && k >= 0 && k < NCOLOR) {
+		if (shift_cmap && k >= 0 && k < ncolor) {
 			/* kludge to copy the colors to higher pixel values */
 			screen->colourMap.data.shorts[k*3+0] = color[i].red;
 			screen->colourMap.data.shorts[k*3+1] = color[i].green;
@@ -285,7 +308,7 @@ if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 			    "with uninitialized clients.\n");
 		}
 		if (shift_cmap) {
-			rfbSetClientColourMaps(screen, 0, NCOLOR);
+			rfbSetClientColourMaps(screen, 0, ncolor);
 		} else {
 			rfbSetClientColourMaps(screen, 0, ncells);
 		}
@@ -297,7 +320,8 @@ if (0) fprintf(stderr, "unset_colormap: %d\n", reset);
 
 static void debug_colormap(XImage *fb) {
 	static int debug_cmap = -1;
-	int i, k, histo[NCOLOR];
+	int i, k, *histo;
+	int ncolor;
 
 	if (debug_cmap < 0) {
 		if (getenv("DEBUG_CMAP") != NULL) {
@@ -312,11 +336,13 @@ static void debug_colormap(XImage *fb) {
 	if (! fb) {
 		return;
 	}
-	if (fb->bits_per_pixel > 8) {
+	if (fb->bits_per_pixel > 16) {
 		return;
 	}
+	ncolor = screen->colourMap.count;
+	histo = (int *) calloc(ncolor * sizeof(int), 1);
 
-	for (i=0; i < NCOLOR; i++) {
+	for (i=0; i < ncolor; i++) {
 		histo[i] = 0;
 	}
 	for (k = 0; k < fb->width * fb->height; k++) {
@@ -327,7 +353,7 @@ static void debug_colormap(XImage *fb) {
 		histo[n]++;
 	}
 	fprintf(stderr, "\nColormap histogram for current screen contents:\n");
-	for (i=0; i < NCOLOR; i++) {
+	for (i=0; i < ncolor; i++) {
 		unsigned short r = screen->colourMap.data.shorts[i*3+0];
 		unsigned short g = screen->colourMap.data.shorts[i*3+1];
 		unsigned short b = screen->colourMap.data.shorts[i*3+2];
@@ -338,6 +364,7 @@ static void debug_colormap(XImage *fb) {
 			fprintf(stderr, "\n");
 		}
 	}
+	free(histo);
 	fprintf(stderr, "\n");
 }
 
@@ -1931,12 +1958,38 @@ XImage *initialize_xdisplay_fb(void) {
 	if (xform24to32) {
 		if (DefaultDepth(dpy, scr) == 24) {
 			vis_str = strdup("TrueColor:32");
-			rfbLog("initialize_xdisplay_fb: vis_str set to: %s ",
+			rfbLog("initialize_xdisplay_fb: vis_str set to: %s\n",
 			    vis_str);
 			visual_id = (VisualID) 0;
 			visual_depth = 0;
 			set_visual_str_to_something = 1;
 		}
+	} else if (DefaultDepth(dpy, scr) < 8) {
+		/* check very low bpp case, e.g. mono or vga16 */
+		Screen *s = DefaultScreenOfDisplay(dpy);
+		XImage *xi = XGetImage_wr(dpy, DefaultRootWindow(dpy), 0, 0, 2, 2, AllPlanes,
+		    ZPixmap);
+		if (xi && xi->bits_per_pixel < 8) {
+			int lowbpp = xi->bits_per_pixel; 
+			if (!vis_str) {
+				char tmp[32];
+				sprintf(tmp, "0x%x:8", (int) s->root_visual->visualid);
+				vis_str = strdup(tmp);
+				rfbLog("initialize_xdisplay_fb: low bpp[%d], vis_str "
+				    "set to: %s\n", lowbpp, vis_str);
+			}
+			if (using_shm) {
+				using_shm = 0;
+				rfbLog("initialize_xdisplay_fb: low bpp[%d], "
+				    "disabling shm\n", lowbpp);
+			}
+			visual_id = (VisualID) 0;
+			visual_depth = 0;
+			set_visual_str_to_something = 1;
+		}
+		if (xi) {
+			XDestroyImage(xi);
+		}
 	}
 
 	if (vis_str != NULL) {
@@ -2420,13 +2473,13 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 				    " bpp != 32 with depth == 24\n");
 				cmap8to24 = 0;
 			}
-		} else if (depth == 8) {
+		} else if (depth <= 16) {
 			/* need to cook up the screen fb to be depth 24 */
 			fb_bpp = 32;
 			fb_depth = 24;
 		} else {
 			if (!quiet) rfbLog("disabling -8to24 mode:"
-			    " default depth not 24 or 8\n");
+			    " default depth not 16 or less\n");
 			cmap8to24 = 0;
 		}
 	}
@@ -2489,9 +2542,14 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 	}
 #endif
 
-	if (cmap8to24 && depth == 8) {
+	if (cmap8to24) {
+		if (depth <= 8) {
 			rfb_bytes_per_line *= 4;
 			rot_bytes_per_line *= 4;
+		} else if (depth <= 16) {
+			rfb_bytes_per_line *= 2;
+			rot_bytes_per_line *= 2;
+		}
 	}
 
 	/*
@@ -2596,7 +2654,7 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 		have_masks = 0;
 	}
 
-	if (cmap8to24 && depth == 8 && dpy) {
+	if (cmap8to24 && depth <= 16 && dpy) {
 		XVisualInfo vinfo;
 		vinfo.red_mask = 0;
 		vinfo.green_mask = 0;
@@ -2638,13 +2696,14 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 		free(fmt);
 	}
 
-	if (! have_masks && screen->serverFormat.bitsPerPixel == 8
+	if (! have_masks && screen->serverFormat.bitsPerPixel <= 16
 	    && dpy && CellsOfScreen(ScreenOfDisplay(dpy, scr))) {
-		/* indexed color */
+		/* indexed color. we let 1 <= bpp <= 16, but it is normally 8 */
 		if (!quiet) {
 			rfbLog("\n");
-			rfbLog("X display %s is 8bpp indexed color\n",
-			    DisplayString(dpy));
+			rfbLog("X display %s is %dbpp indexed color, depth=%d\n",
+			    DisplayString(dpy), screen->serverFormat.bitsPerPixel,
+			    screen->serverFormat.depth);
 			if (! flash_cmap && ! overlay) {
 				rfbLog("\n");
 				rfbLog("In 8bpp PseudoColor mode if you "
@@ -2656,6 +2715,18 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 				rfbLog("\n");
 			}
 		}
+		if (screen->serverFormat.depth < 8) {
+			rfbLog("resetting serverFormat.depth %d -> 8.\n",
+			    screen->serverFormat.depth);
+			rfbLog("resetting serverFormat.bpp   %d -> 8.\n",
+			    screen->serverFormat.bitsPerPixel);
+			screen->serverFormat.depth = 8;
+			screen->serverFormat.bitsPerPixel = 8;
+		}
+		if (screen->serverFormat.depth > 8) {
+			rfbLog("WARNING: indexed color depth > 8, Tight encoding will crash.\n");
+		}
+
 		screen->serverFormat.trueColour = FALSE;
 		indexed_color = 1;
 		set_colormap(1);
@@ -2688,6 +2759,29 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 
 		indexed_color = 0;
 
+		if (!have_masks) {
+			int M, B = bits_per_color;
+
+			if (3*B > fb_bpp) B--;
+			if (B < 1) B = 1;
+			M = (1 << B) - 1;
+
+			rfbLog("WARNING: all TrueColor RGB masks are zero!\n");
+			rfbLog("WARNING: cooking something up for VNC fb...  B=%d M=%d\n", B, M);
+			main_red_mask   = M;
+			main_green_mask = M;
+			main_blue_mask  = M;
+			main_red_mask   = main_red_mask   << (2*B);
+			main_green_mask = main_green_mask << (1*B);
+			main_blue_mask  = main_blue_mask  << (0*B);
+			fprintf(stderr, " red_mask:   0x%08lx  %s\n", main_red_mask,
+			    bitprint(main_red_mask, 32));
+			fprintf(stderr, " green_mask: 0x%08lx  %s\n", main_green_mask,
+			    bitprint(main_green_mask, 32));
+			fprintf(stderr, " blue_mask:  0x%08lx  %s\n", main_blue_mask,
+			    bitprint(main_blue_mask, 32));
+		}
+
 		/* convert masks to bit shifts and max # colors */
 		if (main_red_mask) {
 			while (! (main_red_mask
@@ -2821,8 +2915,10 @@ void initialize_screen(int *argc, char **argv, XImage *fb) {
 
 		if (cmap8to24) {
 			int n = main_bytes_per_line * fb->height;
-			if (depth == 8) {
+			if (depth <= 8) {
 				n *= 4;
+			} else if (depth <= 16) {
+				n *= 2;
 			}
 			cmap8to24_fb = (char *) malloc(n);
 			memset(cmap8to24_fb, 0, n);

x11vnc/userinput.c

@@ -790,8 +790,10 @@ static void draw_box(int x, int y, int w, int h, int restore) {
 	if (cmap8to24 && cmap8to24_fb) {
 		use_fb = cmap8to24_fb;
 		pixelsize = 4;
-		if (depth == 8) {
+		if (depth <= 8) {
 			use_Bpl *= 4;
+		} else if (depth <= 16) {
+			use_Bpl *= 2;
 		}
 	}
 
@@ -1882,9 +1884,14 @@ if (0 || debug_scroll > 1) fprintf(stderr, "<<<-rfbDoCopyRect req: %d mod: %d cp
 					}
 				}
 if (0) fprintf(stderr, "copyrect: cmap8to24_fb: mode=%d\n", mode);
-				if (cmap8to24 && depth == 8) {
+				if (cmap8to24) {
+					if (depth <= 8) {
 						Bpp    = 4 * Bpp0;
 						stride = 4 * stride0;
+					} else if (depth <= 16) {
+						Bpp    = 2 * Bpp0;
+						stride = 2 * stride0;
+					}
 				}
 				dst = cmap8to24_fb + y1*stride + x1*Bpp;
 				src = cmap8to24_fb + (y1-dy)*stride + (x1-dx)*Bpp;
@@ -3566,6 +3573,17 @@ void check_fixscreen(void) {
 			didfull = 1;
 		}
 	}
+	if (advertise_truecolor && advertise_truecolor_reset && indexed_color) {
+		/* this will reset framebuffer to correct colors, if needed */
+		static dlast = 0.0;
+		now = dnow();
+		if (now > last_client + 1.0 && now < last_client + 3.0 && now > dlast + 5.0) {
+			rfbLog("advertise truecolor reset framebuffer\n");
+			do_new_fb(1);
+			dlast = dnow();
+			return;
+		}
+	}
 }
 
 static int wireframe_mod_state() {
@@ -8083,8 +8101,12 @@ void scale_mark_xrootpmap(void) {
 
 	if (cmap8to24 && cmap8to24_fb) {
 		src_fb = cmap8to24_fb;
-		if (scaling && depth == 8) {
+		if (scaling) {
+			if (depth <= 8) {
 				fac = 4;
+			} else if (depth <= 16) {
+				fac = 2;
+			}
 		}
 	}
 	dst_fb = rfb_fb;

x11vnc/x11vnc.1

@@ -2,7 +2,7 @@
 .TH X11VNC "1" "May 2008" "x11vnc " "User Commands"
 .SH NAME
 x11vnc - allow VNC connections to real X11 displays
-         version: 0.9.4, lastmod: 2008-05-10
+         version: 0.9.4, lastmod: 2008-05-31
 .SH SYNOPSIS
 .B x11vnc
 [OPTION]...
@@ -161,6 +161,13 @@ a colormap histogram.  Example: \fB-shiftcmap\fR 240
 For 8bpp displays, force indexed color (i.e. a colormap)
 even if it looks like 8bpp TrueColor (rare problem).
 .PP
+\fB-advertise_truecolor\fR
+.IP
+If the X11 display is indexed color, lie to clients
+when they first connect by telling them it is truecolor.
+To workaround RealVNC: inPF has colourMap but not 8bpp
+Use '-advertise_truecolor reset' to reset client fb too.
+.PP
 \fB-visual\fR \fIn\fR
 .IP
 This option probably does not do what you think.
@@ -172,8 +179,10 @@ Run
 .IR xdpyinfo (1)
 for the values.  One may also use
 "TrueColor", etc. see <X11/X.h> for a list.  If the
-string ends in ":m" then for better or for worse the
-visual depth is forced to be m.
+string ends in ":m" then for better or for worse
+the visual depth is forced to be m.  You may want to
+use \fB-noshm\fR when using this option (so XGetImage may
+automatically translate the pixel data).
 .PP
 \fB-overlay\fR
 .IP

x11vnc/x11vnc.c

@@ -2514,6 +2514,17 @@ int main(int argc, char* argv[]) {
 			shift_cmap = atoi(argv[++i]);
 		} else if (!strcmp(arg, "-notruecolor")) {
 			force_indexed_color = 1;
+		} else if (!strcmp(arg, "-advertise_truecolor")) {
+			advertise_truecolor = 1;
+			if (i < argc-1) {
+				char *s = argv[i+1];
+				if (s[0] != '-') {
+					if (strstr(s, "reset")) {
+						advertise_truecolor_reset = 1;
+					}
+					i++;
+				}
+			}
 		} else if (!strcmp(arg, "-overlay")) {
 			overlay = 1;
 		} else if (!strcmp(arg, "-overlay_nocursor")) {

x11vnc/x11vnc_defs.c

@@ -15,7 +15,7 @@ int xtrap_base_event_type = 0;
 int xdamage_base_event_type = 0;
 
 /*               date +'lastmod: %Y-%m-%d' */
-char lastmod[] = "0.9.4 lastmod: 2008-05-10";
+char lastmod[] = "0.9.4 lastmod: 2008-05-31";
 
 /* X display info */
 

x11vnc/xinerama.c

@@ -263,19 +263,19 @@ static void initialize_xinerama (void) {
 	rfbLog("\n");
 	rfbLog("Xinerama is present and active (e.g. multi-head).\n");
 
-	if (! use_xwarppointer && ! got_noxwarppointer) {
-		rfbLog("Xinerama: enabling -xwarppointer mode to try to correct\n");
-		rfbLog("Xinerama: mouse pointer motion. XTEST+XINERAMA bug.\n");
-		rfbLog("Xinerama: Use -noxwarppointer to force XTEST.\n");
-		use_xwarppointer = 1;
-	}
-
 	/* n.b. change to XineramaGetData() someday */
 	xineramas = XineramaQueryScreens(dpy, &n);
 	if (verbose) {
 		rfbLog("Xinerama: number of sub-screens: %d\n", n);
 	}
 
+	if (! use_xwarppointer && ! got_noxwarppointer && n > 1) {
+		rfbLog("Xinerama: enabling -xwarppointer mode to try to correct\n");
+		rfbLog("Xinerama: mouse pointer motion. XTEST+XINERAMA bug.\n");
+		rfbLog("Xinerama: Use -noxwarppointer to force XTEST.\n");
+		use_xwarppointer = 1;
+	}
+
 	if (n == 1) {
 		if (verbose) {
 			rfbLog("Xinerama: no blackouts needed (only one"