Bug#414045: debugging graphicsmagick-1.1.7 and/or libx11-1.0.3

[ldoolitt@recycle.lbl.gov]

On Sat, Mar 24, 2007 at 04:29:23PM +0100, Julien Cristau wrote:
> So you're saying the remaining problem is in graphicsmagick, not Xlib?

I previously posted a patch for graphicsmagick that fixes broken.xwd.
Here is a patch for libx11 that fixes broken2.xwd.

I thought about possible ways to fixing broken.xwd in libx11, or
broken2.xwd in graphicsmagick, and I don't think it's possible or
desirable.  So I guess this bug needs to be split, and each package
patched.

With this patch included (replaces 022_XInitImage_input_validate.diff),

$ gm convert broken2.xwd temp.png
gm convert: Unrecognized XWD header (broken2.xwd) [No such file or directory].

Other than adding an extraneous errno = EINVAL to userspace,
I don't see how to avoid the inapplicable error string.

Please test, and forward upstream.

    - Larry

Add more input validation to XInitImage(), to avoid buffer overflow in
XGetPixel(), which assumes sane values.
Debian bug #414045.

This patch by Daniel Kobras <kobras@debian.org>
and Larry Doolittle <ldoolitt@recycle.lbl.gov>


--- libx11.orig/src/ImUtil.c	2007-03-09 02:21:29.000000000 +0100
+++ libx11/src/ImUtil.c	2007-03-25 10:33:48.000000000 -0700
@@ -327,12 +327,13 @@
 {
 	register XImage *image;
 	int bits_per_pixel = 1;
+	int min_bytes_per_line;
 
 	if (depth == 0 || depth > 32 ||
 	    (format != XYBitmap && format != XYPixmap && format != ZPixmap) ||
 	    (format == XYBitmap && depth != 1) ||
 	    (xpad != 8 && xpad != 16 && xpad != 32) ||
-	    offset < 0 || image_bytes_per_line < 0)
+	    offset < 0)
 	    return (XImage *) NULL;
 	if ((image = (XImage *) Xcalloc(1, (unsigned) sizeof(XImage))) == NULL)
 	    return (XImage *) NULL;
@@ -363,16 +364,21 @@
 	/*
 	 * compute per line accelerator.
 	 */
-	if (image_bytes_per_line == 0)
 	{
 	if (format == ZPixmap)
-	    image->bytes_per_line = 
+	    min_bytes_per_line = 
 	       ROUNDUP((bits_per_pixel * width), image->bitmap_pad);
 	else
-	    image->bytes_per_line =
+	    min_bytes_per_line =
 	        ROUNDUP((width + offset), image->bitmap_pad);
 	}
-	else image->bytes_per_line = image_bytes_per_line;
+	if (image_bytes_per_line == 0) {
+	    image->bytes_per_line = min_bytes_per_line;
+	} else if (image_bytes_per_line < min_bytes_per_line) {
+	    return 0;
+	} else {
+	    image->bytes_per_line = image_bytes_per_line;
+	}
 
 	image->bits_per_pixel = bits_per_pixel;
 	image->obdata = NULL;
@@ -384,7 +390,10 @@
 Status XInitImage (image)
     XImage *image;
 {
+	int min_bytes_per_line;
 	if (image->depth == 0 || image->depth > 32 ||
+	    image->bits_per_pixel > 32 || image->bitmap_unit > 32 ||
+	    image->bits_per_pixel < 0 || image->bitmap_unit < 0 ||
 	    (image->format != XYBitmap &&
 	     image->format != XYPixmap &&
 	     image->format != ZPixmap) ||
@@ -392,22 +401,26 @@
 	    (image->bitmap_pad != 8 &&
 	     image->bitmap_pad != 16 &&
 	     image->bitmap_pad != 32) ||
-	    image->xoffset < 0 || image->bytes_per_line < 0)
+	    image->xoffset < 0)
 	    return 0;
 
 	/*
 	 * compute per line accelerator.
 	 */
-	if (image->bytes_per_line == 0)
 	{
 	if (image->format == ZPixmap)
-	    image->bytes_per_line = 
+	    min_bytes_per_line = 
 	       ROUNDUP((image->bits_per_pixel * image->width),
 		       image->bitmap_pad);
 	else
-	    image->bytes_per_line =
+	    min_bytes_per_line =
 	        ROUNDUP((image->width + image->xoffset), image->bitmap_pad);
 	}
+	if (image->bytes_per_line == 0) {
+	    image->bytes_per_line = min_bytes_per_line;
+	} else if (image->bytes_per_line < min_bytes_per_line) {
+	    return 0;
+	}
 
 	_XInitImageFuncPtrs (image);