Bug#414045: debugging graphicsmagick-1.1.7 and/or libx11-1.0.3

[ldoolitt@recycle.lbl.gov]

The root problem is integer overflow in the multiplication at
line 292 of graphicsmagick-1.1.7/coders/xwd.c.  With the appended
patch, the two test cases result in the following on my amd64 sid
box:

$ gm convert broken.xwd test.png
gm convert: Memory allocation failed (broken.xwd).
$ echo $?
1
$ gm convert broken2.xwd test.png
gm convert: Unexpected end-of-file (broken2.xwd).
$ echo $?
1
$

--- xwd.c	2007-03-23 09:11:52.000000000 -0700
+++ xwd-fixed.c	2007-03-23 12:18:06.000000000 -0700
@@ -288,11 +288,23 @@
   /*
     Allocate the pixel buffer.
   */
-  if (ximage->format == ZPixmap)
-    length=ximage->bytes_per_line*ximage->height;
-  else
-    length=ximage->bytes_per_line*ximage->height*ximage->depth;
-  ximage->data=MagickAllocateMemory(char *,length);
+  {
+#define OVERFLOW(c,a,b) ((b) != 0 && ((c)/(b) != (a)))
+  int overflow=0;
+  length=ximage->bytes_per_line*ximage->height;
+  if (OVERFLOW(length, ximage->bytes_per_line, ximage->height)) overflow=1;
+  if (ximage->format != ZPixmap) {
+    size_t l1=length*ximage->depth;
+    if (OVERFLOW(l1, length, ximage->depth)) overflow=1;
+    length=l1;
+  }
+  if (overflow) {
+    ximage->data = (char *) NULL;
+  } else {
+    ximage->data=MagickAllocateMemory(char *,length);
+  }
+#undef OVERFLOW
+  }
   if (ximage->data == (char *) NULL)
     ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,image);
   count=ReadBlob(image,length,ximage->data);