GraphicsMagick/CVE-2020-10938.patch

HuffmanDecodeImage(): Fix signed overflow on range check which leads to heap overflow in 32-bit applications. Ascii85Tuple(): Fix thread safety issue.
(CVE-2020-10938)

refers to http://hg.code.sf.net/p/graphicsmagick/code/rev/95abc2b694ce

diff -r 751e9e822b09 -r 95abc2b694ce magick/compress.c
--- a/magick/compress.c	Sun Nov 10 13:33:34 2019 -0600
+++ b/magick/compress.c	Sat Nov 16 10:31:37 2019 -0600
@@ -1,5 +1,5 @@
 /*
-% Copyright (C) 2003 - 2015 GraphicsMagick Group
+% Copyright (C) 2003-2019 GraphicsMagick Group
 % Copyright (C) 2002 ImageMagick Studio
 % Copyright 1991-1999 E. I. du Pont de Nemours and Company
 %
@@ -53,21 +53,26 @@
 */
 typedef struct HuffmanTable
 {
+  unsigned int
+    id;
+
   int
-    id,
-    code,
+    code;
+
+  unsigned int
     length,
     count;
+
 } HuffmanTable;
 
 /*
   Huffman coding declarations.
 */
-#define TWId  23
-#define MWId  24
-#define TBId  25
-#define MBId  26
-#define EXId  27
+#define TWId  23U
+#define MWId  24U
+#define TBId  25U
+#define MBId  26U
+#define EXId  27U
 
 static const HuffmanTable
   MBTable[]=
@@ -202,37 +207,38 @@
 */
 #define MaxLineExtent  36
 
-static char *Ascii85Tuple(unsigned char *data)
+static char *Ascii85Tuple(char tuple[6], const unsigned char * restrict data)
 {
-  static char
-    tuple[6];
+  magick_uint32_t
+    code;
 
-  register long
-    i,
-    x;
-
-  unsigned long
-    code,
-    quantum;
-
-  code=((((unsigned long) data[0] << 8) | (unsigned long) data[1]) << 16) |
-    ((unsigned long) data[2] << 8) | (unsigned long) data[3];
-  if (code == 0L)
+  code=((((magick_uint32_t) data[0] << 8) | (magick_uint32_t) data[1]) << 16) |
+    ((magick_uint32_t) data[2] << 8) | (magick_uint32_t) data[3];
+  if (code == 0)
     {
       tuple[0]='z';
       tuple[1]='\0';
-      return(tuple);
     }
-  quantum=85UL*85UL*85UL*85UL;
-  for (i=0; i < 4; i++)
-  {
-    x=(long) (code/quantum);
-    code-=quantum*x;
-    tuple[i]=(char) (x+(int) '!');
-    quantum/=85L;
-  }
-  tuple[4]=(char) ((code % 85L)+(int) '!');
-  tuple[5]='\0';
+  else
+    {
+      register magick_int32_t
+        i,
+        x;
+
+      magick_uint32_t
+        quantum;
+
+      quantum=85U*85U*85U*85U;
+      for (i=0; i < 4; i++)
+        {
+          x=(magick_int32_t) (code/quantum);
+          code-=quantum*x;
+          tuple[i]=(char) (x+(int) '!');
+          quantum/=85;
+        }
+      tuple[4]=(char) ((code % 85)+(int) '!');
+      tuple[5]='\0';
+    }
   return(tuple);
 }
 
@@ -255,6 +261,9 @@
 
 MagickExport void Ascii85Flush(Image *image)
 {
+  char
+    tuple_buff[6];
+
   register char
     *tuple;
 
@@ -266,7 +275,7 @@
       image->ascii85->buffer[image->ascii85->offset]=0;
       image->ascii85->buffer[image->ascii85->offset+1]=0;
       image->ascii85->buffer[image->ascii85->offset+2]=0;
-      tuple=Ascii85Tuple(image->ascii85->buffer);
+      tuple=Ascii85Tuple(tuple_buff, image->ascii85->buffer);
       (void) WriteBlob(image,image->ascii85->offset+1,
         *tuple == 'z' ? "!!!!" : tuple);
     }
@@ -286,6 +295,9 @@
   register unsigned char
     *p;
 
+  char
+    tuple_buff[6];
+
   assert(image != (Image *) NULL);
   assert(image->signature == MagickSignature);
   assert(image->ascii85 != (Ascii85Info *) NULL);
@@ -296,7 +308,7 @@
   p=image->ascii85->buffer;
   for (n=image->ascii85->offset; n >= 4; n-=4)
   {
-    for (q=Ascii85Tuple(p); *q; q++)
+    for (q=Ascii85Tuple(tuple_buff,p); *q; q++)
     {
       image->ascii85->line_break--;
       if ((image->ascii85->line_break < 0) && (*q != '%'))
@@ -355,11 +367,11 @@
 %
 %
 */
-#define HashSize  1021
-#define MBHashA  293
-#define MBHashB  2695
-#define MWHashA  3510
-#define MWHashB  1178
+#define HashSize 1021U
+#define MBHashA  293U
+#define MBHashB  2695U
+#define MWHashA  3510U
+#define MWHashB  1178U
 
 #define InitializeHashTable(hash,table,a,b) \
 { \
@@ -401,26 +413,30 @@
     byte,
     code,
     color,
-    length,
     null_lines,
     runlength;
 
   unsigned int
     bit,
     index,
+    length,
     mask;
 
   long
-    count,
+    count;
+
+  unsigned long
     y;
 
   register IndexPacket
     *indexes;
 
-  register long
-    i,
+  register unsigned long
     x;
 
+  unsigned int
+    i;
+
   register PixelPacket
     *q;
 
@@ -481,13 +497,13 @@
   image->x_resolution=204.0;
   image->y_resolution=196.0;
   image->units=PixelsPerInchResolution;
-  for (y=0; ((y < (long) image->rows) && (null_lines < 3)); )
+  for (y=0; ((y < image->rows) && (null_lines < 3)); )
   {
     /*
       Initialize scanline to white.
     */
     p=scanline;
-    for (x=0; x < (long) image->columns; x++)
+    for (x=0; x < image->columns; x++)
       *p++=0;
     /*
       Decode Huffman encoded scanline.
@@ -502,7 +518,7 @@
     {
       if (byte == EOF)
         break;
-      if (x >= (long) image->columns)
+      if (x >= image->columns)
         {
           while (runlength < 11)
             InputBit(bit);
@@ -563,7 +579,7 @@
         case TBId:
         {
           count+=entry->count;
-          if ((x+count) > (long) image->columns)
+          if ((x+(unsigned long) count) > image->columns)
             count=(long) image->columns-x;
           if (count > 0)
             {
@@ -603,7 +619,7 @@
         break;
       }
     indexes=AccessMutableIndexes(image);
-    for (x=0; x < (long) image->columns; x++)
+    for (x=0; x < image->columns; x++)
     {
       index=(unsigned int) (*p++);
       indexes[x]=index;
@@ -695,7 +711,9 @@
     runlength;
 
   long
-    n,
+    n;
+
+  unsigned long
     y;
 
   Image
@@ -704,8 +722,10 @@
   register const IndexPacket
     *indexes;
 
-  register long
-    i,
+  unsigned long
+    i;
+
+  register unsigned long
     x;
 
   register const PixelPacket
@@ -772,10 +792,10 @@
     polarity=(PixelIntensityToQuantum(&huffman_image->colormap[0]) <
       PixelIntensityToQuantum(&huffman_image->colormap[1]) ? 0x00 : 0x01);
   q=scanline;
-  for (i=(long) width; i > 0; i--)
+  for (i=0; i < width; i++) /* was: for (i=(long) width; i > 0; i--) */
     *q++=(unsigned char) polarity;
   q=scanline;
-  for (y=0; y < (long) huffman_image->rows; y++)
+  for (y=0; y < huffman_image->rows; y++)
   {
     p=AcquireImagePixels(huffman_image,0,y,huffman_image->columns,1,
       &huffman_image->exception);
@@ -785,7 +805,7 @@
         break;
       }
     indexes=AccessImmutableIndexes(huffman_image);
-    for (x=0; x < (long) huffman_image->columns; x++)
+    for (x=0; x < huffman_image->columns; x++)
     {
       *q=(unsigned char) (indexes[x] == polarity ? !polarity : polarity);
       q++;