incorporate a fix for possible fax3 decoding overflow (CVE-2011-0192)

ChangeLog

@@ -1,3 +1,14 @@
+2011-03-02  Frank Warmerdam  <warmerdam@pobox.com>
+
+	* libtiff/tif_fax3.h: Protect against a fax VL(n) codeword commanding 
+	a move left.  Without this, a malicious input file can generate an 
+	indefinitely large series of runs without a0 ever reaching the right 
+	margin, thus overrunning our buffer of run lengths.  Per CVE-2011-0192.
+	This is a modified version of a patch proposed by Drew Yao of Apple 
+	Product Security.  It adds an unexpected() report, and disallows the 
+	equality case, since emitting a run without increasing a0 still allows 
+	buffer overrun.
+
 2011-02-23  Frank Warmerdam  <warmerdam@pobox.com>
 
 	* libtiff/tif_jpeg.c: avoid divide by zero in degenerate case (#2296)

libtiff/tif_fax3.h

@@ -1,4 +1,4 @@
-/* $Id: tif_fax3.h,v 1.7 2010-03-10 18:56:48 bfriesen Exp $ */
+/* $Id: tif_fax3.h,v 1.8 2011-03-03 01:38:25 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1990-1997 Sam Leffler
@@ -478,6 +478,10 @@ done1d:									\
 	    break;							\
 	case S_VL:							\
 	    CHECK_b1;							\
+	    if (b1 <= (int) (a0 + TabEnt->Param)) {			\
+		unexpected("VL", a0);					\
+		goto eol2d;						\
+	    }								\
 	    SETVALUE(b1 - a0 - TabEnt->Param);				\
 	    b1 -= *--pb;						\
 	    break;							\