poppler/backport-CVE-2022-37052.patch

From 8677500399fc2548fa816b619580c2c07915a98c Mon Sep 17 00:00:00 2001
From: Albert Astals Cid <aacid@kde.org>
Date: Fri, 29 Jul 2022 23:28:35 +0200
Subject: [PATCH] pdfseparate: Account for XRef::add failing because we run out
 of memory

Fixes #1278
---
 poppler/PDFDoc.cc | 63 ++++++++++++++++++++++++++++++++++++-----------
 poppler/PDFDoc.h  |  6 ++---
 poppler/XRef.cc   | 11 +++++++--
 poppler/XRef.h    |  4 +--
 4 files changed, 62 insertions(+), 22 deletions(-)

diff --git a/poppler/PDFDoc.cc b/poppler/PDFDoc.cc
index 43de80e..fcc17a4 100644
--- a/poppler/PDFDoc.cc
+++ b/poppler/PDFDoc.cc
@@ -962,7 +962,14 @@ int PDFDoc::savePageAs(const GooString *name, int pageNo)
   Object resourcesObj = pagesDict->lookup("Resources");
   if (resourcesObj.isDict())
     markPageObjects(resourcesObj.getDict(), yRef, countRef, 0, refPage->num, rootNum + 2);
-  markPageObjects(catDict, yRef, countRef, 0, refPage->num, rootNum + 2);
+  if (!markPageObjects(catDict, yRef, countRef, 0, refPage->num, rootNum + 2)) {
+      fclose(f);
+      delete yRef;
+      delete countRef;
+      delete outStr;
+      error(errSyntaxError, -1, "markPageObjects failed");
+      return errDamaged;
+  }
 
   Dict *pageDict = page.getDict();
   if (resourcesObj.isNull() && !pageDict->hasKey("Resources")) {
@@ -1681,7 +1688,7 @@ void PDFDoc::writeHeader(OutStream *outStr, int major, int minor)
    outStr->printf("%%%c%c%c%c\n", 0xE2, 0xE3, 0xCF, 0xD3);
 }
 
-void PDFDoc::markDictionnary (Dict* dict, XRef * xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts)
+bool PDFDoc::markDictionnary (Dict* dict, XRef * xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts)
 {
   bool deleteSet = false;
   if (!alreadyMarkedDicts) {
@@ -1692,7 +1699,7 @@ void PDFDoc::markDictionnary (Dict* dict, XRef * xRef, XRef *countRef, unsigned
   if (alreadyMarkedDicts->find(dict) != alreadyMarkedDicts->end()) {
     error(errSyntaxWarning, -1, "PDFDoc::markDictionnary: Found recursive dicts");
     if (deleteSet) delete alreadyMarkedDicts;
-    return;
+    return true;
   } else {
     alreadyMarkedDicts->insert(dict);
   }
@@ -1701,7 +1708,10 @@ void PDFDoc::markDictionnary (Dict* dict, XRef * xRef, XRef *countRef, unsigned
     const char *key = dict->getKey(i);
     if (strcmp(key, "Annots") != 0) {
       Object obj1 = dict->getValNF(i).copy();
-      markObject(&obj1, xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+      const bool success = markObject(&obj1, xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+      if (unlikely(!success)) {
+          return false;
+      }
     } else {
       Object annotsObj = dict->getValNF(i).copy();
       if (!annotsObj.isNull()) {
@@ -1713,9 +1723,11 @@ void PDFDoc::markDictionnary (Dict* dict, XRef * xRef, XRef *countRef, unsigned
   if (deleteSet) {
     delete alreadyMarkedDicts;
   }
+
+  return true;
 }
 
-void PDFDoc::markObject (Object* obj, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts)
+bool PDFDoc::markObject (Object* obj, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts)
 {
   Array *array;
 
@@ -1724,25 +1736,37 @@ void PDFDoc::markObject (Object* obj, XRef *xRef, XRef *countRef, unsigned int n
       array = obj->getArray();
       for (int i=0; i<array->getLength(); i++) {
         Object obj1 = array->getNF(i).copy();
-        markObject(&obj1, xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+        const bool success = markObject(&obj1, xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+        if (unlikely(!success)) {
+            return false;
+        }
       }
       break;
-    case objDict:
-      markDictionnary (obj->getDict(), xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
-      break;
+    case objDict: {
+      const bool success = markDictionnary(obj->getDict(), xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+      if (unlikely(!success)) {
+          return false;
+      }
+    } break;
     case objStream: 
       {
         Stream *stream = obj->getStream();
-        markDictionnary (stream->getDict(), xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+        const bool success = markDictionnary(stream->getDict(), xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+        if (unlikely(!success)) {
+            return false;
+        }
       }
       break;
     case objRef:
       {
         if (obj->getRef().num + (int) numOffset >= xRef->getNumObjects() || xRef->getEntry(obj->getRef().num + numOffset)->type == xrefEntryFree) {
           if (getXRef()->getEntry(obj->getRef().num)->type == xrefEntryFree) {
-            return;  // already marked as free => should be replaced
+            return true; // already marked as free => should be replaced
+          }
+          const bool success = xRef->add(obj->getRef().num + numOffset, obj->getRef().gen, 0, true);
+          if (unlikely(!success)) {
+              return false;
           }
-          xRef->add(obj->getRef().num + numOffset, obj->getRef().gen, 0, true);
           if (getXRef()->getEntry(obj->getRef().num)->type == xrefEntryCompressed) {
             xRef->getEntry(obj->getRef().num + numOffset)->type = xrefEntryCompressed;
           }
@@ -1758,12 +1782,17 @@ void PDFDoc::markObject (Object* obj, XRef *xRef, XRef *countRef, unsigned int n
             break;
         } 
         Object obj1 = getXRef()->fetch(obj->getRef());
-        markObject(&obj1, xRef, countRef, numOffset, oldRefNum, newRefNum);
+        const bool success = markObject(&obj1, xRef, countRef, numOffset, oldRefNum, newRefNum);
+        if (unlikely(!success)) {
+            return false;
+        }
       }
       break;
     default:
       break;
   }
+
+  return true;
 }
 
 void PDFDoc::replacePageDict(int pageNo, int rotate,
@@ -1803,7 +1832,7 @@ void PDFDoc::replacePageDict(int pageNo, int rotate,
   getXRef()->setModifiedObject(&page, *refPage);
 }
 
-void PDFDoc::markPageObjects(Dict *pageDict, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts)
+bool PDFDoc::markPageObjects(Dict *pageDict, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts)
 {
   pageDict->remove("OpenAction");
   pageDict->remove("Outlines");
@@ -1818,9 +1847,13 @@ void PDFDoc::markPageObjects(Dict *pageDict, XRef *xRef, XRef *countRef, unsigne
 	      strcmp(key, "Annots") != 0 &&
 	      strcmp(key, "P") != 0 &&
         strcmp(key, "Root") != 0) {
-      markObject(&value, xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+      const bool success = markObject(&value, xRef, countRef, numOffset, oldRefNum, newRefNum, alreadyMarkedDicts);
+      if (unlikely(!success)) {
+          return false;
+      }
     }
   }
+  return true;
 }
 
 bool PDFDoc::markAnnotations(Object *annotsObj, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldPageNum, int newPageNum, std::set<Dict*> *alreadyMarkedDicts) {
diff --git a/poppler/PDFDoc.h b/poppler/PDFDoc.h
index 80b6d60..b504004 100644
--- a/poppler/PDFDoc.h
+++ b/poppler/PDFDoc.h
@@ -333,7 +333,7 @@ public:
 
   // rewrite pageDict with MediaBox, CropBox and new page CTM
   void replacePageDict(int pageNo, int rotate, const PDFRectangle *mediaBox, const PDFRectangle *cropBox);
-  void markPageObjects(Dict *pageDict, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts = nullptr);
+  bool markPageObjects(Dict *pageDict, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts = nullptr);
   bool markAnnotations(Object *annots, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldPageNum, int newPageNum, std::set<Dict*> *alreadyMarkedDicts = nullptr);
   void markAcroForm(Object *afObj, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum);
   // write all objects used by pageDict to outStr
@@ -355,8 +355,8 @@ public:
 
 private:
   // insert referenced objects in XRef
-  void markDictionnary (Dict* dict, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts);
-  void markObject (Object *obj, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts = nullptr);
+  bool markDictionnary (Dict* dict, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts);
+  bool markObject (Object *obj, XRef *xRef, XRef *countRef, unsigned int numOffset, int oldRefNum, int newRefNum, std::set<Dict*> *alreadyMarkedDicts = nullptr);
   static void writeDictionnary (Dict* dict, OutStream* outStr, XRef *xRef, unsigned int numOffset, unsigned char *fileKey,
                                 CryptAlgorithm encAlgorithm, int keyLength, Ref ref, std::set<Dict*> *alreadyWrittenDicts);
 
diff --git a/poppler/XRef.cc b/poppler/XRef.cc
index 9d6b80f..5943bdd 100644
--- a/poppler/XRef.cc
+++ b/poppler/XRef.cc
@@ -1298,11 +1298,17 @@ void XRef::add(Ref ref, Goffset offs, bool used)
     add(ref.num, ref.gen, offs, used);
 }
 
-void XRef::add(int num, int gen, Goffset offs, bool used) {
+bool XRef::add(int num, int gen, Goffset offs, bool used) {
   xrefLocker();
   if (num >= size) {
     if (num >= capacity) {
-      entries = (XRefEntry *)greallocn(entries, num + 1, sizeof(XRefEntry));
+      entries = (XRefEntry *)greallocn_checkoverflow(entries, num + 1, sizeof(XRefEntry));
+      if (unlikely(entries == nullptr)) {
+          size = 0;
+          capacity = 0;
+          return false;
+      }
+
       capacity = num + 1;
     }
     for (int i = size; i < num + 1; ++i) {
@@ -1325,6 +1331,7 @@ void XRef::add(int num, int gen, Goffset offs, bool used) {
     e->type = xrefEntryFree;
     e->offset = 0;
   }
+  return true;
 }
 
 void XRef::setModifiedObject (const Object* o, Ref r) {
diff --git a/poppler/XRef.h b/poppler/XRef.h
index 5c0238b..207f02a 100644
--- a/poppler/XRef.h
+++ b/poppler/XRef.h
@@ -14,7 +14,7 @@
 // under GPL version 2 or later
 //
 // Copyright (C) 2005 Brad Hards <bradh@frogmouth.net>
-// Copyright (C) 2006, 2008, 2010-2013, 2017-2020 Albert Astals Cid <aacid@kde.org>
+// Copyright (C) 2006, 2008, 2010-2013, 2017-2022 Albert Astals Cid <aacid@kde.org>
 // Copyright (C) 2007-2008 Julien Rebetez <julienr@svn.gnome.org>
 // Copyright (C) 2007 Carlos Garcia Campos <carlosgc@gnome.org>
 // Copyright (C) 2010 Ilya Gorenbein <igorenbein@finjan.com>
@@ -196,7 +196,7 @@ public:
   void setModifiedObject(const Object* o, Ref r);
   Ref addIndirectObject (const Object* o);
   void removeIndirectObject(Ref r);
-  void add(int num, int gen,  Goffset offs, bool used);
+  bool add(int num, int gen,  Goffset offs, bool used);
   void add(Ref ref, Goffset offs, bool used);
 
   // Output XRef table to stream
-- 
2.33.0