From aeac3040cc99eeaff1e1171a822114c857b9dca9 Mon Sep 17 00:00:00 2001
From: Armin Novak <anovak@thincast.com>
Date: Sat, 13 Jan 2024 21:01:55 +0100
Subject: [PATCH] [codec,planar] check resolution for overflow

Origin: https://github.com/FreeRDP/FreeRDP/commit/aeac3040cc99eeaff1e1171a822114c857b9dca9

If the codec resolution is too large return an error as the internal
buffers would otherwise overflow.

(cherry picked from commit 44edab1deae4f8c901c00a00683f888cef36d853)
---
 libfreerdp/codec/planar.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c
index b4815a632309..0a5ec581c6cc 100644
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -1496,7 +1496,13 @@ BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* context, UINT32
 	context->bgr = FALSE;
 	context->maxWidth = PLANAR_ALIGN(width, 4);
 	context->maxHeight = PLANAR_ALIGN(height, 4);
-	context->maxPlaneSize = context->maxWidth * context->maxHeight;
+	const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
+	if (tmp > UINT32_MAX)
+		return FALSE;
+	context->maxPlaneSize = tmp;
+
+	if (context->maxWidth > UINT32_MAX / 4)
+		return FALSE;
 	context->nTempStep = context->maxWidth * 4;
 	free(context->planesBuffer);
 	free(context->pTempData);
