From 32492093722636596018a799c438bfc04c343b40 Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@openssl.org>
Date: Mon, 6 Mar 2017 09:54:17 -0500
Subject: [PATCH] Fix an endless loop in rsa_builtin_keygen.

Cherry-picked by Matt Caswell from 69795831.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4670)
---
 crypto/rsa/rsa_gen.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/Cryptlib/OpenSSL/crypto/rsa/rsa_gen.c b/Cryptlib/OpenSSL/crypto/rsa/rsa_gen.c
index 082c8da2efc..a85493d6097 100644
--- a/Cryptlib/OpenSSL/crypto/rsa/rsa_gen.c
+++ b/Cryptlib/OpenSSL/crypto/rsa/rsa_gen.c
@@ -110,6 +110,16 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
     int bitsp, bitsq, ok = -1, n = 0;
     BN_CTX *ctx = NULL;
 
+    /*
+     * When generating ridiculously small keys, we can get stuck
+     * continually regenerating the same prime values.
+     */
+    if (bits < 16) {
+        ok = 0;             /* we set our own err */
+        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
+        goto err;
+    }
+
     ctx = BN_CTX_new();
     if (ctx == NULL)
         goto err;
@@ -161,21 +171,10 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
     if (!BN_GENCB_call(cb, 3, 0))
         goto err;
     for (;;) {
-        /*
-         * When generating ridiculously small keys, we can get stuck
-         * continually regenerating the same prime values. Check for this and
-         * bail if it happens 3 times.
-         */
-        unsigned int degenerate = 0;
         do {
             if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                 goto err;
-        } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
-        if (degenerate == 3) {
-            ok = 0;             /* we set our own err */
-            RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
-            goto err;
-        }
+        } while (BN_cmp(rsa->p, rsa->q) == 0);
         if (!BN_sub(r2, rsa->q, BN_value_one()))
             goto err;
         if (!BN_gcd(r1, r2, rsa->e, ctx))
