DO NOT MERGE

Always use Fermat's Little Theorem in ecdsa_sign_setup.

The case where ec_group_get_mont_data is NULL is only for arbitrary groups
which we now require to be prime order. BN_mod_exp_mont is fine with a NULL
BN_MONT_CTX. It will just compute it. Saves a bit of special-casing.

Also don't mark p-2 as BN_FLG_CONSTTIME as the exponent is public anyway.

(cherry picked from commit 8cf79af7d1497c07bd684764b96c9659e7b32ae1)

Bug: 33752052
Change-Id: Ia57cf8b68415742233b5c70e9da901b0f2954e7a
(cherry picked from commit c5f118738a61b533d90b01841664e6480e0692fe)

2 files changed, 20 insertions, 22 deletions

diff --git a/src/crypto/ec/ec.c b/src/crypto/ec/ec.c
index 1bc2e26..d2be59d 100644
--- a/src/crypto/ec/ec.c
+++ b/src/crypto/ec/ec.c
@@ -361,6 +361,12 @@ int EC_GROUP_set_generator(EC_GROUP *group, const EC_POINT *generator,
     return 0;
   }
 
+  /* Require a cofactor of one for custom curves, which implies prime order. */
+  if (!BN_is_one(cofactor)) {
+    OPENSSL_PUT_ERROR(EC, EC_GROUP_new_curve_GFp, EC_R_WRONG_CURVE_PARAMETERS);
+    return 0;
+  }
+
   if (group->generator == NULL) {
     group->generator = EC_POINT_new(group);
     if (group->generator == NULL) {
diff --git a/src/crypto/ecdsa/ecdsa.c b/src/crypto/ecdsa/ecdsa.c
index 86e41bb..4c35983 100644
--- a/src/crypto/ecdsa/ecdsa.c
+++ b/src/crypto/ecdsa/ecdsa.c
@@ -235,7 +235,7 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp,
                             BIGNUM **rp, const uint8_t *digest,
                             size_t digest_len) {
   BN_CTX *ctx = NULL;
-  BIGNUM *k = NULL, *r = NULL, *order = NULL, *X = NULL;
+  BIGNUM *k = NULL, *r = NULL, *order = NULL, *tmp = NULL;
   EC_POINT *tmp_point = NULL;
   const EC_GROUP *group;
   int ret = 0;
@@ -257,8 +257,8 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp,
   k = BN_new(); /* this value is later returned in *kinvp */
   r = BN_new(); /* this value is later returned in *rp    */
   order = BN_new();
-  X = BN_new();
-  if (!k || !r || !order || !X) {
+  tmp = BN_new();
+  if (!k || !r || !order || !tmp) {
     OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_MALLOC_FAILURE);
     goto err;
   }
@@ -310,33 +310,25 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp,
       OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_EC_LIB);
       goto err;
     }
-    if (!EC_POINT_get_affine_coordinates_GFp(group, tmp_point, X, NULL, ctx)) {
+    if (!EC_POINT_get_affine_coordinates_GFp(group, tmp_point, tmp, NULL,
+                                             ctx)) {
       OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_EC_LIB);
       goto err;
     }
 
-    if (!BN_nnmod(r, X, order, ctx)) {
+    if (!BN_nnmod(r, tmp, order, ctx)) {
       OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_BN_LIB);
       goto err;
     }
   } while (BN_is_zero(r));
 
-  /* compute the inverse of k */
-  if (ec_group_get_mont_data(group) != NULL) {
-    /* We want inverse in constant time, therefore we use that the order must
-     * be prime and thus we can use Fermat's Little Theorem. */
-    if (!BN_set_word(X, 2) ||
-        !BN_sub(X, order, X)) {
-      OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_BN_LIB);
-      goto err;
-    }
-    BN_set_flags(X, BN_FLG_CONSTTIME);
-    if (!BN_mod_exp_mont_consttime(k, k, X, order, ctx,
-                                   ec_group_get_mont_data(group))) {
-      OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_BN_LIB);
-      goto err;
-    }
-  } else if (!BN_mod_inverse(k, k, order, ctx)) {
+  /* Compute the inverse of k. The order is a prime, so use Fermat's Little
+   * Theorem. */
+  if (!BN_set_word(tmp, 2) ||
+      !BN_sub(tmp, order, tmp) ||
+      /* Note |ec_group_get_mont_data| may return NULL but |BN_mod_exp_mont|
+       * allows it to be. */
+      !BN_mod_exp_mont(k, k, tmp, order, ctx, ec_group_get_mont_data(group))) {
     OPENSSL_PUT_ERROR(ECDSA, ecdsa_sign_setup, ERR_R_BN_LIB);
     goto err;
   }
@@ -359,7 +351,7 @@ err:
   }
   BN_free(order);
   EC_POINT_free(tmp_point);
-  BN_clear_free(X);
+  BN_clear_free(tmp);
   return ret;
 }