Return correct error from keymaster0engine for large RSA input

Also, ensure that we always put some error on the OpenSSL error queue
whenever a wrapped keymaster0 operation fails.  Higher layers will look
a the last entry on the queue and use it to determine what error code to
return.  Not putting any error on the queue means that those higher
layers will get whatever error was last enqueued, making the result
effectively random.  Non-determinism bad.

(cherry-picked from commit 22d2355b7edc470949c163e47ba8e837a1a87f47)

Bug: 25337630
Change-Id: I701ab735dd089f5258b2252f543906d9f3baa7a2

1 files changed, 23 insertions, 0 deletions

diff --git a/android_keymaster_test.cpp b/android_keymaster_test.cpp
index a340206..5fb3699 100644
--- a/android_keymaster_test.cpp
+++ b/android_keymaster_test.cpp
@@ -795,6 +795,29 @@ TEST_P(SigningOperationsTest, RsaSignWithEncryptionKey) {
         EXPECT_EQ(2, GetParam()->keymaster0_calls());
 }
 
+TEST_P(SigningOperationsTest, RsaSignTooLargeMessage) {
+    ASSERT_EQ(KM_ERROR_OK, GenerateKey(AuthorizationSetBuilder()
+                                           .RsaSigningKey(256, 3)
+                                           .Digest(KM_DIGEST_NONE)
+                                           .Padding(KM_PAD_NONE)));
+    string message(256 / 8, static_cast<char>(0xff));
+    string signature;
+    AuthorizationSet begin_params(client_params());
+    begin_params.push_back(TAG_PADDING, KM_PAD_NONE);
+    begin_params.push_back(TAG_DIGEST, KM_DIGEST_NONE);
+    ASSERT_EQ(KM_ERROR_OK, BeginOperation(KM_PURPOSE_SIGN, begin_params));
+    string result;
+    size_t input_consumed;
+    ASSERT_EQ(KM_ERROR_OK, UpdateOperation(message, &result, &input_consumed));
+    ASSERT_EQ(message.size(), input_consumed);
+    string output;
+    ASSERT_EQ(KM_ERROR_INVALID_ARGUMENT, FinishOperation(&output));
+
+
+    if (GetParam()->algorithm_in_km0_hardware(KM_ALGORITHM_RSA))
+        EXPECT_EQ(3, GetParam()->keymaster0_calls());
+}
+
 TEST_P(SigningOperationsTest, EcdsaSuccess) {
     ASSERT_EQ(KM_ERROR_OK,
               GenerateKey(AuthorizationSetBuilder().EcdsaSigningKey(224).Digest(KM_DIGEST_NONE)));