Add RSA encryption and decryption support.

This change was already reviewed, merged and reverted, so I'm skipping
the review step this time.

Change-Id: Ie5b7dba86a7ae7f62eedbdb6eec7b61ef83d0c73

1 files changed, 89 insertions, 4 deletions

diff --git a/rsa_operation.cpp b/rsa_operation.cpp
index a013b78..df39006 100644
--- a/rsa_operation.cpp
+++ b/rsa_operation.cpp
@@ -14,8 +14,11 @@
  * limitations under the License.
  */
 
-#include <openssl/rsa.h>
+#include <limits.h>
+
+#include <openssl/err.h>
 #include <openssl/evp.h>
+#include <openssl/rsa.h>
 
 #include "rsa_operation.h"
 #include "openssl_utils.h"
@@ -37,6 +40,8 @@ keymaster_error_t RsaOperation::Update(const Buffer& input, Buffer* /* output */
         return KM_ERROR_UNIMPLEMENTED;
     case KM_PURPOSE_SIGN:
     case KM_PURPOSE_VERIFY:
+    case KM_PURPOSE_ENCRYPT:
+    case KM_PURPOSE_DECRYPT:
         return StoreData(input);
     }
 }
@@ -61,13 +66,16 @@ keymaster_error_t RsaSignOperation::Finish(const Buffer& /* signature */, Buffer
 
 keymaster_error_t RsaVerifyOperation::Finish(const Buffer& signature, Buffer* /* output */) {
 #if defined(OPENSSL_IS_BORINGSSL)
-    if (data_.available_read() != RSA_size(rsa_key_))
-        return KM_ERROR_INVALID_INPUT_LENGTH;
+    size_t message_size = data_.available_read();
 #else
-    if ((int)data_.available_read() != RSA_size(rsa_key_))
+    if (data_.available_read() > INT_MAX)
         return KM_ERROR_INVALID_INPUT_LENGTH;
+    int message_size = (int)data_.available_read();
 #endif
 
+    if (message_size != RSA_size(rsa_key_))
+        return KM_ERROR_INVALID_INPUT_LENGTH;
+
     if (data_.available_read() != signature.available_read())
         return KM_ERROR_VERIFICATION_FAILED;
 
@@ -83,4 +91,81 @@ keymaster_error_t RsaVerifyOperation::Finish(const Buffer& signature, Buffer* /*
     return KM_ERROR_VERIFICATION_FAILED;
 }
 
+const int OAEP_PADDING_OVERHEAD = 41;
+const int PKCS1_PADDING_OVERHEAD = 11;
+
+keymaster_error_t RsaEncryptOperation::Finish(const Buffer& /* signature */, Buffer* output) {
+    int openssl_padding;
+
+#if defined(OPENSSL_IS_BORINGSSL)
+    size_t message_size = data_.available_read();
+#else
+    if (data_.available_read() > INT_MAX)
+        return KM_ERROR_INVALID_INPUT_LENGTH;
+    int message_size = (int)data_.available_read();
+#endif
+
+    switch (padding_) {
+    case KM_PAD_RSA_OAEP:
+        openssl_padding = RSA_PKCS1_OAEP_PADDING;
+        if (message_size >= RSA_size(rsa_key_) - OAEP_PADDING_OVERHEAD) {
+            logger().error("Cannot encrypt %d bytes with %d-byte key and OAEP padding",
+                           data_.available_read(), RSA_size(rsa_key_));
+            return KM_ERROR_INVALID_INPUT_LENGTH;
+        }
+        break;
+    case KM_PAD_RSA_PKCS1_1_5_ENCRYPT:
+        openssl_padding = RSA_PKCS1_PADDING;
+        if (message_size >= RSA_size(rsa_key_) - PKCS1_PADDING_OVERHEAD) {
+            logger().error("Cannot encrypt %d bytes with %d-byte key and PKCS1 padding",
+                           data_.available_read(), RSA_size(rsa_key_));
+            return KM_ERROR_INVALID_INPUT_LENGTH;
+        }
+        break;
+    default:
+        logger().error("Padding mode %d not supported", padding_);
+        return KM_ERROR_UNSUPPORTED_PADDING_MODE;
+    }
+
+    output->Reinitialize(RSA_size(rsa_key_));
+    int bytes_encrypted = RSA_public_encrypt(data_.available_read(), data_.peek_read(),
+                                             output->peek_write(), rsa_key_, openssl_padding);
+
+    if (bytes_encrypted < 0) {
+        logger().error("Error %d encrypting data with RSA", ERR_get_error());
+        return KM_ERROR_UNKNOWN_ERROR;
+    }
+    assert(bytes_encrypted == RSA_size(rsa_key_));
+    output->advance_write(bytes_encrypted);
+
+    return KM_ERROR_OK;
+}
+
+keymaster_error_t RsaDecryptOperation::Finish(const Buffer& /* signature */, Buffer* output) {
+    int openssl_padding;
+    switch (padding_) {
+    case KM_PAD_RSA_OAEP:
+        openssl_padding = RSA_PKCS1_OAEP_PADDING;
+        break;
+    case KM_PAD_RSA_PKCS1_1_5_ENCRYPT:
+        openssl_padding = RSA_PKCS1_PADDING;
+        break;
+    default:
+        logger().error("Padding mode %d not supported", padding_);
+        return KM_ERROR_UNSUPPORTED_PADDING_MODE;
+    }
+
+    output->Reinitialize(RSA_size(rsa_key_));
+    int bytes_decrypted = RSA_private_decrypt(data_.available_read(), data_.peek_read(),
+                                              output->peek_write(), rsa_key_, openssl_padding);
+
+    if (bytes_decrypted < 0) {
+        logger().error("Error %d decrypting data with RSA", ERR_get_error());
+        return KM_ERROR_UNKNOWN_ERROR;
+    }
+    output->advance_write(bytes_decrypted);
+
+    return KM_ERROR_OK;
+}
+
 }  // namespace keymaster