malloc_debug: fix multiplication overflow in debug_calloc

The over flow check for nmemb * bytes in debug_calloc is incorrect,
use the builtin overflow functions to check for multiplication and
addition overflow.

Change-Id: I3f1c13102621bc5380be1f69caa88dba2118f3cb

1 files changed, 9 insertions, 3 deletions

diff --git a/libc/malloc_debug/malloc_debug.cpp b/libc/malloc_debug/malloc_debug.cpp
index b20d634b2..568192d69 100644
--- a/libc/malloc_debug/malloc_debug.cpp
+++ b/libc/malloc_debug/malloc_debug.cpp
@@ -538,13 +538,19 @@ void* debug_calloc(size_t nmemb, size_t bytes) {
     return g_dispatch->calloc(nmemb, bytes);
   }
 
-  size_t size = nmemb * bytes;
+  size_t size;
+  if (__builtin_mul_overflow(nmemb, bytes, &size)) {
+    // Overflow
+    errno = ENOMEM;
+    return nullptr;
+  }
+
   if (size == 0) {
     size = 1;
   }
 
-  size_t real_size = size + g_debug->extra_bytes();
-  if (real_size < bytes || real_size < nmemb) {
+  size_t real_size;
+  if (__builtin_add_overflow(size, g_debug->extra_bytes(), &real_size)) {
     // Overflow.
     errno = ENOMEM;
     return nullptr;