Bulletproof leak dump against null hash entries

Previously, the malloc leak checking code would crash in qsort()
if null entries existed in its bookkeeping table.  This change
makes the comparison function detect null entries and sort them
to the end safely.

Change-Id: I88244a7df1e289dd9d7992ce29606d505bd63079

1 files changed, 29 insertions, 20 deletions

diff --git a/libc/bionic/malloc_debug_common.c b/libc/bionic/malloc_debug_common.c
index ec5682668..f05576c4f 100644
--- a/libc/bionic/malloc_debug_common.c
+++ b/libc/bionic/malloc_debug_common.c
@@ -60,34 +60,43 @@ HashTable gHashTable;
 
 static int hash_entry_compare(const void* arg1, const void* arg2)
 {
+    int result;
+
     HashEntry* e1 = *(HashEntry**)arg1;
     HashEntry* e2 = *(HashEntry**)arg2;
 
-    size_t nbAlloc1 = e1->allocations;
-    size_t nbAlloc2 = e2->allocations;
-    size_t size1 = e1->size & ~SIZE_FLAG_MASK;
-    size_t size2 = e2->size & ~SIZE_FLAG_MASK;
-    size_t alloc1 = nbAlloc1 * size1;
-    size_t alloc2 = nbAlloc2 * size2;
-
-    // sort in descending order by:
-    // 1) total size
-    // 2) number of allocations
-    //
-    // This is used for sorting, not determination of equality, so we don't
-    // need to compare the bit flags.
-    int result;
-    if (alloc1 > alloc2) {
+    // if one or both arg pointers are null, deal gracefully
+    if (e1 == NULL) {
+        result = (e2 == NULL) ? 0 : 1;
+    } else if (e2 == NULL) {
         result = -1;
-    } else if (alloc1 < alloc2) {
-        result = 1;
     } else {
-        if (nbAlloc1 > nbAlloc2) {
+        size_t nbAlloc1 = e1->allocations;
+        size_t nbAlloc2 = e2->allocations;
+        size_t size1 = e1->size & ~SIZE_FLAG_MASK;
+        size_t size2 = e2->size & ~SIZE_FLAG_MASK;
+        size_t alloc1 = nbAlloc1 * size1;
+        size_t alloc2 = nbAlloc2 * size2;
+
+        // sort in descending order by:
+        // 1) total size
+        // 2) number of allocations
+        //
+        // This is used for sorting, not determination of equality, so we don't
+        // need to compare the bit flags.
+        int result;
+        if (alloc1 > alloc2) {
             result = -1;
-        } else if (nbAlloc1 < nbAlloc2) {
+        } else if (alloc1 < alloc2) {
             result = 1;
         } else {
-            result = 0;
+            if (nbAlloc1 > nbAlloc2) {
+                result = -1;
+            } else if (nbAlloc1 < nbAlloc2) {
+                result = 1;
+            } else {
+                result = 0;
+            }
         }
     }
     return result;