ART: Do not attempt to compile a method with too many registers

The mir_graph has a hard limit on how many ssa registers it can
handle, and will silently fail or fatal out in debug builds when
more SSA registers are necessary. This adds a check in the frontend
that tries to catch some of these cases, and makes the DCHECK a
CHECK to not silently fail.

Bug: 16018097
Change-Id: Iad82178945a8f8c34943cc7573fd73d2b4decade

2 files changed, 13 insertions, 1 deletions

diff --git a/compiler/dex/frontend.cc b/compiler/dex/frontend.cc
index d097500a7..51446f635 100644
--- a/compiler/dex/frontend.cc
+++ b/compiler/dex/frontend.cc
@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+#include <cstdint>
+
 #include "compiler.h"
 #include "compiler_internals.h"
 #include "driver/compiler_driver.h"
@@ -470,6 +472,10 @@ static const size_t kUnsupportedOpcodesSize[] = {
 COMPILE_ASSERT(sizeof(kUnsupportedOpcodesSize) == 8 * sizeof(size_t),
                kUnsupportedOpcodesSize_unexp);
 
+// The maximum amount of Dalvik register in a method for which we will start compiling. Tries to
+// avoid an abort when we need to manage more SSA registers than we can.
+static constexpr size_t kMaxAllowedDalvikRegisters = INT16_MAX / 2;
+
 CompilationUnit::CompilationUnit(ArenaPool* pool)
   : compiler_driver(nullptr),
     class_linker(nullptr),
@@ -548,6 +554,12 @@ static bool CanCompileShorty(const char* shorty, InstructionSet instruction_set)
 // Skip the method that we do not support currently.
 static bool CanCompileMethod(uint32_t method_idx, const DexFile& dex_file,
                              CompilationUnit& cu) {
+  // This is a limitation in mir_graph. See MirGraph::SetNumSSARegs.
+  if (cu.num_dalvik_registers > kMaxAllowedDalvikRegisters) {
+    VLOG(compiler) << "Too many dalvik registers : " << cu.num_dalvik_registers;
+    return false;
+  }
+
   // Check whether we do have limitations at all.
   if (kSupportedTypes[cu.instruction_set] == nullptr &&
       kUnsupportedOpcodesSize[cu.instruction_set] == 0U) {
diff --git a/compiler/dex/mir_graph.h b/compiler/dex/mir_graph.h
index 1556a19da..79b3edf50 100644
--- a/compiler/dex/mir_graph.h
+++ b/compiler/dex/mir_graph.h
@@ -727,7 +727,7 @@ class MIRGraph {
       * would be filtered out with current settings.  When orig_sreg field is removed
       * from RegLocation, expand s_reg_low to handle all possible cases and remove DCHECK().
       */
-    DCHECK_EQ(new_num, static_cast<int16_t>(new_num));
+    CHECK_EQ(new_num, static_cast<int16_t>(new_num));
     num_ssa_regs_ = new_num;
   }