Initial checkin of GCC 4.9.0 from trunk (r208799).

Change-Id: I48a3c08bb98542aa215912a75f03c0890e497dba

1 files changed, 906 insertions, 0 deletions

diff --git a/gcc-4.9/gcc/internal-fn.c b/gcc-4.9/gcc/internal-fn.c
new file mode 100644
index 000000000..1062ea847
--- /dev/null
+++ b/gcc-4.9/gcc/internal-fn.c
@@ -0,0 +1,906 @@
+/* Internal functions.
+   Copyright (C) 2011-2014 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "internal-fn.h"
+#include "tree.h"
+#include "stor-layout.h"
+#include "expr.h"
+#include "optabs.h"
+#include "basic-block.h"
+#include "tree-ssa-alias.h"
+#include "internal-fn.h"
+#include "gimple-expr.h"
+#include "is-a.h"
+#include "gimple.h"
+#include "ubsan.h"
+#include "target.h"
+#include "predict.h"
+#include "stringpool.h"
+#include "tree-ssanames.h"
+#include "diagnostic-core.h"
+
+/* The names of each internal function, indexed by function number.  */
+const char *const internal_fn_name_array[] = {
+#define DEF_INTERNAL_FN(CODE, FLAGS) #CODE,
+#include "internal-fn.def"
+#undef DEF_INTERNAL_FN
+  "<invalid-fn>"
+};
+
+/* The ECF_* flags of each internal function, indexed by function number.  */
+const int internal_fn_flags_array[] = {
+#define DEF_INTERNAL_FN(CODE, FLAGS) FLAGS,
+#include "internal-fn.def"
+#undef DEF_INTERNAL_FN
+  0
+};
+
+/* ARRAY_TYPE is an array of vector modes.  Return the associated insn
+   for load-lanes-style optab OPTAB.  The insn must exist.  */
+
+static enum insn_code
+get_multi_vector_move (tree array_type, convert_optab optab)
+{
+  enum insn_code icode;
+  enum machine_mode imode;
+  enum machine_mode vmode;
+
+  gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
+  imode = TYPE_MODE (array_type);
+  vmode = TYPE_MODE (TREE_TYPE (array_type));
+
+  icode = convert_optab_handler (optab, imode, vmode);
+  gcc_assert (icode != CODE_FOR_nothing);
+  return icode;
+}
+
+/* Expand LOAD_LANES call STMT.  */
+
+static void
+expand_LOAD_LANES (gimple stmt)
+{
+  struct expand_operand ops[2];
+  tree type, lhs, rhs;
+  rtx target, mem;
+
+  lhs = gimple_call_lhs (stmt);
+  rhs = gimple_call_arg (stmt, 0);
+  type = TREE_TYPE (lhs);
+
+  target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  mem = expand_normal (rhs);
+
+  gcc_assert (MEM_P (mem));
+  PUT_MODE (mem, TYPE_MODE (type));
+
+  create_output_operand (&ops[0], target, TYPE_MODE (type));
+  create_fixed_operand (&ops[1], mem);
+  expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops);
+}
+
+/* Expand STORE_LANES call STMT.  */
+
+static void
+expand_STORE_LANES (gimple stmt)
+{
+  struct expand_operand ops[2];
+  tree type, lhs, rhs;
+  rtx target, reg;
+
+  lhs = gimple_call_lhs (stmt);
+  rhs = gimple_call_arg (stmt, 0);
+  type = TREE_TYPE (rhs);
+
+  target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  reg = expand_normal (rhs);
+
+  gcc_assert (MEM_P (target));
+  PUT_MODE (target, TYPE_MODE (type));
+
+  create_fixed_operand (&ops[0], target);
+  create_input_operand (&ops[1], reg, TYPE_MODE (type));
+  expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops);
+}
+
+static void
+expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED)
+{
+  gcc_unreachable ();
+}
+
+/* This should get expanded in adjust_simduid_builtins.  */
+
+static void
+expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED)
+{
+  gcc_unreachable ();
+}
+
+/* This should get expanded in adjust_simduid_builtins.  */
+
+static void
+expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED)
+{
+  gcc_unreachable ();
+}
+
+/* This should get expanded in adjust_simduid_builtins.  */
+
+static void
+expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED)
+{
+  gcc_unreachable ();
+}
+
+/* This should get expanded in the sanopt pass.  */
+
+static void
+expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED)
+{
+  gcc_unreachable ();
+}
+
+/* Add sub/add overflow checking to the statement STMT.
+   CODE says whether the operation is +, or -.  */
+
+void
+ubsan_expand_si_overflow_addsub_check (tree_code code, gimple stmt)
+{
+  rtx res, op0, op1;
+  tree lhs, fn, arg0, arg1;
+  rtx done_label, do_error, target = NULL_RTX;
+
+  lhs = gimple_call_lhs (stmt);
+  arg0 = gimple_call_arg (stmt, 0);
+  arg1 = gimple_call_arg (stmt, 1);
+  done_label = gen_label_rtx ();
+  do_error = gen_label_rtx ();
+  do_pending_stack_adjust ();
+  op0 = expand_normal (arg0);
+  op1 = expand_normal (arg1);
+
+  enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
+  if (lhs)
+    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+
+  enum insn_code icode
+    = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode);
+  if (icode != CODE_FOR_nothing)
+    {
+      struct expand_operand ops[4];
+      rtx last = get_last_insn ();
+
+      res = gen_reg_rtx (mode);
+      create_output_operand (&ops[0], res, mode);
+      create_input_operand (&ops[1], op0, mode);
+      create_input_operand (&ops[2], op1, mode);
+      create_fixed_operand (&ops[3], do_error);
+      if (maybe_expand_insn (icode, 4, ops))
+	{
+	  last = get_last_insn ();
+	  if (profile_status_for_fn (cfun) != PROFILE_ABSENT
+	      && JUMP_P (last)
+	      && any_condjump_p (last)
+	      && !find_reg_note (last, REG_BR_PROB, 0))
+	    add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
+	  emit_jump (done_label);
+        }
+      else
+	{
+	  delete_insns_since (last);
+	  icode = CODE_FOR_nothing;
+	}
+    }
+
+  if (icode == CODE_FOR_nothing)
+    {
+      rtx sub_check = gen_label_rtx ();
+      int pos_neg = 3;
+
+      /* Compute the operation.  On RTL level, the addition is always
+	 unsigned.  */
+      res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
+			  op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
+
+      /* If we can prove one of the arguments (for MINUS_EXPR only
+	 the second operand, as subtraction is not commutative) is always
+	 non-negative or always negative, we can do just one comparison
+	 and conditional jump instead of 2 at runtime, 3 present in the
+	 emitted code.  If one of the arguments is CONST_INT, all we
+	 need is to make sure it is op1, then the first
+	 emit_cmp_and_jump_insns will be just folded.  Otherwise try
+	 to use range info if available.  */
+      if (code == PLUS_EXPR && CONST_INT_P (op0))
+	{
+	  rtx tem = op0;
+	  op0 = op1;
+	  op1 = tem;
+	}
+      else if (CONST_INT_P (op1))
+	;
+      else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME)
+	{
+	  double_int arg0_min, arg0_max;
+	  if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE)
+	    {
+	      if (!arg0_min.is_negative ())
+		pos_neg = 1;
+	      else if (arg0_max.is_negative ())
+		pos_neg = 2;
+	    }
+	  if (pos_neg != 3)
+	    {
+	      rtx tem = op0;
+	      op0 = op1;
+	      op1 = tem;
+	    }
+	}
+      if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME)
+	{
+	  double_int arg1_min, arg1_max;
+	  if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE)
+	    {
+	      if (!arg1_min.is_negative ())
+		pos_neg = 1;
+	      else if (arg1_max.is_negative ())
+		pos_neg = 2;
+	    }
+	}
+
+      /* If the op1 is negative, we have to use a different check.  */
+      if (pos_neg == 3)
+	emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode,
+				 false, sub_check, PROB_EVEN);
+
+      /* Compare the result of the operation with one of the operands.  */
+      if (pos_neg & 1)
+	emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE,
+				 NULL_RTX, mode, false, done_label,
+				 PROB_VERY_LIKELY);
+
+      /* If we get here, we have to print the error.  */
+      if (pos_neg == 3)
+	{
+	  emit_jump (do_error);
+
+	  emit_label (sub_check);
+	}
+
+      /* We have k = a + b for b < 0 here.  k <= a must hold.  */
+      if (pos_neg & 2)
+	emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE,
+				 NULL_RTX, mode, false, done_label,
+				 PROB_VERY_LIKELY);
+    }
+
+  emit_label (do_error);
+  /* Expand the ubsan builtin call.  */
+  push_temp_slots ();
+  fn = ubsan_build_overflow_builtin (code, gimple_location (stmt),
+				     TREE_TYPE (arg0), arg0, arg1);
+  expand_normal (fn);
+  pop_temp_slots ();
+  do_pending_stack_adjust ();
+
+  /* We're done.  */
+  emit_label (done_label);
+
+  if (lhs)
+    emit_move_insn (target, res);
+}
+
+/* Add negate overflow checking to the statement STMT.  */
+
+void
+ubsan_expand_si_overflow_neg_check (gimple stmt)
+{
+  rtx res, op1;
+  tree lhs, fn, arg1;
+  rtx done_label, do_error, target = NULL_RTX;
+
+  lhs = gimple_call_lhs (stmt);
+  arg1 = gimple_call_arg (stmt, 1);
+  done_label = gen_label_rtx ();
+  do_error = gen_label_rtx ();
+
+  do_pending_stack_adjust ();
+  op1 = expand_normal (arg1);
+
+  enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
+  if (lhs)
+    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+
+  enum insn_code icode = optab_handler (negv3_optab, mode);
+  if (icode != CODE_FOR_nothing)
+    {
+      struct expand_operand ops[3];
+      rtx last = get_last_insn ();
+
+      res = gen_reg_rtx (mode);
+      create_output_operand (&ops[0], res, mode);
+      create_input_operand (&ops[1], op1, mode);
+      create_fixed_operand (&ops[2], do_error);
+      if (maybe_expand_insn (icode, 3, ops))
+	{
+	  last = get_last_insn ();
+	  if (profile_status_for_fn (cfun) != PROFILE_ABSENT
+	      && JUMP_P (last)
+	      && any_condjump_p (last)
+	      && !find_reg_note (last, REG_BR_PROB, 0))
+	    add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
+	  emit_jump (done_label);
+        }
+      else
+	{
+	  delete_insns_since (last);
+	  icode = CODE_FOR_nothing;
+	}
+    }
+
+  if (icode == CODE_FOR_nothing)
+    {
+      /* Compute the operation.  On RTL level, the addition is always
+	 unsigned.  */
+      res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
+
+      /* Compare the operand with the most negative value.  */
+      rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
+      emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false,
+			       done_label, PROB_VERY_LIKELY);
+    }
+
+  emit_label (do_error);
+  /* Expand the ubsan builtin call.  */
+  push_temp_slots ();
+  fn = ubsan_build_overflow_builtin (NEGATE_EXPR, gimple_location (stmt),
+				     TREE_TYPE (arg1), arg1, NULL_TREE);
+  expand_normal (fn);
+  pop_temp_slots ();
+  do_pending_stack_adjust ();
+
+  /* We're done.  */
+  emit_label (done_label);
+
+  if (lhs)
+    emit_move_insn (target, res);
+}
+
+/* Add mul overflow checking to the statement STMT.  */
+
+void
+ubsan_expand_si_overflow_mul_check (gimple stmt)
+{
+  rtx res, op0, op1;
+  tree lhs, fn, arg0, arg1;
+  rtx done_label, do_error, target = NULL_RTX;
+
+  lhs = gimple_call_lhs (stmt);
+  arg0 = gimple_call_arg (stmt, 0);
+  arg1 = gimple_call_arg (stmt, 1);
+  done_label = gen_label_rtx ();
+  do_error = gen_label_rtx ();
+
+  do_pending_stack_adjust ();
+  op0 = expand_normal (arg0);
+  op1 = expand_normal (arg1);
+
+  enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
+  if (lhs)
+    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+
+  enum insn_code icode = optab_handler (mulv4_optab, mode);
+  if (icode != CODE_FOR_nothing)
+    {
+      struct expand_operand ops[4];
+      rtx last = get_last_insn ();
+
+      res = gen_reg_rtx (mode);
+      create_output_operand (&ops[0], res, mode);
+      create_input_operand (&ops[1], op0, mode);
+      create_input_operand (&ops[2], op1, mode);
+      create_fixed_operand (&ops[3], do_error);
+      if (maybe_expand_insn (icode, 4, ops))
+	{
+	  last = get_last_insn ();
+	  if (profile_status_for_fn (cfun) != PROFILE_ABSENT
+	      && JUMP_P (last)
+	      && any_condjump_p (last)
+	      && !find_reg_note (last, REG_BR_PROB, 0))
+	    add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
+	  emit_jump (done_label);
+        }
+      else
+	{
+	  delete_insns_since (last);
+	  icode = CODE_FOR_nothing;
+	}
+    }
+
+  if (icode == CODE_FOR_nothing)
+    {
+      struct separate_ops ops;
+      enum machine_mode hmode
+	= mode_for_size (GET_MODE_PRECISION (mode) / 2, MODE_INT, 1);
+      ops.op0 = arg0;
+      ops.op1 = arg1;
+      ops.op2 = NULL_TREE;
+      ops.location = gimple_location (stmt);
+      if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
+	  && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
+	{
+	  enum machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
+	  ops.code = WIDEN_MULT_EXPR;
+	  ops.type
+	    = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0);
+
+	  res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
+	  rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res,
+				     GET_MODE_PRECISION (mode), NULL_RTX, 0);
+	  hipart = gen_lowpart (mode, hipart);
+	  res = gen_lowpart (mode, res);
+	  rtx signbit = expand_shift (RSHIFT_EXPR, mode, res,
+				      GET_MODE_PRECISION (mode) - 1,
+				      NULL_RTX, 0);
+	  /* RES is low half of the double width result, HIPART
+	     the high half.  There was overflow if
+	     HIPART is different from RES < 0 ? -1 : 0.  */
+	  emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode,
+				   false, done_label, PROB_VERY_LIKELY);
+	}
+      else if (hmode != BLKmode
+	       && 2 * GET_MODE_PRECISION (hmode) == GET_MODE_PRECISION (mode))
+	{
+	  rtx large_op0 = gen_label_rtx ();
+	  rtx small_op0_large_op1 = gen_label_rtx ();
+	  rtx one_small_one_large = gen_label_rtx ();
+	  rtx both_ops_large = gen_label_rtx ();
+	  rtx after_hipart_neg = gen_label_rtx ();
+	  rtx after_lopart_neg = gen_label_rtx ();
+	  rtx do_overflow = gen_label_rtx ();
+	  rtx hipart_different = gen_label_rtx ();
+
+	  int hprec = GET_MODE_PRECISION (hmode);
+	  rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
+				      NULL_RTX, 0);
+	  hipart0 = gen_lowpart (hmode, hipart0);
+	  rtx lopart0 = gen_lowpart (hmode, op0);
+	  rtx signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
+				       NULL_RTX, 0);
+	  rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
+				      NULL_RTX, 0);
+	  hipart1 = gen_lowpart (hmode, hipart1);
+	  rtx lopart1 = gen_lowpart (hmode, op1);
+	  rtx signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
+				       NULL_RTX, 0);
+
+	  res = gen_reg_rtx (mode);
+
+	  /* True if op0 resp. op1 are known to be in the range of
+	     halfstype.  */
+	  bool op0_small_p = false;
+	  bool op1_small_p = false;
+	  /* True if op0 resp. op1 are known to have all zeros or all ones
+	     in the upper half of bits, but are not known to be
+	     op{0,1}_small_p.  */
+	  bool op0_medium_p = false;
+	  bool op1_medium_p = false;
+	  /* -1 if op{0,1} is known to be negative, 0 if it is known to be
+	     nonnegative, 1 if unknown.  */
+	  int op0_sign = 1;
+	  int op1_sign = 1;
+
+	  if (TREE_CODE (arg0) == SSA_NAME)
+	    {
+	      double_int arg0_min, arg0_max;
+	      if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE)
+		{
+		  if (arg0_max.sle (double_int::max_value (hprec, false))
+		      && double_int::min_value (hprec, false).sle (arg0_min))
+		    op0_small_p = true;
+		  else if (arg0_max.sle (double_int::max_value (hprec, true))
+			   && (~double_int::max_value (hprec,
+						       true)).sle (arg0_min))
+		    op0_medium_p = true;
+		  if (!arg0_min.is_negative ())
+		    op0_sign = 0;
+		  else if (arg0_max.is_negative ())
+		    op0_sign = -1;
+		}
+	    }
+	  if (TREE_CODE (arg1) == SSA_NAME)
+	    {
+	      double_int arg1_min, arg1_max;
+	      if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE)
+		{
+		  if (arg1_max.sle (double_int::max_value (hprec, false))
+		      && double_int::min_value (hprec, false).sle (arg1_min))
+		    op1_small_p = true;
+		  else if (arg1_max.sle (double_int::max_value (hprec, true))
+			   && (~double_int::max_value (hprec,
+						       true)).sle (arg1_min))
+		    op1_medium_p = true;
+		  if (!arg1_min.is_negative ())
+		    op1_sign = 0;
+		  else if (arg1_max.is_negative ())
+		    op1_sign = -1;
+		}
+	    }
+
+	  int smaller_sign = 1;
+	  int larger_sign = 1;
+	  if (op0_small_p)
+	    {
+	      smaller_sign = op0_sign;
+	      larger_sign = op1_sign;
+	    }
+	  else if (op1_small_p)
+	    {
+	      smaller_sign = op1_sign;
+	      larger_sign = op0_sign;
+	    }
+	  else if (op0_sign == op1_sign)
+	    {
+	      smaller_sign = op0_sign;
+	      larger_sign = op0_sign;
+	    }
+
+	  if (!op0_small_p)
+	    emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode,
+				     false, large_op0, PROB_UNLIKELY);
+
+	  if (!op1_small_p)
+	    emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
+				     false, small_op0_large_op1,
+				     PROB_UNLIKELY);
+
+	  /* If both op0 and op1 are sign extended from hmode to mode,
+	     the multiplication will never overflow.  We can do just one
+	     hmode x hmode => mode widening multiplication.  */
+	  if (GET_CODE (lopart0) == SUBREG)
+	    {
+	      SUBREG_PROMOTED_VAR_P (lopart0) = 1;
+	      SUBREG_PROMOTED_UNSIGNED_SET (lopart0, 0);
+	    }
+	  if (GET_CODE (lopart1) == SUBREG)
+	    {
+	      SUBREG_PROMOTED_VAR_P (lopart1) = 1;
+	      SUBREG_PROMOTED_UNSIGNED_SET (lopart1, 0);
+	    }
+	  tree halfstype = build_nonstandard_integer_type (hprec, 0);
+	  ops.op0 = make_tree (halfstype, lopart0);
+	  ops.op1 = make_tree (halfstype, lopart1);
+	  ops.code = WIDEN_MULT_EXPR;
+	  ops.type = TREE_TYPE (arg0);
+	  rtx thisres
+	    = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
+	  emit_move_insn (res, thisres);
+	  emit_jump (done_label);
+
+	  emit_label (small_op0_large_op1);
+
+	  /* If op0 is sign extended from hmode to mode, but op1 is not,
+	     just swap the arguments and handle it as op1 sign extended,
+	     op0 not.  */
+	  rtx larger = gen_reg_rtx (mode);
+	  rtx hipart = gen_reg_rtx (hmode);
+	  rtx lopart = gen_reg_rtx (hmode);
+	  emit_move_insn (larger, op1);
+	  emit_move_insn (hipart, hipart1);
+	  emit_move_insn (lopart, lopart0);
+	  emit_jump (one_small_one_large);
+
+	  emit_label (large_op0);
+
+	  if (!op1_small_p)
+	    emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
+				     false, both_ops_large, PROB_UNLIKELY);
+
+	  /* If op1 is sign extended from hmode to mode, but op0 is not,
+	     prepare larger, hipart and lopart pseudos and handle it together
+	     with small_op0_large_op1.  */
+	  emit_move_insn (larger, op0);
+	  emit_move_insn (hipart, hipart0);
+	  emit_move_insn (lopart, lopart1);
+
+	  emit_label (one_small_one_large);
+
+	  /* lopart is the low part of the operand that is sign extended
+	     to mode, larger is the the other operand, hipart is the
+	     high part of larger and lopart0 and lopart1 are the low parts
+	     of both operands.
+	     We perform lopart0 * lopart1 and lopart * hipart widening
+	     multiplications.  */
+	  tree halfutype = build_nonstandard_integer_type (hprec, 1);
+	  ops.op0 = make_tree (halfutype, lopart0);
+	  ops.op1 = make_tree (halfutype, lopart1);
+	  rtx lo0xlo1
+	    = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
+
+	  ops.op0 = make_tree (halfutype, lopart);
+	  ops.op1 = make_tree (halfutype, hipart);
+	  rtx loxhi = gen_reg_rtx (mode);
+	  rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
+	  emit_move_insn (loxhi, tem);
+
+	  /* if (hipart < 0) loxhi -= lopart << (bitsize / 2);  */
+	  if (larger_sign == 0)
+	    emit_jump (after_hipart_neg);
+	  else if (larger_sign != -1)
+	    emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX, hmode,
+				     false, after_hipart_neg, PROB_EVEN);
+
+	  tem = convert_modes (mode, hmode, lopart, 1);
+	  tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
+	  tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
+				     1, OPTAB_DIRECT);
+	  emit_move_insn (loxhi, tem);
+
+	  emit_label (after_hipart_neg);
+
+	  /* if (lopart < 0) loxhi -= larger;  */
+	  if (smaller_sign == 0)
+	    emit_jump (after_lopart_neg);
+	  else if (smaller_sign != -1)
+	    emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX, hmode,
+				     false, after_lopart_neg, PROB_EVEN);
+
+	  tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
+				     1, OPTAB_DIRECT);
+	  emit_move_insn (loxhi, tem);
+
+	  emit_label (after_lopart_neg);
+
+	  /* loxhi += (uns) lo0xlo1 >> (bitsize / 2);  */
+	  tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
+	  tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
+				     1, OPTAB_DIRECT);
+	  emit_move_insn (loxhi, tem);
+
+	  /* if (loxhi >> (bitsize / 2)
+		 == (hmode) loxhi >> (bitsize / 2 - 1))  */
+	  rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
+					  NULL_RTX, 0);
+	  hipartloxhi = gen_lowpart (hmode, hipartloxhi);
+	  rtx lopartloxhi = gen_lowpart (hmode, loxhi);
+	  rtx signbitloxhi = expand_shift (RSHIFT_EXPR, hmode, lopartloxhi,
+					   hprec - 1, NULL_RTX, 0);
+
+	  emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX,
+				   hmode, false, do_overflow,
+				   PROB_VERY_UNLIKELY);
+
+	  /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1;  */
+	  rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
+					   NULL_RTX, 1);
+	  tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1);
+
+	  tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
+				     1, OPTAB_DIRECT);
+	  if (tem != res)
+	    emit_move_insn (res, tem);
+	  emit_jump (done_label);
+
+	  emit_label (both_ops_large);
+
+	  /* If both operands are large (not sign extended from hmode),
+	     then perform the full multiplication which will be the result
+	     of the operation.  The only cases which don't overflow are
+	     some cases where both hipart0 and highpart1 are 0 or -1.  */
+	  ops.code = MULT_EXPR;
+	  ops.op0 = make_tree (TREE_TYPE (arg0), op0);
+	  ops.op1 = make_tree (TREE_TYPE (arg0), op1);
+	  tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
+	  emit_move_insn (res, tem);
+
+	  if (!op0_medium_p)
+	    {
+	      tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
+					 NULL_RTX, 1, OPTAB_DIRECT);
+	      emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode,
+				       true, do_error, PROB_VERY_UNLIKELY);
+	    }
+
+	  if (!op1_medium_p)
+	    {
+	      tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
+					 NULL_RTX, 1, OPTAB_DIRECT);
+	      emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode,
+				       true, do_error, PROB_VERY_UNLIKELY);
+	    }
+
+	  /* At this point hipart{0,1} are both in [-1, 0].  If they are the
+	     same, overflow happened if res is negative, if they are different,
+	     overflow happened if res is positive.  */
+	  if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
+	    emit_jump (hipart_different);
+	  else if (op0_sign == 1 || op1_sign == 1)
+	    emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode,
+				     true, hipart_different, PROB_EVEN);
+
+	  emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false,
+				   do_error, PROB_VERY_UNLIKELY);
+	  emit_jump (done_label);
+
+	  emit_label (hipart_different);
+
+	  emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false,
+				   do_error, PROB_VERY_UNLIKELY);
+	  emit_jump (done_label);
+
+	  emit_label (do_overflow);
+
+	  /* Overflow, do full multiplication and fallthru into do_error.  */
+	  ops.op0 = make_tree (TREE_TYPE (arg0), op0);
+	  ops.op1 = make_tree (TREE_TYPE (arg0), op1);
+	  tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
+	  emit_move_insn (res, tem);
+	}
+      else
+	{
+	  ops.code = MULT_EXPR;
+	  ops.type = TREE_TYPE (arg0);
+	  res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
+	  emit_jump (done_label);
+	}
+    }
+
+  emit_label (do_error);
+  /* Expand the ubsan builtin call.  */
+  push_temp_slots ();
+  fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt),
+				     TREE_TYPE (arg0), arg0, arg1);
+  expand_normal (fn);
+  pop_temp_slots ();
+  do_pending_stack_adjust ();
+
+  /* We're done.  */
+  emit_label (done_label);
+
+  if (lhs)
+    emit_move_insn (target, res);
+}
+
+/* Expand UBSAN_CHECK_ADD call STMT.  */
+
+static void
+expand_UBSAN_CHECK_ADD (gimple stmt)
+{
+  ubsan_expand_si_overflow_addsub_check (PLUS_EXPR, stmt);
+}
+
+/* Expand UBSAN_CHECK_SUB call STMT.  */
+
+static void
+expand_UBSAN_CHECK_SUB (gimple stmt)
+{
+  if (integer_zerop (gimple_call_arg (stmt, 0)))
+    ubsan_expand_si_overflow_neg_check (stmt);
+  else
+    ubsan_expand_si_overflow_addsub_check (MINUS_EXPR, stmt);
+}
+
+/* Expand UBSAN_CHECK_MUL call STMT.  */
+
+static void
+expand_UBSAN_CHECK_MUL (gimple stmt)
+{
+  ubsan_expand_si_overflow_mul_check (stmt);
+}
+
+/* This should get folded in tree-vectorizer.c.  */
+
+static void
+expand_LOOP_VECTORIZED (gimple stmt ATTRIBUTE_UNUSED)
+{
+  gcc_unreachable ();
+}
+
+static void
+expand_MASK_LOAD (gimple stmt)
+{
+  struct expand_operand ops[3];
+  tree type, lhs, rhs, maskt;
+  rtx mem, target, mask;
+
+  maskt = gimple_call_arg (stmt, 2);
+  lhs = gimple_call_lhs (stmt);
+  if (lhs == NULL_TREE)
+    return;
+  type = TREE_TYPE (lhs);
+  rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
+		     gimple_call_arg (stmt, 1));
+
+  mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  gcc_assert (MEM_P (mem));
+  mask = expand_normal (maskt);
+  target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  create_output_operand (&ops[0], target, TYPE_MODE (type));
+  create_fixed_operand (&ops[1], mem);
+  create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
+  expand_insn (optab_handler (maskload_optab, TYPE_MODE (type)), 3, ops);
+}
+
+static void
+expand_MASK_STORE (gimple stmt)
+{
+  struct expand_operand ops[3];
+  tree type, lhs, rhs, maskt;
+  rtx mem, reg, mask;
+
+  maskt = gimple_call_arg (stmt, 2);
+  rhs = gimple_call_arg (stmt, 3);
+  type = TREE_TYPE (rhs);
+  lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
+		     gimple_call_arg (stmt, 1));
+
+  mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  gcc_assert (MEM_P (mem));
+  mask = expand_normal (maskt);
+  reg = expand_normal (rhs);
+  create_fixed_operand (&ops[0], mem);
+  create_input_operand (&ops[1], reg, TYPE_MODE (type));
+  create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
+  expand_insn (optab_handler (maskstore_optab, TYPE_MODE (type)), 3, ops);
+}
+
+static void
+expand_ABNORMAL_DISPATCHER (gimple)
+{
+}
+
+static void
+expand_BUILTIN_EXPECT (gimple stmt)
+{
+  /* When guessing was done, the hints should be already stripped away.  */
+  gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
+
+  rtx target;
+  tree lhs = gimple_call_lhs (stmt);
+  if (lhs)
+    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  else
+    target = const0_rtx;
+  rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
+  if (lhs && val != target)
+    emit_move_insn (target, val);
+}
+
+/* Routines to expand each internal function, indexed by function number.
+   Each routine has the prototype:
+
+       expand_<NAME> (gimple stmt)
+
+   where STMT is the statement that performs the call. */
+static void (*const internal_fn_expanders[]) (gimple) = {
+#define DEF_INTERNAL_FN(CODE, FLAGS) expand_##CODE,
+#include "internal-fn.def"
+#undef DEF_INTERNAL_FN
+  0
+};
+
+/* Expand STMT, which is a call to internal function FN.  */
+
+void
+expand_internal_call (gimple stmt)
+{
+  internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt);
+}