diff options
author | Ben Cheng <bccheng@google.com> | 2014-03-25 22:37:19 -0700 |
---|---|---|
committer | Ben Cheng <bccheng@google.com> | 2014-03-25 22:37:19 -0700 |
commit | 1bc5aee63eb72b341f506ad058502cd0361f0d10 (patch) | |
tree | c607e8252f3405424ff15bc2d00aa38dadbb2518 /gcc-4.9/gcc/internal-fn.c | |
parent | 283a0bf58fcf333c58a2a92c3ebbc41fb9eb1fdb (diff) | |
download | toolchain_gcc-1bc5aee63eb72b341f506ad058502cd0361f0d10.tar.gz toolchain_gcc-1bc5aee63eb72b341f506ad058502cd0361f0d10.tar.bz2 toolchain_gcc-1bc5aee63eb72b341f506ad058502cd0361f0d10.zip |
Initial checkin of GCC 4.9.0 from trunk (r208799).
Change-Id: I48a3c08bb98542aa215912a75f03c0890e497dba
Diffstat (limited to 'gcc-4.9/gcc/internal-fn.c')
-rw-r--r-- | gcc-4.9/gcc/internal-fn.c | 906 |
1 files changed, 906 insertions, 0 deletions
diff --git a/gcc-4.9/gcc/internal-fn.c b/gcc-4.9/gcc/internal-fn.c new file mode 100644 index 000000000..1062ea847 --- /dev/null +++ b/gcc-4.9/gcc/internal-fn.c @@ -0,0 +1,906 @@ +/* Internal functions. + Copyright (C) 2011-2014 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with GCC; see the file COPYING3. If not see +<http://www.gnu.org/licenses/>. */ + +#include "config.h" +#include "system.h" +#include "coretypes.h" +#include "internal-fn.h" +#include "tree.h" +#include "stor-layout.h" +#include "expr.h" +#include "optabs.h" +#include "basic-block.h" +#include "tree-ssa-alias.h" +#include "internal-fn.h" +#include "gimple-expr.h" +#include "is-a.h" +#include "gimple.h" +#include "ubsan.h" +#include "target.h" +#include "predict.h" +#include "stringpool.h" +#include "tree-ssanames.h" +#include "diagnostic-core.h" + +/* The names of each internal function, indexed by function number. */ +const char *const internal_fn_name_array[] = { +#define DEF_INTERNAL_FN(CODE, FLAGS) #CODE, +#include "internal-fn.def" +#undef DEF_INTERNAL_FN + "<invalid-fn>" +}; + +/* The ECF_* flags of each internal function, indexed by function number. */ +const int internal_fn_flags_array[] = { +#define DEF_INTERNAL_FN(CODE, FLAGS) FLAGS, +#include "internal-fn.def" +#undef DEF_INTERNAL_FN + 0 +}; + +/* ARRAY_TYPE is an array of vector modes. Return the associated insn + for load-lanes-style optab OPTAB. The insn must exist. */ + +static enum insn_code +get_multi_vector_move (tree array_type, convert_optab optab) +{ + enum insn_code icode; + enum machine_mode imode; + enum machine_mode vmode; + + gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE); + imode = TYPE_MODE (array_type); + vmode = TYPE_MODE (TREE_TYPE (array_type)); + + icode = convert_optab_handler (optab, imode, vmode); + gcc_assert (icode != CODE_FOR_nothing); + return icode; +} + +/* Expand LOAD_LANES call STMT. */ + +static void +expand_LOAD_LANES (gimple stmt) +{ + struct expand_operand ops[2]; + tree type, lhs, rhs; + rtx target, mem; + + lhs = gimple_call_lhs (stmt); + rhs = gimple_call_arg (stmt, 0); + type = TREE_TYPE (lhs); + + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + mem = expand_normal (rhs); + + gcc_assert (MEM_P (mem)); + PUT_MODE (mem, TYPE_MODE (type)); + + create_output_operand (&ops[0], target, TYPE_MODE (type)); + create_fixed_operand (&ops[1], mem); + expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops); +} + +/* Expand STORE_LANES call STMT. */ + +static void +expand_STORE_LANES (gimple stmt) +{ + struct expand_operand ops[2]; + tree type, lhs, rhs; + rtx target, reg; + + lhs = gimple_call_lhs (stmt); + rhs = gimple_call_arg (stmt, 0); + type = TREE_TYPE (rhs); + + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + reg = expand_normal (rhs); + + gcc_assert (MEM_P (target)); + PUT_MODE (target, TYPE_MODE (type)); + + create_fixed_operand (&ops[0], target); + create_input_operand (&ops[1], reg, TYPE_MODE (type)); + expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops); +} + +static void +expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED) +{ + gcc_unreachable (); +} + +/* This should get expanded in adjust_simduid_builtins. */ + +static void +expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED) +{ + gcc_unreachable (); +} + +/* This should get expanded in adjust_simduid_builtins. */ + +static void +expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED) +{ + gcc_unreachable (); +} + +/* This should get expanded in adjust_simduid_builtins. */ + +static void +expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED) +{ + gcc_unreachable (); +} + +/* This should get expanded in the sanopt pass. */ + +static void +expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED) +{ + gcc_unreachable (); +} + +/* Add sub/add overflow checking to the statement STMT. + CODE says whether the operation is +, or -. */ + +void +ubsan_expand_si_overflow_addsub_check (tree_code code, gimple stmt) +{ + rtx res, op0, op1; + tree lhs, fn, arg0, arg1; + rtx done_label, do_error, target = NULL_RTX; + + lhs = gimple_call_lhs (stmt); + arg0 = gimple_call_arg (stmt, 0); + arg1 = gimple_call_arg (stmt, 1); + done_label = gen_label_rtx (); + do_error = gen_label_rtx (); + do_pending_stack_adjust (); + op0 = expand_normal (arg0); + op1 = expand_normal (arg1); + + enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0)); + if (lhs) + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + + enum insn_code icode + = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode); + if (icode != CODE_FOR_nothing) + { + struct expand_operand ops[4]; + rtx last = get_last_insn (); + + res = gen_reg_rtx (mode); + create_output_operand (&ops[0], res, mode); + create_input_operand (&ops[1], op0, mode); + create_input_operand (&ops[2], op1, mode); + create_fixed_operand (&ops[3], do_error); + if (maybe_expand_insn (icode, 4, ops)) + { + last = get_last_insn (); + if (profile_status_for_fn (cfun) != PROFILE_ABSENT + && JUMP_P (last) + && any_condjump_p (last) + && !find_reg_note (last, REG_BR_PROB, 0)) + add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + emit_jump (done_label); + } + else + { + delete_insns_since (last); + icode = CODE_FOR_nothing; + } + } + + if (icode == CODE_FOR_nothing) + { + rtx sub_check = gen_label_rtx (); + int pos_neg = 3; + + /* Compute the operation. On RTL level, the addition is always + unsigned. */ + res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab, + op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); + + /* If we can prove one of the arguments (for MINUS_EXPR only + the second operand, as subtraction is not commutative) is always + non-negative or always negative, we can do just one comparison + and conditional jump instead of 2 at runtime, 3 present in the + emitted code. If one of the arguments is CONST_INT, all we + need is to make sure it is op1, then the first + emit_cmp_and_jump_insns will be just folded. Otherwise try + to use range info if available. */ + if (code == PLUS_EXPR && CONST_INT_P (op0)) + { + rtx tem = op0; + op0 = op1; + op1 = tem; + } + else if (CONST_INT_P (op1)) + ; + else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME) + { + double_int arg0_min, arg0_max; + if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE) + { + if (!arg0_min.is_negative ()) + pos_neg = 1; + else if (arg0_max.is_negative ()) + pos_neg = 2; + } + if (pos_neg != 3) + { + rtx tem = op0; + op0 = op1; + op1 = tem; + } + } + if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME) + { + double_int arg1_min, arg1_max; + if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE) + { + if (!arg1_min.is_negative ()) + pos_neg = 1; + else if (arg1_max.is_negative ()) + pos_neg = 2; + } + } + + /* If the op1 is negative, we have to use a different check. */ + if (pos_neg == 3) + emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode, + false, sub_check, PROB_EVEN); + + /* Compare the result of the operation with one of the operands. */ + if (pos_neg & 1) + emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE, + NULL_RTX, mode, false, done_label, + PROB_VERY_LIKELY); + + /* If we get here, we have to print the error. */ + if (pos_neg == 3) + { + emit_jump (do_error); + + emit_label (sub_check); + } + + /* We have k = a + b for b < 0 here. k <= a must hold. */ + if (pos_neg & 2) + emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE, + NULL_RTX, mode, false, done_label, + PROB_VERY_LIKELY); + } + + emit_label (do_error); + /* Expand the ubsan builtin call. */ + push_temp_slots (); + fn = ubsan_build_overflow_builtin (code, gimple_location (stmt), + TREE_TYPE (arg0), arg0, arg1); + expand_normal (fn); + pop_temp_slots (); + do_pending_stack_adjust (); + + /* We're done. */ + emit_label (done_label); + + if (lhs) + emit_move_insn (target, res); +} + +/* Add negate overflow checking to the statement STMT. */ + +void +ubsan_expand_si_overflow_neg_check (gimple stmt) +{ + rtx res, op1; + tree lhs, fn, arg1; + rtx done_label, do_error, target = NULL_RTX; + + lhs = gimple_call_lhs (stmt); + arg1 = gimple_call_arg (stmt, 1); + done_label = gen_label_rtx (); + do_error = gen_label_rtx (); + + do_pending_stack_adjust (); + op1 = expand_normal (arg1); + + enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg1)); + if (lhs) + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + + enum insn_code icode = optab_handler (negv3_optab, mode); + if (icode != CODE_FOR_nothing) + { + struct expand_operand ops[3]; + rtx last = get_last_insn (); + + res = gen_reg_rtx (mode); + create_output_operand (&ops[0], res, mode); + create_input_operand (&ops[1], op1, mode); + create_fixed_operand (&ops[2], do_error); + if (maybe_expand_insn (icode, 3, ops)) + { + last = get_last_insn (); + if (profile_status_for_fn (cfun) != PROFILE_ABSENT + && JUMP_P (last) + && any_condjump_p (last) + && !find_reg_note (last, REG_BR_PROB, 0)) + add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + emit_jump (done_label); + } + else + { + delete_insns_since (last); + icode = CODE_FOR_nothing; + } + } + + if (icode == CODE_FOR_nothing) + { + /* Compute the operation. On RTL level, the addition is always + unsigned. */ + res = expand_unop (mode, neg_optab, op1, NULL_RTX, false); + + /* Compare the operand with the most negative value. */ + rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1))); + emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false, + done_label, PROB_VERY_LIKELY); + } + + emit_label (do_error); + /* Expand the ubsan builtin call. */ + push_temp_slots (); + fn = ubsan_build_overflow_builtin (NEGATE_EXPR, gimple_location (stmt), + TREE_TYPE (arg1), arg1, NULL_TREE); + expand_normal (fn); + pop_temp_slots (); + do_pending_stack_adjust (); + + /* We're done. */ + emit_label (done_label); + + if (lhs) + emit_move_insn (target, res); +} + +/* Add mul overflow checking to the statement STMT. */ + +void +ubsan_expand_si_overflow_mul_check (gimple stmt) +{ + rtx res, op0, op1; + tree lhs, fn, arg0, arg1; + rtx done_label, do_error, target = NULL_RTX; + + lhs = gimple_call_lhs (stmt); + arg0 = gimple_call_arg (stmt, 0); + arg1 = gimple_call_arg (stmt, 1); + done_label = gen_label_rtx (); + do_error = gen_label_rtx (); + + do_pending_stack_adjust (); + op0 = expand_normal (arg0); + op1 = expand_normal (arg1); + + enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0)); + if (lhs) + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + + enum insn_code icode = optab_handler (mulv4_optab, mode); + if (icode != CODE_FOR_nothing) + { + struct expand_operand ops[4]; + rtx last = get_last_insn (); + + res = gen_reg_rtx (mode); + create_output_operand (&ops[0], res, mode); + create_input_operand (&ops[1], op0, mode); + create_input_operand (&ops[2], op1, mode); + create_fixed_operand (&ops[3], do_error); + if (maybe_expand_insn (icode, 4, ops)) + { + last = get_last_insn (); + if (profile_status_for_fn (cfun) != PROFILE_ABSENT + && JUMP_P (last) + && any_condjump_p (last) + && !find_reg_note (last, REG_BR_PROB, 0)) + add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + emit_jump (done_label); + } + else + { + delete_insns_since (last); + icode = CODE_FOR_nothing; + } + } + + if (icode == CODE_FOR_nothing) + { + struct separate_ops ops; + enum machine_mode hmode + = mode_for_size (GET_MODE_PRECISION (mode) / 2, MODE_INT, 1); + ops.op0 = arg0; + ops.op1 = arg1; + ops.op2 = NULL_TREE; + ops.location = gimple_location (stmt); + if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode + && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode))) + { + enum machine_mode wmode = GET_MODE_2XWIDER_MODE (mode); + ops.code = WIDEN_MULT_EXPR; + ops.type + = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0); + + res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL); + rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, + GET_MODE_PRECISION (mode), NULL_RTX, 0); + hipart = gen_lowpart (mode, hipart); + res = gen_lowpart (mode, res); + rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, + GET_MODE_PRECISION (mode) - 1, + NULL_RTX, 0); + /* RES is low half of the double width result, HIPART + the high half. There was overflow if + HIPART is different from RES < 0 ? -1 : 0. */ + emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode, + false, done_label, PROB_VERY_LIKELY); + } + else if (hmode != BLKmode + && 2 * GET_MODE_PRECISION (hmode) == GET_MODE_PRECISION (mode)) + { + rtx large_op0 = gen_label_rtx (); + rtx small_op0_large_op1 = gen_label_rtx (); + rtx one_small_one_large = gen_label_rtx (); + rtx both_ops_large = gen_label_rtx (); + rtx after_hipart_neg = gen_label_rtx (); + rtx after_lopart_neg = gen_label_rtx (); + rtx do_overflow = gen_label_rtx (); + rtx hipart_different = gen_label_rtx (); + + int hprec = GET_MODE_PRECISION (hmode); + rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec, + NULL_RTX, 0); + hipart0 = gen_lowpart (hmode, hipart0); + rtx lopart0 = gen_lowpart (hmode, op0); + rtx signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1, + NULL_RTX, 0); + rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec, + NULL_RTX, 0); + hipart1 = gen_lowpart (hmode, hipart1); + rtx lopart1 = gen_lowpart (hmode, op1); + rtx signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1, + NULL_RTX, 0); + + res = gen_reg_rtx (mode); + + /* True if op0 resp. op1 are known to be in the range of + halfstype. */ + bool op0_small_p = false; + bool op1_small_p = false; + /* True if op0 resp. op1 are known to have all zeros or all ones + in the upper half of bits, but are not known to be + op{0,1}_small_p. */ + bool op0_medium_p = false; + bool op1_medium_p = false; + /* -1 if op{0,1} is known to be negative, 0 if it is known to be + nonnegative, 1 if unknown. */ + int op0_sign = 1; + int op1_sign = 1; + + if (TREE_CODE (arg0) == SSA_NAME) + { + double_int arg0_min, arg0_max; + if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE) + { + if (arg0_max.sle (double_int::max_value (hprec, false)) + && double_int::min_value (hprec, false).sle (arg0_min)) + op0_small_p = true; + else if (arg0_max.sle (double_int::max_value (hprec, true)) + && (~double_int::max_value (hprec, + true)).sle (arg0_min)) + op0_medium_p = true; + if (!arg0_min.is_negative ()) + op0_sign = 0; + else if (arg0_max.is_negative ()) + op0_sign = -1; + } + } + if (TREE_CODE (arg1) == SSA_NAME) + { + double_int arg1_min, arg1_max; + if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE) + { + if (arg1_max.sle (double_int::max_value (hprec, false)) + && double_int::min_value (hprec, false).sle (arg1_min)) + op1_small_p = true; + else if (arg1_max.sle (double_int::max_value (hprec, true)) + && (~double_int::max_value (hprec, + true)).sle (arg1_min)) + op1_medium_p = true; + if (!arg1_min.is_negative ()) + op1_sign = 0; + else if (arg1_max.is_negative ()) + op1_sign = -1; + } + } + + int smaller_sign = 1; + int larger_sign = 1; + if (op0_small_p) + { + smaller_sign = op0_sign; + larger_sign = op1_sign; + } + else if (op1_small_p) + { + smaller_sign = op1_sign; + larger_sign = op0_sign; + } + else if (op0_sign == op1_sign) + { + smaller_sign = op0_sign; + larger_sign = op0_sign; + } + + if (!op0_small_p) + emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode, + false, large_op0, PROB_UNLIKELY); + + if (!op1_small_p) + emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode, + false, small_op0_large_op1, + PROB_UNLIKELY); + + /* If both op0 and op1 are sign extended from hmode to mode, + the multiplication will never overflow. We can do just one + hmode x hmode => mode widening multiplication. */ + if (GET_CODE (lopart0) == SUBREG) + { + SUBREG_PROMOTED_VAR_P (lopart0) = 1; + SUBREG_PROMOTED_UNSIGNED_SET (lopart0, 0); + } + if (GET_CODE (lopart1) == SUBREG) + { + SUBREG_PROMOTED_VAR_P (lopart1) = 1; + SUBREG_PROMOTED_UNSIGNED_SET (lopart1, 0); + } + tree halfstype = build_nonstandard_integer_type (hprec, 0); + ops.op0 = make_tree (halfstype, lopart0); + ops.op1 = make_tree (halfstype, lopart1); + ops.code = WIDEN_MULT_EXPR; + ops.type = TREE_TYPE (arg0); + rtx thisres + = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); + emit_move_insn (res, thisres); + emit_jump (done_label); + + emit_label (small_op0_large_op1); + + /* If op0 is sign extended from hmode to mode, but op1 is not, + just swap the arguments and handle it as op1 sign extended, + op0 not. */ + rtx larger = gen_reg_rtx (mode); + rtx hipart = gen_reg_rtx (hmode); + rtx lopart = gen_reg_rtx (hmode); + emit_move_insn (larger, op1); + emit_move_insn (hipart, hipart1); + emit_move_insn (lopart, lopart0); + emit_jump (one_small_one_large); + + emit_label (large_op0); + + if (!op1_small_p) + emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode, + false, both_ops_large, PROB_UNLIKELY); + + /* If op1 is sign extended from hmode to mode, but op0 is not, + prepare larger, hipart and lopart pseudos and handle it together + with small_op0_large_op1. */ + emit_move_insn (larger, op0); + emit_move_insn (hipart, hipart0); + emit_move_insn (lopart, lopart1); + + emit_label (one_small_one_large); + + /* lopart is the low part of the operand that is sign extended + to mode, larger is the the other operand, hipart is the + high part of larger and lopart0 and lopart1 are the low parts + of both operands. + We perform lopart0 * lopart1 and lopart * hipart widening + multiplications. */ + tree halfutype = build_nonstandard_integer_type (hprec, 1); + ops.op0 = make_tree (halfutype, lopart0); + ops.op1 = make_tree (halfutype, lopart1); + rtx lo0xlo1 + = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); + + ops.op0 = make_tree (halfutype, lopart); + ops.op1 = make_tree (halfutype, hipart); + rtx loxhi = gen_reg_rtx (mode); + rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); + emit_move_insn (loxhi, tem); + + /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */ + if (larger_sign == 0) + emit_jump (after_hipart_neg); + else if (larger_sign != -1) + emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX, hmode, + false, after_hipart_neg, PROB_EVEN); + + tem = convert_modes (mode, hmode, lopart, 1); + tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1); + tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX, + 1, OPTAB_DIRECT); + emit_move_insn (loxhi, tem); + + emit_label (after_hipart_neg); + + /* if (lopart < 0) loxhi -= larger; */ + if (smaller_sign == 0) + emit_jump (after_lopart_neg); + else if (smaller_sign != -1) + emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX, hmode, + false, after_lopart_neg, PROB_EVEN); + + tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX, + 1, OPTAB_DIRECT); + emit_move_insn (loxhi, tem); + + emit_label (after_lopart_neg); + + /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */ + tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1); + tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX, + 1, OPTAB_DIRECT); + emit_move_insn (loxhi, tem); + + /* if (loxhi >> (bitsize / 2) + == (hmode) loxhi >> (bitsize / 2 - 1)) */ + rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec, + NULL_RTX, 0); + hipartloxhi = gen_lowpart (hmode, hipartloxhi); + rtx lopartloxhi = gen_lowpart (hmode, loxhi); + rtx signbitloxhi = expand_shift (RSHIFT_EXPR, hmode, lopartloxhi, + hprec - 1, NULL_RTX, 0); + + emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX, + hmode, false, do_overflow, + PROB_VERY_UNLIKELY); + + /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */ + rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec, + NULL_RTX, 1); + tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1); + + tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res, + 1, OPTAB_DIRECT); + if (tem != res) + emit_move_insn (res, tem); + emit_jump (done_label); + + emit_label (both_ops_large); + + /* If both operands are large (not sign extended from hmode), + then perform the full multiplication which will be the result + of the operation. The only cases which don't overflow are + some cases where both hipart0 and highpart1 are 0 or -1. */ + ops.code = MULT_EXPR; + ops.op0 = make_tree (TREE_TYPE (arg0), op0); + ops.op1 = make_tree (TREE_TYPE (arg0), op1); + tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); + emit_move_insn (res, tem); + + if (!op0_medium_p) + { + tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx, + NULL_RTX, 1, OPTAB_DIRECT); + emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode, + true, do_error, PROB_VERY_UNLIKELY); + } + + if (!op1_medium_p) + { + tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx, + NULL_RTX, 1, OPTAB_DIRECT); + emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode, + true, do_error, PROB_VERY_UNLIKELY); + } + + /* At this point hipart{0,1} are both in [-1, 0]. If they are the + same, overflow happened if res is negative, if they are different, + overflow happened if res is positive. */ + if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign) + emit_jump (hipart_different); + else if (op0_sign == 1 || op1_sign == 1) + emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode, + true, hipart_different, PROB_EVEN); + + emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false, + do_error, PROB_VERY_UNLIKELY); + emit_jump (done_label); + + emit_label (hipart_different); + + emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false, + do_error, PROB_VERY_UNLIKELY); + emit_jump (done_label); + + emit_label (do_overflow); + + /* Overflow, do full multiplication and fallthru into do_error. */ + ops.op0 = make_tree (TREE_TYPE (arg0), op0); + ops.op1 = make_tree (TREE_TYPE (arg0), op1); + tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); + emit_move_insn (res, tem); + } + else + { + ops.code = MULT_EXPR; + ops.type = TREE_TYPE (arg0); + res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); + emit_jump (done_label); + } + } + + emit_label (do_error); + /* Expand the ubsan builtin call. */ + push_temp_slots (); + fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt), + TREE_TYPE (arg0), arg0, arg1); + expand_normal (fn); + pop_temp_slots (); + do_pending_stack_adjust (); + + /* We're done. */ + emit_label (done_label); + + if (lhs) + emit_move_insn (target, res); +} + +/* Expand UBSAN_CHECK_ADD call STMT. */ + +static void +expand_UBSAN_CHECK_ADD (gimple stmt) +{ + ubsan_expand_si_overflow_addsub_check (PLUS_EXPR, stmt); +} + +/* Expand UBSAN_CHECK_SUB call STMT. */ + +static void +expand_UBSAN_CHECK_SUB (gimple stmt) +{ + if (integer_zerop (gimple_call_arg (stmt, 0))) + ubsan_expand_si_overflow_neg_check (stmt); + else + ubsan_expand_si_overflow_addsub_check (MINUS_EXPR, stmt); +} + +/* Expand UBSAN_CHECK_MUL call STMT. */ + +static void +expand_UBSAN_CHECK_MUL (gimple stmt) +{ + ubsan_expand_si_overflow_mul_check (stmt); +} + +/* This should get folded in tree-vectorizer.c. */ + +static void +expand_LOOP_VECTORIZED (gimple stmt ATTRIBUTE_UNUSED) +{ + gcc_unreachable (); +} + +static void +expand_MASK_LOAD (gimple stmt) +{ + struct expand_operand ops[3]; + tree type, lhs, rhs, maskt; + rtx mem, target, mask; + + maskt = gimple_call_arg (stmt, 2); + lhs = gimple_call_lhs (stmt); + if (lhs == NULL_TREE) + return; + type = TREE_TYPE (lhs); + rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), + gimple_call_arg (stmt, 1)); + + mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + gcc_assert (MEM_P (mem)); + mask = expand_normal (maskt); + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + create_output_operand (&ops[0], target, TYPE_MODE (type)); + create_fixed_operand (&ops[1], mem); + create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt))); + expand_insn (optab_handler (maskload_optab, TYPE_MODE (type)), 3, ops); +} + +static void +expand_MASK_STORE (gimple stmt) +{ + struct expand_operand ops[3]; + tree type, lhs, rhs, maskt; + rtx mem, reg, mask; + + maskt = gimple_call_arg (stmt, 2); + rhs = gimple_call_arg (stmt, 3); + type = TREE_TYPE (rhs); + lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), + gimple_call_arg (stmt, 1)); + + mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + gcc_assert (MEM_P (mem)); + mask = expand_normal (maskt); + reg = expand_normal (rhs); + create_fixed_operand (&ops[0], mem); + create_input_operand (&ops[1], reg, TYPE_MODE (type)); + create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt))); + expand_insn (optab_handler (maskstore_optab, TYPE_MODE (type)), 3, ops); +} + +static void +expand_ABNORMAL_DISPATCHER (gimple) +{ +} + +static void +expand_BUILTIN_EXPECT (gimple stmt) +{ + /* When guessing was done, the hints should be already stripped away. */ + gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ()); + + rtx target; + tree lhs = gimple_call_lhs (stmt); + if (lhs) + target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); + else + target = const0_rtx; + rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL); + if (lhs && val != target) + emit_move_insn (target, val); +} + +/* Routines to expand each internal function, indexed by function number. + Each routine has the prototype: + + expand_<NAME> (gimple stmt) + + where STMT is the statement that performs the call. */ +static void (*const internal_fn_expanders[]) (gimple) = { +#define DEF_INTERNAL_FN(CODE, FLAGS) expand_##CODE, +#include "internal-fn.def" +#undef DEF_INTERNAL_FN + 0 +}; + +/* Expand STMT, which is a call to internal function FN. */ + +void +expand_internal_call (gimple stmt) +{ + internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt); +} |