diff options
Diffstat (limited to 'gcc-4.8.1/gcc/config/aarch64/aarch64-simd.md')
| -rw-r--r-- | gcc-4.8.1/gcc/config/aarch64/aarch64-simd.md | 3716 |
1 files changed, 0 insertions, 3716 deletions
diff --git a/gcc-4.8.1/gcc/config/aarch64/aarch64-simd.md b/gcc-4.8.1/gcc/config/aarch64/aarch64-simd.md deleted file mode 100644 index 79c309331..000000000 --- a/gcc-4.8.1/gcc/config/aarch64/aarch64-simd.md +++ /dev/null @@ -1,3716 +0,0 @@ -;; Machine description for AArch64 AdvSIMD architecture. -;; Copyright (C) 2011-2013 Free Software Foundation, Inc. -;; Contributed by ARM Ltd. -;; -;; This file is part of GCC. -;; -;; GCC is free software; you can redistribute it and/or modify it -;; under the terms of the GNU General Public License as published by -;; the Free Software Foundation; either version 3, or (at your option) -;; any later version. -;; -;; GCC is distributed in the hope that it will be useful, but -;; WITHOUT ANY WARRANTY; without even the implied warranty of -;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -;; General Public License for more details. -;; -;; You should have received a copy of the GNU General Public License -;; along with GCC; see the file COPYING3. If not see -;; <http://www.gnu.org/licenses/>. - - -; Main data types used by the insntructions - -(define_attr "simd_mode" "unknown,none,V8QI,V16QI,V4HI,V8HI,V2SI,V4SI,V2DI,V2SF,V4SF,V2DF,OI,CI,XI,DI,DF,SI,HI,QI" - (const_string "unknown")) - - -; Classification of AdvSIMD instructions for scheduling purposes. -; Do not set this attribute and the "v8type" attribute together in -; any instruction pattern. - -; simd_abd integer absolute difference and accumulate. -; simd_abdl integer absolute difference and accumulate (long). -; simd_adal integer add and accumulate (long). -; simd_add integer addition/subtraction. -; simd_addl integer addition/subtraction (long). -; simd_addlv across lanes integer sum (long). -; simd_addn integer addition/subtraction (narrow). -; simd_addn2 integer addition/subtraction (narrow, high). -; simd_addv across lanes integer sum. -; simd_cls count leading sign/zero bits. -; simd_cmp compare / create mask. -; simd_cnt population count. -; simd_dup duplicate element. -; simd_dupgp duplicate general purpose register. -; simd_ext bitwise extract from pair. -; simd_fadd floating point add/sub. -; simd_fcmp floating point compare. -; simd_fcvti floating point convert to integer. -; simd_fcvtl floating-point convert upsize. -; simd_fcvtn floating-point convert downsize (narrow). -; simd_fcvtn2 floating-point convert downsize (narrow, high). -; simd_fdiv floating point division. -; simd_fminmax floating point min/max. -; simd_fminmaxv across lanes floating point min/max. -; simd_fmla floating point multiply-add. -; simd_fmla_elt floating point multiply-add (by element). -; simd_fmul floating point multiply. -; simd_fmul_elt floating point multiply (by element). -; simd_fnegabs floating point neg/abs. -; simd_frcpe floating point reciprocal estimate. -; simd_frcps floating point reciprocal step. -; simd_frecx floating point reciprocal exponent. -; simd_frint floating point round to integer. -; simd_fsqrt floating point square root. -; simd_icvtf integer convert to floating point. -; simd_ins insert element. -; simd_insgp insert general purpose register. -; simd_load1 load multiple structures to one register (LD1). -; simd_load1r load single structure to all lanes of one register (LD1R). -; simd_load1s load single structure to one lane of one register (LD1 [index]). -; simd_load2 load multiple structures to two registers (LD1, LD2). -; simd_load2r load single structure to all lanes of two registers (LD1R, LD2R). -; simd_load2s load single structure to one lane of two registers (LD2 [index]). -; simd_load3 load multiple structures to three registers (LD1, LD3). -; simd_load3r load single structure to all lanes of three registers (LD3R). -; simd_load3s load single structure to one lane of three registers (LD3 [index]). -; simd_load4 load multiple structures to four registers (LD1, LD2, LD4). -; simd_load4r load single structure to all lanes of four registers (LD4R). -; simd_load4s load single structure to one lane of four registers (LD4 [index]). -; simd_logic logical operation. -; simd_logic_imm logcial operation (immediate). -; simd_minmax integer min/max. -; simd_minmaxv across lanes integer min/max, -; simd_mla integer multiply-accumulate. -; simd_mla_elt integer multiply-accumulate (by element). -; simd_mlal integer multiply-accumulate (long). -; simd_mlal_elt integer multiply-accumulate (by element, long). -; simd_move move register. -; simd_move_imm move immediate. -; simd_movgp move element to general purpose register. -; simd_mul integer multiply. -; simd_mul_elt integer multiply (by element). -; simd_mull integer multiply (long). -; simd_mull_elt integer multiply (by element, long). -; simd_negabs integer negate/absolute. -; simd_rbit bitwise reverse. -; simd_rcpe integer reciprocal estimate. -; simd_rcps integer reciprocal square root. -; simd_rev element reverse. -; simd_sat_add integer saturating addition/subtraction. -; simd_sat_mlal integer saturating multiply-accumulate (long). -; simd_sat_mlal_elt integer saturating multiply-accumulate (by element, long). -; simd_sat_mul integer saturating multiply. -; simd_sat_mul_elt integer saturating multiply (by element). -; simd_sat_mull integer saturating multiply (long). -; simd_sat_mull_elt integer saturating multiply (by element, long). -; simd_sat_negabs integer saturating negate/absolute. -; simd_sat_shift integer saturating shift. -; simd_sat_shift_imm integer saturating shift (immediate). -; simd_sat_shiftn_imm integer saturating shift (narrow, immediate). -; simd_sat_shiftn2_imm integer saturating shift (narrow, high, immediate). -; simd_shift shift register/vector. -; simd_shift_acc shift accumulate. -; simd_shift_imm shift immediate. -; simd_shift_imm_acc shift immediate and accumualte. -; simd_shiftl shift register/vector (long). -; simd_shiftl_imm shift register/vector (long, immediate). -; simd_shiftn_imm shift register/vector (narrow, immediate). -; simd_shiftn2_imm shift register/vector (narrow, high, immediate). -; simd_store1 store multiple structures from one register (ST1). -; simd_store1s store single structure from one lane of one register (ST1 [index]). -; simd_store2 store multiple structures from two registers (ST1, ST2). -; simd_store2s store single structure from one lane of two registers (ST2 [index]). -; simd_store3 store multiple structures from three registers (ST1, ST3). -; simd_store3s store single structure from one lane of three register (ST3 [index]). -; simd_store4 store multiple structures from four registers (ST1, ST2, ST4). -; simd_store4s store single structure from one lane for four registers (ST4 [index]). -; simd_tbl table lookup. -; simd_trn transpose. -; simd_uzp unzip. -; simd_zip zip. - -(define_attr "simd_type" - "simd_abd,\ - simd_abdl,\ - simd_adal,\ - simd_add,\ - simd_addl,\ - simd_addlv,\ - simd_addn,\ - simd_addn2,\ - simd_addv,\ - simd_cls,\ - simd_cmp,\ - simd_cnt,\ - simd_dup,\ - simd_dupgp,\ - simd_ext,\ - simd_fadd,\ - simd_fcmp,\ - simd_fcvti,\ - simd_fcvtl,\ - simd_fcvtn,\ - simd_fcvtn2,\ - simd_fdiv,\ - simd_fminmax,\ - simd_fminmaxv,\ - simd_fmla,\ - simd_fmla_elt,\ - simd_fmul,\ - simd_fmul_elt,\ - simd_fnegabs,\ - simd_frcpe,\ - simd_frcps,\ - simd_frecx,\ - simd_frint,\ - simd_fsqrt,\ - simd_icvtf,\ - simd_ins,\ - simd_insgp,\ - simd_load1,\ - simd_load1r,\ - simd_load1s,\ - simd_load2,\ - simd_load2r,\ - simd_load2s,\ - simd_load3,\ - simd_load3r,\ - simd_load3s,\ - simd_load4,\ - simd_load4r,\ - simd_load4s,\ - simd_logic,\ - simd_logic_imm,\ - simd_minmax,\ - simd_minmaxv,\ - simd_mla,\ - simd_mla_elt,\ - simd_mlal,\ - simd_mlal_elt,\ - simd_movgp,\ - simd_move,\ - simd_move_imm,\ - simd_mul,\ - simd_mul_elt,\ - simd_mull,\ - simd_mull_elt,\ - simd_negabs,\ - simd_rbit,\ - simd_rcpe,\ - simd_rcps,\ - simd_rev,\ - simd_sat_add,\ - simd_sat_mlal,\ - simd_sat_mlal_elt,\ - simd_sat_mul,\ - simd_sat_mul_elt,\ - simd_sat_mull,\ - simd_sat_mull_elt,\ - simd_sat_negabs,\ - simd_sat_shift,\ - simd_sat_shift_imm,\ - simd_sat_shiftn_imm,\ - simd_sat_shiftn2_imm,\ - simd_shift,\ - simd_shift_acc,\ - simd_shift_imm,\ - simd_shift_imm_acc,\ - simd_shiftl,\ - simd_shiftl_imm,\ - simd_shiftn_imm,\ - simd_shiftn2_imm,\ - simd_store1,\ - simd_store1s,\ - simd_store2,\ - simd_store2s,\ - simd_store3,\ - simd_store3s,\ - simd_store4,\ - simd_store4s,\ - simd_tbl,\ - simd_trn,\ - simd_uzp,\ - simd_zip,\ - none" - (const_string "none")) - - -; The "neon_type" attribute is used by the AArch32 backend. Below is a mapping -; from "simd_type" to "neon_type". - -(define_attr "neon_type" - "neon_int_1,neon_int_2,neon_int_3,neon_int_4,neon_int_5,neon_vqneg_vqabs, - neon_vmov,neon_vaba,neon_vsma,neon_vaba_qqq, - neon_mul_ddd_8_16_qdd_16_8_long_32_16_long,neon_mul_qqq_8_16_32_ddd_32, - neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar, - neon_mla_ddd_8_16_qdd_16_8_long_32_16_long,neon_mla_qqq_8_16, - neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long, - neon_mla_qqq_32_qqd_32_scalar,neon_mul_ddd_16_scalar_32_16_long_scalar, - neon_mul_qqd_32_scalar,neon_mla_ddd_16_scalar_qdd_32_16_long_scalar, - neon_shift_1,neon_shift_2,neon_shift_3,neon_vshl_ddd, - neon_vqshl_vrshl_vqrshl_qqq,neon_vsra_vrsra,neon_fp_vadd_ddd_vabs_dd, - neon_fp_vadd_qqq_vabs_qq,neon_fp_vsum,neon_fp_vmul_ddd,neon_fp_vmul_qqd, - neon_fp_vmla_ddd,neon_fp_vmla_qqq,neon_fp_vmla_ddd_scalar, - neon_fp_vmla_qqq_scalar,neon_fp_vrecps_vrsqrts_ddd, - neon_fp_vrecps_vrsqrts_qqq,neon_bp_simple,neon_bp_2cycle,neon_bp_3cycle, - neon_ldr,neon_str,neon_vld1_1_2_regs,neon_vld1_3_4_regs, - neon_vld2_2_regs_vld1_vld2_all_lanes,neon_vld2_4_regs,neon_vld3_vld4, - neon_vst1_1_2_regs_vst2_2_regs,neon_vst1_3_4_regs, - neon_vst2_4_regs_vst3_vst4,neon_vst3_vst4,neon_vld1_vld2_lane, - neon_vld3_vld4_lane,neon_vst1_vst2_lane,neon_vst3_vst4_lane, - neon_vld3_vld4_all_lanes,neon_mcr,neon_mcr_2_mcrr,neon_mrc,neon_mrrc, - neon_ldm_2,neon_stm_2,none,unknown" - (cond [ - (eq_attr "simd_type" "simd_dup") (const_string "neon_bp_simple") - (eq_attr "simd_type" "simd_movgp") (const_string "neon_bp_simple") - (eq_attr "simd_type" "simd_add,simd_logic,simd_logic_imm") (const_string "neon_int_1") - (eq_attr "simd_type" "simd_negabs,simd_addlv") (const_string "neon_int_3") - (eq_attr "simd_type" "simd_addn,simd_addn2,simd_addl,simd_sat_add,simd_sat_negabs") (const_string "neon_int_4") - (eq_attr "simd_type" "simd_move") (const_string "neon_vmov") - (eq_attr "simd_type" "simd_ins") (const_string "neon_mcr") - (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") - (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V2SI,V8QI,V16QI,V2SI")) (const_string "neon_mul_qqq_8_16_32_ddd_32") - (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") - (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar") - (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") - (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V2SI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long") - (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V16QI,V8HI")) (const_string "neon_mla_qqq_8_16") - (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V4SI")) (const_string "neon_mla_qqq_32_qqd_32_scalar") - (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") - (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long") - (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd") - (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq") - (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd_scalar") - (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq_scalar") - (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmul_ddd") - (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmul_qqd") - (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd") - (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq") - (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd") - (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq") - (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V8QI,V4HI,V2SI")) (const_string "neon_vshl_ddd") - (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V16QI,V8HI,V4SI,V2DI")) (const_string "neon_shift_3") - (eq_attr "simd_type" "simd_minmax,simd_minmaxv") (const_string "neon_int_5") - (eq_attr "simd_type" "simd_shiftn_imm,simd_shiftn2_imm,simd_shiftl_imm,") (const_string "neon_shift_1") - (eq_attr "simd_type" "simd_load1,simd_load2") (const_string "neon_vld1_1_2_regs") - (eq_attr "simd_type" "simd_load3,simd_load3") (const_string "neon_vld1_3_4_regs") - (eq_attr "simd_type" "simd_load1r,simd_load2r,simd_load3r,simd_load4r") (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes") - (eq_attr "simd_type" "simd_load1s,simd_load2s") (const_string "neon_vld1_vld2_lane") - (eq_attr "simd_type" "simd_load3s,simd_load4s") (const_string "neon_vld3_vld4_lane") - (eq_attr "simd_type" "simd_store1,simd_store2") (const_string "neon_vst1_1_2_regs_vst2_2_regs") - (eq_attr "simd_type" "simd_store3,simd_store4") (const_string "neon_vst1_3_4_regs") - (eq_attr "simd_type" "simd_store1s,simd_store2s") (const_string "neon_vst1_vst2_lane") - (eq_attr "simd_type" "simd_store3s,simd_store4s") (const_string "neon_vst3_vst4_lane") - (and (eq_attr "simd_type" "simd_frcpe,simd_frcps") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vrecps_vrsqrts_ddd") - (and (eq_attr "simd_type" "simd_frcpe,simd_frcps") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vrecps_vrsqrts_qqq") - (eq_attr "simd_type" "none") (const_string "none") - ] - (const_string "unknown"))) - - -(define_expand "mov<mode>" - [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "") - (match_operand:VALL 1 "aarch64_simd_general_operand" ""))] - "TARGET_SIMD" - " - if (GET_CODE (operands[0]) == MEM) - operands[1] = force_reg (<MODE>mode, operands[1]); - " -) - -(define_expand "movmisalign<mode>" - [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "") - (match_operand:VALL 1 "aarch64_simd_general_operand" ""))] - "TARGET_SIMD" -{ - /* This pattern is not permitted to fail during expansion: if both arguments - are non-registers (e.g. memory := constant, which can be created by the - auto-vectorizer), force operand 1 into a register. */ - if (!register_operand (operands[0], <MODE>mode) - && !register_operand (operands[1], <MODE>mode)) - operands[1] = force_reg (<MODE>mode, operands[1]); -}) - -(define_insn "aarch64_simd_dup<mode>" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r")))] - "TARGET_SIMD" - "dup\\t%0.<Vtype>, %<vw>1" - [(set_attr "simd_type" "simd_dupgp") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_dup_lane<mode>" - [(set (match_operand:VDQ_I 0 "register_operand" "=w") - (vec_duplicate:VDQ_I - (vec_select:<VEL> - (match_operand:<VCON> 1 "register_operand" "w") - (parallel [(match_operand:SI 2 "immediate_operand" "i")]) - )))] - "TARGET_SIMD" - "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]" - [(set_attr "simd_type" "simd_dup") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_dup_lane<mode>" - [(set (match_operand:SDQ_I 0 "register_operand" "=w") - (vec_select:<VEL> - (match_operand:<VCON> 1 "register_operand" "w") - (parallel [(match_operand:SI 2 "immediate_operand" "i")]) - ))] - "TARGET_SIMD" - "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]" - [(set_attr "simd_type" "simd_dup") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_dup<mode>" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))] - "TARGET_SIMD" - "dup\\t%0.<Vtype>, %1.<Vetype>[0]" - [(set_attr "simd_type" "simd_dup") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "*aarch64_simd_mov<mode>" - [(set (match_operand:VD 0 "aarch64_simd_nonimmediate_operand" - "=w, Utv, w, ?r, ?w, ?r, w") - (match_operand:VD 1 "aarch64_simd_general_operand" - "Utv, w, w, w, r, r, Dn"))] - "TARGET_SIMD - && (register_operand (operands[0], <MODE>mode) - || register_operand (operands[1], <MODE>mode))" -{ - switch (which_alternative) - { - case 0: return "ld1\t{%0.<Vtype>}, %1"; - case 1: return "st1\t{%1.<Vtype>}, %0"; - case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>"; - case 3: return "umov\t%0, %1.d[0]"; - case 4: return "ins\t%0.d[0], %1"; - case 5: return "mov\t%0, %1"; - case 6: - return aarch64_output_simd_mov_immediate (&operands[1], - <MODE>mode, 64); - default: gcc_unreachable (); - } -} - [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "*aarch64_simd_mov<mode>" - [(set (match_operand:VQ 0 "aarch64_simd_nonimmediate_operand" - "=w, Utv, w, ?r, ?w, ?r, w") - (match_operand:VQ 1 "aarch64_simd_general_operand" - "Utv, w, w, w, r, r, Dn"))] - "TARGET_SIMD - && (register_operand (operands[0], <MODE>mode) - || register_operand (operands[1], <MODE>mode))" -{ - switch (which_alternative) - { - case 0: return "ld1\t{%0.<Vtype>}, %1"; - case 1: return "st1\t{%1.<Vtype>}, %0"; - case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>"; - case 3: return "umov\t%0, %1.d[0]\;umov\t%H0, %1.d[1]"; - case 4: return "ins\t%0.d[0], %1\;ins\t%0.d[1], %H1"; - case 5: return "#"; - case 6: - return aarch64_output_simd_mov_immediate (&operands[1], - <MODE>mode, 128); - default: gcc_unreachable (); - } -} - [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm") - (set_attr "simd_mode" "<MODE>") - (set_attr "length" "4,4,4,8,8,8,4")] -) - -(define_split - [(set (match_operand:VQ 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" ""))] - "TARGET_SIMD && reload_completed - && GP_REGNUM_P (REGNO (operands[0])) - && GP_REGNUM_P (REGNO (operands[1]))" - [(set (match_dup 0) (match_dup 1)) - (set (match_dup 2) (match_dup 3))] -{ - int rdest = REGNO (operands[0]); - int rsrc = REGNO (operands[1]); - rtx dest[2], src[2]; - - dest[0] = gen_rtx_REG (DImode, rdest); - src[0] = gen_rtx_REG (DImode, rsrc); - dest[1] = gen_rtx_REG (DImode, rdest + 1); - src[1] = gen_rtx_REG (DImode, rsrc + 1); - - aarch64_simd_disambiguate_copy (operands, dest, src, 2); -}) - -(define_insn "orn<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w")) - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>" - [(set_attr "simd_type" "simd_logic") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "bic<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w")) - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>" - [(set_attr "simd_type" "simd_logic") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "add<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (plus:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "sub<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (minus:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "mul<mode>3" - [(set (match_operand:VDQM 0 "register_operand" "=w") - (mult:VDQM (match_operand:VDQM 1 "register_operand" "w") - (match_operand:VDQM 2 "register_operand" "w")))] - "TARGET_SIMD" - "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "neg<mode>2" - [(set (match_operand:VDQM 0 "register_operand" "=w") - (neg:VDQM (match_operand:VDQM 1 "register_operand" "w")))] - "TARGET_SIMD" - "neg\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_negabs") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "abs<mode>2" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))] - "TARGET_SIMD" - "abs\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_negabs") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "and<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (and:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>" - [(set_attr "simd_type" "simd_logic") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "ior<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (ior:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>" - [(set_attr "simd_type" "simd_logic") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "xor<mode>3" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (xor:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>" - [(set_attr "simd_type" "simd_logic") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "one_cmpl<mode>2" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))] - "TARGET_SIMD" - "not\t%0.<Vbtype>, %1.<Vbtype>" - [(set_attr "simd_type" "simd_logic") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_vec_set<mode>" - [(set (match_operand:VQ_S 0 "register_operand" "=w") - (vec_merge:VQ_S - (vec_duplicate:VQ_S - (match_operand:<VEL> 1 "register_operand" "r")) - (match_operand:VQ_S 3 "register_operand" "0") - (match_operand:SI 2 "immediate_operand" "i")))] - "TARGET_SIMD" - "ins\t%0.<Vetype>[%p2], %w1"; - [(set_attr "simd_type" "simd_insgp") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_lshr<mode>" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))] - "TARGET_SIMD" - "ushr\t%0.<Vtype>, %1.<Vtype>, %2" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_ashr<mode>" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))] - "TARGET_SIMD" - "sshr\t%0.<Vtype>, %1.<Vtype>, %2" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_imm_shl<mode>" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "aarch64_simd_lshift_imm" "Dl")))] - "TARGET_SIMD" - "shl\t%0.<Vtype>, %1.<Vtype>, %2" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_reg_sshl<mode>" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")))] - "TARGET_SIMD" - "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_shift") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_reg_shl<mode>_unsigned" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")] - UNSPEC_ASHIFT_UNSIGNED))] - "TARGET_SIMD" - "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_shift") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_reg_shl<mode>_signed" - [(set (match_operand:VDQ 0 "register_operand" "=w") - (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w") - (match_operand:VDQ 2 "register_operand" "w")] - UNSPEC_ASHIFT_SIGNED))] - "TARGET_SIMD" - "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_shift") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "ashl<mode>3" - [(match_operand:VDQ 0 "register_operand" "") - (match_operand:VDQ 1 "register_operand" "") - (match_operand:SI 2 "general_operand" "")] - "TARGET_SIMD" -{ - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - int shift_amount; - - if (CONST_INT_P (operands[2])) - { - shift_amount = INTVAL (operands[2]); - if (shift_amount >= 0 && shift_amount < bit_width) - { - rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode, - shift_amount); - emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0], - operands[1], - tmp)); - DONE; - } - else - { - operands[2] = force_reg (SImode, operands[2]); - } - } - else if (MEM_P (operands[2])) - { - operands[2] = force_reg (SImode, operands[2]); - } - - if (REG_P (operands[2])) - { - rtx tmp = gen_reg_rtx (<MODE>mode); - emit_insn (gen_aarch64_simd_dup<mode> (tmp, - convert_to_mode (<VEL>mode, - operands[2], - 0))); - emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1], - tmp)); - DONE; - } - else - FAIL; -} -) - -(define_expand "lshr<mode>3" - [(match_operand:VDQ 0 "register_operand" "") - (match_operand:VDQ 1 "register_operand" "") - (match_operand:SI 2 "general_operand" "")] - "TARGET_SIMD" -{ - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - int shift_amount; - - if (CONST_INT_P (operands[2])) - { - shift_amount = INTVAL (operands[2]); - if (shift_amount > 0 && shift_amount <= bit_width) - { - rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode, - shift_amount); - emit_insn (gen_aarch64_simd_lshr<mode> (operands[0], - operands[1], - tmp)); - DONE; - } - else - operands[2] = force_reg (SImode, operands[2]); - } - else if (MEM_P (operands[2])) - { - operands[2] = force_reg (SImode, operands[2]); - } - - if (REG_P (operands[2])) - { - rtx tmp = gen_reg_rtx (SImode); - rtx tmp1 = gen_reg_rtx (<MODE>mode); - emit_insn (gen_negsi2 (tmp, operands[2])); - emit_insn (gen_aarch64_simd_dup<mode> (tmp1, - convert_to_mode (<VEL>mode, - tmp, 0))); - emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], - operands[1], - tmp1)); - DONE; - } - else - FAIL; -} -) - -(define_expand "ashr<mode>3" - [(match_operand:VDQ 0 "register_operand" "") - (match_operand:VDQ 1 "register_operand" "") - (match_operand:SI 2 "general_operand" "")] - "TARGET_SIMD" -{ - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - int shift_amount; - - if (CONST_INT_P (operands[2])) - { - shift_amount = INTVAL (operands[2]); - if (shift_amount > 0 && shift_amount <= bit_width) - { - rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode, - shift_amount); - emit_insn (gen_aarch64_simd_ashr<mode> (operands[0], - operands[1], - tmp)); - DONE; - } - else - operands[2] = force_reg (SImode, operands[2]); - } - else if (MEM_P (operands[2])) - { - operands[2] = force_reg (SImode, operands[2]); - } - - if (REG_P (operands[2])) - { - rtx tmp = gen_reg_rtx (SImode); - rtx tmp1 = gen_reg_rtx (<MODE>mode); - emit_insn (gen_negsi2 (tmp, operands[2])); - emit_insn (gen_aarch64_simd_dup<mode> (tmp1, - convert_to_mode (<VEL>mode, - tmp, 0))); - emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], - operands[1], - tmp1)); - DONE; - } - else - FAIL; -} -) - -(define_expand "vashl<mode>3" - [(match_operand:VDQ 0 "register_operand" "") - (match_operand:VDQ 1 "register_operand" "") - (match_operand:VDQ 2 "register_operand" "")] - "TARGET_SIMD" -{ - emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1], - operands[2])); - DONE; -}) - -;; Using mode VQ_S as there is no V2DImode neg! -;; Negating individual lanes most certainly offsets the -;; gain from vectorization. -(define_expand "vashr<mode>3" - [(match_operand:VQ_S 0 "register_operand" "") - (match_operand:VQ_S 1 "register_operand" "") - (match_operand:VQ_S 2 "register_operand" "")] - "TARGET_SIMD" -{ - rtx neg = gen_reg_rtx (<MODE>mode); - emit (gen_neg<mode>2 (neg, operands[2])); - emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1], - neg)); - DONE; -}) - -(define_expand "vlshr<mode>3" - [(match_operand:VQ_S 0 "register_operand" "") - (match_operand:VQ_S 1 "register_operand" "") - (match_operand:VQ_S 2 "register_operand" "")] - "TARGET_SIMD" -{ - rtx neg = gen_reg_rtx (<MODE>mode); - emit (gen_neg<mode>2 (neg, operands[2])); - emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1], - neg)); - DONE; -}) - -(define_expand "vec_set<mode>" - [(match_operand:VQ_S 0 "register_operand" "+w") - (match_operand:<VEL> 1 "register_operand" "r") - (match_operand:SI 2 "immediate_operand" "")] - "TARGET_SIMD" - { - HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]); - emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1], - GEN_INT (elem), operands[0])); - DONE; - } -) - -(define_insn "aarch64_simd_vec_setv2di" - [(set (match_operand:V2DI 0 "register_operand" "=w") - (vec_merge:V2DI - (vec_duplicate:V2DI - (match_operand:DI 1 "register_operand" "r")) - (match_operand:V2DI 3 "register_operand" "0") - (match_operand:SI 2 "immediate_operand" "i")))] - "TARGET_SIMD" - "ins\t%0.d[%p2], %1"; - [(set_attr "simd_type" "simd_insgp") - (set_attr "simd_mode" "V2DI")] -) - -(define_expand "vec_setv2di" - [(match_operand:V2DI 0 "register_operand" "+w") - (match_operand:DI 1 "register_operand" "r") - (match_operand:SI 2 "immediate_operand" "")] - "TARGET_SIMD" - { - HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]); - emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1], - GEN_INT (elem), operands[0])); - DONE; - } -) - -(define_insn "aarch64_simd_vec_set<mode>" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (vec_merge:VDQF - (vec_duplicate:VDQF - (match_operand:<VEL> 1 "register_operand" "w")) - (match_operand:VDQF 3 "register_operand" "0") - (match_operand:SI 2 "immediate_operand" "i")))] - "TARGET_SIMD" - "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]"; - [(set_attr "simd_type" "simd_ins") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "vec_set<mode>" - [(match_operand:VDQF 0 "register_operand" "+w") - (match_operand:<VEL> 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "")] - "TARGET_SIMD" - { - HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]); - emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1], - GEN_INT (elem), operands[0])); - DONE; - } -) - - -(define_insn "aarch64_mla<mode>" - [(set (match_operand:VQ_S 0 "register_operand" "=w") - (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w") - (match_operand:VQ_S 3 "register_operand" "w")) - (match_operand:VQ_S 1 "register_operand" "0")))] - "TARGET_SIMD" - "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>" - [(set_attr "simd_type" "simd_mla") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_mls<mode>" - [(set (match_operand:VQ_S 0 "register_operand" "=w") - (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0") - (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w") - (match_operand:VQ_S 3 "register_operand" "w"))))] - "TARGET_SIMD" - "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>" - [(set_attr "simd_type" "simd_mla") - (set_attr "simd_mode" "<MODE>")] -) - -;; Max/Min operations. -(define_insn "<maxmin><mode>3" - [(set (match_operand:VQ_S 0 "register_operand" "=w") - (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w") - (match_operand:VQ_S 2 "register_operand" "w")))] - "TARGET_SIMD" - "<maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_minmax") - (set_attr "simd_mode" "<MODE>")] -) - -;; Move into low-half clearing high half to 0. - -(define_insn "move_lo_quad_<mode>" - [(set (match_operand:VQ 0 "register_operand" "=w") - (vec_concat:VQ - (match_operand:<VHALF> 1 "register_operand" "w") - (vec_duplicate:<VHALF> (const_int 0))))] - "TARGET_SIMD" - "mov\\t%d0, %d1"; - [(set_attr "simd_type" "simd_dup") - (set_attr "simd_mode" "<MODE>")] -) - -;; Move into high-half. - -(define_insn "aarch64_simd_move_hi_quad_<mode>" - [(set (match_operand:VQ 0 "register_operand" "+w") - (vec_concat:VQ - (vec_select:<VHALF> - (match_dup 0) - (match_operand:VQ 2 "vect_par_cnst_lo_half" "")) - (match_operand:<VHALF> 1 "register_operand" "w")))] - "TARGET_SIMD" - "ins\\t%0.d[1], %1.d[0]"; - [(set_attr "simd_type" "simd_ins") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "move_hi_quad_<mode>" - [(match_operand:VQ 0 "register_operand" "") - (match_operand:<VHALF> 1 "register_operand" "")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false); - emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0], - operands[1], p)); - DONE; -}) - -;; Narrowing operations. - -;; For doubles. -(define_insn "aarch64_simd_vec_pack_trunc_<mode>" - [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") - (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))] - "TARGET_SIMD" - "xtn\\t%0.<Vntype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_shiftn_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "vec_pack_trunc_<mode>" - [(match_operand:<VNARROWD> 0 "register_operand" "") - (match_operand:VDN 1 "register_operand" "") - (match_operand:VDN 2 "register_operand" "")] - "TARGET_SIMD" -{ - rtx tempreg = gen_reg_rtx (<VDBL>mode); - - emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[1])); - emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[2])); - emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg)); - DONE; -}) - -;; For quads. - -(define_insn "vec_pack_trunc_<mode>" - [(set (match_operand:<VNARROWQ2> 0 "register_operand" "+&w") - (vec_concat:<VNARROWQ2> - (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")) - (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))] - "TARGET_SIMD" - "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_shiftn2_imm") - (set_attr "simd_mode" "<MODE>") - (set_attr "length" "8")] -) - -;; Widening operations. - -(define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 2 "vect_par_cnst_lo_half" "") - )))] - "TARGET_SIMD" - "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0" - [(set_attr "simd_type" "simd_shiftl_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 2 "vect_par_cnst_hi_half" "") - )))] - "TARGET_SIMD" - "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0" - [(set_attr "simd_type" "simd_shiftl_imm") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "vec_unpack<su>_hi_<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "") - (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))] - "TARGET_SIMD" - { - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0], - operands[1], p)); - DONE; - } -) - -(define_expand "vec_unpack<su>_lo_<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "") - (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))] - "TARGET_SIMD" - { - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false); - emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0], - operands[1], p)); - DONE; - } -) - -;; Widening arithmetic. - -(define_insn "aarch64_simd_vec_<su>mult_lo_<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 3 "vect_par_cnst_lo_half" ""))) - (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 2 "register_operand" "w") - (match_dup 3)))))] - "TARGET_SIMD" - "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>" - [(set_attr "simd_type" "simd_mull") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "vec_widen_<su>mult_lo_<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "") - (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" "")) - (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))] - "TARGET_SIMD" - { - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false); - emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0], - operands[1], - operands[2], p)); - DONE; - } -) - -(define_insn "aarch64_simd_vec_<su>mult_hi_<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 3 "vect_par_cnst_hi_half" ""))) - (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 2 "register_operand" "w") - (match_dup 3)))))] - "TARGET_SIMD" - "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_mull") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "vec_widen_<su>mult_hi_<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "") - (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" "")) - (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))] - "TARGET_SIMD" - { - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0], - operands[1], - operands[2], p)); - DONE; - - } -) - -;; FP vector operations. -;; AArch64 AdvSIMD supports single-precision (32-bit) and -;; double-precision (64-bit) floating-point data types and arithmetic as -;; defined by the IEEE 754-2008 standard. This makes them vectorizable -;; without the need for -ffast-math or -funsafe-math-optimizations. -;; -;; Floating-point operations can raise an exception. Vectorizing such -;; operations are safe because of reasons explained below. -;; -;; ARMv8 permits an extension to enable trapped floating-point -;; exception handling, however this is an optional feature. In the -;; event of a floating-point exception being raised by vectorised -;; code then: -;; 1. If trapped floating-point exceptions are available, then a trap -;; will be taken when any lane raises an enabled exception. A trap -;; handler may determine which lane raised the exception. -;; 2. Alternatively a sticky exception flag is set in the -;; floating-point status register (FPSR). Software may explicitly -;; test the exception flags, in which case the tests will either -;; prevent vectorisation, allowing precise identification of the -;; failing operation, or if tested outside of vectorisable regions -;; then the specific operation and lane are not of interest. - -;; FP arithmetic operations. - -(define_insn "add<mode>3" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (plus:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")))] - "TARGET_SIMD" - "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fadd") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "sub<mode>3" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (minus:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")))] - "TARGET_SIMD" - "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fadd") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "mul<mode>3" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (mult:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")))] - "TARGET_SIMD" - "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fmul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "div<mode>3" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (div:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")))] - "TARGET_SIMD" - "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fdiv") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "neg<mode>2" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))] - "TARGET_SIMD" - "fneg\\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_fnegabs") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "abs<mode>2" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))] - "TARGET_SIMD" - "fabs\\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_fnegabs") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "fma<mode>4" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (fma:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w") - (match_operand:VDQF 3 "register_operand" "0")))] - "TARGET_SIMD" - "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fmla") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_frint<frint_suffix><mode>" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")] - FRINT))] - "TARGET_SIMD" - "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_frint") - (set_attr "simd_mode" "<MODE>")] -) - -;; Vector versions of the floating-point frint patterns. -;; Expands to btrunc, ceil, floor, nearbyint, rint, round. -(define_expand "<frint_pattern><mode>2" - [(set (match_operand:VDQF 0 "register_operand") - (unspec:VDQF [(match_operand:VDQF 1 "register_operand")] - FRINT))] - "TARGET_SIMD" - {}) - -(define_insn "aarch64_fcvt<frint_suffix><su><mode>" - [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w") - (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET> - [(match_operand:VDQF 1 "register_operand" "w")] - FCVT)))] - "TARGET_SIMD" - "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_fcvti") - (set_attr "simd_mode" "<MODE>")] -) - -;; Vector versions of the fcvt standard patterns. -;; Expands to lbtrunc, lround, lceil, lfloor -(define_expand "l<fcvt_pattern><su_optab><fcvt_target><VDQF:mode>2" - [(set (match_operand:<FCVT_TARGET> 0 "register_operand") - (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET> - [(match_operand:VDQF 1 "register_operand")] - FCVT)))] - "TARGET_SIMD" - {}) - -(define_insn "aarch64_vmls<mode>" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (minus:VDQF (match_operand:VDQF 1 "register_operand" "0") - (mult:VDQF (match_operand:VDQF 2 "register_operand" "w") - (match_operand:VDQF 3 "register_operand" "w"))))] - "TARGET_SIMD" - "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>" - [(set_attr "simd_type" "simd_fmla") - (set_attr "simd_mode" "<MODE>")] -) - -;; FP Max/Min -;; Max/Min are introduced by idiom recognition by GCC's mid-end. An -;; expression like: -;; a = (b < c) ? b : c; -;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled -;; either explicitly or indirectly via -ffast-math. -;; -;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL. -;; The 'smax' and 'smin' RTL standard pattern names do not specify which -;; operand will be returned when both operands are zero (i.e. they may not -;; honour signed zeroes), or when either operand is NaN. Therefore GCC -;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring -;; NaNs. - -(define_insn "smax<mode>3" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (smax:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")))] - "TARGET_SIMD" - "fmaxnm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fminmax") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "smin<mode>3" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (smin:VDQF (match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")))] - "TARGET_SIMD" - "fminnm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fminmax") - (set_attr "simd_mode" "<MODE>")] -) - -;; FP 'across lanes' max and min ops. - -(define_insn "reduc_s<fmaxminv>_v4sf" - [(set (match_operand:V4SF 0 "register_operand" "=w") - (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")] - FMAXMINV))] - "TARGET_SIMD" - "f<fmaxminv>nmv\\t%s0, %1.4s"; - [(set_attr "simd_type" "simd_fminmaxv") - (set_attr "simd_mode" "V4SF")] -) - -(define_insn "reduc_s<fmaxminv>_<mode>" - [(set (match_operand:V2F 0 "register_operand" "=w") - (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")] - FMAXMINV))] - "TARGET_SIMD" - "f<fmaxminv>nmp\\t%0.<Vtype>, %1.<Vtype>, %1.<Vtype>"; - [(set_attr "simd_type" "simd_fminmax") - (set_attr "simd_mode" "<MODE>")] -) - -;; FP 'across lanes' add. - -(define_insn "aarch64_addvv4sf" - [(set (match_operand:V4SF 0 "register_operand" "=w") - (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")] - UNSPEC_FADDV))] - "TARGET_SIMD" - "faddp\\t%0.4s, %1.4s, %1.4s" - [(set_attr "simd_type" "simd_fadd") - (set_attr "simd_mode" "V4SF")] -) - -(define_expand "reduc_uplus_v4sf" - [(set (match_operand:V4SF 0 "register_operand" "=w") - (match_operand:V4SF 1 "register_operand" "w"))] - "TARGET_SIMD" -{ - rtx tmp = gen_reg_rtx (V4SFmode); - emit_insn (gen_aarch64_addvv4sf (tmp, operands[1])); - emit_insn (gen_aarch64_addvv4sf (operands[0], tmp)); - DONE; -}) - -(define_expand "reduc_splus_v4sf" - [(set (match_operand:V4SF 0 "register_operand" "=w") - (match_operand:V4SF 1 "register_operand" "w"))] - "TARGET_SIMD" -{ - rtx tmp = gen_reg_rtx (V4SFmode); - emit_insn (gen_aarch64_addvv4sf (tmp, operands[1])); - emit_insn (gen_aarch64_addvv4sf (operands[0], tmp)); - DONE; -}) - -(define_insn "aarch64_addv<mode>" - [(set (match_operand:V2F 0 "register_operand" "=w") - (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")] - UNSPEC_FADDV))] - "TARGET_SIMD" - "faddp\\t%<Vetype>0, %1.<Vtype>" - [(set_attr "simd_type" "simd_fadd") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "reduc_uplus_<mode>" - [(set (match_operand:V2F 0 "register_operand" "=w") - (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")] - UNSPEC_FADDV))] - "TARGET_SIMD" - "" -) - -(define_expand "reduc_splus_<mode>" - [(set (match_operand:V2F 0 "register_operand" "=w") - (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")] - UNSPEC_FADDV))] - "TARGET_SIMD" - "" -) - -;; Reduction across lanes. - -(define_insn "aarch64_addv<mode>" - [(set (match_operand:VDQV 0 "register_operand" "=w") - (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "addv\\t%<Vetype>0, %1.<Vtype>" - [(set_attr "simd_type" "simd_addv") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "reduc_splus_<mode>" - [(set (match_operand:VDQV 0 "register_operand" "=w") - (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "" -) - -(define_expand "reduc_uplus_<mode>" - [(set (match_operand:VDQV 0 "register_operand" "=w") - (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "" -) - -(define_insn "aarch64_addvv2di" - [(set (match_operand:V2DI 0 "register_operand" "=w") - (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "addp\\t%d0, %1.2d" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "V2DI")] -) - -(define_expand "reduc_uplus_v2di" - [(set (match_operand:V2DI 0 "register_operand" "=w") - (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "" -) - -(define_expand "reduc_splus_v2di" - [(set (match_operand:V2DI 0 "register_operand" "=w") - (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "" -) - -(define_insn "aarch64_addvv2si" - [(set (match_operand:V2SI 0 "register_operand" "=w") - (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "addp\\t%0.2s, %1.2s, %1.2s" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "V2SI")] -) - -(define_expand "reduc_uplus_v2si" - [(set (match_operand:V2SI 0 "register_operand" "=w") - (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "" -) - -(define_expand "reduc_splus_v2si" - [(set (match_operand:V2SI 0 "register_operand" "=w") - (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")] - UNSPEC_ADDV))] - "TARGET_SIMD" - "" -) - -(define_insn "reduc_<maxminv>_<mode>" - [(set (match_operand:VDQV 0 "register_operand" "=w") - (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")] - MAXMINV))] - "TARGET_SIMD" - "<maxminv>v\\t%<Vetype>0, %1.<Vtype>" - [(set_attr "simd_type" "simd_minmaxv") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "reduc_<maxminv>_v2si" - [(set (match_operand:V2SI 0 "register_operand" "=w") - (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")] - MAXMINV))] - "TARGET_SIMD" - "<maxminv>p\\t%0.2s, %1.2s, %1.2s" - [(set_attr "simd_type" "simd_minmax") - (set_attr "simd_mode" "V2SI")] -) - -;; vbsl_* intrinsics may compile to any of bsl/bif/bit depending on register -;; allocation. For an intrinsic of form: -;; vD = bsl_* (vS, vN, vM) -;; We can use any of: -;; bsl vS, vN, vM (if D = S) -;; bit vD, vN, vS (if D = M, so 1-bits in vS choose bits from vN, else vM) -;; bif vD, vM, vS (if D = N, so 0-bits in vS choose bits from vM, else vN) - -(define_insn "aarch64_simd_bsl<mode>_internal" - [(set (match_operand:VALL 0 "register_operand" "=w,w,w") - (unspec:VALL - [(match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w") - (match_operand:VALL 2 "register_operand" " w,w,0") - (match_operand:VALL 3 "register_operand" " w,0,w")] - UNSPEC_BSL))] - "TARGET_SIMD" - "@ - bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype> - bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype> - bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>" -) - -(define_expand "aarch64_simd_bsl<mode>" - [(set (match_operand:VALL 0 "register_operand") - (unspec:VALL [(match_operand:<V_cmp_result> 1 "register_operand") - (match_operand:VALL 2 "register_operand") - (match_operand:VALL 3 "register_operand")] - UNSPEC_BSL))] - "TARGET_SIMD" -{ - /* We can't alias operands together if they have different modes. */ - operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]); -}) - -(define_expand "aarch64_vcond_internal<mode>" - [(set (match_operand:VDQ 0 "register_operand") - (if_then_else:VDQ - (match_operator 3 "comparison_operator" - [(match_operand:VDQ 4 "register_operand") - (match_operand:VDQ 5 "nonmemory_operand")]) - (match_operand:VDQ 1 "register_operand") - (match_operand:VDQ 2 "register_operand")))] - "TARGET_SIMD" -{ - int inverse = 0, has_zero_imm_form = 0; - rtx mask = gen_reg_rtx (<MODE>mode); - - switch (GET_CODE (operands[3])) - { - case LE: - case LT: - case NE: - inverse = 1; - /* Fall through. */ - case GE: - case GT: - case EQ: - has_zero_imm_form = 1; - break; - case LEU: - case LTU: - inverse = 1; - break; - default: - break; - } - - if (!REG_P (operands[5]) - && (operands[5] != CONST0_RTX (<MODE>mode) || !has_zero_imm_form)) - operands[5] = force_reg (<MODE>mode, operands[5]); - - switch (GET_CODE (operands[3])) - { - case LT: - case GE: - emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5])); - break; - - case LE: - case GT: - emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5])); - break; - - case LTU: - case GEU: - emit_insn (gen_aarch64_cmhs<mode> (mask, operands[4], operands[5])); - break; - - case LEU: - case GTU: - emit_insn (gen_aarch64_cmhi<mode> (mask, operands[4], operands[5])); - break; - - case NE: - case EQ: - emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5])); - break; - - default: - gcc_unreachable (); - } - - if (inverse) - emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[2], - operands[1])); - else - emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[1], - operands[2])); - - DONE; -}) - -(define_expand "aarch64_vcond_internal<mode>" - [(set (match_operand:VDQF 0 "register_operand") - (if_then_else:VDQF - (match_operator 3 "comparison_operator" - [(match_operand:VDQF 4 "register_operand") - (match_operand:VDQF 5 "nonmemory_operand")]) - (match_operand:VDQF 1 "register_operand") - (match_operand:VDQF 2 "register_operand")))] - "TARGET_SIMD" -{ - int inverse = 0; - int use_zero_form = 0; - int swap_bsl_operands = 0; - rtx mask = gen_reg_rtx (<V_cmp_result>mode); - rtx tmp = gen_reg_rtx (<V_cmp_result>mode); - - rtx (*base_comparison) (rtx, rtx, rtx); - rtx (*complimentary_comparison) (rtx, rtx, rtx); - - switch (GET_CODE (operands[3])) - { - case GE: - case GT: - case LE: - case LT: - case EQ: - if (operands[5] == CONST0_RTX (<MODE>mode)) - { - use_zero_form = 1; - break; - } - /* Fall through. */ - default: - if (!REG_P (operands[5])) - operands[5] = force_reg (<MODE>mode, operands[5]); - } - - switch (GET_CODE (operands[3])) - { - case LT: - case UNLT: - inverse = 1; - /* Fall through. */ - case GE: - case UNGE: - case ORDERED: - case UNORDERED: - base_comparison = gen_aarch64_cmge<mode>; - complimentary_comparison = gen_aarch64_cmgt<mode>; - break; - case LE: - case UNLE: - inverse = 1; - /* Fall through. */ - case GT: - case UNGT: - base_comparison = gen_aarch64_cmgt<mode>; - complimentary_comparison = gen_aarch64_cmge<mode>; - break; - case EQ: - case NE: - case UNEQ: - base_comparison = gen_aarch64_cmeq<mode>; - complimentary_comparison = gen_aarch64_cmeq<mode>; - break; - default: - gcc_unreachable (); - } - - switch (GET_CODE (operands[3])) - { - case LT: - case LE: - case GT: - case GE: - case EQ: - /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ. - As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are: - a GE b -> a GE b - a GT b -> a GT b - a LE b -> b GE a - a LT b -> b GT a - a EQ b -> a EQ b - Note that there also exist direct comparison against 0 forms, - so catch those as a special case. */ - if (use_zero_form) - { - inverse = 0; - switch (GET_CODE (operands[3])) - { - case LT: - base_comparison = gen_aarch64_cmlt<mode>; - break; - case LE: - base_comparison = gen_aarch64_cmle<mode>; - break; - default: - /* Do nothing, other zero form cases already have the correct - base_comparison. */ - break; - } - } - - if (!inverse) - emit_insn (base_comparison (mask, operands[4], operands[5])); - else - emit_insn (complimentary_comparison (mask, operands[5], operands[4])); - break; - case UNLT: - case UNLE: - case UNGT: - case UNGE: - case NE: - /* FCM returns false for lanes which are unordered, so if we use - the inverse of the comparison we actually want to emit, then - swap the operands to BSL, we will end up with the correct result. - Note that a NE NaN and NaN NE b are true for all a, b. - - Our transformations are: - a GE b -> !(b GT a) - a GT b -> !(b GE a) - a LE b -> !(a GT b) - a LT b -> !(a GE b) - a NE b -> !(a EQ b) */ - - if (inverse) - emit_insn (base_comparison (mask, operands[4], operands[5])); - else - emit_insn (complimentary_comparison (mask, operands[5], operands[4])); - - swap_bsl_operands = 1; - break; - case UNEQ: - /* We check (a > b || b > a). combining these comparisons give us - true iff !(a != b && a ORDERED b), swapping the operands to BSL - will then give us (a == b || a UNORDERED b) as intended. */ - - emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5])); - emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[5], operands[4])); - emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp)); - swap_bsl_operands = 1; - break; - case UNORDERED: - /* Operands are ORDERED iff (a > b || b >= a). - Swapping the operands to BSL will give the UNORDERED case. */ - swap_bsl_operands = 1; - /* Fall through. */ - case ORDERED: - emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[4], operands[5])); - emit_insn (gen_aarch64_cmge<mode> (mask, operands[5], operands[4])); - emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp)); - break; - default: - gcc_unreachable (); - } - - if (swap_bsl_operands) - emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[2], - operands[1])); - else - emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[1], - operands[2])); - DONE; -}) - -(define_expand "vcond<mode><mode>" - [(set (match_operand:VALL 0 "register_operand") - (if_then_else:VALL - (match_operator 3 "comparison_operator" - [(match_operand:VALL 4 "register_operand") - (match_operand:VALL 5 "nonmemory_operand")]) - (match_operand:VALL 1 "register_operand") - (match_operand:VALL 2 "register_operand")))] - "TARGET_SIMD" -{ - emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1], - operands[2], operands[3], - operands[4], operands[5])); - DONE; -}) - - -(define_expand "vcondu<mode><mode>" - [(set (match_operand:VDQ 0 "register_operand") - (if_then_else:VDQ - (match_operator 3 "comparison_operator" - [(match_operand:VDQ 4 "register_operand") - (match_operand:VDQ 5 "nonmemory_operand")]) - (match_operand:VDQ 1 "register_operand") - (match_operand:VDQ 2 "register_operand")))] - "TARGET_SIMD" -{ - emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1], - operands[2], operands[3], - operands[4], operands[5])); - DONE; -}) - -;; Patterns for AArch64 SIMD Intrinsics. - -(define_expand "aarch64_create<mode>" - [(match_operand:VD_RE 0 "register_operand" "") - (match_operand:DI 1 "general_operand" "")] - "TARGET_SIMD" -{ - rtx src = gen_lowpart (<MODE>mode, operands[1]); - emit_move_insn (operands[0], src); - DONE; -}) - -(define_insn "aarch64_get_lane_signed<mode>" - [(set (match_operand:<VEL> 0 "register_operand" "=r") - (sign_extend:<VEL> - (vec_select:<VEL> - (match_operand:VQ_S 1 "register_operand" "w") - (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] - "TARGET_SIMD" - "smov\\t%0, %1.<Vetype>[%2]" - [(set_attr "simd_type" "simd_movgp") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_get_lane_unsigned<mode>" - [(set (match_operand:<VEL> 0 "register_operand" "=r") - (zero_extend:<VEL> - (vec_select:<VEL> - (match_operand:VDQ 1 "register_operand" "w") - (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] - "TARGET_SIMD" - "umov\\t%<vw>0, %1.<Vetype>[%2]" - [(set_attr "simd_type" "simd_movgp") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_get_lane<mode>" - [(set (match_operand:<VEL> 0 "register_operand" "=w") - (vec_select:<VEL> - (match_operand:VDQF 1 "register_operand" "w") - (parallel [(match_operand:SI 2 "immediate_operand" "i")])))] - "TARGET_SIMD" - "mov\\t%0.<Vetype>[0], %1.<Vetype>[%2]" - [(set_attr "simd_type" "simd_ins") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_get_lanedi" - [(match_operand:DI 0 "register_operand" "=r") - (match_operand:DI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[2], 0, 1); - emit_move_insn (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv8qi<mode>" - [(match_operand:V8QI 0 "register_operand" "") - (match_operand:VDC 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv4hi<mode>" - [(match_operand:V4HI 0 "register_operand" "") - (match_operand:VDC 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv2si<mode>" - [(match_operand:V2SI 0 "register_operand" "") - (match_operand:VDC 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv2sf<mode>" - [(match_operand:V2SF 0 "register_operand" "") - (match_operand:VDC 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretdi<mode>" - [(match_operand:DI 0 "register_operand" "") - (match_operand:VD_RE 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv16qi<mode>" - [(match_operand:V16QI 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv8hi<mode>" - [(match_operand:V8HI 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv4si<mode>" - [(match_operand:V4SI 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv4sf<mode>" - [(match_operand:V4SF 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv2di<mode>" - [(match_operand:V2DI 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -(define_expand "aarch64_reinterpretv2df<mode>" - [(match_operand:V2DF 0 "register_operand" "") - (match_operand:VQ 1 "register_operand" "")] - "TARGET_SIMD" -{ - aarch64_simd_reinterpret (operands[0], operands[1]); - DONE; -}) - -;; In this insn, operand 1 should be low, and operand 2 the high part of the -;; dest vector. - -(define_insn "*aarch64_combinez<mode>" - [(set (match_operand:<VDBL> 0 "register_operand" "=&w") - (vec_concat:<VDBL> - (match_operand:VDIC 1 "register_operand" "w") - (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))] - "TARGET_SIMD" - "mov\\t%0.8b, %1.8b" - [(set_attr "simd_type" "simd_move") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_combine<mode>" - [(set (match_operand:<VDBL> 0 "register_operand" "=&w") - (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w") - (match_operand:VDC 2 "register_operand" "w")))] - "TARGET_SIMD" - "mov\\t%0.d[0], %1.d[0]\;ins\\t%0.d[1], %2.d[0]" - [(set_attr "simd_type" "simd_ins") - (set_attr "simd_mode" "<MODE>")] -) - -;; <su><addsub>l<q>. - -(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l2<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 3 "vect_par_cnst_hi_half" ""))) - (ANY_EXTEND:<VWIDE> (vec_select:<VHALF> - (match_operand:VQW 2 "register_operand" "w") - (match_dup 3)))))] - "TARGET_SIMD" - "<ANY_EXTEND:su><ADDSUB:optab>l2 %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_addl") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_saddl2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_saddl2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -(define_expand "aarch64_uaddl2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_uaddl2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -(define_expand "aarch64_ssubl2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_ssubl2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -(define_expand "aarch64_usubl2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQW 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_usubl2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> - (match_operand:VDW 1 "register_operand" "w")) - (ANY_EXTEND:<VWIDE> - (match_operand:VDW 2 "register_operand" "w"))))] - "TARGET_SIMD" - "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_addl") - (set_attr "simd_mode" "<MODE>")] -) - -;; <su><addsub>w<q>. - -(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w") - (ANY_EXTEND:<VWIDE> - (match_operand:VDW 2 "register_operand" "w"))))] - "TARGET_SIMD" - "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_addl") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w") - (ANY_EXTEND:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQW 2 "register_operand" "w") - (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))] - "TARGET_SIMD" - "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_addl") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_saddw2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -(define_expand "aarch64_uaddw2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - - -(define_expand "aarch64_ssubw2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -(define_expand "aarch64_usubw2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQW 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -;; <su><r>h<addsub>. - -(define_insn "aarch64_<sur>h<addsub><mode>" - [(set (match_operand:VQ_S 0 "register_operand" "=w") - (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w") - (match_operand:VQ_S 2 "register_operand" "w")] - HADDSUB))] - "TARGET_SIMD" - "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "<MODE>")] -) - -;; <r><addsub>hn<q>. - -(define_insn "aarch64_<sur><addsub>hn<mode>" - [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") - (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w") - (match_operand:VQN 2 "register_operand" "w")] - ADDSUBHN))] - "TARGET_SIMD" - "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_addn") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_<sur><addsub>hn2<mode>" - [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w") - (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0") - (match_operand:VQN 2 "register_operand" "w") - (match_operand:VQN 3 "register_operand" "w")] - ADDSUBHN2))] - "TARGET_SIMD" - "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>" - [(set_attr "simd_type" "simd_addn2") - (set_attr "simd_mode" "<MODE>")] -) - -;; pmul. - -(define_insn "aarch64_pmul<mode>" - [(set (match_operand:VB 0 "register_operand" "=w") - (unspec:VB [(match_operand:VB 1 "register_operand" "w") - (match_operand:VB 2 "register_operand" "w")] - UNSPEC_PMUL))] - "TARGET_SIMD" - "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_mul") - (set_attr "simd_mode" "<MODE>")] -) - -;; <su>q<addsub> - -(define_insn "aarch64_<su_optab><optab><mode>" - [(set (match_operand:VSDQ_I 0 "register_operand" "=w") - (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w") - (match_operand:VSDQ_I 2 "register_operand" "w")))] - "TARGET_SIMD" - "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "<MODE>")] -) - -;; suqadd and usqadd - -(define_insn "aarch64_<sur>qadd<mode>" - [(set (match_operand:VSDQ_I 0 "register_operand" "=w") - (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0") - (match_operand:VSDQ_I 2 "register_operand" "w")] - USSUQADD))] - "TARGET_SIMD" - "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_sat_add") - (set_attr "simd_mode" "<MODE>")] -) - -;; sqmovun - -(define_insn "aarch64_sqmovun<mode>" - [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") - (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")] - UNSPEC_SQXTUN))] - "TARGET_SIMD" - "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>" - [(set_attr "simd_type" "simd_sat_shiftn_imm") - (set_attr "simd_mode" "<MODE>")] - ) - -;; sqmovn and uqmovn - -(define_insn "aarch64_<sur>qmovn<mode>" - [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") - (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")] - SUQMOVN))] - "TARGET_SIMD" - "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>" - [(set_attr "simd_type" "simd_sat_shiftn_imm") - (set_attr "simd_mode" "<MODE>")] - ) - -;; <su>q<absneg> - -(define_insn "aarch64_s<optab><mode>" - [(set (match_operand:VSDQ_I_BHSI 0 "register_operand" "=w") - (UNQOPS:VSDQ_I_BHSI - (match_operand:VSDQ_I_BHSI 1 "register_operand" "w")))] - "TARGET_SIMD" - "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>" - [(set_attr "simd_type" "simd_sat_negabs") - (set_attr "simd_mode" "<MODE>")] -) - -;; sq<r>dmulh. - -(define_insn "aarch64_sq<r>dmulh<mode>" - [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w") - (unspec:VSDQ_HSI - [(match_operand:VSDQ_HSI 1 "register_operand" "w") - (match_operand:VSDQ_HSI 2 "register_operand" "w")] - VQDMULH))] - "TARGET_SIMD" - "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -;; sq<r>dmulh_lane - -(define_insn "aarch64_sq<r>dmulh_lane<mode>" - [(set (match_operand:VDQHS 0 "register_operand" "=w") - (unspec:VDQHS - [(match_operand:VDQHS 1 "register_operand" "w") - (vec_select:<VEL> - (match_operand:<VCOND> 2 "register_operand" "<vwx>") - (parallel [(match_operand:SI 3 "immediate_operand" "i")]))] - VQDMULH))] - "TARGET_SIMD" - "* - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode)); - return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_sq<r>dmulh_laneq<mode>" - [(set (match_operand:VDQHS 0 "register_operand" "=w") - (unspec:VDQHS - [(match_operand:VDQHS 1 "register_operand" "w") - (vec_select:<VEL> - (match_operand:<VCONQ> 2 "register_operand" "<vwx>") - (parallel [(match_operand:SI 3 "immediate_operand" "i")]))] - VQDMULH))] - "TARGET_SIMD" - "* - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode)); - return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_sq<r>dmulh_lane<mode>" - [(set (match_operand:SD_HSI 0 "register_operand" "=w") - (unspec:SD_HSI - [(match_operand:SD_HSI 1 "register_operand" "w") - (vec_select:<VEL> - (match_operand:<VCONQ> 2 "register_operand" "<vwx>") - (parallel [(match_operand:SI 3 "immediate_operand" "i")]))] - VQDMULH))] - "TARGET_SIMD" - "* - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode)); - return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -;; vqdml[sa]l - -(define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:VSD_HSI 2 "register_operand" "w")) - (sign_extend:<VWIDE> - (match_operand:VSD_HSI 3 "register_operand" "w"))) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -;; vqdml[sa]l_lane - -(define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:VD_HSI 2 "register_operand" "w")) - (sign_extend:<VWIDE> - (vec_duplicate:VD_HSI - (vec_select:<VEL> - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (parallel [(match_operand:SI 4 "immediate_operand" "i")]))) - )) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:SD_HSI 2 "register_operand" "w")) - (sign_extend:<VWIDE> - (vec_select:<VEL> - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (parallel [(match_operand:SI 4 "immediate_operand" "i")]))) - ) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmlal_lane<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "0") - (match_operand:VSD_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2); - emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4])); - DONE; -}) - -(define_expand "aarch64_sqdmlal_laneq<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "0") - (match_operand:VSD_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode)); - emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4])); - DONE; -}) - -(define_expand "aarch64_sqdmlsl_lane<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "0") - (match_operand:VSD_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2); - emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4])); - DONE; -}) - -(define_expand "aarch64_sqdmlsl_laneq<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "0") - (match_operand:VSD_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode)); - emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4])); - DONE; -}) - -;; vqdml[sa]l_n - -(define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:VD_HSI 2 "register_operand" "w")) - (sign_extend:<VWIDE> - (vec_duplicate:VD_HSI - (match_operand:<VEL> 3 "register_operand" "w")))) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -;; sqdml[as]l2 - -(define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" ""))) - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 3 "register_operand" "w") - (match_dup 4)))) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmlal2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:VQ_HSI 3 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1], - operands[2], operands[3], p)); - DONE; -}) - -(define_expand "aarch64_sqdmlsl2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:VQ_HSI 3 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1], - operands[2], operands[3], p)); - DONE; -}) - -;; vqdml[sa]l2_lane - -(define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" ""))) - (sign_extend:<VWIDE> - (vec_duplicate:<VHALF> - (vec_select:<VEL> - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (parallel [(match_operand:SI 4 "immediate_operand" "i")]) - )))) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmlal2_lane<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2); - emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4], p)); - DONE; -}) - -(define_expand "aarch64_sqdmlal2_laneq<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); - emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4], p)); - DONE; -}) - -(define_expand "aarch64_sqdmlsl2_lane<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2); - emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4], p)); - DONE; -}) - -(define_expand "aarch64_sqdmlsl2_laneq<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:<VCON> 3 "register_operand" "<vwx>") - (match_operand:SI 4 "immediate_operand" "i")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); - emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - operands[4], p)); - DONE; -}) - -(define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (SBINQOPS:<VWIDE> - (match_operand:<VWIDE> 1 "register_operand" "0") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" ""))) - (sign_extend:<VWIDE> - (vec_duplicate:<VHALF> - (match_operand:<VEL> 3 "register_operand" "w")))) - (const_int 1))))] - "TARGET_SIMD" - "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]" - [(set_attr "simd_type" "simd_sat_mlal") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmlal2_n<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:<VEL> 3 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - p)); - DONE; -}) - -(define_expand "aarch64_sqdmlsl2_n<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:<VWIDE> 1 "register_operand" "w") - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_operand:<VEL> 3 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - p)); - DONE; -}) - -;; vqdmull - -(define_insn "aarch64_sqdmull<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:VSD_HSI 1 "register_operand" "w")) - (sign_extend:<VWIDE> - (match_operand:VSD_HSI 2 "register_operand" "w"))) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -;; vqdmull_lane - -(define_insn "aarch64_sqdmull_lane<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:VD_HSI 1 "register_operand" "w")) - (sign_extend:<VWIDE> - (vec_duplicate:VD_HSI - (vec_select:<VEL> - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (parallel [(match_operand:SI 3 "immediate_operand" "i")]))) - )) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_sqdmull_lane<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:SD_HSI 1 "register_operand" "w")) - (sign_extend:<VWIDE> - (vec_select:<VEL> - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (parallel [(match_operand:SI 3 "immediate_operand" "i")])) - )) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmull_lane<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VSD_HSI 1 "register_operand" "w") - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (match_operand:SI 3 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode) / 2); - emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3])); - DONE; -}) - -(define_expand "aarch64_sqdmull_laneq<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VD_HSI 1 "register_operand" "w") - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (match_operand:SI 3 "immediate_operand" "i")] - "TARGET_SIMD" -{ - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode)); - emit_insn (gen_aarch64_sqdmull_lane<mode>_internal - (operands[0], operands[1], operands[2], operands[3])); - DONE; -}) - -;; vqdmull_n - -(define_insn "aarch64_sqdmull_n<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (match_operand:VD_HSI 1 "register_operand" "w")) - (sign_extend:<VWIDE> - (vec_duplicate:VD_HSI - (match_operand:<VEL> 2 "register_operand" "w"))) - ) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -;; vqdmull2 - - - -(define_insn "aarch64_sqdmull2<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" ""))) - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 2 "register_operand" "w") - (match_dup 3))) - ) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmull2<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:<VCON> 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -;; vqdmull2_lane - -(define_insn "aarch64_sqdmull2_lane<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" ""))) - (sign_extend:<VWIDE> - (vec_duplicate:<VHALF> - (vec_select:<VEL> - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (parallel [(match_operand:SI 3 "immediate_operand" "i")]))) - )) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmull2_lane<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (match_operand:SI 3 "immediate_operand" "i")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode) / 2); - emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - p)); - DONE; -}) - -(define_expand "aarch64_sqdmull2_laneq<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:<VCON> 2 "register_operand" "<vwx>") - (match_operand:SI 3 "immediate_operand" "i")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); - emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1], - operands[2], operands[3], - p)); - DONE; -}) - -;; vqdmull2_n - -(define_insn "aarch64_sqdmull2_n<mode>_internal" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (ss_ashift:<VWIDE> - (mult:<VWIDE> - (sign_extend:<VWIDE> - (vec_select:<VHALF> - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" ""))) - (sign_extend:<VWIDE> - (vec_duplicate:<VHALF> - (match_operand:<VEL> 2 "register_operand" "w"))) - ) - (const_int 1)))] - "TARGET_SIMD" - "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]" - [(set_attr "simd_type" "simd_sat_mul") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqdmull2_n<mode>" - [(match_operand:<VWIDE> 0 "register_operand" "=w") - (match_operand:VQ_HSI 1 "register_operand" "w") - (match_operand:<VEL> 2 "register_operand" "w")] - "TARGET_SIMD" -{ - rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true); - emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1], - operands[2], p)); - DONE; -}) - -;; vshl - -(define_insn "aarch64_<sur>shl<mode>" - [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (unspec:VSDQ_I_DI - [(match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:VSDQ_I_DI 2 "register_operand" "w")] - VSHL))] - "TARGET_SIMD" - "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"; - [(set_attr "simd_type" "simd_shift") - (set_attr "simd_mode" "<MODE>")] -) - - -;; vqshl - -(define_insn "aarch64_<sur>q<r>shl<mode>" - [(set (match_operand:VSDQ_I 0 "register_operand" "=w") - (unspec:VSDQ_I - [(match_operand:VSDQ_I 1 "register_operand" "w") - (match_operand:VSDQ_I 2 "register_operand" "w")] - VQSHL))] - "TARGET_SIMD" - "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"; - [(set_attr "simd_type" "simd_sat_shift") - (set_attr "simd_mode" "<MODE>")] -) - -;; vshl_n - -(define_expand "aarch64_sshl_n<mode>" - [(match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - emit_insn (gen_ashl<mode>3 (operands[0], operands[1], operands[2])); - DONE; -}) - -(define_expand "aarch64_ushl_n<mode>" - [(match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - emit_insn (gen_ashl<mode>3 (operands[0], operands[1], operands[2])); - DONE; -}) - -;; vshll_n - -(define_insn "aarch64_<sur>shll_n<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - VSHLL))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[2], 0, bit_width + 1); - if (INTVAL (operands[2]) == bit_width) - { - return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\"; - } - else { - return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\"; - }" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -;; vshll_high_n - -(define_insn "aarch64_<sur>shll2_n<mode>" - [(set (match_operand:<VWIDE> 0 "register_operand" "=w") - (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - VSHLL))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[2], 0, bit_width + 1); - if (INTVAL (operands[2]) == bit_width) - { - return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\"; - } - else { - return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\"; - }" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -;; vshr_n - -(define_expand "aarch64_sshr_n<mode>" - [(match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - emit_insn (gen_ashr<mode>3 (operands[0], operands[1], operands[2])); - DONE; -}) - -(define_expand "aarch64_ushr_n<mode>" - [(match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - emit_insn (gen_lshr<mode>3 (operands[0], operands[1], operands[2])); - DONE; -}) - -;; vrshr_n - -(define_insn "aarch64_<sur>shr_n<mode>" - [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - VRSHR_N))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[2], 1, bit_width + 1); - return \"<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -;; v(r)sra_n - -(define_insn "aarch64_<sur>sra_n<mode>" - [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0") - (match_operand:VSDQ_I_DI 2 "register_operand" "w") - (match_operand:SI 3 "immediate_operand" "i")] - VSRA))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[3], 1, bit_width + 1); - return \"<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";" - [(set_attr "simd_type" "simd_shift_imm_acc") - (set_attr "simd_mode" "<MODE>")] -) - -;; vs<lr>i_n - -(define_insn "aarch64_<sur>s<lr>i_n<mode>" - [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w") - (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0") - (match_operand:VSDQ_I_DI 2 "register_operand" "w") - (match_operand:SI 3 "immediate_operand" "i")] - VSLRI))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[3], 1 - <VSLRI:offsetlr>, - bit_width - <VSLRI:offsetlr> + 1); - return \"s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";" - [(set_attr "simd_type" "simd_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - -;; vqshl(u) - -(define_insn "aarch64_<sur>qshl<u>_n<mode>" - [(set (match_operand:VSDQ_I 0 "register_operand" "=w") - (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - VQSHL_N))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[2], 0, bit_width); - return \"<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";" - [(set_attr "simd_type" "simd_sat_shift_imm") - (set_attr "simd_mode" "<MODE>")] -) - - -;; vq(r)shr(u)n_n - -(define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>" - [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") - (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - VQSHRN_N))] - "TARGET_SIMD" - "* - int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT; - aarch64_simd_const_bounds (operands[2], 1, bit_width + 1); - return \"<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2\";" - [(set_attr "simd_type" "simd_sat_shiftn_imm") - (set_attr "simd_mode" "<MODE>")] -) - - -;; cm(eq|ge|le|lt|gt) - -(define_insn "aarch64_cm<cmp><mode>" - [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w") - (unspec:<V_cmp_result> - [(match_operand:VSDQ_I_DI 1 "register_operand" "w,w") - (match_operand:VSDQ_I_DI 2 "aarch64_simd_reg_or_zero" "w,Z")] - VCMP_S))] - "TARGET_SIMD" - "@ - cm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype> - cm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0" - [(set_attr "simd_type" "simd_cmp") - (set_attr "simd_mode" "<MODE>")] -) - -;; cm(hs|hi|tst) - -(define_insn "aarch64_cm<cmp><mode>" - [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w") - (unspec:<V_cmp_result> - [(match_operand:VSDQ_I_DI 1 "register_operand" "w") - (match_operand:VSDQ_I_DI 2 "register_operand" "w")] - VCMP_U))] - "TARGET_SIMD" - "cm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_cmp") - (set_attr "simd_mode" "<MODE>")] -) - -;; fcm(eq|ge|le|lt|gt) - -(define_insn "aarch64_cm<cmp><mode>" - [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w") - (unspec:<V_cmp_result> - [(match_operand:VDQF 1 "register_operand" "w,w") - (match_operand:VDQF 2 "aarch64_simd_reg_or_zero" "w,Dz")] - VCMP_S))] - "TARGET_SIMD" - "@ - fcm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype> - fcm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0" - [(set_attr "simd_type" "simd_fcmp") - (set_attr "simd_mode" "<MODE>")] -) - -;; addp - -(define_insn "aarch64_addp<mode>" - [(set (match_operand:VD_BHSI 0 "register_operand" "=w") - (unspec:VD_BHSI - [(match_operand:VD_BHSI 1 "register_operand" "w") - (match_operand:VD_BHSI 2 "register_operand" "w")] - UNSPEC_ADDP))] - "TARGET_SIMD" - "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_addpdi" - [(set (match_operand:DI 0 "register_operand" "=w") - (unspec:DI - [(match_operand:V2DI 1 "register_operand" "w")] - UNSPEC_ADDP))] - "TARGET_SIMD" - "addp\t%d0, %1.2d" - [(set_attr "simd_type" "simd_add") - (set_attr "simd_mode" "DI")] -) - -;; v(max|min) - -(define_expand "aarch64_<maxmin><mode>" - [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w") - (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w") - (match_operand:VDQ_BHSI 2 "register_operand" "w")))] - "TARGET_SIMD" -{ - emit_insn (gen_<maxmin><mode>3 (operands[0], operands[1], operands[2])); - DONE; -}) - - -(define_insn "aarch64_<fmaxmin><mode>" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w") - (match_operand:VDQF 2 "register_operand" "w")] - FMAXMIN))] - "TARGET_SIMD" - "<fmaxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_fminmax") - (set_attr "simd_mode" "<MODE>")] -) - -;; sqrt - -(define_insn "sqrt<mode>2" - [(set (match_operand:VDQF 0 "register_operand" "=w") - (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))] - "TARGET_SIMD" - "fsqrt\\t%0.<Vtype>, %1.<Vtype>" - [(set_attr "simd_type" "simd_fsqrt") - (set_attr "simd_mode" "<MODE>")] -) - -(define_expand "aarch64_sqrt<mode>" - [(match_operand:VDQF 0 "register_operand" "=w") - (match_operand:VDQF 1 "register_operand" "w")] - "TARGET_SIMD" -{ - emit_insn (gen_sqrt<mode>2 (operands[0], operands[1])); - DONE; -}) - - -;; Patterns for vector struct loads and stores. - -(define_insn "vec_load_lanesoi<mode>" - [(set (match_operand:OI 0 "register_operand" "=w") - (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_LD2))] - "TARGET_SIMD" - "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load2") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "vec_store_lanesoi<mode>" - [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:OI [(match_operand:OI 1 "register_operand" "w") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST2))] - "TARGET_SIMD" - "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0" - [(set_attr "simd_type" "simd_store2") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "vec_load_lanesci<mode>" - [(set (match_operand:CI 0 "register_operand" "=w") - (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_LD3))] - "TARGET_SIMD" - "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load3") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "vec_store_lanesci<mode>" - [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:CI [(match_operand:CI 1 "register_operand" "w") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST3))] - "TARGET_SIMD" - "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0" - [(set_attr "simd_type" "simd_store3") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "vec_load_lanesxi<mode>" - [(set (match_operand:XI 0 "register_operand" "=w") - (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_LD4))] - "TARGET_SIMD" - "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load4") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "vec_store_lanesxi<mode>" - [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:XI [(match_operand:XI 1 "register_operand" "w") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST4))] - "TARGET_SIMD" - "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0" - [(set_attr "simd_type" "simd_store4") - (set_attr "simd_mode" "<MODE>")]) - -;; Reload patterns for AdvSIMD register list operands. - -(define_expand "mov<mode>" - [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "") - (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))] - "TARGET_SIMD" -{ - if (can_create_pseudo_p ()) - { - if (GET_CODE (operands[0]) != REG) - operands[1] = force_reg (<MODE>mode, operands[1]); - } -}) - -(define_insn "*aarch64_mov<mode>" - [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w") - (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))] - "TARGET_SIMD - && (register_operand (operands[0], <MODE>mode) - || register_operand (operands[1], <MODE>mode))" - -{ - switch (which_alternative) - { - case 0: return "#"; - case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0"; - case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"; - default: gcc_unreachable (); - } -} - [(set_attr "simd_type" "simd_move,simd_store<nregs>,simd_load<nregs>") - (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)")) - (set_attr "simd_mode" "<MODE>")]) - -(define_split - [(set (match_operand:OI 0 "register_operand" "") - (match_operand:OI 1 "register_operand" ""))] - "TARGET_SIMD && reload_completed" - [(set (match_dup 0) (match_dup 1)) - (set (match_dup 2) (match_dup 3))] -{ - int rdest = REGNO (operands[0]); - int rsrc = REGNO (operands[1]); - rtx dest[2], src[2]; - - dest[0] = gen_rtx_REG (TFmode, rdest); - src[0] = gen_rtx_REG (TFmode, rsrc); - dest[1] = gen_rtx_REG (TFmode, rdest + 1); - src[1] = gen_rtx_REG (TFmode, rsrc + 1); - - aarch64_simd_disambiguate_copy (operands, dest, src, 2); -}) - -(define_split - [(set (match_operand:CI 0 "register_operand" "") - (match_operand:CI 1 "register_operand" ""))] - "TARGET_SIMD && reload_completed" - [(set (match_dup 0) (match_dup 1)) - (set (match_dup 2) (match_dup 3)) - (set (match_dup 4) (match_dup 5))] -{ - int rdest = REGNO (operands[0]); - int rsrc = REGNO (operands[1]); - rtx dest[3], src[3]; - - dest[0] = gen_rtx_REG (TFmode, rdest); - src[0] = gen_rtx_REG (TFmode, rsrc); - dest[1] = gen_rtx_REG (TFmode, rdest + 1); - src[1] = gen_rtx_REG (TFmode, rsrc + 1); - dest[2] = gen_rtx_REG (TFmode, rdest + 2); - src[2] = gen_rtx_REG (TFmode, rsrc + 2); - - aarch64_simd_disambiguate_copy (operands, dest, src, 3); -}) - -(define_split - [(set (match_operand:XI 0 "register_operand" "") - (match_operand:XI 1 "register_operand" ""))] - "TARGET_SIMD && reload_completed" - [(set (match_dup 0) (match_dup 1)) - (set (match_dup 2) (match_dup 3)) - (set (match_dup 4) (match_dup 5)) - (set (match_dup 6) (match_dup 7))] -{ - int rdest = REGNO (operands[0]); - int rsrc = REGNO (operands[1]); - rtx dest[4], src[4]; - - dest[0] = gen_rtx_REG (TFmode, rdest); - src[0] = gen_rtx_REG (TFmode, rsrc); - dest[1] = gen_rtx_REG (TFmode, rdest + 1); - src[1] = gen_rtx_REG (TFmode, rsrc + 1); - dest[2] = gen_rtx_REG (TFmode, rdest + 2); - src[2] = gen_rtx_REG (TFmode, rsrc + 2); - dest[3] = gen_rtx_REG (TFmode, rdest + 3); - src[3] = gen_rtx_REG (TFmode, rsrc + 3); - - aarch64_simd_disambiguate_copy (operands, dest, src, 4); -}) - -(define_insn "aarch64_ld2<mode>_dreg" - [(set (match_operand:OI 0 "register_operand" "=w") - (subreg:OI - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")] - UNSPEC_LD2) - (vec_duplicate:VD (const_int 0))) - (vec_concat:<VDBL> - (unspec:VD [(match_dup 1)] - UNSPEC_LD2) - (vec_duplicate:VD (const_int 0)))) 0))] - "TARGET_SIMD" - "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load2") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_ld2<mode>_dreg" - [(set (match_operand:OI 0 "register_operand" "=w") - (subreg:OI - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")] - UNSPEC_LD2) - (const_int 0)) - (vec_concat:<VDBL> - (unspec:DX [(match_dup 1)] - UNSPEC_LD2) - (const_int 0))) 0))] - "TARGET_SIMD" - "ld1\\t{%S0.1d - %T0.1d}, %1" - [(set_attr "simd_type" "simd_load2") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_ld3<mode>_dreg" - [(set (match_operand:CI 0 "register_operand" "=w") - (subreg:CI - (vec_concat:<VRL3> - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")] - UNSPEC_LD3) - (vec_duplicate:VD (const_int 0))) - (vec_concat:<VDBL> - (unspec:VD [(match_dup 1)] - UNSPEC_LD3) - (vec_duplicate:VD (const_int 0)))) - (vec_concat:<VDBL> - (unspec:VD [(match_dup 1)] - UNSPEC_LD3) - (vec_duplicate:VD (const_int 0)))) 0))] - "TARGET_SIMD" - "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load3") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_ld3<mode>_dreg" - [(set (match_operand:CI 0 "register_operand" "=w") - (subreg:CI - (vec_concat:<VRL3> - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")] - UNSPEC_LD3) - (const_int 0)) - (vec_concat:<VDBL> - (unspec:DX [(match_dup 1)] - UNSPEC_LD3) - (const_int 0))) - (vec_concat:<VDBL> - (unspec:DX [(match_dup 1)] - UNSPEC_LD3) - (const_int 0))) 0))] - "TARGET_SIMD" - "ld1\\t{%S0.1d - %U0.1d}, %1" - [(set_attr "simd_type" "simd_load3") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_ld4<mode>_dreg" - [(set (match_operand:XI 0 "register_operand" "=w") - (subreg:XI - (vec_concat:<VRL4> - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")] - UNSPEC_LD4) - (vec_duplicate:VD (const_int 0))) - (vec_concat:<VDBL> - (unspec:VD [(match_dup 1)] - UNSPEC_LD4) - (vec_duplicate:VD (const_int 0)))) - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:VD [(match_dup 1)] - UNSPEC_LD4) - (vec_duplicate:VD (const_int 0))) - (vec_concat:<VDBL> - (unspec:VD [(match_dup 1)] - UNSPEC_LD4) - (vec_duplicate:VD (const_int 0))))) 0))] - "TARGET_SIMD" - "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load4") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_ld4<mode>_dreg" - [(set (match_operand:XI 0 "register_operand" "=w") - (subreg:XI - (vec_concat:<VRL4> - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")] - UNSPEC_LD4) - (const_int 0)) - (vec_concat:<VDBL> - (unspec:DX [(match_dup 1)] - UNSPEC_LD4) - (const_int 0))) - (vec_concat:<VRL2> - (vec_concat:<VDBL> - (unspec:DX [(match_dup 1)] - UNSPEC_LD4) - (const_int 0)) - (vec_concat:<VDBL> - (unspec:DX [(match_dup 1)] - UNSPEC_LD4) - (const_int 0)))) 0))] - "TARGET_SIMD" - "ld1\\t{%S0.1d - %V0.1d}, %1" - [(set_attr "simd_type" "simd_load4") - (set_attr "simd_mode" "<MODE>")]) - -(define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>" - [(match_operand:VSTRUCT 0 "register_operand" "=w") - (match_operand:DI 1 "register_operand" "r") - (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - "TARGET_SIMD" -{ - enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode; - rtx mem = gen_rtx_MEM (mode, operands[1]); - - emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem)); - DONE; -}) - -(define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>" - [(match_operand:VSTRUCT 0 "register_operand" "=w") - (match_operand:DI 1 "register_operand" "r") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - "TARGET_SIMD" -{ - enum machine_mode mode = <VSTRUCT:MODE>mode; - rtx mem = gen_rtx_MEM (mode, operands[1]); - - emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem)); - DONE; -}) - -;; Expanders for builtins to extract vector registers from large -;; opaque integer modes. - -;; D-register list. - -(define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>" - [(match_operand:VDC 0 "register_operand" "=w") - (match_operand:VSTRUCT 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - int part = INTVAL (operands[2]); - rtx temp = gen_reg_rtx (<VDC:VDBL>mode); - int offset = part * 16; - - emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset)); - emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp)); - DONE; -}) - -;; Q-register list. - -(define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>" - [(match_operand:VQ 0 "register_operand" "=w") - (match_operand:VSTRUCT 1 "register_operand" "w") - (match_operand:SI 2 "immediate_operand" "i")] - "TARGET_SIMD" -{ - int part = INTVAL (operands[2]); - int offset = part * 16; - - emit_move_insn (operands[0], - gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset)); - DONE; -}) - -;; Permuted-store expanders for neon intrinsics. - -;; Permute instructions - -;; vec_perm support - -(define_expand "vec_perm_const<mode>" - [(match_operand:VALL 0 "register_operand") - (match_operand:VALL 1 "register_operand") - (match_operand:VALL 2 "register_operand") - (match_operand:<V_cmp_result> 3)] - "TARGET_SIMD" -{ - if (aarch64_expand_vec_perm_const (operands[0], operands[1], - operands[2], operands[3])) - DONE; - else - FAIL; -}) - -(define_expand "vec_perm<mode>" - [(match_operand:VB 0 "register_operand") - (match_operand:VB 1 "register_operand") - (match_operand:VB 2 "register_operand") - (match_operand:VB 3 "register_operand")] - "TARGET_SIMD" -{ - aarch64_expand_vec_perm (operands[0], operands[1], - operands[2], operands[3]); - DONE; -}) - -(define_insn "aarch64_tbl1<mode>" - [(set (match_operand:VB 0 "register_operand" "=w") - (unspec:VB [(match_operand:V16QI 1 "register_operand" "w") - (match_operand:VB 2 "register_operand" "w")] - UNSPEC_TBL))] - "TARGET_SIMD" - "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>" - [(set_attr "simd_type" "simd_tbl") - (set_attr "simd_mode" "<MODE>")] -) - -;; Two source registers. - -(define_insn "aarch64_tbl2v16qi" - [(set (match_operand:V16QI 0 "register_operand" "=w") - (unspec:V16QI [(match_operand:OI 1 "register_operand" "w") - (match_operand:V16QI 2 "register_operand" "w")] - UNSPEC_TBL))] - "TARGET_SIMD" - "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b" - [(set_attr "simd_type" "simd_tbl") - (set_attr "simd_mode" "V16QI")] -) - -(define_insn_and_split "aarch64_combinev16qi" - [(set (match_operand:OI 0 "register_operand" "=w") - (unspec:OI [(match_operand:V16QI 1 "register_operand" "w") - (match_operand:V16QI 2 "register_operand" "w")] - UNSPEC_CONCAT))] - "TARGET_SIMD" - "#" - "&& reload_completed" - [(const_int 0)] -{ - aarch64_split_combinev16qi (operands); - DONE; -}) - -(define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>" - [(set (match_operand:VALL 0 "register_operand" "=w") - (unspec:VALL [(match_operand:VALL 1 "register_operand" "w") - (match_operand:VALL 2 "register_operand" "w")] - PERMUTE))] - "TARGET_SIMD" - "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>" - [(set_attr "simd_type" "simd_<PERMUTE:perm_insn>") - (set_attr "simd_mode" "<MODE>")] -) - -(define_insn "aarch64_st2<mode>_dreg" - [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:TI [(match_operand:OI 1 "register_operand" "w") - (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST2))] - "TARGET_SIMD" - "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0" - [(set_attr "simd_type" "simd_store2") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_st2<mode>_dreg" - [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:TI [(match_operand:OI 1 "register_operand" "w") - (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST2))] - "TARGET_SIMD" - "st1\\t{%S1.1d - %T1.1d}, %0" - [(set_attr "simd_type" "simd_store2") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_st3<mode>_dreg" - [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:EI [(match_operand:CI 1 "register_operand" "w") - (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST3))] - "TARGET_SIMD" - "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0" - [(set_attr "simd_type" "simd_store3") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_st3<mode>_dreg" - [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:EI [(match_operand:CI 1 "register_operand" "w") - (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST3))] - "TARGET_SIMD" - "st1\\t{%S1.1d - %U1.1d}, %0" - [(set_attr "simd_type" "simd_store3") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_st4<mode>_dreg" - [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:OI [(match_operand:XI 1 "register_operand" "w") - (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST4))] - "TARGET_SIMD" - "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0" - [(set_attr "simd_type" "simd_store4") - (set_attr "simd_mode" "<MODE>")]) - -(define_insn "aarch64_st4<mode>_dreg" - [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv") - (unspec:OI [(match_operand:XI 1 "register_operand" "w") - (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - UNSPEC_ST4))] - "TARGET_SIMD" - "st1\\t{%S1.1d - %V1.1d}, %0" - [(set_attr "simd_type" "simd_store4") - (set_attr "simd_mode" "<MODE>")]) - -(define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>" - [(match_operand:DI 0 "register_operand" "r") - (match_operand:VSTRUCT 1 "register_operand" "w") - (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - "TARGET_SIMD" -{ - enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode; - rtx mem = gen_rtx_MEM (mode, operands[0]); - - emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1])); - DONE; -}) - -(define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>" - [(match_operand:DI 0 "register_operand" "r") - (match_operand:VSTRUCT 1 "register_operand" "w") - (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] - "TARGET_SIMD" -{ - enum machine_mode mode = <VSTRUCT:MODE>mode; - rtx mem = gen_rtx_MEM (mode, operands[0]); - - emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1])); - DONE; -}) - -;; Expander for builtins to insert vector registers into large -;; opaque integer modes. - -;; Q-register list. We don't need a D-reg inserter as we zero -;; extend them in arm_neon.h and insert the resulting Q-regs. - -(define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>" - [(match_operand:VSTRUCT 0 "register_operand" "+w") - (match_operand:VSTRUCT 1 "register_operand" "0") - (match_operand:VQ 2 "register_operand" "w") - (match_operand:SI 3 "immediate_operand" "i")] - "TARGET_SIMD" -{ - int part = INTVAL (operands[3]); - int offset = part * 16; - - emit_move_insn (operands[0], operands[1]); - emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset), - operands[2]); - DONE; -}) - -;; Standard pattern name vec_init<mode>. - -(define_expand "vec_init<mode>" - [(match_operand:VALL 0 "register_operand" "") - (match_operand 1 "" "")] - "TARGET_SIMD" -{ - aarch64_expand_vector_init (operands[0], operands[1]); - DONE; -}) - -(define_insn "*aarch64_simd_ld1r<mode>" - [(set (match_operand:VALLDI 0 "register_operand" "=w") - (vec_duplicate:VALLDI - (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))] - "TARGET_SIMD" - "ld1r\\t{%0.<Vtype>}, %1" - [(set_attr "simd_type" "simd_load1r") - (set_attr "simd_mode" "<MODE>")]) |