diff options
Diffstat (limited to 'gcc-4.9/gcc/config/arm/neon.ml')
| -rw-r--r-- | gcc-4.9/gcc/config/arm/neon.ml | 2355 |
1 files changed, 2355 insertions, 0 deletions
diff --git a/gcc-4.9/gcc/config/arm/neon.ml b/gcc-4.9/gcc/config/arm/neon.ml new file mode 100644 index 000000000..4289b8ce0 --- /dev/null +++ b/gcc-4.9/gcc/config/arm/neon.ml @@ -0,0 +1,2355 @@ +(* Common code for ARM NEON header file, documentation and test case + generators. + + Copyright (C) 2006-2014 Free Software Foundation, Inc. + Contributed by CodeSourcery. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation; either version 3, or (at your option) any later + version. + + GCC is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with GCC; see the file COPYING3. If not see + <http://www.gnu.org/licenses/>. *) + +(* Shorthand types for vector elements. *) +type elts = S8 | S16 | S32 | S64 | F16 | F32 | U8 | U16 | U32 | U64 | P8 | P16 + | P64 | P128 | I8 | I16 | I32 | I64 | B8 | B16 | B32 | B64 | Conv of elts * elts + | Cast of elts * elts | NoElts + +type eltclass = Signed | Unsigned | Float | Poly | Int | Bits + | ConvClass of eltclass * eltclass | NoType + +(* These vector types correspond directly to C types. *) +type vectype = T_int8x8 | T_int8x16 + | T_int16x4 | T_int16x8 + | T_int32x2 | T_int32x4 + | T_int64x1 | T_int64x2 + | T_uint8x8 | T_uint8x16 + | T_uint16x4 | T_uint16x8 + | T_uint32x2 | T_uint32x4 + | T_uint64x1 | T_uint64x2 + | T_float16x4 + | T_float32x2 | T_float32x4 + | T_poly8x8 | T_poly8x16 + | T_poly16x4 | T_poly16x8 + | T_immediate of int * int + | T_int8 | T_int16 + | T_int32 | T_int64 + | T_uint8 | T_uint16 + | T_uint32 | T_uint64 + | T_poly8 | T_poly16 + | T_poly64 | T_poly64x1 + | T_poly64x2 | T_poly128 + | T_float16 | T_float32 + | T_arrayof of int * vectype + | T_ptrto of vectype | T_const of vectype + | T_void | T_intQI + | T_intHI | T_intSI + | T_intDI | T_intTI + | T_floatHF | T_floatSF + +(* The meanings of the following are: + TImode : "Tetra", two registers (four words). + EImode : "hExa", three registers (six words). + OImode : "Octa", four registers (eight words). + CImode : "dodeCa", six registers (twelve words). + XImode : "heXadeca", eight registers (sixteen words). +*) + +type inttype = B_TImode | B_EImode | B_OImode | B_CImode | B_XImode + +type shape_elt = Dreg | Qreg | Corereg | Immed | VecArray of int * shape_elt + | PtrTo of shape_elt | CstPtrTo of shape_elt + (* These next ones are used only in the test generator. *) + | Element_of_dreg (* Used for "lane" variants. *) + | Element_of_qreg (* Likewise. *) + | All_elements_of_dreg (* Used for "dup" variants. *) + | Alternatives of shape_elt list (* Used for multiple valid operands *) + +type shape_form = All of int * shape_elt + | Long + | Long_noreg of shape_elt + | Wide + | Wide_noreg of shape_elt + | Narrow + | Long_imm + | Narrow_imm + | Binary_imm of shape_elt + | Use_operands of shape_elt array + | By_scalar of shape_elt + | Unary_scalar of shape_elt + | Wide_lane + | Wide_scalar + | Pair_result of shape_elt + +type arity = Arity0 of vectype + | Arity1 of vectype * vectype + | Arity2 of vectype * vectype * vectype + | Arity3 of vectype * vectype * vectype * vectype + | Arity4 of vectype * vectype * vectype * vectype * vectype + +type vecmode = V8QI | V4HI | V4HF |V2SI | V2SF | DI + | V16QI | V8HI | V4SI | V4SF | V2DI | TI + | QI | HI | SI | SF + +type opcode = + (* Binary ops. *) + Vadd + | Vmul + | Vmla + | Vmls + | Vfma + | Vfms + | Vsub + | Vceq + | Vcge + | Vcgt + | Vcle + | Vclt + | Vcage + | Vcagt + | Vcale + | Vcalt + | Vtst + | Vabd + | Vaba + | Vmax + | Vmin + | Vpadd + | Vpada + | Vpmax + | Vpmin + | Vrecps + | Vrsqrts + | Vshl + | Vshr_n + | Vshl_n + | Vsra_n + | Vsri + | Vsli + (* Logic binops. *) + | Vand + | Vorr + | Veor + | Vbic + | Vorn + | Vbsl + (* Ops with scalar. *) + | Vmul_lane + | Vmla_lane + | Vmls_lane + | Vmul_n + | Vmla_n + | Vmls_n + | Vmull_n + | Vmull_lane + | Vqdmull_n + | Vqdmull_lane + | Vqdmulh_n + | Vqdmulh_lane + (* Unary ops. *) + | Vrintn + | Vrinta + | Vrintp + | Vrintm + | Vrintz + | Vabs + | Vneg + | Vcls + | Vclz + | Vcnt + | Vrecpe + | Vrsqrte + | Vmvn + (* Vector extract. *) + | Vext + (* Reverse elements. *) + | Vrev64 + | Vrev32 + | Vrev16 + (* Transposition ops. *) + | Vtrn + | Vzip + | Vuzp + (* Loads and stores (VLD1/VST1/VLD2...), elements and structures. *) + | Vldx of int + | Vstx of int + | Vldx_lane of int + | Vldx_dup of int + | Vstx_lane of int + (* Set/extract lanes from a vector. *) + | Vget_lane + | Vset_lane + (* Initialize vector from bit pattern. *) + | Vcreate + (* Set all lanes to same value. *) + | Vdup_n + | Vmov_n (* Is this the same? *) + (* Duplicate scalar to all lanes of vector. *) + | Vdup_lane + (* Combine vectors. *) + | Vcombine + (* Get quadword high/low parts. *) + | Vget_high + | Vget_low + (* Convert vectors. *) + | Vcvt + | Vcvt_n + (* Narrow/lengthen vectors. *) + | Vmovn + | Vmovl + (* Table lookup. *) + | Vtbl of int + | Vtbx of int + (* Reinterpret casts. *) + | Vreinterp + +let rev_elems revsize elsize nelts _ = + let mask = (revsize / elsize) - 1 in + let arr = Array.init nelts + (fun i -> i lxor mask) in + Array.to_list arr + +let permute_range i stride nelts increment = + let rec build i = function + 0 -> [] + | nelts -> i :: (i + stride) :: build (i + increment) (pred nelts) in + build i nelts + +(* Generate a list of integers suitable for vzip. *) +let zip_range i stride nelts = permute_range i stride nelts 1 + +(* Generate a list of integers suitable for vunzip. *) +let uzip_range i stride nelts = permute_range i stride nelts 4 + +(* Generate a list of integers suitable for trn. *) +let trn_range i stride nelts = permute_range i stride nelts 2 + +let zip_elems _ nelts part = + match part with + `lo -> zip_range 0 nelts (nelts / 2) + | `hi -> zip_range (nelts / 2) nelts (nelts / 2) + +let uzip_elems _ nelts part = + match part with + `lo -> uzip_range 0 2 (nelts / 2) + | `hi -> uzip_range 1 2 (nelts / 2) + +let trn_elems _ nelts part = + match part with + `lo -> trn_range 0 nelts (nelts / 2) + | `hi -> trn_range 1 nelts (nelts / 2) + +(* Features used for documentation, to distinguish between some instruction + variants, and to signal special requirements (e.g. swapping arguments). *) + +type features = + Halving + | Rounding + | Saturating + | Dst_unsign + | High_half + | Doubling + | Flipped of string (* Builtin name to use with flipped arguments. *) + | InfoWord (* Pass an extra word for signage/rounding etc. (always passed + for All _, Long, Wide, Narrow shape_forms. *) + (* Implement builtin as shuffle. The parameter is a function which returns + masks suitable for __builtin_shuffle: arguments are (element size, + number of elements, high/low part selector). *) + | Use_shuffle of (int -> int -> [`lo|`hi] -> int list) + (* A specification as to the shape of instruction expected upon + disassembly, used if it differs from the shape used to build the + intrinsic prototype. Multiple entries in the constructor's argument + indicate that the intrinsic expands to more than one assembly + instruction, each with a corresponding shape specified here. *) + | Disassembles_as of shape_form list + | Builtin_name of string (* Override the name of the builtin. *) + (* Override the name of the instruction. If more than one name + is specified, it means that the instruction can have any of those + names. *) + | Instruction_name of string list + (* Mark that the intrinsic yields no instructions, or expands to yield + behavior that the test generator cannot test. *) + | No_op + (* Mark that the intrinsic has constant arguments that cannot be set + to the defaults (zero for pointers and one otherwise) in the test + cases. The function supplied must return the integer to be written + into the testcase for the argument number (0-based) supplied to it. *) + | Const_valuator of (int -> int) + | Fixed_vector_reg + | Fixed_core_reg + (* Mark that the intrinsic requires __ARM_FEATURE_string to be defined. *) + | Requires_feature of string + (* Mark that the intrinsic requires a particular architecture version. *) + | Requires_arch of int + (* Mark that the intrinsic requires a particular bit in __ARM_FP to + be set. *) + | Requires_FP_bit of int + +exception MixedMode of elts * elts + +let rec elt_width = function + S8 | U8 | P8 | I8 | B8 -> 8 + | S16 | U16 | P16 | I16 | B16 | F16 -> 16 + | S32 | F32 | U32 | I32 | B32 -> 32 + | S64 | U64 | P64 | I64 | B64 -> 64 + | P128 -> 128 + | Conv (a, b) -> + let wa = elt_width a and wb = elt_width b in + if wa = wb then wa else raise (MixedMode (a, b)) + | Cast (a, b) -> raise (MixedMode (a, b)) + | NoElts -> failwith "No elts" + +let rec elt_class = function + S8 | S16 | S32 | S64 -> Signed + | U8 | U16 | U32 | U64 -> Unsigned + | P8 | P16 | P64 | P128 -> Poly + | F16 | F32 -> Float + | I8 | I16 | I32 | I64 -> Int + | B8 | B16 | B32 | B64 -> Bits + | Conv (a, b) | Cast (a, b) -> ConvClass (elt_class a, elt_class b) + | NoElts -> NoType + +let elt_of_class_width c w = + match c, w with + Signed, 8 -> S8 + | Signed, 16 -> S16 + | Signed, 32 -> S32 + | Signed, 64 -> S64 + | Float, 16 -> F16 + | Float, 32 -> F32 + | Unsigned, 8 -> U8 + | Unsigned, 16 -> U16 + | Unsigned, 32 -> U32 + | Unsigned, 64 -> U64 + | Poly, 8 -> P8 + | Poly, 16 -> P16 + | Poly, 64 -> P64 + | Poly, 128 -> P128 + | Int, 8 -> I8 + | Int, 16 -> I16 + | Int, 32 -> I32 + | Int, 64 -> I64 + | Bits, 8 -> B8 + | Bits, 16 -> B16 + | Bits, 32 -> B32 + | Bits, 64 -> B64 + | _ -> failwith "Bad element type" + +(* Return unsigned integer element the same width as argument. *) +let unsigned_of_elt elt = + elt_of_class_width Unsigned (elt_width elt) + +let signed_of_elt elt = + elt_of_class_width Signed (elt_width elt) + +(* Return untyped bits element the same width as argument. *) +let bits_of_elt elt = + elt_of_class_width Bits (elt_width elt) + +let non_signed_variant = function + S8 -> I8 + | S16 -> I16 + | S32 -> I32 + | S64 -> I64 + | U8 -> I8 + | U16 -> I16 + | U32 -> I32 + | U64 -> I64 + | x -> x + +let poly_unsigned_variant v = + let elclass = match elt_class v with + Poly -> Unsigned + | x -> x in + elt_of_class_width elclass (elt_width v) + +let widen_elt elt = + let w = elt_width elt + and c = elt_class elt in + elt_of_class_width c (w * 2) + +let narrow_elt elt = + let w = elt_width elt + and c = elt_class elt in + elt_of_class_width c (w / 2) + +(* If we're trying to find a mode from a "Use_operands" instruction, use the + last vector operand as the dominant mode used to invoke the correct builtin. + We must stick to this rule in neon.md. *) +let find_key_operand operands = + let rec scan opno = + match operands.(opno) with + Qreg -> Qreg + | Dreg -> Dreg + | VecArray (_, Qreg) -> Qreg + | VecArray (_, Dreg) -> Dreg + | _ -> scan (opno-1) + in + scan ((Array.length operands) - 1) + +(* Find a vecmode from a shape_elt ELT for an instruction with shape_form + SHAPE. For a Use_operands shape, if ARGPOS is passed then return the mode + for the given argument position, else determine which argument to return a + mode for automatically. *) + +let rec mode_of_elt ?argpos elt shape = + let flt = match elt_class elt with + Float | ConvClass(_, Float) -> true | _ -> false in + let idx = + match elt_width elt with + 8 -> 0 | 16 -> 1 | 32 -> 2 | 64 -> 3 | 128 -> 4 + | _ -> failwith "Bad element width" + in match shape with + All (_, Dreg) | By_scalar Dreg | Pair_result Dreg | Unary_scalar Dreg + | Binary_imm Dreg | Long_noreg Dreg | Wide_noreg Dreg -> + if flt then + [| V8QI; V4HF; V2SF; DI |].(idx) + else + [| V8QI; V4HI; V2SI; DI |].(idx) + | All (_, Qreg) | By_scalar Qreg | Pair_result Qreg | Unary_scalar Qreg + | Binary_imm Qreg | Long_noreg Qreg | Wide_noreg Qreg -> + [| V16QI; V8HI; if flt then V4SF else V4SI; V2DI; TI|].(idx) + | All (_, (Corereg | PtrTo _ | CstPtrTo _)) -> + [| QI; HI; if flt then SF else SI; DI |].(idx) + | Long | Wide | Wide_lane | Wide_scalar + | Long_imm -> + [| V8QI; V4HI; V2SI; DI |].(idx) + | Narrow | Narrow_imm -> [| V16QI; V8HI; V4SI; V2DI |].(idx) + | Use_operands ops -> + begin match argpos with + None -> mode_of_elt ?argpos elt (All (0, (find_key_operand ops))) + | Some pos -> mode_of_elt ?argpos elt (All (0, ops.(pos))) + end + | _ -> failwith "invalid shape" + +(* Modify an element type dependent on the shape of the instruction and the + operand number. *) + +let shapemap shape no = + let ident = fun x -> x in + match shape with + All _ | Use_operands _ | By_scalar _ | Pair_result _ | Unary_scalar _ + | Binary_imm _ -> ident + | Long | Long_noreg _ | Wide_scalar | Long_imm -> + [| widen_elt; ident; ident |].(no) + | Wide | Wide_noreg _ -> [| widen_elt; widen_elt; ident |].(no) + | Wide_lane -> [| widen_elt; ident; ident; ident |].(no) + | Narrow | Narrow_imm -> [| narrow_elt; ident; ident |].(no) + +(* Register type (D/Q) of an operand, based on shape and operand number. *) + +let regmap shape no = + match shape with + All (_, reg) | Long_noreg reg | Wide_noreg reg -> reg + | Long -> [| Qreg; Dreg; Dreg |].(no) + | Wide -> [| Qreg; Qreg; Dreg |].(no) + | Narrow -> [| Dreg; Qreg; Qreg |].(no) + | Wide_lane -> [| Qreg; Dreg; Dreg; Immed |].(no) + | Wide_scalar -> [| Qreg; Dreg; Corereg |].(no) + | By_scalar reg -> [| reg; reg; Dreg; Immed |].(no) + | Unary_scalar reg -> [| reg; Dreg; Immed |].(no) + | Pair_result reg -> [| VecArray (2, reg); reg; reg |].(no) + | Binary_imm reg -> [| reg; reg; Immed |].(no) + | Long_imm -> [| Qreg; Dreg; Immed |].(no) + | Narrow_imm -> [| Dreg; Qreg; Immed |].(no) + | Use_operands these -> these.(no) + +let type_for_elt shape elt no = + let elt = (shapemap shape no) elt in + let reg = regmap shape no in + let rec type_for_reg_elt reg elt = + match reg with + Dreg -> + begin match elt with + S8 -> T_int8x8 + | S16 -> T_int16x4 + | S32 -> T_int32x2 + | S64 -> T_int64x1 + | U8 -> T_uint8x8 + | U16 -> T_uint16x4 + | U32 -> T_uint32x2 + | U64 -> T_uint64x1 + | P64 -> T_poly64x1 + | P128 -> T_poly128 + | F16 -> T_float16x4 + | F32 -> T_float32x2 + | P8 -> T_poly8x8 + | P16 -> T_poly16x4 + | _ -> failwith "Bad elt type for Dreg" + end + | Qreg -> + begin match elt with + S8 -> T_int8x16 + | S16 -> T_int16x8 + | S32 -> T_int32x4 + | S64 -> T_int64x2 + | U8 -> T_uint8x16 + | U16 -> T_uint16x8 + | U32 -> T_uint32x4 + | U64 -> T_uint64x2 + | F32 -> T_float32x4 + | P8 -> T_poly8x16 + | P16 -> T_poly16x8 + | P64 -> T_poly64x2 + | P128 -> T_poly128 + | _ -> failwith "Bad elt type for Qreg" + end + | Corereg -> + begin match elt with + S8 -> T_int8 + | S16 -> T_int16 + | S32 -> T_int32 + | S64 -> T_int64 + | U8 -> T_uint8 + | U16 -> T_uint16 + | U32 -> T_uint32 + | U64 -> T_uint64 + | P8 -> T_poly8 + | P16 -> T_poly16 + | P64 -> T_poly64 + | P128 -> T_poly128 + | F32 -> T_float32 + | _ -> failwith "Bad elt type for Corereg" + end + | Immed -> + T_immediate (0, 0) + | VecArray (num, sub) -> + T_arrayof (num, type_for_reg_elt sub elt) + | PtrTo x -> + T_ptrto (type_for_reg_elt x elt) + | CstPtrTo x -> + T_ptrto (T_const (type_for_reg_elt x elt)) + (* Anything else is solely for the use of the test generator. *) + | _ -> assert false + in + type_for_reg_elt reg elt + +(* Return size of a vector type, in bits. *) +let vectype_size = function + T_int8x8 | T_int16x4 | T_int32x2 | T_int64x1 + | T_uint8x8 | T_uint16x4 | T_uint32x2 | T_uint64x1 + | T_float32x2 | T_poly8x8 | T_poly64x1 | T_poly16x4 | T_float16x4 -> 64 + | T_int8x16 | T_int16x8 | T_int32x4 | T_int64x2 + | T_uint8x16 | T_uint16x8 | T_uint32x4 | T_uint64x2 + | T_float32x4 | T_poly8x16 | T_poly64x2 | T_poly16x8 -> 128 + | _ -> raise Not_found + +let inttype_for_array num elttype = + let eltsize = vectype_size elttype in + let numwords = (num * eltsize) / 32 in + match numwords with + 4 -> B_TImode + | 6 -> B_EImode + | 8 -> B_OImode + | 12 -> B_CImode + | 16 -> B_XImode + | _ -> failwith ("no int type for size " ^ string_of_int numwords) + +(* These functions return pairs of (internal, external) types, where "internal" + types are those seen by GCC, and "external" are those seen by the assembler. + These types aren't necessarily the same, since the intrinsics can munge more + than one C type into each assembler opcode. *) + +let make_sign_invariant func shape elt = + let arity, elt' = func shape elt in + arity, non_signed_variant elt' + +(* Don't restrict any types. *) + +let elts_same make_arity shape elt = + let vtype = type_for_elt shape elt in + make_arity vtype, elt + +(* As sign_invar_*, but when sign matters. *) +let elts_same_io_lane = + elts_same (fun vtype -> Arity4 (vtype 0, vtype 0, vtype 1, vtype 2, vtype 3)) + +let elts_same_io = + elts_same (fun vtype -> Arity3 (vtype 0, vtype 0, vtype 1, vtype 2)) + +let elts_same_2_lane = + elts_same (fun vtype -> Arity3 (vtype 0, vtype 1, vtype 2, vtype 3)) + +let elts_same_3 = elts_same_2_lane + +let elts_same_2 = + elts_same (fun vtype -> Arity2 (vtype 0, vtype 1, vtype 2)) + +let elts_same_1 = + elts_same (fun vtype -> Arity1 (vtype 0, vtype 1)) + +(* Use for signed/unsigned invariant operations (i.e. where the operation + doesn't depend on the sign of the data. *) + +let sign_invar_io_lane = make_sign_invariant elts_same_io_lane +let sign_invar_io = make_sign_invariant elts_same_io +let sign_invar_2_lane = make_sign_invariant elts_same_2_lane +let sign_invar_2 = make_sign_invariant elts_same_2 +let sign_invar_1 = make_sign_invariant elts_same_1 + +(* Sign-sensitive comparison. *) + +let cmp_sign_matters shape elt = + let vtype = type_for_elt shape elt + and rtype = type_for_elt shape (unsigned_of_elt elt) 0 in + Arity2 (rtype, vtype 1, vtype 2), elt + +(* Signed/unsigned invariant comparison. *) + +let cmp_sign_invar shape elt = + let shape', elt' = cmp_sign_matters shape elt in + let elt'' = + match non_signed_variant elt' with + P8 -> I8 + | x -> x + in + shape', elt'' + +(* Comparison (VTST) where only the element width matters. *) + +let cmp_bits shape elt = + let vtype = type_for_elt shape elt + and rtype = type_for_elt shape (unsigned_of_elt elt) 0 + and bits_only = bits_of_elt elt in + Arity2 (rtype, vtype 1, vtype 2), bits_only + +let reg_shift shape elt = + let vtype = type_for_elt shape elt + and op2type = type_for_elt shape (signed_of_elt elt) 2 in + Arity2 (vtype 0, vtype 1, op2type), elt + +(* Genericised constant-shift type-generating function. *) + +let const_shift mkimm ?arity ?result shape elt = + let op2type = (shapemap shape 2) elt in + let op2width = elt_width op2type in + let op2 = mkimm op2width + and op1 = type_for_elt shape elt 1 + and r_elt = + match result with + None -> elt + | Some restriction -> restriction elt in + let rtype = type_for_elt shape r_elt 0 in + match arity with + None -> Arity2 (rtype, op1, op2), elt + | Some mkarity -> mkarity rtype op1 op2, elt + +(* Use for immediate right-shifts. *) + +let shift_right shape elt = + const_shift (fun imm -> T_immediate (1, imm)) shape elt + +let shift_right_acc shape elt = + const_shift (fun imm -> T_immediate (1, imm)) + ~arity:(fun dst op1 op2 -> Arity3 (dst, dst, op1, op2)) shape elt + +(* Use for immediate right-shifts when the operation doesn't care about + signedness. *) + +let shift_right_sign_invar = + make_sign_invariant shift_right + +(* Immediate right-shift; result is unsigned even when operand is signed. *) + +let shift_right_to_uns shape elt = + const_shift (fun imm -> T_immediate (1, imm)) ~result:unsigned_of_elt + shape elt + +(* Immediate left-shift. *) + +let shift_left shape elt = + const_shift (fun imm -> T_immediate (0, imm - 1)) shape elt + +(* Immediate left-shift, unsigned result. *) + +let shift_left_to_uns shape elt = + const_shift (fun imm -> T_immediate (0, imm - 1)) ~result:unsigned_of_elt + shape elt + +(* Immediate left-shift, don't care about signs. *) + +let shift_left_sign_invar = + make_sign_invariant shift_left + +(* Shift left/right and insert: only element size matters. *) + +let shift_insert shape elt = + let arity, elt = + const_shift (fun imm -> T_immediate (1, imm)) + ~arity:(fun dst op1 op2 -> Arity3 (dst, dst, op1, op2)) shape elt in + arity, bits_of_elt elt + +(* Get/set lane. *) + +let get_lane shape elt = + let vtype = type_for_elt shape elt in + Arity2 (vtype 0, vtype 1, vtype 2), + (match elt with P8 -> U8 | P16 -> U16 | S32 | U32 | F32 -> B32 | x -> x) + +let set_lane shape elt = + let vtype = type_for_elt shape elt in + Arity3 (vtype 0, vtype 1, vtype 2, vtype 3), bits_of_elt elt + +let set_lane_notype shape elt = + let vtype = type_for_elt shape elt in + Arity3 (vtype 0, vtype 1, vtype 2, vtype 3), NoElts + +let create_vector shape elt = + let vtype = type_for_elt shape U64 1 + and rtype = type_for_elt shape elt 0 in + Arity1 (rtype, vtype), elt + +let conv make_arity shape elt = + let edest, esrc = match elt with + Conv (edest, esrc) | Cast (edest, esrc) -> edest, esrc + | _ -> failwith "Non-conversion element in conversion" in + let vtype = type_for_elt shape esrc + and rtype = type_for_elt shape edest 0 in + make_arity rtype vtype, elt + +let conv_1 = conv (fun rtype vtype -> Arity1 (rtype, vtype 1)) +let conv_2 = conv (fun rtype vtype -> Arity2 (rtype, vtype 1, vtype 2)) + +(* Operation has an unsigned result even if operands are signed. *) + +let dst_unsign make_arity shape elt = + let vtype = type_for_elt shape elt + and rtype = type_for_elt shape (unsigned_of_elt elt) 0 in + make_arity rtype vtype, elt + +let dst_unsign_1 = dst_unsign (fun rtype vtype -> Arity1 (rtype, vtype 1)) + +let make_bits_only func shape elt = + let arity, elt' = func shape elt in + arity, bits_of_elt elt' + +(* Extend operation. *) + +let extend shape elt = + let vtype = type_for_elt shape elt in + Arity3 (vtype 0, vtype 1, vtype 2, vtype 3), bits_of_elt elt + +(* Table look-up operations. Operand 2 is signed/unsigned for signed/unsigned + integer ops respectively, or unsigned for polynomial ops. *) + +let table mkarity shape elt = + let vtype = type_for_elt shape elt in + let op2 = type_for_elt shape (poly_unsigned_variant elt) 2 in + mkarity vtype op2, bits_of_elt elt + +let table_2 = table (fun vtype op2 -> Arity2 (vtype 0, vtype 1, op2)) +let table_io = table (fun vtype op2 -> Arity3 (vtype 0, vtype 0, vtype 1, op2)) + +(* Operations where only bits matter. *) + +let bits_1 = make_bits_only elts_same_1 +let bits_2 = make_bits_only elts_same_2 +let bits_3 = make_bits_only elts_same_3 + +(* Store insns. *) +let store_1 shape elt = + let vtype = type_for_elt shape elt in + Arity2 (T_void, vtype 0, vtype 1), bits_of_elt elt + +let store_3 shape elt = + let vtype = type_for_elt shape elt in + Arity3 (T_void, vtype 0, vtype 1, vtype 2), bits_of_elt elt + +let make_notype func shape elt = + let arity, _ = func shape elt in + arity, NoElts + +let notype_1 = make_notype elts_same_1 +let notype_2 = make_notype elts_same_2 +let notype_3 = make_notype elts_same_3 + +(* Bit-select operations (first operand is unsigned int). *) + +let bit_select shape elt = + let vtype = type_for_elt shape elt + and itype = type_for_elt shape (unsigned_of_elt elt) in + Arity3 (vtype 0, itype 1, vtype 2, vtype 3), NoElts + +(* Common lists of supported element types. *) + +let s_8_32 = [S8; S16; S32] +let u_8_32 = [U8; U16; U32] +let su_8_32 = [S8; S16; S32; U8; U16; U32] +let su_8_64 = S64 :: U64 :: su_8_32 +let su_16_64 = [S16; S32; S64; U16; U32; U64] +let pf_su_8_16 = [P8; P16; S8; S16; U8; U16] +let pf_su_8_32 = P8 :: P16 :: F32 :: su_8_32 +let pf_su_8_64 = P8 :: P16 :: F32 :: su_8_64 +let suf_32 = [S32; U32; F32] + +let ops = + [ + (* Addition. *) + Vadd, [], All (3, Dreg), "vadd", sign_invar_2, F32 :: su_8_32; + Vadd, [No_op], All (3, Dreg), "vadd", sign_invar_2, [S64; U64]; + Vadd, [], All (3, Qreg), "vaddQ", sign_invar_2, F32 :: su_8_64; + Vadd, [], Long, "vaddl", elts_same_2, su_8_32; + Vadd, [], Wide, "vaddw", elts_same_2, su_8_32; + Vadd, [Halving], All (3, Dreg), "vhadd", elts_same_2, su_8_32; + Vadd, [Halving], All (3, Qreg), "vhaddQ", elts_same_2, su_8_32; + Vadd, [Instruction_name ["vrhadd"]; Rounding; Halving], + All (3, Dreg), "vRhadd", elts_same_2, su_8_32; + Vadd, [Instruction_name ["vrhadd"]; Rounding; Halving], + All (3, Qreg), "vRhaddQ", elts_same_2, su_8_32; + Vadd, [Saturating], All (3, Dreg), "vqadd", elts_same_2, su_8_64; + Vadd, [Saturating], All (3, Qreg), "vqaddQ", elts_same_2, su_8_64; + Vadd, [High_half], Narrow, "vaddhn", sign_invar_2, su_16_64; + Vadd, [Instruction_name ["vraddhn"]; Rounding; High_half], + Narrow, "vRaddhn", sign_invar_2, su_16_64; + + (* Multiplication. *) + Vmul, [], All (3, Dreg), "vmul", sign_invar_2, P8 :: F32 :: su_8_32; + Vmul, [], All (3, Qreg), "vmulQ", sign_invar_2, P8 :: F32 :: su_8_32; + Vmul, [Saturating; Doubling; High_half], All (3, Dreg), "vqdmulh", + elts_same_2, [S16; S32]; + Vmul, [Saturating; Doubling; High_half], All (3, Qreg), "vqdmulhQ", + elts_same_2, [S16; S32]; + Vmul, + [Saturating; Rounding; Doubling; High_half; + Instruction_name ["vqrdmulh"]], + All (3, Dreg), "vqRdmulh", + elts_same_2, [S16; S32]; + Vmul, + [Saturating; Rounding; Doubling; High_half; + Instruction_name ["vqrdmulh"]], + All (3, Qreg), "vqRdmulhQ", + elts_same_2, [S16; S32]; + Vmul, [], Long, "vmull", elts_same_2, P8 :: su_8_32; + Vmul, [Saturating; Doubling], Long, "vqdmull", elts_same_2, [S16; S32]; + + (* Multiply-accumulate. *) + Vmla, [], All (3, Dreg), "vmla", sign_invar_io, F32 :: su_8_32; + Vmla, [], All (3, Qreg), "vmlaQ", sign_invar_io, F32 :: su_8_32; + Vmla, [], Long, "vmlal", elts_same_io, su_8_32; + Vmla, [Saturating; Doubling], Long, "vqdmlal", elts_same_io, [S16; S32]; + + (* Multiply-subtract. *) + Vmls, [], All (3, Dreg), "vmls", sign_invar_io, F32 :: su_8_32; + Vmls, [], All (3, Qreg), "vmlsQ", sign_invar_io, F32 :: su_8_32; + Vmls, [], Long, "vmlsl", elts_same_io, su_8_32; + Vmls, [Saturating; Doubling], Long, "vqdmlsl", elts_same_io, [S16; S32]; + + (* Fused-multiply-accumulate. *) + Vfma, [Requires_feature "FMA"], All (3, Dreg), "vfma", elts_same_io, [F32]; + Vfma, [Requires_feature "FMA"], All (3, Qreg), "vfmaQ", elts_same_io, [F32]; + Vfms, [Requires_feature "FMA"], All (3, Dreg), "vfms", elts_same_io, [F32]; + Vfms, [Requires_feature "FMA"], All (3, Qreg), "vfmsQ", elts_same_io, [F32]; + + (* Round to integral. *) + Vrintn, [Builtin_name "vrintn"; Requires_arch 8], Use_operands [| Dreg; Dreg |], + "vrndn", elts_same_1, [F32]; + Vrintn, [Builtin_name "vrintn"; Requires_arch 8], Use_operands [| Qreg; Qreg |], + "vrndqn", elts_same_1, [F32]; + Vrinta, [Builtin_name "vrinta"; Requires_arch 8], Use_operands [| Dreg; Dreg |], + "vrnda", elts_same_1, [F32]; + Vrinta, [Builtin_name "vrinta"; Requires_arch 8], Use_operands [| Qreg; Qreg |], + "vrndqa", elts_same_1, [F32]; + Vrintp, [Builtin_name "vrintp"; Requires_arch 8], Use_operands [| Dreg; Dreg |], + "vrndp", elts_same_1, [F32]; + Vrintp, [Builtin_name "vrintp"; Requires_arch 8], Use_operands [| Qreg; Qreg |], + "vrndqp", elts_same_1, [F32]; + Vrintm, [Builtin_name "vrintm"; Requires_arch 8], Use_operands [| Dreg; Dreg |], + "vrndm", elts_same_1, [F32]; + Vrintm, [Builtin_name "vrintm"; Requires_arch 8], Use_operands [| Qreg; Qreg |], + "vrndqm", elts_same_1, [F32]; + Vrintz, [Builtin_name "vrintz"; Requires_arch 8], Use_operands [| Dreg; Dreg |], + "vrnd", elts_same_1, [F32]; + Vrintz, [Builtin_name "vrintz"; Requires_arch 8], Use_operands [| Qreg; Qreg |], + "vrndq", elts_same_1, [F32]; + (* Subtraction. *) + Vsub, [], All (3, Dreg), "vsub", sign_invar_2, F32 :: su_8_32; + Vsub, [No_op], All (3, Dreg), "vsub", sign_invar_2, [S64; U64]; + Vsub, [], All (3, Qreg), "vsubQ", sign_invar_2, F32 :: su_8_64; + Vsub, [], Long, "vsubl", elts_same_2, su_8_32; + Vsub, [], Wide, "vsubw", elts_same_2, su_8_32; + Vsub, [Halving], All (3, Dreg), "vhsub", elts_same_2, su_8_32; + Vsub, [Halving], All (3, Qreg), "vhsubQ", elts_same_2, su_8_32; + Vsub, [Saturating], All (3, Dreg), "vqsub", elts_same_2, su_8_64; + Vsub, [Saturating], All (3, Qreg), "vqsubQ", elts_same_2, su_8_64; + Vsub, [High_half], Narrow, "vsubhn", sign_invar_2, su_16_64; + Vsub, [Instruction_name ["vrsubhn"]; Rounding; High_half], + Narrow, "vRsubhn", sign_invar_2, su_16_64; + + (* Comparison, equal. *) + Vceq, [], All (3, Dreg), "vceq", cmp_sign_invar, P8 :: F32 :: su_8_32; + Vceq, [], All (3, Qreg), "vceqQ", cmp_sign_invar, P8 :: F32 :: su_8_32; + + (* Comparison, greater-than or equal. *) + Vcge, [], All (3, Dreg), "vcge", cmp_sign_matters, F32 :: s_8_32; + Vcge, [Instruction_name ["vcge"]; Builtin_name "vcgeu"], + All (3, Dreg), "vcge", cmp_sign_matters, + u_8_32; + Vcge, [], All (3, Qreg), "vcgeQ", cmp_sign_matters, F32 :: s_8_32; + Vcge, [Instruction_name ["vcge"]; Builtin_name "vcgeu"], + All (3, Qreg), "vcgeQ", cmp_sign_matters, + u_8_32; + + (* Comparison, less-than or equal. *) + Vcle, [Flipped "vcge"], All (3, Dreg), "vcle", cmp_sign_matters, + F32 :: s_8_32; + Vcle, [Instruction_name ["vcge"]; Flipped "vcgeu"], + All (3, Dreg), "vcle", cmp_sign_matters, + u_8_32; + Vcle, [Instruction_name ["vcge"]; Flipped "vcgeQ"], + All (3, Qreg), "vcleQ", cmp_sign_matters, + F32 :: s_8_32; + Vcle, [Instruction_name ["vcge"]; Flipped "vcgeuQ"], + All (3, Qreg), "vcleQ", cmp_sign_matters, + u_8_32; + + (* Comparison, greater-than. *) + Vcgt, [], All (3, Dreg), "vcgt", cmp_sign_matters, F32 :: s_8_32; + Vcgt, [Instruction_name ["vcgt"]; Builtin_name "vcgtu"], + All (3, Dreg), "vcgt", cmp_sign_matters, + u_8_32; + Vcgt, [], All (3, Qreg), "vcgtQ", cmp_sign_matters, F32 :: s_8_32; + Vcgt, [Instruction_name ["vcgt"]; Builtin_name "vcgtu"], + All (3, Qreg), "vcgtQ", cmp_sign_matters, + u_8_32; + + (* Comparison, less-than. *) + Vclt, [Flipped "vcgt"], All (3, Dreg), "vclt", cmp_sign_matters, + F32 :: s_8_32; + Vclt, [Instruction_name ["vcgt"]; Flipped "vcgtu"], + All (3, Dreg), "vclt", cmp_sign_matters, + u_8_32; + Vclt, [Instruction_name ["vcgt"]; Flipped "vcgtQ"], + All (3, Qreg), "vcltQ", cmp_sign_matters, + F32 :: s_8_32; + Vclt, [Instruction_name ["vcgt"]; Flipped "vcgtuQ"], + All (3, Qreg), "vcltQ", cmp_sign_matters, + u_8_32; + + (* Compare absolute greater-than or equal. *) + Vcage, [Instruction_name ["vacge"]], + All (3, Dreg), "vcage", cmp_sign_matters, [F32]; + Vcage, [Instruction_name ["vacge"]], + All (3, Qreg), "vcageQ", cmp_sign_matters, [F32]; + + (* Compare absolute less-than or equal. *) + Vcale, [Instruction_name ["vacge"]; Flipped "vcage"], + All (3, Dreg), "vcale", cmp_sign_matters, [F32]; + Vcale, [Instruction_name ["vacge"]; Flipped "vcageQ"], + All (3, Qreg), "vcaleQ", cmp_sign_matters, [F32]; + + (* Compare absolute greater-than or equal. *) + Vcagt, [Instruction_name ["vacgt"]], + All (3, Dreg), "vcagt", cmp_sign_matters, [F32]; + Vcagt, [Instruction_name ["vacgt"]], + All (3, Qreg), "vcagtQ", cmp_sign_matters, [F32]; + + (* Compare absolute less-than or equal. *) + Vcalt, [Instruction_name ["vacgt"]; Flipped "vcagt"], + All (3, Dreg), "vcalt", cmp_sign_matters, [F32]; + Vcalt, [Instruction_name ["vacgt"]; Flipped "vcagtQ"], + All (3, Qreg), "vcaltQ", cmp_sign_matters, [F32]; + + (* Test bits. *) + Vtst, [], All (3, Dreg), "vtst", cmp_bits, P8 :: su_8_32; + Vtst, [], All (3, Qreg), "vtstQ", cmp_bits, P8 :: su_8_32; + + (* Absolute difference. *) + Vabd, [], All (3, Dreg), "vabd", elts_same_2, F32 :: su_8_32; + Vabd, [], All (3, Qreg), "vabdQ", elts_same_2, F32 :: su_8_32; + Vabd, [], Long, "vabdl", elts_same_2, su_8_32; + + (* Absolute difference and accumulate. *) + Vaba, [], All (3, Dreg), "vaba", elts_same_io, su_8_32; + Vaba, [], All (3, Qreg), "vabaQ", elts_same_io, su_8_32; + Vaba, [], Long, "vabal", elts_same_io, su_8_32; + + (* Max. *) + Vmax, [], All (3, Dreg), "vmax", elts_same_2, F32 :: su_8_32; + Vmax, [], All (3, Qreg), "vmaxQ", elts_same_2, F32 :: su_8_32; + + (* Min. *) + Vmin, [], All (3, Dreg), "vmin", elts_same_2, F32 :: su_8_32; + Vmin, [], All (3, Qreg), "vminQ", elts_same_2, F32 :: su_8_32; + + (* Pairwise add. *) + Vpadd, [], All (3, Dreg), "vpadd", sign_invar_2, F32 :: su_8_32; + Vpadd, [], Long_noreg Dreg, "vpaddl", elts_same_1, su_8_32; + Vpadd, [], Long_noreg Qreg, "vpaddlQ", elts_same_1, su_8_32; + + (* Pairwise add, widen and accumulate. *) + Vpada, [], Wide_noreg Dreg, "vpadal", elts_same_2, su_8_32; + Vpada, [], Wide_noreg Qreg, "vpadalQ", elts_same_2, su_8_32; + + (* Folding maximum, minimum. *) + Vpmax, [], All (3, Dreg), "vpmax", elts_same_2, F32 :: su_8_32; + Vpmin, [], All (3, Dreg), "vpmin", elts_same_2, F32 :: su_8_32; + + (* Reciprocal step. *) + Vrecps, [], All (3, Dreg), "vrecps", elts_same_2, [F32]; + Vrecps, [], All (3, Qreg), "vrecpsQ", elts_same_2, [F32]; + Vrsqrts, [], All (3, Dreg), "vrsqrts", elts_same_2, [F32]; + Vrsqrts, [], All (3, Qreg), "vrsqrtsQ", elts_same_2, [F32]; + + (* Vector shift left. *) + Vshl, [], All (3, Dreg), "vshl", reg_shift, su_8_64; + Vshl, [], All (3, Qreg), "vshlQ", reg_shift, su_8_64; + Vshl, [Instruction_name ["vrshl"]; Rounding], + All (3, Dreg), "vRshl", reg_shift, su_8_64; + Vshl, [Instruction_name ["vrshl"]; Rounding], + All (3, Qreg), "vRshlQ", reg_shift, su_8_64; + Vshl, [Saturating], All (3, Dreg), "vqshl", reg_shift, su_8_64; + Vshl, [Saturating], All (3, Qreg), "vqshlQ", reg_shift, su_8_64; + Vshl, [Instruction_name ["vqrshl"]; Saturating; Rounding], + All (3, Dreg), "vqRshl", reg_shift, su_8_64; + Vshl, [Instruction_name ["vqrshl"]; Saturating; Rounding], + All (3, Qreg), "vqRshlQ", reg_shift, su_8_64; + + (* Vector shift right by constant. *) + Vshr_n, [], Binary_imm Dreg, "vshr_n", shift_right, su_8_64; + Vshr_n, [], Binary_imm Qreg, "vshrQ_n", shift_right, su_8_64; + Vshr_n, [Instruction_name ["vrshr"]; Rounding], Binary_imm Dreg, + "vRshr_n", shift_right, su_8_64; + Vshr_n, [Instruction_name ["vrshr"]; Rounding], Binary_imm Qreg, + "vRshrQ_n", shift_right, su_8_64; + Vshr_n, [], Narrow_imm, "vshrn_n", shift_right_sign_invar, su_16_64; + Vshr_n, [Instruction_name ["vrshrn"]; Rounding], Narrow_imm, "vRshrn_n", + shift_right_sign_invar, su_16_64; + Vshr_n, [Saturating], Narrow_imm, "vqshrn_n", shift_right, su_16_64; + Vshr_n, [Instruction_name ["vqrshrn"]; Saturating; Rounding], Narrow_imm, + "vqRshrn_n", shift_right, su_16_64; + Vshr_n, [Saturating; Dst_unsign], Narrow_imm, "vqshrun_n", + shift_right_to_uns, [S16; S32; S64]; + Vshr_n, [Instruction_name ["vqrshrun"]; Saturating; Dst_unsign; Rounding], + Narrow_imm, "vqRshrun_n", shift_right_to_uns, [S16; S32; S64]; + + (* Vector shift left by constant. *) + Vshl_n, [], Binary_imm Dreg, "vshl_n", shift_left_sign_invar, su_8_64; + Vshl_n, [], Binary_imm Qreg, "vshlQ_n", shift_left_sign_invar, su_8_64; + Vshl_n, [Saturating], Binary_imm Dreg, "vqshl_n", shift_left, su_8_64; + Vshl_n, [Saturating], Binary_imm Qreg, "vqshlQ_n", shift_left, su_8_64; + Vshl_n, [Saturating; Dst_unsign], Binary_imm Dreg, "vqshlu_n", + shift_left_to_uns, [S8; S16; S32; S64]; + Vshl_n, [Saturating; Dst_unsign], Binary_imm Qreg, "vqshluQ_n", + shift_left_to_uns, [S8; S16; S32; S64]; + Vshl_n, [], Long_imm, "vshll_n", shift_left, su_8_32; + + (* Vector shift right by constant and accumulate. *) + Vsra_n, [], Binary_imm Dreg, "vsra_n", shift_right_acc, su_8_64; + Vsra_n, [], Binary_imm Qreg, "vsraQ_n", shift_right_acc, su_8_64; + Vsra_n, [Instruction_name ["vrsra"]; Rounding], Binary_imm Dreg, + "vRsra_n", shift_right_acc, su_8_64; + Vsra_n, [Instruction_name ["vrsra"]; Rounding], Binary_imm Qreg, + "vRsraQ_n", shift_right_acc, su_8_64; + + (* Vector shift right and insert. *) + Vsri, [Requires_feature "CRYPTO"], Use_operands [| Dreg; Dreg; Immed |], "vsri_n", shift_insert, + [P64]; + Vsri, [], Use_operands [| Dreg; Dreg; Immed |], "vsri_n", shift_insert, + P8 :: P16 :: su_8_64; + Vsri, [Requires_feature "CRYPTO"], Use_operands [| Qreg; Qreg; Immed |], "vsriQ_n", shift_insert, + [P64]; + Vsri, [], Use_operands [| Qreg; Qreg; Immed |], "vsriQ_n", shift_insert, + P8 :: P16 :: su_8_64; + + (* Vector shift left and insert. *) + Vsli, [Requires_feature "CRYPTO"], Use_operands [| Dreg; Dreg; Immed |], "vsli_n", shift_insert, + [P64]; + Vsli, [], Use_operands [| Dreg; Dreg; Immed |], "vsli_n", shift_insert, + P8 :: P16 :: su_8_64; + Vsli, [Requires_feature "CRYPTO"], Use_operands [| Qreg; Qreg; Immed |], "vsliQ_n", shift_insert, + [P64]; + Vsli, [], Use_operands [| Qreg; Qreg; Immed |], "vsliQ_n", shift_insert, + P8 :: P16 :: su_8_64; + + (* Absolute value. *) + Vabs, [], All (2, Dreg), "vabs", elts_same_1, [S8; S16; S32; F32]; + Vabs, [], All (2, Qreg), "vabsQ", elts_same_1, [S8; S16; S32; F32]; + Vabs, [Saturating], All (2, Dreg), "vqabs", elts_same_1, [S8; S16; S32]; + Vabs, [Saturating], All (2, Qreg), "vqabsQ", elts_same_1, [S8; S16; S32]; + + (* Negate. *) + Vneg, [], All (2, Dreg), "vneg", elts_same_1, [S8; S16; S32; F32]; + Vneg, [], All (2, Qreg), "vnegQ", elts_same_1, [S8; S16; S32; F32]; + Vneg, [Saturating], All (2, Dreg), "vqneg", elts_same_1, [S8; S16; S32]; + Vneg, [Saturating], All (2, Qreg), "vqnegQ", elts_same_1, [S8; S16; S32]; + + (* Bitwise not. *) + Vmvn, [], All (2, Dreg), "vmvn", notype_1, P8 :: su_8_32; + Vmvn, [], All (2, Qreg), "vmvnQ", notype_1, P8 :: su_8_32; + + (* Count leading sign bits. *) + Vcls, [], All (2, Dreg), "vcls", elts_same_1, [S8; S16; S32]; + Vcls, [], All (2, Qreg), "vclsQ", elts_same_1, [S8; S16; S32]; + + (* Count leading zeros. *) + Vclz, [], All (2, Dreg), "vclz", sign_invar_1, su_8_32; + Vclz, [], All (2, Qreg), "vclzQ", sign_invar_1, su_8_32; + + (* Count number of set bits. *) + Vcnt, [], All (2, Dreg), "vcnt", bits_1, [P8; S8; U8]; + Vcnt, [], All (2, Qreg), "vcntQ", bits_1, [P8; S8; U8]; + + (* Reciprocal estimate. *) + Vrecpe, [], All (2, Dreg), "vrecpe", elts_same_1, [U32; F32]; + Vrecpe, [], All (2, Qreg), "vrecpeQ", elts_same_1, [U32; F32]; + + (* Reciprocal square-root estimate. *) + Vrsqrte, [], All (2, Dreg), "vrsqrte", elts_same_1, [U32; F32]; + Vrsqrte, [], All (2, Qreg), "vrsqrteQ", elts_same_1, [U32; F32]; + + (* Get lanes from a vector. *) + Vget_lane, + [InfoWord; Disassembles_as [Use_operands [| Corereg; Element_of_dreg |]]; + Instruction_name ["vmov"]], + Use_operands [| Corereg; Dreg; Immed |], + "vget_lane", get_lane, pf_su_8_32; + Vget_lane, + [No_op; + InfoWord; + Disassembles_as [Use_operands [| Corereg; Corereg; Dreg |]]; + Instruction_name ["vmov"]; Const_valuator (fun _ -> 0)], + Use_operands [| Corereg; Dreg; Immed |], + "vget_lane", notype_2, [S64; U64]; + Vget_lane, + [InfoWord; Disassembles_as [Use_operands [| Corereg; Element_of_dreg |]]; + Instruction_name ["vmov"]], + Use_operands [| Corereg; Qreg; Immed |], + "vgetQ_lane", get_lane, pf_su_8_32; + Vget_lane, + [InfoWord; + Disassembles_as [Use_operands [| Corereg; Corereg; Dreg |]]; + Instruction_name ["vmov"; "fmrrd"]; Const_valuator (fun _ -> 0); + Fixed_core_reg], + Use_operands [| Corereg; Qreg; Immed |], + "vgetQ_lane", notype_2, [S64; U64]; + + (* Set lanes in a vector. *) + Vset_lane, [Disassembles_as [Use_operands [| Element_of_dreg; Corereg |]]; + Instruction_name ["vmov"]], + Use_operands [| Dreg; Corereg; Dreg; Immed |], "vset_lane", + set_lane, pf_su_8_32; + Vset_lane, [No_op; + Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]]; + Instruction_name ["vmov"]; Const_valuator (fun _ -> 0)], + Use_operands [| Dreg; Corereg; Dreg; Immed |], "vset_lane", + set_lane_notype, [S64; U64]; + Vset_lane, [Disassembles_as [Use_operands [| Element_of_dreg; Corereg |]]; + Instruction_name ["vmov"]], + Use_operands [| Qreg; Corereg; Qreg; Immed |], "vsetQ_lane", + set_lane, pf_su_8_32; + Vset_lane, [Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]]; + Instruction_name ["vmov"]; Const_valuator (fun _ -> 0)], + Use_operands [| Qreg; Corereg; Qreg; Immed |], "vsetQ_lane", + set_lane_notype, [S64; U64]; + + (* Create vector from literal bit pattern. *) + Vcreate, + [Requires_feature "CRYPTO"; No_op], (* Not really, but it can yield various things that are too + hard for the test generator at this time. *) + Use_operands [| Dreg; Corereg |], "vcreate", create_vector, + [P64]; + Vcreate, + [No_op], (* Not really, but it can yield various things that are too + hard for the test generator at this time. *) + Use_operands [| Dreg; Corereg |], "vcreate", create_vector, + pf_su_8_64; + + (* Set all lanes to the same value. *) + Vdup_n, + [Disassembles_as [Use_operands [| Dreg; + Alternatives [ Corereg; + Element_of_dreg ] |]]], + Use_operands [| Dreg; Corereg |], "vdup_n", bits_1, + pf_su_8_32; + Vdup_n, + [No_op; Requires_feature "CRYPTO"; + Instruction_name ["vmov"]; + Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]]], + Use_operands [| Dreg; Corereg |], "vdup_n", notype_1, + [P64]; + Vdup_n, + [No_op; + Instruction_name ["vmov"]; + Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]]], + Use_operands [| Dreg; Corereg |], "vdup_n", notype_1, + [S64; U64]; + Vdup_n, + [No_op; Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| Qreg; + Alternatives [ Corereg; + Element_of_dreg ] |]]], + Use_operands [| Qreg; Corereg |], "vdupQ_n", bits_1, + [P64]; + Vdup_n, + [Disassembles_as [Use_operands [| Qreg; + Alternatives [ Corereg; + Element_of_dreg ] |]]], + Use_operands [| Qreg; Corereg |], "vdupQ_n", bits_1, + pf_su_8_32; + Vdup_n, + [No_op; + Instruction_name ["vmov"]; + Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]; + Use_operands [| Dreg; Corereg; Corereg |]]], + Use_operands [| Qreg; Corereg |], "vdupQ_n", notype_1, + [S64; U64]; + + (* These are just aliases for the above. *) + Vmov_n, + [Builtin_name "vdup_n"; + Disassembles_as [Use_operands [| Dreg; + Alternatives [ Corereg; + Element_of_dreg ] |]]], + Use_operands [| Dreg; Corereg |], + "vmov_n", bits_1, pf_su_8_32; + Vmov_n, + [No_op; + Builtin_name "vdup_n"; + Instruction_name ["vmov"]; + Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]]], + Use_operands [| Dreg; Corereg |], + "vmov_n", notype_1, [S64; U64]; + Vmov_n, + [Builtin_name "vdupQ_n"; + Disassembles_as [Use_operands [| Qreg; + Alternatives [ Corereg; + Element_of_dreg ] |]]], + Use_operands [| Qreg; Corereg |], + "vmovQ_n", bits_1, pf_su_8_32; + Vmov_n, + [No_op; + Builtin_name "vdupQ_n"; + Instruction_name ["vmov"]; + Disassembles_as [Use_operands [| Dreg; Corereg; Corereg |]; + Use_operands [| Dreg; Corereg; Corereg |]]], + Use_operands [| Qreg; Corereg |], + "vmovQ_n", notype_1, [S64; U64]; + + (* Duplicate, lane version. We can't use Use_operands here because the + rightmost register (always Dreg) would be picked up by find_key_operand, + when we want the leftmost register to be used in this case (otherwise + the modes are indistinguishable in neon.md, etc. *) + Vdup_lane, + [Disassembles_as [Use_operands [| Dreg; Element_of_dreg |]]], + Unary_scalar Dreg, "vdup_lane", bits_2, pf_su_8_32; + Vdup_lane, + [No_op; Requires_feature "CRYPTO"; Const_valuator (fun _ -> 0)], + Unary_scalar Dreg, "vdup_lane", bits_2, [P64]; + Vdup_lane, + [No_op; Const_valuator (fun _ -> 0)], + Unary_scalar Dreg, "vdup_lane", bits_2, [S64; U64]; + Vdup_lane, + [Disassembles_as [Use_operands [| Qreg; Element_of_dreg |]]], + Unary_scalar Qreg, "vdupQ_lane", bits_2, pf_su_8_32; + Vdup_lane, + [No_op; Requires_feature "CRYPTO"; Const_valuator (fun _ -> 0)], + Unary_scalar Qreg, "vdupQ_lane", bits_2, [P64]; + Vdup_lane, + [No_op; Const_valuator (fun _ -> 0)], + Unary_scalar Qreg, "vdupQ_lane", bits_2, [S64; U64]; + + (* Combining vectors. *) + Vcombine, [Requires_feature "CRYPTO"; No_op], + Use_operands [| Qreg; Dreg; Dreg |], "vcombine", notype_2, + [P64]; + Vcombine, [No_op], + Use_operands [| Qreg; Dreg; Dreg |], "vcombine", notype_2, + pf_su_8_64; + + (* Splitting vectors. *) + Vget_high, [Requires_feature "CRYPTO"; No_op], + Use_operands [| Dreg; Qreg |], "vget_high", + notype_1, [P64]; + Vget_high, [No_op], + Use_operands [| Dreg; Qreg |], "vget_high", + notype_1, pf_su_8_64; + Vget_low, [Instruction_name ["vmov"]; + Disassembles_as [Use_operands [| Dreg; Dreg |]]; + Fixed_vector_reg], + Use_operands [| Dreg; Qreg |], "vget_low", + notype_1, pf_su_8_32; + Vget_low, [Requires_feature "CRYPTO"; No_op], + Use_operands [| Dreg; Qreg |], "vget_low", + notype_1, [P64]; + Vget_low, [No_op], + Use_operands [| Dreg; Qreg |], "vget_low", + notype_1, [S64; U64]; + + (* Conversions. *) + Vcvt, [InfoWord], All (2, Dreg), "vcvt", conv_1, + [Conv (S32, F32); Conv (U32, F32); Conv (F32, S32); Conv (F32, U32)]; + Vcvt, [InfoWord], All (2, Qreg), "vcvtQ", conv_1, + [Conv (S32, F32); Conv (U32, F32); Conv (F32, S32); Conv (F32, U32)]; + Vcvt, [Builtin_name "vcvt" ; Requires_FP_bit 1], + Use_operands [| Dreg; Qreg; |], "vcvt", conv_1, [Conv (F16, F32)]; + Vcvt, [Builtin_name "vcvt" ; Requires_FP_bit 1], + Use_operands [| Qreg; Dreg; |], "vcvt", conv_1, [Conv (F32, F16)]; + Vcvt_n, [InfoWord], Use_operands [| Dreg; Dreg; Immed |], "vcvt_n", conv_2, + [Conv (S32, F32); Conv (U32, F32); Conv (F32, S32); Conv (F32, U32)]; + Vcvt_n, [InfoWord], Use_operands [| Qreg; Qreg; Immed |], "vcvtQ_n", conv_2, + [Conv (S32, F32); Conv (U32, F32); Conv (F32, S32); Conv (F32, U32)]; + + (* Move, narrowing. *) + Vmovn, [Disassembles_as [Use_operands [| Dreg; Qreg |]]], + Narrow, "vmovn", sign_invar_1, su_16_64; + Vmovn, [Disassembles_as [Use_operands [| Dreg; Qreg |]]; Saturating], + Narrow, "vqmovn", elts_same_1, su_16_64; + Vmovn, + [Disassembles_as [Use_operands [| Dreg; Qreg |]]; Saturating; Dst_unsign], + Narrow, "vqmovun", dst_unsign_1, + [S16; S32; S64]; + + (* Move, long. *) + Vmovl, [Disassembles_as [Use_operands [| Qreg; Dreg |]]], + Long, "vmovl", elts_same_1, su_8_32; + + (* Table lookup. *) + Vtbl 1, + [Instruction_name ["vtbl"]; + Disassembles_as [Use_operands [| Dreg; VecArray (1, Dreg); Dreg |]]], + Use_operands [| Dreg; Dreg; Dreg |], "vtbl1", table_2, [U8; S8; P8]; + Vtbl 2, [Instruction_name ["vtbl"]], + Use_operands [| Dreg; VecArray (2, Dreg); Dreg |], "vtbl2", table_2, + [U8; S8; P8]; + Vtbl 3, [Instruction_name ["vtbl"]], + Use_operands [| Dreg; VecArray (3, Dreg); Dreg |], "vtbl3", table_2, + [U8; S8; P8]; + Vtbl 4, [Instruction_name ["vtbl"]], + Use_operands [| Dreg; VecArray (4, Dreg); Dreg |], "vtbl4", table_2, + [U8; S8; P8]; + + (* Extended table lookup. *) + Vtbx 1, + [Instruction_name ["vtbx"]; + Disassembles_as [Use_operands [| Dreg; VecArray (1, Dreg); Dreg |]]], + Use_operands [| Dreg; Dreg; Dreg |], "vtbx1", table_io, [U8; S8; P8]; + Vtbx 2, [Instruction_name ["vtbx"]], + Use_operands [| Dreg; VecArray (2, Dreg); Dreg |], "vtbx2", table_io, + [U8; S8; P8]; + Vtbx 3, [Instruction_name ["vtbx"]], + Use_operands [| Dreg; VecArray (3, Dreg); Dreg |], "vtbx3", table_io, + [U8; S8; P8]; + Vtbx 4, [Instruction_name ["vtbx"]], + Use_operands [| Dreg; VecArray (4, Dreg); Dreg |], "vtbx4", table_io, + [U8; S8; P8]; + + (* Multiply, lane. (note: these were undocumented at the time of + writing). *) + Vmul_lane, [], By_scalar Dreg, "vmul_lane", sign_invar_2_lane, + [S16; S32; U16; U32; F32]; + Vmul_lane, [], By_scalar Qreg, "vmulQ_lane", sign_invar_2_lane, + [S16; S32; U16; U32; F32]; + + (* Multiply-accumulate, lane. *) + Vmla_lane, [], By_scalar Dreg, "vmla_lane", sign_invar_io_lane, + [S16; S32; U16; U32; F32]; + Vmla_lane, [], By_scalar Qreg, "vmlaQ_lane", sign_invar_io_lane, + [S16; S32; U16; U32; F32]; + Vmla_lane, [], Wide_lane, "vmlal_lane", elts_same_io_lane, + [S16; S32; U16; U32]; + Vmla_lane, [Saturating; Doubling], Wide_lane, "vqdmlal_lane", + elts_same_io_lane, [S16; S32]; + + (* Multiply-subtract, lane. *) + Vmls_lane, [], By_scalar Dreg, "vmls_lane", sign_invar_io_lane, + [S16; S32; U16; U32; F32]; + Vmls_lane, [], By_scalar Qreg, "vmlsQ_lane", sign_invar_io_lane, + [S16; S32; U16; U32; F32]; + Vmls_lane, [], Wide_lane, "vmlsl_lane", elts_same_io_lane, + [S16; S32; U16; U32]; + Vmls_lane, [Saturating; Doubling], Wide_lane, "vqdmlsl_lane", + elts_same_io_lane, [S16; S32]; + + (* Long multiply, lane. *) + Vmull_lane, [], + Wide_lane, "vmull_lane", elts_same_2_lane, [S16; S32; U16; U32]; + + (* Saturating doubling long multiply, lane. *) + Vqdmull_lane, [Saturating; Doubling], + Wide_lane, "vqdmull_lane", elts_same_2_lane, [S16; S32]; + + (* Saturating doubling long multiply high, lane. *) + Vqdmulh_lane, [Saturating; Halving], + By_scalar Qreg, "vqdmulhQ_lane", elts_same_2_lane, [S16; S32]; + Vqdmulh_lane, [Saturating; Halving], + By_scalar Dreg, "vqdmulh_lane", elts_same_2_lane, [S16; S32]; + Vqdmulh_lane, [Saturating; Halving; Rounding; + Instruction_name ["vqrdmulh"]], + By_scalar Qreg, "vqRdmulhQ_lane", elts_same_2_lane, [S16; S32]; + Vqdmulh_lane, [Saturating; Halving; Rounding; + Instruction_name ["vqrdmulh"]], + By_scalar Dreg, "vqRdmulh_lane", elts_same_2_lane, [S16; S32]; + + (* Vector multiply by scalar. *) + Vmul_n, [InfoWord; + Disassembles_as [Use_operands [| Dreg; Dreg; Element_of_dreg |]]], + Use_operands [| Dreg; Dreg; Corereg |], "vmul_n", + sign_invar_2, [S16; S32; U16; U32; F32]; + Vmul_n, [InfoWord; + Disassembles_as [Use_operands [| Qreg; Qreg; Element_of_dreg |]]], + Use_operands [| Qreg; Qreg; Corereg |], "vmulQ_n", + sign_invar_2, [S16; S32; U16; U32; F32]; + + (* Vector long multiply by scalar. *) + Vmull_n, [Instruction_name ["vmull"]; + Disassembles_as [Use_operands [| Qreg; Dreg; Element_of_dreg |]]], + Wide_scalar, "vmull_n", + elts_same_2, [S16; S32; U16; U32]; + + (* Vector saturating doubling long multiply by scalar. *) + Vqdmull_n, [Saturating; Doubling; + Disassembles_as [Use_operands [| Qreg; Dreg; + Element_of_dreg |]]], + Wide_scalar, "vqdmull_n", + elts_same_2, [S16; S32]; + + (* Vector saturating doubling long multiply high by scalar. *) + Vqdmulh_n, + [Saturating; Halving; InfoWord; + Disassembles_as [Use_operands [| Qreg; Qreg; Element_of_dreg |]]], + Use_operands [| Qreg; Qreg; Corereg |], + "vqdmulhQ_n", elts_same_2, [S16; S32]; + Vqdmulh_n, + [Saturating; Halving; InfoWord; + Disassembles_as [Use_operands [| Dreg; Dreg; Element_of_dreg |]]], + Use_operands [| Dreg; Dreg; Corereg |], + "vqdmulh_n", elts_same_2, [S16; S32]; + Vqdmulh_n, + [Saturating; Halving; Rounding; InfoWord; + Instruction_name ["vqrdmulh"]; + Disassembles_as [Use_operands [| Qreg; Qreg; Element_of_dreg |]]], + Use_operands [| Qreg; Qreg; Corereg |], + "vqRdmulhQ_n", elts_same_2, [S16; S32]; + Vqdmulh_n, + [Saturating; Halving; Rounding; InfoWord; + Instruction_name ["vqrdmulh"]; + Disassembles_as [Use_operands [| Dreg; Dreg; Element_of_dreg |]]], + Use_operands [| Dreg; Dreg; Corereg |], + "vqRdmulh_n", elts_same_2, [S16; S32]; + + (* Vector multiply-accumulate by scalar. *) + Vmla_n, [InfoWord; + Disassembles_as [Use_operands [| Dreg; Dreg; Element_of_dreg |]]], + Use_operands [| Dreg; Dreg; Corereg |], "vmla_n", + sign_invar_io, [S16; S32; U16; U32; F32]; + Vmla_n, [InfoWord; + Disassembles_as [Use_operands [| Qreg; Qreg; Element_of_dreg |]]], + Use_operands [| Qreg; Qreg; Corereg |], "vmlaQ_n", + sign_invar_io, [S16; S32; U16; U32; F32]; + Vmla_n, [], Wide_scalar, "vmlal_n", elts_same_io, [S16; S32; U16; U32]; + Vmla_n, [Saturating; Doubling], Wide_scalar, "vqdmlal_n", elts_same_io, + [S16; S32]; + + (* Vector multiply subtract by scalar. *) + Vmls_n, [InfoWord; + Disassembles_as [Use_operands [| Dreg; Dreg; Element_of_dreg |]]], + Use_operands [| Dreg; Dreg; Corereg |], "vmls_n", + sign_invar_io, [S16; S32; U16; U32; F32]; + Vmls_n, [InfoWord; + Disassembles_as [Use_operands [| Qreg; Qreg; Element_of_dreg |]]], + Use_operands [| Qreg; Qreg; Corereg |], "vmlsQ_n", + sign_invar_io, [S16; S32; U16; U32; F32]; + Vmls_n, [], Wide_scalar, "vmlsl_n", elts_same_io, [S16; S32; U16; U32]; + Vmls_n, [Saturating; Doubling], Wide_scalar, "vqdmlsl_n", elts_same_io, + [S16; S32]; + + (* Vector extract. *) + Vext, [Requires_feature "CRYPTO"; Const_valuator (fun _ -> 0)], + Use_operands [| Dreg; Dreg; Dreg; Immed |], "vext", extend, + [P64]; + Vext, [Const_valuator (fun _ -> 0)], + Use_operands [| Dreg; Dreg; Dreg; Immed |], "vext", extend, + pf_su_8_64; + Vext, [Requires_feature "CRYPTO"; Const_valuator (fun _ -> 0)], + Use_operands [| Qreg; Qreg; Qreg; Immed |], "vextQ", extend, + [P64]; + Vext, [Const_valuator (fun _ -> 0)], + Use_operands [| Qreg; Qreg; Qreg; Immed |], "vextQ", extend, + pf_su_8_64; + + (* Reverse elements. *) + Vrev64, [Use_shuffle (rev_elems 64)], All (2, Dreg), "vrev64", bits_1, + P8 :: P16 :: F32 :: su_8_32; + Vrev64, [Use_shuffle (rev_elems 64)], All (2, Qreg), "vrev64Q", bits_1, + P8 :: P16 :: F32 :: su_8_32; + Vrev32, [Use_shuffle (rev_elems 32)], All (2, Dreg), "vrev32", bits_1, + [P8; P16; S8; U8; S16; U16]; + Vrev32, [Use_shuffle (rev_elems 32)], All (2, Qreg), "vrev32Q", bits_1, + [P8; P16; S8; U8; S16; U16]; + Vrev16, [Use_shuffle (rev_elems 16)], All (2, Dreg), "vrev16", bits_1, + [P8; S8; U8]; + Vrev16, [Use_shuffle (rev_elems 16)], All (2, Qreg), "vrev16Q", bits_1, + [P8; S8; U8]; + + (* Bit selection. *) + Vbsl, + [Requires_feature "CRYPTO"; Instruction_name ["vbsl"; "vbit"; "vbif"]; + Disassembles_as [Use_operands [| Dreg; Dreg; Dreg |]]], + Use_operands [| Dreg; Dreg; Dreg; Dreg |], "vbsl", bit_select, + [P64]; + Vbsl, + [Instruction_name ["vbsl"; "vbit"; "vbif"]; + Disassembles_as [Use_operands [| Dreg; Dreg; Dreg |]]], + Use_operands [| Dreg; Dreg; Dreg; Dreg |], "vbsl", bit_select, + pf_su_8_64; + Vbsl, + [Requires_feature "CRYPTO"; Instruction_name ["vbsl"; "vbit"; "vbif"]; + Disassembles_as [Use_operands [| Qreg; Qreg; Qreg |]]], + Use_operands [| Qreg; Qreg; Qreg; Qreg |], "vbslQ", bit_select, + [P64]; + Vbsl, + [Instruction_name ["vbsl"; "vbit"; "vbif"]; + Disassembles_as [Use_operands [| Qreg; Qreg; Qreg |]]], + Use_operands [| Qreg; Qreg; Qreg; Qreg |], "vbslQ", bit_select, + pf_su_8_64; + + Vtrn, [Use_shuffle trn_elems], Pair_result Dreg, "vtrn", bits_2, pf_su_8_16; + Vtrn, [Use_shuffle trn_elems; Instruction_name ["vuzp"]], Pair_result Dreg, "vtrn", bits_2, suf_32; + Vtrn, [Use_shuffle trn_elems], Pair_result Qreg, "vtrnQ", bits_2, pf_su_8_32; + (* Zip elements. *) + Vzip, [Use_shuffle zip_elems], Pair_result Dreg, "vzip", bits_2, pf_su_8_16; + Vzip, [Use_shuffle zip_elems; Instruction_name ["vuzp"]], Pair_result Dreg, "vzip", bits_2, suf_32; + Vzip, [Use_shuffle zip_elems], Pair_result Qreg, "vzipQ", bits_2, pf_su_8_32; + + (* Unzip elements. *) + Vuzp, [Use_shuffle uzip_elems], Pair_result Dreg, "vuzp", bits_2, + pf_su_8_32; + Vuzp, [Use_shuffle uzip_elems], Pair_result Qreg, "vuzpQ", bits_2, + pf_su_8_32; + + (* Element/structure loads. VLD1 variants. *) + Vldx 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Dreg; CstPtrTo Corereg |], "vld1", bits_1, + [P64]; + Vldx 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Dreg; CstPtrTo Corereg |], "vld1", bits_1, + pf_su_8_64; + Vldx 1, [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (2, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg |], "vld1Q", bits_1, + [P64]; + Vldx 1, [Disassembles_as [Use_operands [| VecArray (2, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg |], "vld1Q", bits_1, + pf_su_8_64; + + Vldx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| Dreg; CstPtrTo Corereg; Dreg; Immed |], + "vld1_lane", bits_3, pf_su_8_32; + Vldx_lane 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]; + Const_valuator (fun _ -> 0)], + Use_operands [| Dreg; CstPtrTo Corereg; Dreg; Immed |], + "vld1_lane", bits_3, [P64]; + Vldx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]; + Const_valuator (fun _ -> 0)], + Use_operands [| Dreg; CstPtrTo Corereg; Dreg; Immed |], + "vld1_lane", bits_3, [S64; U64]; + Vldx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg; Qreg; Immed |], + "vld1Q_lane", bits_3, pf_su_8_32; + Vldx_lane 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg; Qreg; Immed |], + "vld1Q_lane", bits_3, [P64]; + Vldx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg; Qreg; Immed |], + "vld1Q_lane", bits_3, [S64; U64]; + + Vldx_dup 1, + [Disassembles_as [Use_operands [| VecArray (1, All_elements_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| Dreg; CstPtrTo Corereg |], "vld1_dup", + bits_1, pf_su_8_32; + Vldx_dup 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Dreg; CstPtrTo Corereg |], "vld1_dup", + bits_1, [P64]; + Vldx_dup 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Dreg; CstPtrTo Corereg |], "vld1_dup", + bits_1, [S64; U64]; + Vldx_dup 1, + [Disassembles_as [Use_operands [| VecArray (2, All_elements_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg |], "vld1Q_dup", + bits_1, pf_su_8_32; + (* Treated identically to vld1_dup above as we now + do a single load followed by a duplicate. *) + Vldx_dup 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg |], "vld1Q_dup", + bits_1, [P64]; + Vldx_dup 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| Qreg; CstPtrTo Corereg |], "vld1Q_dup", + bits_1, [S64; U64]; + + (* VST1 variants. *) + Vstx 1, [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Dreg |], "vst1", + store_1, [P64]; + Vstx 1, [Disassembles_as [Use_operands [| VecArray (1, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Dreg |], "vst1", + store_1, pf_su_8_64; + Vstx 1, [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Qreg |], "vst1Q", + store_1, [P64]; + Vstx 1, [Disassembles_as [Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Qreg |], "vst1Q", + store_1, pf_su_8_64; + + Vstx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Dreg; Immed |], + "vst1_lane", store_3, pf_su_8_32; + Vstx_lane 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]; + Const_valuator (fun _ -> 0)], + Use_operands [| PtrTo Corereg; Dreg; Immed |], + "vst1_lane", store_3, [P64]; + Vstx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]; + Const_valuator (fun _ -> 0)], + Use_operands [| PtrTo Corereg; Dreg; Immed |], + "vst1_lane", store_3, [U64; S64]; + Vstx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Qreg; Immed |], + "vst1Q_lane", store_3, pf_su_8_32; + Vstx_lane 1, + [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Qreg; Immed |], + "vst1Q_lane", store_3, [P64]; + Vstx_lane 1, + [Disassembles_as [Use_operands [| VecArray (1, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; Qreg; Immed |], + "vst1Q_lane", store_3, [U64; S64]; + + (* VLD2 variants. *) + Vldx 2, [], Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg |], + "vld2", bits_1, pf_su_8_32; + Vldx 2, [Requires_feature "CRYPTO"; Instruction_name ["vld1"]], + Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg |], + "vld2", bits_1, [P64]; + Vldx 2, [Instruction_name ["vld1"]], + Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg |], + "vld2", bits_1, [S64; U64]; + Vldx 2, [Disassembles_as [Use_operands [| VecArray (2, Dreg); + CstPtrTo Corereg |]; + Use_operands [| VecArray (2, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (2, Qreg); CstPtrTo Corereg |], + "vld2Q", bits_1, pf_su_8_32; + + Vldx_lane 2, + [Disassembles_as [Use_operands + [| VecArray (2, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg; + VecArray (2, Dreg); Immed |], + "vld2_lane", bits_3, P8 :: P16 :: F32 :: su_8_32; + Vldx_lane 2, + [Disassembles_as [Use_operands + [| VecArray (2, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (2, Qreg); CstPtrTo Corereg; + VecArray (2, Qreg); Immed |], + "vld2Q_lane", bits_3, [P16; F32; U16; U32; S16; S32]; + + Vldx_dup 2, + [Disassembles_as [Use_operands + [| VecArray (2, All_elements_of_dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg |], + "vld2_dup", bits_1, pf_su_8_32; + Vldx_dup 2, + [Requires_feature "CRYPTO"; + Instruction_name ["vld1"]; Disassembles_as [Use_operands + [| VecArray (2, Dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg |], + "vld2_dup", bits_1, [P64]; + Vldx_dup 2, + [Instruction_name ["vld1"]; Disassembles_as [Use_operands + [| VecArray (2, Dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (2, Dreg); CstPtrTo Corereg |], + "vld2_dup", bits_1, [S64; U64]; + + (* VST2 variants. *) + Vstx 2, [Disassembles_as [Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (2, Dreg) |], "vst2", + store_1, pf_su_8_32; + Vstx 2, [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]]; + Instruction_name ["vst1"]], + Use_operands [| PtrTo Corereg; VecArray (2, Dreg) |], "vst2", + store_1, [P64]; + Vstx 2, [Disassembles_as [Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]]; + Instruction_name ["vst1"]], + Use_operands [| PtrTo Corereg; VecArray (2, Dreg) |], "vst2", + store_1, [S64; U64]; + Vstx 2, [Disassembles_as [Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]; + Use_operands [| VecArray (2, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (2, Qreg) |], "vst2Q", + store_1, pf_su_8_32; + + Vstx_lane 2, + [Disassembles_as [Use_operands + [| VecArray (2, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (2, Dreg); Immed |], "vst2_lane", + store_3, P8 :: P16 :: F32 :: su_8_32; + Vstx_lane 2, + [Disassembles_as [Use_operands + [| VecArray (2, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (2, Qreg); Immed |], "vst2Q_lane", + store_3, [P16; F32; U16; U32; S16; S32]; + + (* VLD3 variants. *) + Vldx 3, [], Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg |], + "vld3", bits_1, pf_su_8_32; + Vldx 3, [Requires_feature "CRYPTO"; Instruction_name ["vld1"]], + Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg |], + "vld3", bits_1, [P64]; + Vldx 3, [Instruction_name ["vld1"]], + Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg |], + "vld3", bits_1, [S64; U64]; + Vldx 3, [Disassembles_as [Use_operands [| VecArray (3, Dreg); + CstPtrTo Corereg |]; + Use_operands [| VecArray (3, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (3, Qreg); CstPtrTo Corereg |], + "vld3Q", bits_1, P8 :: P16 :: F32 :: su_8_32; + + Vldx_lane 3, + [Disassembles_as [Use_operands + [| VecArray (3, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg; + VecArray (3, Dreg); Immed |], + "vld3_lane", bits_3, P8 :: P16 :: F32 :: su_8_32; + Vldx_lane 3, + [Disassembles_as [Use_operands + [| VecArray (3, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (3, Qreg); CstPtrTo Corereg; + VecArray (3, Qreg); Immed |], + "vld3Q_lane", bits_3, [P16; F32; U16; U32; S16; S32]; + + Vldx_dup 3, + [Disassembles_as [Use_operands + [| VecArray (3, All_elements_of_dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg |], + "vld3_dup", bits_1, pf_su_8_32; + Vldx_dup 3, + [Requires_feature "CRYPTO"; + Instruction_name ["vld1"]; Disassembles_as [Use_operands + [| VecArray (3, Dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg |], + "vld3_dup", bits_1, [P64]; + Vldx_dup 3, + [Instruction_name ["vld1"]; Disassembles_as [Use_operands + [| VecArray (3, Dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (3, Dreg); CstPtrTo Corereg |], + "vld3_dup", bits_1, [S64; U64]; + + (* VST3 variants. *) + Vstx 3, [Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (3, Dreg) |], "vst3", + store_1, pf_su_8_32; + Vstx 3, [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]; + Instruction_name ["vst1"]], + Use_operands [| PtrTo Corereg; VecArray (3, Dreg) |], "vst3", + store_1, [P64]; + Vstx 3, [Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]; + Instruction_name ["vst1"]], + Use_operands [| PtrTo Corereg; VecArray (3, Dreg) |], "vst3", + store_1, [S64; U64]; + Vstx 3, [Disassembles_as [Use_operands [| VecArray (3, Dreg); + PtrTo Corereg |]; + Use_operands [| VecArray (3, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (3, Qreg) |], "vst3Q", + store_1, pf_su_8_32; + + Vstx_lane 3, + [Disassembles_as [Use_operands + [| VecArray (3, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (3, Dreg); Immed |], "vst3_lane", + store_3, P8 :: P16 :: F32 :: su_8_32; + Vstx_lane 3, + [Disassembles_as [Use_operands + [| VecArray (3, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (3, Qreg); Immed |], "vst3Q_lane", + store_3, [P16; F32; U16; U32; S16; S32]; + + (* VLD4/VST4 variants. *) + Vldx 4, [], Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg |], + "vld4", bits_1, pf_su_8_32; + Vldx 4, [Requires_feature "CRYPTO"; Instruction_name ["vld1"]], + Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg |], + "vld4", bits_1, [P64]; + Vldx 4, [Instruction_name ["vld1"]], + Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg |], + "vld4", bits_1, [S64; U64]; + Vldx 4, [Disassembles_as [Use_operands [| VecArray (4, Dreg); + CstPtrTo Corereg |]; + Use_operands [| VecArray (4, Dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (4, Qreg); CstPtrTo Corereg |], + "vld4Q", bits_1, P8 :: P16 :: F32 :: su_8_32; + + Vldx_lane 4, + [Disassembles_as [Use_operands + [| VecArray (4, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg; + VecArray (4, Dreg); Immed |], + "vld4_lane", bits_3, P8 :: P16 :: F32 :: su_8_32; + Vldx_lane 4, + [Disassembles_as [Use_operands + [| VecArray (4, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| VecArray (4, Qreg); CstPtrTo Corereg; + VecArray (4, Qreg); Immed |], + "vld4Q_lane", bits_3, [P16; F32; U16; U32; S16; S32]; + + Vldx_dup 4, + [Disassembles_as [Use_operands + [| VecArray (4, All_elements_of_dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg |], + "vld4_dup", bits_1, pf_su_8_32; + Vldx_dup 4, + [Requires_feature "CRYPTO"; + Instruction_name ["vld1"]; Disassembles_as [Use_operands + [| VecArray (4, Dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg |], + "vld4_dup", bits_1, [P64]; + Vldx_dup 4, + [Instruction_name ["vld1"]; Disassembles_as [Use_operands + [| VecArray (4, Dreg); CstPtrTo Corereg |]]], + Use_operands [| VecArray (4, Dreg); CstPtrTo Corereg |], + "vld4_dup", bits_1, [S64; U64]; + + Vstx 4, [Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (4, Dreg) |], "vst4", + store_1, pf_su_8_32; + Vstx 4, [Requires_feature "CRYPTO"; + Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]; + Instruction_name ["vst1"]], + Use_operands [| PtrTo Corereg; VecArray (4, Dreg) |], "vst4", + store_1, [P64]; + Vstx 4, [Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]; + Instruction_name ["vst1"]], + Use_operands [| PtrTo Corereg; VecArray (4, Dreg) |], "vst4", + store_1, [S64; U64]; + Vstx 4, [Disassembles_as [Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]; + Use_operands [| VecArray (4, Dreg); + PtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (4, Qreg) |], "vst4Q", + store_1, pf_su_8_32; + + Vstx_lane 4, + [Disassembles_as [Use_operands + [| VecArray (4, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (4, Dreg); Immed |], "vst4_lane", + store_3, P8 :: P16 :: F32 :: su_8_32; + Vstx_lane 4, + [Disassembles_as [Use_operands + [| VecArray (4, Element_of_dreg); + CstPtrTo Corereg |]]], + Use_operands [| PtrTo Corereg; VecArray (4, Qreg); Immed |], "vst4Q_lane", + store_3, [P16; F32; U16; U32; S16; S32]; + + (* Logical operations. And. *) + Vand, [], All (3, Dreg), "vand", notype_2, su_8_32; + Vand, [No_op], All (3, Dreg), "vand", notype_2, [S64; U64]; + Vand, [], All (3, Qreg), "vandQ", notype_2, su_8_64; + + (* Or. *) + Vorr, [], All (3, Dreg), "vorr", notype_2, su_8_32; + Vorr, [No_op], All (3, Dreg), "vorr", notype_2, [S64; U64]; + Vorr, [], All (3, Qreg), "vorrQ", notype_2, su_8_64; + + (* Eor. *) + Veor, [], All (3, Dreg), "veor", notype_2, su_8_32; + Veor, [No_op], All (3, Dreg), "veor", notype_2, [S64; U64]; + Veor, [], All (3, Qreg), "veorQ", notype_2, su_8_64; + + (* Bic (And-not). *) + Vbic, [], All (3, Dreg), "vbic", notype_2, su_8_32; + Vbic, [No_op], All (3, Dreg), "vbic", notype_2, [S64; U64]; + Vbic, [], All (3, Qreg), "vbicQ", notype_2, su_8_64; + + (* Or-not. *) + Vorn, [], All (3, Dreg), "vorn", notype_2, su_8_32; + Vorn, [No_op], All (3, Dreg), "vorn", notype_2, [S64; U64]; + Vorn, [], All (3, Qreg), "vornQ", notype_2, su_8_64; + ] + +let type_in_crypto_only t + = (t == P64) or (t == P128) + +let cross_product s1 s2 + = List.filter (fun (e, e') -> e <> e') + (List.concat (List.map (fun e1 -> List.map (fun e2 -> (e1,e2)) s1) s2)) + +let reinterp = + let elems = P8 :: P16 :: F32 :: P64 :: su_8_64 in + let casts = cross_product elems elems in + List.map + (fun (convto, convfrom) -> + Vreinterp, (if (type_in_crypto_only convto) or (type_in_crypto_only convfrom) + then [Requires_feature "CRYPTO"] else []) @ [No_op], Use_operands [| Dreg; Dreg |], + "vreinterpret", conv_1, [Cast (convto, convfrom)]) + casts + +let reinterpq = + let elems = P8 :: P16 :: F32 :: P64 :: P128 :: su_8_64 in + let casts = cross_product elems elems in + List.map + (fun (convto, convfrom) -> + Vreinterp, (if (type_in_crypto_only convto) or (type_in_crypto_only convfrom) + then [Requires_feature "CRYPTO"] else []) @ [No_op], Use_operands [| Qreg; Qreg |], + "vreinterpretQ", conv_1, [Cast (convto, convfrom)]) + casts + +(* Output routines. *) + +let rec string_of_elt = function + S8 -> "s8" | S16 -> "s16" | S32 -> "s32" | S64 -> "s64" + | U8 -> "u8" | U16 -> "u16" | U32 -> "u32" | U64 -> "u64" + | I8 -> "i8" | I16 -> "i16" | I32 -> "i32" | I64 -> "i64" + | B8 -> "8" | B16 -> "16" | B32 -> "32" | B64 -> "64" + | F16 -> "f16" | F32 -> "f32" | P8 -> "p8" | P16 -> "p16" + | P64 -> "p64" | P128 -> "p128" + | Conv (a, b) | Cast (a, b) -> string_of_elt a ^ "_" ^ string_of_elt b + | NoElts -> failwith "No elts" + +let string_of_elt_dots elt = + match elt with + Conv (a, b) | Cast (a, b) -> string_of_elt a ^ "." ^ string_of_elt b + | _ -> string_of_elt elt + +let string_of_vectype vt = + let rec name affix = function + T_int8x8 -> affix "int8x8" + | T_int8x16 -> affix "int8x16" + | T_int16x4 -> affix "int16x4" + | T_int16x8 -> affix "int16x8" + | T_int32x2 -> affix "int32x2" + | T_int32x4 -> affix "int32x4" + | T_int64x1 -> affix "int64x1" + | T_int64x2 -> affix "int64x2" + | T_uint8x8 -> affix "uint8x8" + | T_uint8x16 -> affix "uint8x16" + | T_uint16x4 -> affix "uint16x4" + | T_uint16x8 -> affix "uint16x8" + | T_uint32x2 -> affix "uint32x2" + | T_uint32x4 -> affix "uint32x4" + | T_uint64x1 -> affix "uint64x1" + | T_uint64x2 -> affix "uint64x2" + | T_float16x4 -> affix "float16x4" + | T_float32x2 -> affix "float32x2" + | T_float32x4 -> affix "float32x4" + | T_poly8x8 -> affix "poly8x8" + | T_poly8x16 -> affix "poly8x16" + | T_poly16x4 -> affix "poly16x4" + | T_poly16x8 -> affix "poly16x8" + | T_int8 -> affix "int8" + | T_int16 -> affix "int16" + | T_int32 -> affix "int32" + | T_int64 -> affix "int64" + | T_uint8 -> affix "uint8" + | T_uint16 -> affix "uint16" + | T_uint32 -> affix "uint32" + | T_uint64 -> affix "uint64" + | T_poly8 -> affix "poly8" + | T_poly16 -> affix "poly16" + | T_poly64 -> affix "poly64" + | T_poly64x1 -> affix "poly64x1" + | T_poly64x2 -> affix "poly64x2" + | T_poly128 -> affix "poly128" + | T_float16 -> affix "float16" + | T_float32 -> affix "float32" + | T_immediate _ -> "const int" + | T_void -> "void" + | T_intQI -> "__builtin_neon_qi" + | T_intHI -> "__builtin_neon_hi" + | T_intSI -> "__builtin_neon_si" + | T_intDI -> "__builtin_neon_di" + | T_intTI -> "__builtin_neon_ti" + | T_floatHF -> "__builtin_neon_hf" + | T_floatSF -> "__builtin_neon_sf" + | T_arrayof (num, base) -> + let basename = name (fun x -> x) base in + affix (Printf.sprintf "%sx%d" basename num) + | T_ptrto x -> + let basename = name affix x in + Printf.sprintf "%s *" basename + | T_const x -> + let basename = name affix x in + Printf.sprintf "const %s" basename + in + name (fun x -> x ^ "_t") vt + +let string_of_inttype = function + B_TImode -> "__builtin_neon_ti" + | B_EImode -> "__builtin_neon_ei" + | B_OImode -> "__builtin_neon_oi" + | B_CImode -> "__builtin_neon_ci" + | B_XImode -> "__builtin_neon_xi" + +let string_of_mode = function + V8QI -> "v8qi" | V4HI -> "v4hi" | V4HF -> "v4hf" | V2SI -> "v2si" + | V2SF -> "v2sf" | DI -> "di" | V16QI -> "v16qi" | V8HI -> "v8hi" + | V4SI -> "v4si" | V4SF -> "v4sf" | V2DI -> "v2di" | QI -> "qi" + | HI -> "hi" | SI -> "si" | SF -> "sf" | TI -> "ti" + +(* Use uppercase chars for letters which form part of the intrinsic name, but + should be omitted from the builtin name (the info is passed in an extra + argument, instead). *) +let intrinsic_name name = String.lowercase name + +(* Allow the name of the builtin to be overridden by things (e.g. Flipped) + found in the features list. *) +let builtin_name features name = + let name = List.fold_right + (fun el name -> + match el with + Flipped x | Builtin_name x -> x + | _ -> name) + features name in + let islower x = let str = String.make 1 x in (String.lowercase str) = str + and buf = Buffer.create (String.length name) in + String.iter (fun c -> if islower c then Buffer.add_char buf c) name; + Buffer.contents buf + +(* Transform an arity into a list of strings. *) +let strings_of_arity a = + match a with + | Arity0 vt -> [string_of_vectype vt] + | Arity1 (vt1, vt2) -> [string_of_vectype vt1; string_of_vectype vt2] + | Arity2 (vt1, vt2, vt3) -> [string_of_vectype vt1; + string_of_vectype vt2; + string_of_vectype vt3] + | Arity3 (vt1, vt2, vt3, vt4) -> [string_of_vectype vt1; + string_of_vectype vt2; + string_of_vectype vt3; + string_of_vectype vt4] + | Arity4 (vt1, vt2, vt3, vt4, vt5) -> [string_of_vectype vt1; + string_of_vectype vt2; + string_of_vectype vt3; + string_of_vectype vt4; + string_of_vectype vt5] + +(* Suffixes on the end of builtin names that are to be stripped in order + to obtain the name used as an instruction. They are only stripped if + preceded immediately by an underscore. *) +let suffixes_to_strip = [ "n"; "lane"; "dup" ] + +(* Get the possible names of an instruction corresponding to a "name" from the + ops table. This is done by getting the equivalent builtin name and + stripping any suffixes from the list at the top of this file, unless + the features list presents with an Instruction_name entry, in which + case that is used; or unless the features list presents with a Flipped + entry, in which case that is used. If both such entries are present, + the first in the list will be chosen. *) +let get_insn_names features name = + let names = try + begin + match List.find (fun feature -> match feature with + Instruction_name _ -> true + | Flipped _ -> true + | _ -> false) features + with + Instruction_name names -> names + | Flipped name -> [name] + | _ -> assert false + end + with Not_found -> [builtin_name features name] + in + begin + List.map (fun name' -> + try + let underscore = String.rindex name' '_' in + let our_suffix = String.sub name' (underscore + 1) + ((String.length name') - underscore - 1) + in + let rec strip remaining_suffixes = + match remaining_suffixes with + [] -> name' + | s::ss when our_suffix = s -> String.sub name' 0 underscore + | _::ss -> strip ss + in + strip suffixes_to_strip + with (Not_found | Invalid_argument _) -> name') names + end + +(* Apply a function to each element of a list and then comma-separate + the resulting strings. *) +let rec commas f elts acc = + match elts with + [] -> acc + | [elt] -> acc ^ (f elt) + | elt::elts -> + commas f elts (acc ^ (f elt) ^ ", ") + +(* Given a list of features and the shape specified in the "ops" table, apply + a function to each possible shape that the instruction may have. + By default, this is the "shape" entry in "ops". If the features list + contains a Disassembles_as entry, the shapes contained in that entry are + mapped to corresponding outputs and returned in a list. If there is more + than one Disassembles_as entry, only the first is used. *) +let analyze_all_shapes features shape f = + try + match List.find (fun feature -> + match feature with Disassembles_as _ -> true + | _ -> false) + features with + Disassembles_as shapes -> List.map f shapes + | _ -> assert false + with Not_found -> [f shape] + +(* The crypto intrinsics have unconventional shapes and are not that + numerous to be worth the trouble of encoding here. We implement them + explicitly here. *) +let crypto_intrinsics = +" +#ifdef __ARM_FEATURE_CRYPTO + +__extension__ static __inline poly128_t __attribute__ ((__always_inline__)) +vldrq_p128 (poly128_t const * __ptr) +{ +#ifdef __ARM_BIG_ENDIAN + poly64_t* __ptmp = (poly64_t*) __ptr; + poly64_t __d0 = vld1_p64 (__ptmp); + poly64_t __d1 = vld1_p64 (__ptmp + 1); + return vreinterpretq_p128_p64 (vcombine_p64 (__d1, __d0)); +#else + return vreinterpretq_p128_p64 (vld1q_p64 ((poly64_t*) __ptr)); +#endif +} + +__extension__ static __inline void __attribute__ ((__always_inline__)) +vstrq_p128 (poly128_t * __ptr, poly128_t __val) +{ +#ifdef __ARM_BIG_ENDIAN + poly64x2_t __tmp = vreinterpretq_p64_p128 (__val); + poly64_t __d0 = vget_high_p64 (__tmp); + poly64_t __d1 = vget_low_p64 (__tmp); + vst1q_p64 ((poly64_t*) __ptr, vcombine_p64 (__d0, __d1)); +#else + vst1q_p64 ((poly64_t*) __ptr, vreinterpretq_p64_p128 (__val)); +#endif +} + +/* The vceq_p64 intrinsic does not map to a single instruction. + Instead we emulate it by performing a 32-bit variant of the vceq + and applying a pairwise min reduction to the result. + vceq_u32 will produce two 32-bit halves, each of which will contain either + all ones or all zeros depending on whether the corresponding 32-bit + halves of the poly64_t were equal. The whole poly64_t values are equal + if and only if both halves are equal, i.e. vceq_u32 returns all ones. + If the result is all zeroes for any half then the whole result is zeroes. + This is what the pairwise min reduction achieves. */ + +__extension__ static __inline uint64x1_t __attribute__ ((__always_inline__)) +vceq_p64 (poly64x1_t __a, poly64x1_t __b) +{ + uint32x2_t __t_a = vreinterpret_u32_p64 (__a); + uint32x2_t __t_b = vreinterpret_u32_p64 (__b); + uint32x2_t __c = vceq_u32 (__t_a, __t_b); + uint32x2_t __m = vpmin_u32 (__c, __c); + return vreinterpret_u64_u32 (__m); +} + +/* The vtst_p64 intrinsic does not map to a single instruction. + We emulate it in way similar to vceq_p64 above but here we do + a reduction with max since if any two corresponding bits + in the two poly64_t's match, then the whole result must be all ones. */ + +__extension__ static __inline uint64x1_t __attribute__ ((__always_inline__)) +vtst_p64 (poly64x1_t __a, poly64x1_t __b) +{ + uint32x2_t __t_a = vreinterpret_u32_p64 (__a); + uint32x2_t __t_b = vreinterpret_u32_p64 (__b); + uint32x2_t __c = vtst_u32 (__t_a, __t_b); + uint32x2_t __m = vpmax_u32 (__c, __c); + return vreinterpret_u64_u32 (__m); +} + +__extension__ static __inline uint8x16_t __attribute__ ((__always_inline__)) +vaeseq_u8 (uint8x16_t __data, uint8x16_t __key) +{ + return __builtin_arm_crypto_aese (__data, __key); +} + +__extension__ static __inline uint8x16_t __attribute__ ((__always_inline__)) +vaesdq_u8 (uint8x16_t __data, uint8x16_t __key) +{ + return __builtin_arm_crypto_aesd (__data, __key); +} + +__extension__ static __inline uint8x16_t __attribute__ ((__always_inline__)) +vaesmcq_u8 (uint8x16_t __data) +{ + return __builtin_arm_crypto_aesmc (__data); +} + +__extension__ static __inline uint8x16_t __attribute__ ((__always_inline__)) +vaesimcq_u8 (uint8x16_t __data) +{ + return __builtin_arm_crypto_aesimc (__data); +} + +__extension__ static __inline uint32_t __attribute__ ((__always_inline__)) +vsha1h_u32 (uint32_t __hash_e) +{ + uint32x4_t __t = vdupq_n_u32 (0); + __t = vsetq_lane_u32 (__hash_e, __t, 0); + __t = __builtin_arm_crypto_sha1h (__t); + return vgetq_lane_u32 (__t, 0); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha1cq_u32 (uint32x4_t __hash_abcd, uint32_t __hash_e, uint32x4_t __wk) +{ + uint32x4_t __t = vdupq_n_u32 (0); + __t = vsetq_lane_u32 (__hash_e, __t, 0); + return __builtin_arm_crypto_sha1c (__hash_abcd, __t, __wk); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha1pq_u32 (uint32x4_t __hash_abcd, uint32_t __hash_e, uint32x4_t __wk) +{ + uint32x4_t __t = vdupq_n_u32 (0); + __t = vsetq_lane_u32 (__hash_e, __t, 0); + return __builtin_arm_crypto_sha1p (__hash_abcd, __t, __wk); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha1mq_u32 (uint32x4_t __hash_abcd, uint32_t __hash_e, uint32x4_t __wk) +{ + uint32x4_t __t = vdupq_n_u32 (0); + __t = vsetq_lane_u32 (__hash_e, __t, 0); + return __builtin_arm_crypto_sha1m (__hash_abcd, __t, __wk); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha1su0q_u32 (uint32x4_t __w0_3, uint32x4_t __w4_7, uint32x4_t __w8_11) +{ + return __builtin_arm_crypto_sha1su0 (__w0_3, __w4_7, __w8_11); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha1su1q_u32 (uint32x4_t __tw0_3, uint32x4_t __w12_15) +{ + return __builtin_arm_crypto_sha1su1 (__tw0_3, __w12_15); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha256hq_u32 (uint32x4_t __hash_abcd, uint32x4_t __hash_efgh, uint32x4_t __wk) +{ + return __builtin_arm_crypto_sha256h (__hash_abcd, __hash_efgh, __wk); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha256h2q_u32 (uint32x4_t __hash_abcd, uint32x4_t __hash_efgh, uint32x4_t __wk) +{ + return __builtin_arm_crypto_sha256h2 (__hash_abcd, __hash_efgh, __wk); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha256su0q_u32 (uint32x4_t __w0_3, uint32x4_t __w4_7) +{ + return __builtin_arm_crypto_sha256su0 (__w0_3, __w4_7); +} + +__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__)) +vsha256su1q_u32 (uint32x4_t __tw0_3, uint32x4_t __w8_11, uint32x4_t __w12_15) +{ + return __builtin_arm_crypto_sha256su1 (__tw0_3, __w8_11, __w12_15); +} + +__extension__ static __inline poly128_t __attribute__ ((__always_inline__)) +vmull_p64 (poly64_t __a, poly64_t __b) +{ + return (poly128_t) __builtin_arm_crypto_vmullp64 ((uint64_t) __a, (uint64_t) __b); +} + +__extension__ static __inline poly128_t __attribute__ ((__always_inline__)) +vmull_high_p64 (poly64x2_t __a, poly64x2_t __b) +{ + poly64_t __t1 = vget_high_p64 (__a); + poly64_t __t2 = vget_high_p64 (__b); + + return (poly128_t) __builtin_arm_crypto_vmullp64 ((uint64_t) __t1, (uint64_t) __t2); +} + +#endif +" |