aboutsummaryrefslogtreecommitdiffstats
path: root/gcc-4.9
diff options
context:
space:
mode:
authorAlexander Ivchenko <alexander.ivchenko@intel.com>2015-03-16 07:30:57 (GMT)
committerAlexander Ivchenko <alexander.ivchenko@intel.com>2015-03-17 10:03:08 (GMT)
commit3951a3654b8197466bee3e6732b3bc94e4018f68 (patch)
tree5f71295bf4a3df8c9d8187ae983591466ff82a86 /gcc-4.9
parent8075018d7ad15059179e6ff7d0dd12071e1749b9 (diff)
downloadtoolchain_gcc-3951a3654b8197466bee3e6732b3bc94e4018f68.zip
toolchain_gcc-3951a3654b8197466bee3e6732b3bc94e4018f68.tar.gz
toolchain_gcc-3951a3654b8197466bee3e6732b3bc94e4018f68.tar.bz2
[4.9] Several improvements in code generation for x86. Backport from trunk.
2014-11-21 Evgeny Stupachenko <evstupac@gmail.com> PR target/60451 * config/i386/i386.c (expand_vec_perm_even_odd_pack): New. (expand_vec_perm_even_odd_1): Add new expand for V8HI mode, replace for V16QI, V16HI and V32QI modes. (ix86_expand_vec_perm_const_1): Add new expand. 2014-06-11 Evgeny Stupachenko <evstupac@gmail.com> * tree-vect-data-refs.c (vect_grouped_store_supported): New check for stores group of length 3. (vect_permute_store_chain): New permutations for stores group of length 3. * tree-vect-stmts.c (vect_model_store_cost): Change cost of vec_perm_shuffle for the new permutations. 2014-11-28 Evgeny Stupachenko <evstupac@gmail.com> * tree-vect-data-refs.c (vect_transform_grouped_load): Limit shift permutations to loads group of size 3. 2014-12-18 Bin Cheng <bin.cheng@arm.com> PR tree-optimization/62178 * tree-ssa-loop-ivopts.c (cheaper_cost_with_cand): New function. (iv_ca_replace): New function. (try_improve_iv_set): New parameter try_replace_p. Break local optimal fixed-point by calling iv_ca_replace. (find_optimal_iv_set_1): Pass new argument to try_improve_iv_set. Change-Id: I5dca8236d3807cedc5e09d7eda65f0ccec9f5cb2 Signed-off-by: Alexander Ivchenko <alexander.ivchenko@intel.com>
Diffstat (limited to 'gcc-4.9')
-rw-r--r--gcc-4.9/gcc/config/i386/i386.c153
-rw-r--r--gcc-4.9/gcc/tree-ssa-loop-ivopts.c127
-rw-r--r--gcc-4.9/gcc/tree-vect-data-refs.c222
-rw-r--r--gcc-4.9/gcc/tree-vect-stmts.c6
4 files changed, 429 insertions, 79 deletions
diff --git a/gcc-4.9/gcc/config/i386/i386.c b/gcc-4.9/gcc/config/i386/i386.c
index fcd5f0d..a598b8e 100644
--- a/gcc-4.9/gcc/config/i386/i386.c
+++ b/gcc-4.9/gcc/config/i386/i386.c
@@ -44231,6 +44231,127 @@ expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
return true;
}
+/* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
+ and extract-odd permutations of two V16QI, V8HI, V16HI or V32QI operands
+ with two "and" and "pack" or two "shift" and "pack" insns. We should
+ have already failed all two instruction sequences. */
+
+static bool
+expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
+{
+ rtx op, dop0, dop1, t, rperm[16];
+ unsigned i, odd, c, s, nelt = d->nelt;
+ bool end_perm = false;
+ machine_mode half_mode;
+ rtx (*gen_and) (rtx, rtx, rtx);
+ rtx (*gen_pack) (rtx, rtx, rtx);
+ rtx (*gen_shift) (rtx, rtx, rtx);
+
+ if (d->one_operand_p)
+ return false;
+
+ switch (d->vmode)
+ {
+ case V8HImode:
+ /* Required for "pack". */
+ if (!TARGET_SSE4_1)
+ return false;
+ c = 0xffff;
+ s = 16;
+ half_mode = V4SImode;
+ gen_and = gen_andv4si3;
+ gen_pack = gen_sse4_1_packusdw;
+ gen_shift = gen_lshrv4si3;
+ break;
+ case V16QImode:
+ /* No check as all instructions are SSE2. */
+ c = 0xff;
+ s = 8;
+ half_mode = V8HImode;
+ gen_and = gen_andv8hi3;
+ gen_pack = gen_sse2_packuswb;
+ gen_shift = gen_lshrv8hi3;
+ break;
+ case V16HImode:
+ if (!TARGET_AVX2)
+ return false;
+ c = 0xffff;
+ s = 16;
+ half_mode = V8SImode;
+ gen_and = gen_andv8si3;
+ gen_pack = gen_avx2_packusdw;
+ gen_shift = gen_lshrv8si3;
+ end_perm = true;
+ break;
+ case V32QImode:
+ if (!TARGET_AVX2)
+ return false;
+ c = 0xff;
+ s = 8;
+ half_mode = V16HImode;
+ gen_and = gen_andv16hi3;
+ gen_pack = gen_avx2_packuswb;
+ gen_shift = gen_lshrv16hi3;
+ end_perm = true;
+ break;
+ default:
+ /* Only V8HI, V16QI, V16HI and V32QI modes are more profitable than
+ general shuffles. */
+ return false;
+ }
+
+ /* Check that permutation is even or odd. */
+ odd = d->perm[0];
+ if (odd > 1)
+ return false;
+
+ for (i = 1; i < nelt; ++i)
+ if (d->perm[i] != 2 * i + odd)
+ return false;
+
+ if (d->testing_p)
+ return true;
+
+ dop0 = gen_reg_rtx (half_mode);
+ dop1 = gen_reg_rtx (half_mode);
+ if (odd == 0)
+ {
+ for (i = 0; i < nelt / 2; i++)
+ rperm[i] = GEN_INT (c);
+ t = gen_rtx_CONST_VECTOR (half_mode, gen_rtvec_v (nelt / 2, rperm));
+ t = force_reg (half_mode, t);
+ emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
+ emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
+ }
+ else
+ {
+ emit_insn (gen_shift (dop0,
+ gen_lowpart (half_mode, d->op0),
+ GEN_INT (s)));
+ emit_insn (gen_shift (dop1,
+ gen_lowpart (half_mode, d->op1),
+ GEN_INT (s)));
+ }
+ /* In AVX2 for 256 bit case we need to permute pack result. */
+ if (TARGET_AVX2 && end_perm)
+ {
+ op = gen_reg_rtx (d->vmode);
+ t = gen_reg_rtx (V4DImode);
+ emit_insn (gen_pack (op, dop0, dop1));
+ emit_insn (gen_avx2_permv4di_1 (t,
+ gen_lowpart (V4DImode, op),
+ const0_rtx,
+ const2_rtx,
+ const1_rtx,
+ GEN_INT (3)));
+ emit_move_insn (d->target, gen_lowpart (d->vmode, t));
+ }
+ else
+ emit_insn (gen_pack (d->target, dop0, dop1));
+
+ return true;
+}
+
/* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
and extract-odd permutations. */
@@ -44302,7 +44423,9 @@ expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
gcc_unreachable ();
case V8HImode:
- if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
+ if (TARGET_SSE4_1)
+ return expand_vec_perm_even_odd_pack (d);
+ else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
return expand_vec_perm_pshufb2 (d);
else
{
@@ -44325,32 +44448,11 @@ expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
break;
case V16QImode:
- if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
- return expand_vec_perm_pshufb2 (d);
- else
- {
- if (d->testing_p)
- break;
- t1 = gen_reg_rtx (V16QImode);
- t2 = gen_reg_rtx (V16QImode);
- t3 = gen_reg_rtx (V16QImode);
- emit_insn (gen_vec_interleave_highv16qi (t1, d->op0, d->op1));
- emit_insn (gen_vec_interleave_lowv16qi (d->target, d->op0, d->op1));
- emit_insn (gen_vec_interleave_highv16qi (t2, d->target, t1));
- emit_insn (gen_vec_interleave_lowv16qi (d->target, d->target, t1));
- emit_insn (gen_vec_interleave_highv16qi (t3, d->target, t2));
- emit_insn (gen_vec_interleave_lowv16qi (d->target, d->target, t2));
- if (odd)
- t3 = gen_vec_interleave_highv16qi (d->target, d->target, t3);
- else
- t3 = gen_vec_interleave_lowv16qi (d->target, d->target, t3);
- emit_insn (t3);
- }
- break;
+ return expand_vec_perm_even_odd_pack (d);
case V16HImode:
case V32QImode:
- return expand_vec_perm_vpshufb2_vpermq_even_odd (d);
+ return expand_vec_perm_even_odd_pack (d);
case V4DImode:
if (!TARGET_AVX2)
@@ -44720,6 +44822,9 @@ ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
/* Try sequences of three instructions. */
+ if (expand_vec_perm_even_odd_pack (d))
+ return true;
+
if (expand_vec_perm_2vperm2f128_vshuf (d))
return true;
diff --git a/gcc-4.9/gcc/tree-ssa-loop-ivopts.c b/gcc-4.9/gcc/tree-ssa-loop-ivopts.c
index 78f036e..c5a5dd4 100644
--- a/gcc-4.9/gcc/tree-ssa-loop-ivopts.c
+++ b/gcc-4.9/gcc/tree-ssa-loop-ivopts.c
@@ -5863,6 +5863,108 @@ iv_ca_prune (struct ivopts_data *data, struct iv_ca *ivs,
return best_cost;
}
+/* Check if CAND_IDX is a candidate other than OLD_CAND and has
+ cheaper local cost for USE than BEST_CP. Return pointer to
+ the corresponding cost_pair, otherwise just return BEST_CP. */
+
+static struct cost_pair*
+cheaper_cost_with_cand (struct ivopts_data *data, struct iv_use *use,
+ unsigned int cand_idx, struct iv_cand *old_cand,
+ struct cost_pair *best_cp)
+{
+ struct iv_cand *cand;
+ struct cost_pair *cp;
+
+ gcc_assert (old_cand != NULL && best_cp != NULL);
+ if (cand_idx == old_cand->id)
+ return best_cp;
+
+ cand = iv_cand (data, cand_idx);
+ cp = get_use_iv_cost (data, use, cand);
+ if (cp != NULL && cheaper_cost_pair (cp, best_cp))
+ return cp;
+
+ return best_cp;
+}
+
+/* Try breaking local optimal fixed-point for IVS by replacing candidates
+ which are used by more than one iv uses. For each of those candidates,
+ this function tries to represent iv uses under that candidate using
+ other ones with lower local cost, then tries to prune the new set.
+ If the new set has lower cost, It returns the new cost after recording
+ candidate replacement in list DELTA. */
+
+static comp_cost
+iv_ca_replace (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_ca_delta **delta)
+{
+ bitmap_iterator bi, bj;
+ unsigned int i, j, k;
+ struct iv_use *use;
+ struct iv_cand *cand;
+ comp_cost orig_cost, acost;
+ struct iv_ca_delta *act_delta, *tmp_delta;
+ struct cost_pair *old_cp, *best_cp = NULL;
+
+ *delta = NULL;
+ orig_cost = iv_ca_cost (ivs);
+
+ EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
+ {
+ if (ivs->n_cand_uses[i] == 1
+ || ivs->n_cand_uses[i] > ALWAYS_PRUNE_CAND_SET_BOUND)
+ continue;
+
+ cand = iv_cand (data, i);
+
+ act_delta = NULL;
+ /* Represent uses under current candidate using other ones with
+ lower local cost. */
+ for (j = 0; j < ivs->upto; j++)
+ {
+ use = iv_use (data, j);
+ old_cp = iv_ca_cand_for_use (ivs, use);
+
+ if (old_cp->cand != cand)
+ continue;
+
+ best_cp = old_cp;
+ if (data->consider_all_candidates)
+ for (k = 0; k < n_iv_cands (data); k++)
+ best_cp = cheaper_cost_with_cand (data, use, k,
+ old_cp->cand, best_cp);
+ else
+ EXECUTE_IF_SET_IN_BITMAP (use->related_cands, 0, k, bj)
+ best_cp = cheaper_cost_with_cand (data, use, k,
+ old_cp->cand, best_cp);
+
+ if (best_cp == old_cp)
+ continue;
+
+ act_delta = iv_ca_delta_add (use, old_cp, best_cp, act_delta);
+ }
+ /* No need for further prune. */
+ if (!act_delta)
+ continue;
+
+ /* Prune the new candidate set. */
+ iv_ca_delta_commit (data, ivs, act_delta, true);
+ acost = iv_ca_prune (data, ivs, NULL, &tmp_delta);
+ iv_ca_delta_commit (data, ivs, act_delta, false);
+ act_delta = iv_ca_delta_join (act_delta, tmp_delta);
+
+ if (compare_costs (acost, orig_cost) < 0)
+ {
+ *delta = act_delta;
+ return acost;
+ }
+ else
+ iv_ca_delta_free (&act_delta);
+ }
+
+ return orig_cost;
+}
+
/* Tries to extend the sets IVS in the best possible way in order
to express the USE. If ORIGINALP is true, prefer candidates from
the original set of IVs, otherwise favor important candidates not
@@ -6005,10 +6107,13 @@ get_initial_solution (struct ivopts_data *data, bool originalp)
return ivs;
}
-/* Tries to improve set of induction variables IVS. */
+/* Tries to improve set of induction variables IVS. TRY_REPLACE_P
+ points to a bool variable, this function tries to break local
+ optimal fixed-point by replacing candidates in IVS if it's true. */
static bool
-try_improve_iv_set (struct ivopts_data *data, struct iv_ca *ivs)
+try_improve_iv_set (struct ivopts_data *data,
+ struct iv_ca *ivs, bool *try_replace_p)
{
unsigned i, n_ivs;
comp_cost acost, best_cost = iv_ca_cost (ivs);
@@ -6052,7 +6157,20 @@ try_improve_iv_set (struct ivopts_data *data, struct iv_ca *ivs)
/* Try removing the candidates from the set instead. */
best_cost = iv_ca_prune (data, ivs, NULL, &best_delta);
- /* Nothing more we can do. */
+ if (!best_delta && *try_replace_p)
+ {
+ *try_replace_p = false;
+ /* So far candidate selecting algorithm tends to choose fewer IVs
+ so that it can handle cases in which loops have many variables
+ but the best choice is often to use only one general biv. One
+ weakness is it can't handle opposite cases, in which different
+ candidates should be chosen with respect to each use. To solve
+ the problem, we replace candidates in a manner described by the
+ comments of iv_ca_replace, thus give general algorithm a chance
+ to break local optimal fixed-point in these cases. */
+ best_cost = iv_ca_replace (data, ivs, &best_delta);
+ }
+
if (!best_delta)
return false;
}
@@ -6071,6 +6189,7 @@ static struct iv_ca *
find_optimal_iv_set_1 (struct ivopts_data *data, bool originalp)
{
struct iv_ca *set;
+ bool try_replace_p = true;
/* Get the initial solution. */
set = get_initial_solution (data, originalp);
@@ -6087,7 +6206,7 @@ find_optimal_iv_set_1 (struct ivopts_data *data, bool originalp)
iv_ca_dump (data, dump_file, set);
}
- while (try_improve_iv_set (data, set))
+ while (try_improve_iv_set (data, set, &try_replace_p))
{
if (dump_file && (dump_flags & TDF_DETAILS))
{
diff --git a/gcc-4.9/gcc/tree-vect-data-refs.c b/gcc-4.9/gcc/tree-vect-data-refs.c
index 853b89a..7af32d1 100644
--- a/gcc-4.9/gcc/tree-vect-data-refs.c
+++ b/gcc-4.9/gcc/tree-vect-data-refs.c
@@ -4391,13 +4391,14 @@ vect_grouped_store_supported (tree vectype, unsigned HOST_WIDE_INT count)
{
enum machine_mode mode = TYPE_MODE (vectype);
- /* vect_permute_store_chain requires the group size to be a power of two. */
- if (exact_log2 (count) == -1)
+ /* vect_permute_store_chain requires the group size to be equal to 3 or
+ be a power of two. */
+ if (count != 3 && exact_log2 (count) == -1)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "the size of the group of accesses"
- " is not a power of 2\n");
+ "the size of the group of accesses"
+ " is not a power of 2 or not eqaul to 3\n");
return false;
}
@@ -4406,23 +4407,76 @@ vect_grouped_store_supported (tree vectype, unsigned HOST_WIDE_INT count)
{
unsigned int i, nelt = GET_MODE_NUNITS (mode);
unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
- for (i = 0; i < nelt / 2; i++)
+
+ if (count == 3)
{
- sel[i * 2] = i;
- sel[i * 2 + 1] = i + nelt;
+ unsigned int j0 = 0, j1 = 0, j2 = 0;
+ unsigned int i, j;
+
+ for (j = 0; j < 3; j++)
+ {
+ int nelt0 = ((3 - j) * nelt) % 3;
+ int nelt1 = ((3 - j) * nelt + 1) % 3;
+ int nelt2 = ((3 - j) * nelt + 2) % 3;
+ for (i = 0; i < nelt; i++)
+ {
+ if (3 * i + nelt0 < nelt)
+ sel[3 * i + nelt0] = j0++;
+ if (3 * i + nelt1 < nelt)
+ sel[3 * i + nelt1] = nelt + j1++;
+ if (3 * i + nelt2 < nelt)
+ sel[3 * i + nelt2] = 0;
+ }
+ if (!can_vec_perm_p (mode, false, sel))
+ {
+ if (dump_enabled_p ())
+ dump_printf (MSG_MISSED_OPTIMIZATION,
+ "permutaion op not supported by target.\n");
+ return false;
+ }
+
+ for (i = 0; i < nelt; i++)
+ {
+ if (3 * i + nelt0 < nelt)
+ sel[3 * i + nelt0] = 3 * i + nelt0;
+ if (3 * i + nelt1 < nelt)
+ sel[3 * i + nelt1] = 3 * i + nelt1;
+ if (3 * i + nelt2 < nelt)
+ sel[3 * i + nelt2] = nelt + j2++;
+ }
+ if (!can_vec_perm_p (mode, false, sel))
+ {
+ if (dump_enabled_p ())
+ dump_printf (MSG_MISSED_OPTIMIZATION,
+ "permutaion op not supported by target.\n");
+ return false;
+ }
+ }
+ return true;
}
- if (can_vec_perm_p (mode, false, sel))
+ else
{
- for (i = 0; i < nelt; i++)
- sel[i] += nelt / 2;
- if (can_vec_perm_p (mode, false, sel))
- return true;
+ /* If length is not equal to 3 then only power of 2 is supported. */
+ gcc_assert (exact_log2 (count) != -1);
+
+ for (i = 0; i < nelt / 2; i++)
+ {
+ sel[i * 2] = i;
+ sel[i * 2 + 1] = i + nelt;
+ }
+ if (can_vec_perm_p (mode, false, sel))
+ {
+ for (i = 0; i < nelt; i++)
+ sel[i] += nelt / 2;
+ if (can_vec_perm_p (mode, false, sel))
+ return true;
+ }
}
}
if (dump_enabled_p ())
dump_printf (MSG_MISSED_OPTIMIZATION,
- "interleave op not supported by target.\n");
+ "permutaion op not supported by target.\n");
return false;
}
@@ -4442,9 +4496,9 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count)
/* Function vect_permute_store_chain.
Given a chain of interleaved stores in DR_CHAIN of LENGTH that must be
- a power of 2, generate interleave_high/low stmts to reorder the data
- correctly for the stores. Return the final references for stores in
- RESULT_CHAIN.
+ a power of 2 or equal to 3, generate interleave_high/low stmts to reorder
+ the data correctly for the stores. Return the final references for stores
+ in RESULT_CHAIN.
E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
The input is 4 vectors each containing 8 elements. We assign a number to
@@ -4511,7 +4565,9 @@ vect_permute_store_chain (vec<tree> dr_chain,
gimple perm_stmt;
tree vectype = STMT_VINFO_VECTYPE (vinfo_for_stmt (stmt));
tree perm_mask_low, perm_mask_high;
- unsigned int i, n;
+ tree data_ref;
+ tree perm3_mask_low, perm3_mask_high;
+ unsigned int i, n, log_length = exact_log2 (length);
unsigned int j, nelt = TYPE_VECTOR_SUBPARTS (vectype);
unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
@@ -4519,47 +4575,116 @@ vect_permute_store_chain (vec<tree> dr_chain,
memcpy (result_chain->address (), dr_chain.address (),
length * sizeof (tree));
- for (i = 0, n = nelt / 2; i < n; i++)
+ if (length == 3)
{
- sel[i * 2] = i;
- sel[i * 2 + 1] = i + nelt;
- }
- perm_mask_high = vect_gen_perm_mask (vectype, sel);
- gcc_assert (perm_mask_high != NULL);
+ unsigned int j0 = 0, j1 = 0, j2 = 0;
- for (i = 0; i < nelt; i++)
- sel[i] += nelt / 2;
- perm_mask_low = vect_gen_perm_mask (vectype, sel);
- gcc_assert (perm_mask_low != NULL);
+ for (j = 0; j < 3; j++)
+ {
+ int nelt0 = ((3 - j) * nelt) % 3;
+ int nelt1 = ((3 - j) * nelt + 1) % 3;
+ int nelt2 = ((3 - j) * nelt + 2) % 3;
- for (i = 0, n = exact_log2 (length); i < n; i++)
- {
- for (j = 0; j < length/2; j++)
- {
- vect1 = dr_chain[j];
- vect2 = dr_chain[j+length/2];
+ for (i = 0; i < nelt; i++)
+ {
+ if (3 * i + nelt0 < nelt)
+ sel[3 * i + nelt0] = j0++;
+ if (3 * i + nelt1 < nelt)
+ sel[3 * i + nelt1] = nelt + j1++;
+ if (3 * i + nelt2 < nelt)
+ sel[3 * i + nelt2] = 0;
+ }
+ perm3_mask_low = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm3_mask_low != NULL);
+
+ for (i = 0; i < nelt; i++)
+ {
+ if (3 * i + nelt0 < nelt)
+ sel[3 * i + nelt0] = 3 * i + nelt0;
+ if (3 * i + nelt1 < nelt)
+ sel[3 * i + nelt1] = 3 * i + nelt1;
+ if (3 * i + nelt2 < nelt)
+ sel[3 * i + nelt2] = nelt + j2++;
+ }
+ perm3_mask_high = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm3_mask_high != NULL);
+
+ vect1 = dr_chain[0];
+ vect2 = dr_chain[1];
/* Create interleaving stmt:
- high = VEC_PERM_EXPR <vect1, vect2, {0, nelt, 1, nelt+1, ...}> */
- high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
- perm_stmt
- = gimple_build_assign_with_ops (VEC_PERM_EXPR, high,
- vect1, vect2, perm_mask_high);
+ low = VEC_PERM_EXPR <vect1, vect2,
+ {j, nelt, *, j + 1, nelt + j + 1, *,
+ j + 2, nelt + j + 2, *, ...}> */
+ data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
+ perm_stmt = gimple_build_assign_with_ops (VEC_PERM_EXPR, data_ref,
+ vect1, vect2,
+ perm3_mask_low);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
- (*result_chain)[2*j] = high;
+ vect1 = data_ref;
+ vect2 = dr_chain[2];
/* Create interleaving stmt:
- low = VEC_PERM_EXPR <vect1, vect2, {nelt/2, nelt*3/2, nelt/2+1,
- nelt*3/2+1, ...}> */
- low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
- perm_stmt
- = gimple_build_assign_with_ops (VEC_PERM_EXPR, low,
- vect1, vect2, perm_mask_low);
+ low = VEC_PERM_EXPR <vect1, vect2,
+ {0, 1, nelt + j, 3, 4, nelt + j + 1,
+ 6, 7, nelt + j + 2, ...}> */
+ data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
+ perm_stmt = gimple_build_assign_with_ops (VEC_PERM_EXPR, data_ref,
+ vect1, vect2,
+ perm3_mask_high);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
- (*result_chain)[2*j+1] = low;
+ (*result_chain)[j] = data_ref;
}
- memcpy (dr_chain.address (), result_chain->address (),
- length * sizeof (tree));
+ }
+ else
+ {
+ /* If length is not equal to 3 then only power of 2 is supported. */
+ gcc_assert (exact_log2 (length) != -1);
+
+ for (i = 0, n = nelt / 2; i < n; i++)
+ {
+ sel[i * 2] = i;
+ sel[i * 2 + 1] = i + nelt;
+ }
+ perm_mask_high = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm_mask_high != NULL);
+
+ for (i = 0; i < nelt; i++)
+ sel[i] += nelt / 2;
+ perm_mask_low = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm_mask_low != NULL);
+
+ for (i = 0, n = log_length; i < n; i++)
+ {
+ for (j = 0; j < length/2; j++)
+ {
+ vect1 = dr_chain[j];
+ vect2 = dr_chain[j+length/2];
+
+ /* Create interleaving stmt:
+ high = VEC_PERM_EXPR <vect1, vect2, {0, nelt, 1, nelt+1,
+ ...}> */
+ high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
+ perm_stmt
+ = gimple_build_assign_with_ops (VEC_PERM_EXPR, high,
+ vect1, vect2, perm_mask_high);
+ vect_finish_stmt_generation (stmt, perm_stmt, gsi);
+ (*result_chain)[2*j] = high;
+
+ /* Create interleaving stmt:
+ low = VEC_PERM_EXPR <vect1, vect2,
+ {nelt/2, nelt*3/2, nelt/2+1, nelt*3/2+1,
+ ...}> */
+ low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
+ perm_stmt
+ = gimple_build_assign_with_ops (VEC_PERM_EXPR, low,
+ vect1, vect2, perm_mask_low);
+ vect_finish_stmt_generation (stmt, perm_stmt, gsi);
+ (*result_chain)[2*j+1] = low;
+ }
+ memcpy (dr_chain.address (), result_chain->address (),
+ length * sizeof (tree));
+ }
}
}
@@ -5475,6 +5600,7 @@ vect_transform_grouped_load (gimple stmt, vec<tree> dr_chain, int size,
get chain for loads group using vect_shift_permute_load_chain. */
mode = TYPE_MODE (STMT_VINFO_VECTYPE (vinfo_for_stmt (stmt)));
if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
+ || exact_log2 (size) != -1
|| !vect_shift_permute_load_chain (dr_chain, size, stmt,
gsi, &result_chain))
vect_permute_load_chain (dr_chain, size, stmt, gsi, &result_chain);
diff --git a/gcc-4.9/gcc/tree-vect-stmts.c b/gcc-4.9/gcc/tree-vect-stmts.c
index d208f2d..9677d60 100644
--- a/gcc-4.9/gcc/tree-vect-stmts.c
+++ b/gcc-4.9/gcc/tree-vect-stmts.c
@@ -974,9 +974,9 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
include the cost of the permutes. */
if (!store_lanes_p && group_size > 1)
{
- /* Uses a high and low interleave operation for each needed permute. */
-
- int nstmts = ncopies * exact_log2 (group_size) * group_size;
+ /* Uses a high and low interleave or shuffle operations for each
+ needed permute. */
+ int nstmts = ncopies * ceil_log2 (group_size) * group_size;
inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
stmt_info, 0, vect_body);