diff options
Diffstat (limited to 'gcc-4.9/gcc/value-prof.c')
| -rw-r--r-- | gcc-4.9/gcc/value-prof.c | 454 |
1 files changed, 428 insertions, 26 deletions
diff --git a/gcc-4.9/gcc/value-prof.c b/gcc-4.9/gcc/value-prof.c index 289009347..c0bd06538 100644 --- a/gcc-4.9/gcc/value-prof.c +++ b/gcc-4.9/gcc/value-prof.c @@ -55,7 +55,11 @@ along with GCC; see the file COPYING3. If not see #include "gcov-io.h" #include "timevar.h" #include "dumpfile.h" +#include "langhooks.h" +#include "params.h" +#include "l-ipo.h" #include "profile.h" +#include "ipa-inline.h" #include "data-streamer.h" #include "builtins.h" #include "tree-nested.h" @@ -133,11 +137,10 @@ static bool gimple_divmod_fixed_value_transform (gimple_stmt_iterator *); static bool gimple_mod_pow2_value_transform (gimple_stmt_iterator *); static bool gimple_mod_subtract_transform (gimple_stmt_iterator *); static bool gimple_stringops_transform (gimple_stmt_iterator *); -static bool gimple_ic_transform (gimple_stmt_iterator *); /* Allocate histogram value. */ -static histogram_value +histogram_value gimple_alloc_histogram_value (struct function *fun ATTRIBUTE_UNUSED, enum hist_type type, gimple stmt, tree value) { @@ -349,6 +352,10 @@ dump_histogram_value (FILE *dump_file, histogram_value hist) } fprintf (dump_file, ".\n"); break; + case HIST_TYPE_INDIR_CALL_TOPN: + fprintf (dump_file, "Indirect call -- top N\n"); + /* TODO add more elaborate dumping code. */ + break; case HIST_TYPE_TIME_PROFILE: fprintf (dump_file, "Time profile "); if (hist->hvalue.counters) @@ -434,6 +441,11 @@ stream_in_histogram_value (struct lto_input_block *ib, gimple stmt) case HIST_TYPE_TIME_PROFILE: ncounters = 1; break; + + case HIST_TYPE_INDIR_CALL_TOPN: + ncounters = (GCOV_ICALL_TOPN_VAL << 2) + 1; + break; + case HIST_TYPE_MAX: gcc_unreachable (); } @@ -638,6 +650,70 @@ check_counter (gimple stmt, const char * name, return false; } +/* The overall number of invocations of the counter should match + execution count of basic block. Report it as error rather than + internal error as it might mean that user has misused the profile + somehow. STMT is the indiret call, COUNT1 and COUNT2 are counts + of two top targets, and ALL is the enclosing basic block execution + count. */ + +static bool +check_ic_counter (gimple stmt, gcov_type *count1, gcov_type *count2, + gcov_type all) +{ + location_t locus; + locus = (stmt != NULL) + ? gimple_location (stmt) + : DECL_SOURCE_LOCATION (current_function_decl); + if (*count1 > all && flag_profile_correction) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus, + "Correcting inconsistent value profile: " + "ic (topn) profiler top target count (%ld) exceeds " + "BB count (%ld)", (long)*count1, (long)all); + *count1 = all; + } + if (*count2 > all && flag_profile_correction) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus, + "Correcting inconsistent value profile: " + "ic (topn) profiler second target count (%ld) exceeds " + "BB count (%ld)", (long)*count2, (long)all); + *count2 = all; + } + + if (*count2 > *count1) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus, + "Corrupted topn ic value profile: " + "first target count (%ld) is less than the second " + "target count (%ld)", (long)*count1, (long)*count2); + return true; + } + + if (*count1 + *count2 > all) + { + /* If (COUNT1 + COUNT2) is greater than ALL by less than around 10% then + just fix COUNT2 up so that (COUNT1 + COUNT2) equals ALL. */ + if ((*count1 + *count2 - all) < (all >> 3)) + *count2 = all - *count1; + else + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus, + "Corrupted topn ic value profile: top two " + "targets's total count (%ld) exceeds bb count " + "(%ld)", + (long)(*count1 + *count2), (long)all); + return true; + } + } + return false; +} + /* GIMPLE based transformations. */ @@ -690,9 +766,7 @@ gimple_value_profile_transformations (void) } if (changed) - { - counts_to_freqs (); - } + counts_to_freqs (); return changed; } @@ -1214,6 +1288,9 @@ init_node_map (bool local) struct cgraph_node *n; cgraph_node_map = pointer_map_create (); + if (L_IPO_COMP_MODE) + return; + FOR_EACH_DEFINED_FUNCTION (n) if (cgraph_function_with_gimple_body_p (n) && !cgraph_only_called_directly_p (n)) @@ -1270,6 +1347,9 @@ init_node_map (bool local) void del_node_map (void) { + if (L_IPO_COMP_MODE) + return; + pointer_map_destroy (cgraph_node_map); } @@ -1286,13 +1366,129 @@ find_func_by_profile_id (int profile_id) return NULL; } +/* Initialize map of gids (gid -> cgraph node) */ + +static htab_t gid_map = NULL; + +typedef struct func_gid_entry +{ + struct cgraph_node *node; + unsigned HOST_WIDEST_INT gid; +} func_gid_entry_t; + +/* Hash function for function global unique ids. */ + +static hashval_t +htab_gid_hash (const void * ent) +{ + const func_gid_entry_t *const entry = (const func_gid_entry_t *) ent; + return entry->gid; +} + +/* Hash table equality function for function global unique ids. */ + +static int +htab_gid_eq (const void *ent1, const void * ent2) +{ + const func_gid_entry_t *const entry1 = (const func_gid_entry_t *) ent1; + const func_gid_entry_t *const entry2 = (const func_gid_entry_t *) ent2; + return entry1->gid == entry2->gid; +} + +static void +htab_gid_del (void *ent) +{ + func_gid_entry_t *const entry = (func_gid_entry_t *) ent; + free (entry); +} + +/* Initialize the global unique id map for functions. */ + +static void +init_gid_map (void) +{ + struct cgraph_node *n; + + if (!gid_map) + gid_map = htab_create (10, htab_gid_hash, htab_gid_eq, htab_gid_del); + + FOR_EACH_FUNCTION (n) + { + func_gid_entry_t ent, *entp; + func_gid_entry_t **slot; + struct function *f; + ent.node = n; + f = DECL_STRUCT_FUNCTION (n->decl); + /* Do not care to indirect call promote a function with id. */ + if (!f || DECL_ABSTRACT (n->decl)) + continue; + /* The global function id computed at profile-use time + is slightly different from the one computed in + instrumentation runtime -- for the latter, the intra- + module function ident is 1 based while in profile-use + phase, it is zero based. See get_next_funcdef_no in + function.c. */ + ent.gid = FUNC_DECL_GLOBAL_ID (DECL_STRUCT_FUNCTION (n->decl)); + slot = (func_gid_entry_t **) htab_find_slot (gid_map, &ent, INSERT); + + gcc_assert (!*slot || ((*slot)->gid == ent.gid && (*slot)->node == n)); + if (!*slot) + { + *slot = entp = XCNEW (func_gid_entry_t); + entp->node = n; + entp->gid = ent.gid; + } + else if (cgraph_pre_profiling_inlining_done) + { + (*slot)->node = cgraph_lipo_get_resolved_node (n->decl); + (*slot)->gid = ent.gid; + } + else + { + gcc_assert ((*slot)->gid == ent.gid && (*slot)->node == n); + } + } +} + +/* Initialize the global unique id map for functions. */ + +void +cgraph_init_gid_map (void) +{ + if (!L_IPO_COMP_MODE) + return; + + init_gid_map (); +} + +/* Return cgraph node for function with global id. */ + +struct cgraph_node * +find_func_by_global_id (unsigned HOST_WIDE_INT gid, bool is_auto_fdo) +{ + func_gid_entry_t ent, *entp; + + if (is_auto_fdo) + return cgraph_node_for_asm (get_identifier ((const char *) gid)); + + gcc_assert (gid_map); + + ent.node = NULL; + ent.gid = gid; + entp = (func_gid_entry_t *)htab_find (gid_map, &ent); + if (entp) + return entp->node; + return NULL; +} + + /* Perform sanity check on the indirect call target. Due to race conditions, false function target may be attributed to an indirect call site. If the call expression type mismatches with the target function's type, expand_call may ICE. Here we only do very minimal sanity check just to make compiler happy. Returns true if TARGET is considered ok for call CALL_STMT. */ -static bool +bool check_ic_target (gimple call_stmt, struct cgraph_node *target) { location_t locus; @@ -1456,26 +1652,11 @@ gimple_ic (gimple icall_stmt, struct cgraph_node *direct_call, */ static bool -gimple_ic_transform (gimple_stmt_iterator *gsi) +gimple_ic_transform_single_targ (gimple stmt, histogram_value histogram) { - gimple stmt = gsi_stmt (*gsi); - histogram_value histogram; gcov_type val, count, all, bb_all; struct cgraph_node *direct_call; - if (gimple_code (stmt) != GIMPLE_CALL) - return false; - - if (gimple_call_fndecl (stmt) != NULL_TREE) - return false; - - if (gimple_call_internal_p (stmt)) - return false; - - histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_INDIR_CALL); - if (!histogram) - return false; - val = histogram->hvalue.counters [0]; count = histogram->hvalue.counters [1]; all = histogram->hvalue.counters [2]; @@ -1540,6 +1721,219 @@ gimple_ic_transform (gimple_stmt_iterator *gsi) return true; } +/* Convert indirect function call STMT into guarded direct function + calls. Multiple indirect call targets are supported. HISTOGRAM + is the target distribution for the callsite. */ + +static bool +gimple_ic_transform_mult_targ (gimple stmt, histogram_value histogram) +{ + gcov_type val1, val2, count1, count2, all, bb_all; + gcov_type prob1, prob2; + gimple modify1, modify2; + struct cgraph_node *direct_call1 = 0, *direct_call2 = 0; + int perc_threshold, count_threshold, always_inline; + int use_hotness_heur = false; + location_t locus; + + val1 = histogram->hvalue.counters [1]; + count1 = histogram->hvalue.counters [2]; + val2 = histogram->hvalue.counters [3]; + count2 = histogram->hvalue.counters [4]; + bb_all = flag_auto_profile ? histogram->hvalue.counters[0] + : gimple_bb (stmt)->count; + all = bb_all; + + gimple_remove_histogram_value (cfun, stmt, histogram); + + if (count1 == 0) + return false; + + perc_threshold = PARAM_VALUE (PARAM_ICALL_PROMOTE_PERCENT_THRESHOLD); + count_threshold = PARAM_VALUE (PARAM_ICALL_PROMOTE_COUNT_THRESHOLD); + always_inline = PARAM_VALUE (PARAM_ALWAYS_INLINE_ICALL_TARGET); + use_hotness_heur = PARAM_VALUE (PARAM_ICALL_USE_HOTNESS_HEUR); + + if (!use_hotness_heur) + { + if (100 * count1 < all * perc_threshold || count1 < count_threshold) + return false; + } + else + { + if (!maybe_hot_count_p (cfun, count1)) + return false; + } + + if (check_ic_counter (stmt, &count1, &count2, all)) + return false; + + if (all > 0) + { + prob1 = (count1 * REG_BR_PROB_BASE + all / 2) / all; + if (all - count1 > 0) + prob2 = (count2 * REG_BR_PROB_BASE + + (all - count1) / 2) / (all - count1); + else + prob2 = 0; + } + else + prob1 = prob2 = 0; + + direct_call1 = find_func_by_global_id (val1, flag_auto_profile); + + if (val2 && (100 * count2 >= all * perc_threshold) + && count2 > count_threshold) + direct_call2 = find_func_by_global_id (val2, flag_auto_profile); + + locus = (stmt != NULL) ? gimple_location (stmt) + : DECL_SOURCE_LOCATION (current_function_decl); + if (direct_call1 == NULL + || !check_ic_target (stmt, direct_call1)) + { + if (dump_enabled_p () && !flag_auto_profile) + { + if (!direct_call1) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus, + "Can not find indirect call target decl " + "(%d:%d)[cnt:%u] in current module", + EXTRACT_MODULE_ID_FROM_GLOBAL_ID (val1), + EXTRACT_FUNC_ID_FROM_GLOBAL_ID (val1), + (unsigned) count1); + else + dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus, + "Can not find promote indirect call target decl " + "-- type mismatch (%d:%d)[cnt:%u] in current " + "module", + EXTRACT_MODULE_ID_FROM_GLOBAL_ID (val1), + EXTRACT_FUNC_ID_FROM_GLOBAL_ID (val1), + (unsigned) count1); + } + return false; + } + + /* Don't indirect-call promote if the target is in auxiliary module and + DECL_ARTIFICIAL and not TREE_PUBLIC, because we don't static-promote + DECL_ARTIFICIALs yet. */ + if (cgraph_is_auxiliary (direct_call1->decl) + && DECL_ARTIFICIAL (direct_call1->decl) + && ! TREE_PUBLIC (direct_call1->decl)) + return false; + + modify1 = gimple_ic (stmt, direct_call1, prob1, count1, all); + if (dump_enabled_p ()) + dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, locus, + "Promote indirect call to target (call count:%u) %s", + (unsigned) count1, + lang_hooks.decl_printable_name (direct_call1->decl, + 3)); + + if (always_inline && count1 >= always_inline) + { + /* TODO: should mark the call edge. */ + DECL_DISREGARD_INLINE_LIMITS (direct_call1->decl) = 1; + } + if (dump_file) + { + fprintf (dump_file, "Indirect call -> direct call "); + print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM); + fprintf (dump_file, "=> "); + print_generic_expr (dump_file, direct_call1->decl, TDF_SLIM); + if (flag_auto_profile) + fprintf (dump_file, " (%s)\n", (char *) val1); + else + fprintf (dump_file, " (module_id:%d, func_id:%d)\n", + EXTRACT_MODULE_ID_FROM_GLOBAL_ID (val1), + EXTRACT_FUNC_ID_FROM_GLOBAL_ID (val1)); + fprintf (dump_file, "Transformation on insn:\n"); + print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM); + fprintf (dump_file, "==>\n"); + print_gimple_stmt (dump_file, modify1, 0, TDF_SLIM); + fprintf (dump_file, "hist->count "HOST_WIDEST_INT_PRINT_DEC + " hist->all "HOST_WIDEST_INT_PRINT_DEC"\n", count1, all); + } + + if (direct_call2 && check_ic_target (stmt, direct_call2) + /* Don't indirect-call promote if the target is in auxiliary module and + DECL_ARTIFICIAL and not TREE_PUBLIC, because we don't static-promote + DECL_ARTIFICIALs yet. */ + && ! (cgraph_is_auxiliary (direct_call2->decl) + && DECL_ARTIFICIAL (direct_call2->decl) + && ! TREE_PUBLIC (direct_call2->decl))) + { + modify2 = gimple_ic (stmt, direct_call2, + prob2, count2, all - count1); + + if (dump_enabled_p ()) + dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, locus, + "Promote indirect call to target (call count:%u) %s", + (unsigned) count2, + lang_hooks.decl_printable_name (direct_call2->decl, + 3)); + + if (always_inline && count2 >= always_inline) + { + /* TODO: should mark the call edge. */ + DECL_DISREGARD_INLINE_LIMITS (direct_call2->decl) = 1; + } + if (dump_file) + { + fprintf (dump_file, "Indirect call -> direct call "); + print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM); + fprintf (dump_file, "=> "); + print_generic_expr (dump_file, direct_call2->decl, TDF_SLIM); + if (flag_auto_profile) + fprintf (dump_file, " (%s)\n", (char *) val2); + else + fprintf (dump_file, " (module_id:%d, func_id:%d)\n", + EXTRACT_MODULE_ID_FROM_GLOBAL_ID (val2), + EXTRACT_FUNC_ID_FROM_GLOBAL_ID (val2)); + fprintf (dump_file, "Transformation on insn\n"); + print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM); + fprintf (dump_file, "=>\n"); + print_gimple_stmt (dump_file, modify2, 0, TDF_SLIM); + fprintf (dump_file, "hist->count "HOST_WIDEST_INT_PRINT_DEC + " hist->all "HOST_WIDEST_INT_PRINT_DEC"\n", count2, + all - count1); + } + } + + return true; +} + +/* Perform indirect call (STMT) to guarded direct function call + transformation using value profile data. */ + +bool +gimple_ic_transform (gimple_stmt_iterator *gsi) +{ + gimple stmt = gsi_stmt (*gsi); + histogram_value histogram; + tree callee; + + if (gimple_code (stmt) != GIMPLE_CALL) + return false; + + callee = gimple_call_fn (stmt); + + if (TREE_CODE (callee) == FUNCTION_DECL) + return false; + + histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_INDIR_CALL); + if (!histogram) + { + histogram = gimple_histogram_value_of_type (cfun, stmt, + HIST_TYPE_INDIR_CALL_TOPN); + if (!histogram) + return false; + } + + if (histogram->type == HIST_TYPE_INDIR_CALL) + return gimple_ic_transform_single_targ (stmt, histogram); + else + return gimple_ic_transform_mult_targ (stmt, histogram); +} + /* Return true if the stringop CALL with FNDECL shall be profiled. SIZE_ARG be set to the argument index for the size of the string operation. @@ -1883,9 +2277,13 @@ gimple_indirect_call_to_profile (gimple stmt, histogram_values *values) values->reserve (3); - values->quick_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_INDIR_CALL, - stmt, callee)); - + if (flag_dyn_ipa) + values->quick_push (gimple_alloc_histogram_value (cfun, + HIST_TYPE_INDIR_CALL_TOPN, + stmt, callee)); + else + values->quick_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_INDIR_CALL, + stmt, callee)); return; } @@ -1971,7 +2369,7 @@ gimple_find_values_to_profile (histogram_values *values) break; case HIST_TYPE_INDIR_CALL: - hist->n_counters = 3; + hist->n_counters = 3; break; case HIST_TYPE_TIME_PROFILE: @@ -1986,6 +2384,10 @@ gimple_find_values_to_profile (histogram_values *values) hist->n_counters = 1; break; + case HIST_TYPE_INDIR_CALL_TOPN: + hist->n_counters = (GCOV_ICALL_TOPN_VAL << 2) + 1; + break; + default: gcc_unreachable (); } |