diff options
Diffstat (limited to 'gcc-4.8.1/contrib/compare_two_ftime_report_sets')
| -rwxr-xr-x | gcc-4.8.1/contrib/compare_two_ftime_report_sets | 605 |
1 files changed, 0 insertions, 605 deletions
diff --git a/gcc-4.8.1/contrib/compare_two_ftime_report_sets b/gcc-4.8.1/contrib/compare_two_ftime_report_sets deleted file mode 100755 index 384dfde1d..000000000 --- a/gcc-4.8.1/contrib/compare_two_ftime_report_sets +++ /dev/null @@ -1,605 +0,0 @@ -#!/usr/bin/python - -# Script to statistically compare two sets of log files with -ftime-report -# output embedded within them. - -# Contributed by Lawrence Crowl <crowl@google.com> -# -# Copyright (C) 2012 Free Software Foundation, Inc. -# -# This file is part of GCC. -# -# GCC is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3, or (at your option) -# any later version. -# -# GCC is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GCC; see the file COPYING. If not, write to -# the Free Software Foundation, 51 Franklin Street, Fifth Floor, -# Boston, MA 02110-1301, USA. - - -""" Compare two sets of compile-time performance numbers. - -The intent of this script is to compare compile-time performance of two -different versions of the compiler. Each version of the compiler must be -run at least three times with the -ftime-report option. Each log file -represents a data point, or trial. The set of trials for each compiler -version constitutes a sample. The ouput of the script is a description -of the statistically significant difference between the two version of -the compiler. - -The parameters to the script are: - - Two file patterns that each match a set of log files. You will probably - need to quote the patterns before passing them to the script. - - Each pattern corresponds to a version of the compiler. - - A regular expression that finds interesting lines in the log files. - If you want to match the beginning of the line, you will need to add - the ^ operator. The filtering uses Python regular expression syntax. - - The default is "TOTAL". - - All of the interesting lines in a single log file are summed to produce - a single trial (data point). - - A desired statistical confidence within the range 60% to 99.9%. Due to - the implementation, this confidence will be rounded down to one of 60%, - 70%, 80%, 90%, 95%, 98%, 99%, 99.5%, 99.8%, and 99.9%. - - The default is 95. - - If the computed confidence is lower than desired, the script will - estimate the number of trials needed to meet the desired confidence. - This estimate is not very good, as the variance tends to change as - you increase the number of trials. - -The most common use of the script is total compile-time comparison between -logfiles stored in different directories. - -compare_two_ftime_report_sets "Log1/*perf" "Log2/*perf" - -One can also look at parsing time, but expecting a lower confidence. - -compare_two_ftime_report_sets "Log1/*perf" "Log2/*perf" "^phase parsing" 75 - -""" - - -import os -import sys -import fnmatch -import glob -import re -import math - - -####################################################################### Utility - - -def divide(dividend, divisor): - """ Return the quotient, avoiding division by zero. - """ - if divisor == 0: - return sys.float_info.max - else: - return dividend / divisor - - -################################################################# File and Line - - -# Should you repurpose this script, this code might help. -# -#def find_files(topdir, filepat): -# """ Find a set of file names, under a given directory, -# matching a Unix shell file pattern. -# Returns an iterator over the file names. -# """ -# for path, dirlist, filelist in os.walk(topdir): -# for name in fnmatch.filter(filelist, filepat): -# yield os.path.join(path, name) - - -def match_files(fileglob): - """ Find a set of file names matching a Unix shell glob pattern. - Returns an iterator over the file names. - """ - return glob.iglob(os.path.expanduser(fileglob)) - - -def lines_in_file(filename): - """ Return an iterator over lines in the named file. """ - filedesc = open(filename, "r") - for line in filedesc: - yield line - filedesc.close() - - -def lines_containing_pattern(pattern, lines): - """ Find lines by a Python regular-expression. - Returns an iterator over lines containing the expression. - """ - parser = re.compile(pattern) - for line in lines: - if parser.search(line): - yield line - - -############################################################# Number Formatting - - -def strip_redundant_digits(numrep): - if numrep.find(".") == -1: - return numrep - return numrep.rstrip("0").rstrip(".") - - -def text_number(number): - return strip_redundant_digits("%g" % number) - - -def round_significant(digits, number): - if number == 0: - return 0 - magnitude = abs(number) - significance = math.floor(math.log10(magnitude)) - least_position = int(significance - digits + 1) - return round(number, -least_position) - - -def text_significant(digits, number): - return text_number(round_significant(digits, number)) - - -def text_percent(number): - return text_significant(3, number*100) + "%" - - -################################################################ T-Distribution - - -# This section of code provides functions for using Student's t-distribution. - - -# The functions are implemented using table lookup -# to facilitate implementation of inverse functions. - - -# The table is comprised of row 0 listing the alpha values, -# column 0 listing the degree-of-freedom values, -# and the other entries listing the corresponding t-distribution values. - -t_dist_table = [ -[ 0, 0.200, 0.150, 0.100, 0.050, 0.025, 0.010, 0.005, .0025, 0.001, .0005], -[ 1, 1.376, 1.963, 3.078, 6.314, 12.71, 31.82, 63.66, 127.3, 318.3, 636.6], -[ 2, 1.061, 1.386, 1.886, 2.920, 4.303, 6.965, 9.925, 14.09, 22.33, 31.60], -[ 3, 0.978, 1.250, 1.638, 2.353, 3.182, 4.541, 5.841, 7.453, 10.21, 12.92], -[ 4, 0.941, 1.190, 1.533, 2.132, 2.776, 3.747, 4.604, 5.598, 7.173, 8.610], -[ 5, 0.920, 1.156, 1.476, 2.015, 2.571, 3.365, 4.032, 4.773, 5.894, 6.869], -[ 6, 0.906, 1.134, 1.440, 1.943, 2.447, 3.143, 3.707, 4.317, 5.208, 5.959], -[ 7, 0.896, 1.119, 1.415, 1.895, 2.365, 2.998, 3.499, 4.029, 4.785, 5.408], -[ 8, 0.889, 1.108, 1.397, 1.860, 2.306, 2.896, 3.355, 3.833, 4.501, 5.041], -[ 9, 0.883, 1.100, 1.383, 1.833, 2.262, 2.821, 3.250, 3.690, 4.297, 4.781], -[ 10, 0.879, 1.093, 1.372, 1.812, 2.228, 2.764, 3.169, 3.581, 4.144, 4.587], -[ 11, 0.876, 1.088, 1.363, 1.796, 2.201, 2.718, 3.106, 3.497, 4.025, 4.437], -[ 12, 0.873, 1.083, 1.356, 1.782, 2.179, 2.681, 3.055, 3.428, 3.930, 4.318], -[ 13, 0.870, 1.079, 1.350, 1.771, 2.160, 2.650, 3.012, 3.372, 3.852, 4.221], -[ 14, 0.868, 1.076, 1.345, 1.761, 2.145, 2.624, 2.977, 3.326, 3.787, 4.140], -[ 15, 0.866, 1.074, 1.341, 1.753, 2.131, 2.602, 2.947, 3.286, 3.733, 4.073], -[ 16, 0.865, 1.071, 1.337, 1.746, 2.120, 2.583, 2.921, 3.252, 3.686, 4.015], -[ 17, 0.863, 1.069, 1.333, 1.740, 2.110, 2.567, 2.898, 3.222, 3.646, 3.965], -[ 18, 0.862, 1.067, 1.330, 1.734, 2.101, 2.552, 2.878, 3.197, 3.610, 3.922], -[ 19, 0.861, 1.066, 1.328, 1.729, 2.093, 2.539, 2.861, 3.174, 3.579, 3.883], -[ 20, 0.860, 1.064, 1.325, 1.725, 2.086, 2.528, 2.845, 3.153, 3.552, 3.850], -[ 21, 0.859, 1.063, 1.323, 1.721, 2.080, 2.518, 2.831, 3.135, 3.527, 3.819], -[ 22, 0.858, 1.061, 1.321, 1.717, 2.074, 2.508, 2.819, 3.119, 3.505, 3.792], -[ 23, 0.858, 1.060, 1.319, 1.714, 2.069, 2.500, 2.807, 3.104, 3.485, 3.768], -[ 24, 0.857, 1.059, 1.318, 1.711, 2.064, 2.492, 2.797, 3.091, 3.467, 3.745], -[ 25, 0.856, 1.058, 1.316, 1.708, 2.060, 2.485, 2.787, 3.078, 3.450, 3.725], -[ 26, 0.856, 1.058, 1.315, 1.706, 2.056, 2.479, 2.779, 3.067, 3.435, 3.707], -[ 27, 0.855, 1.057, 1.314, 1.703, 2.052, 2.473, 2.771, 3.057, 3.421, 3.689], -[ 28, 0.855, 1.056, 1.313, 1.701, 2.048, 2.467, 2.763, 3.047, 3.408, 3.674], -[ 29, 0.854, 1.055, 1.311, 1.699, 2.045, 2.462, 2.756, 3.038, 3.396, 3.660], -[ 30, 0.854, 1.055, 1.310, 1.697, 2.042, 2.457, 2.750, 3.030, 3.385, 3.646], -[ 31, 0.853, 1.054, 1.309, 1.696, 2.040, 2.453, 2.744, 3.022, 3.375, 3.633], -[ 32, 0.853, 1.054, 1.309, 1.694, 2.037, 2.449, 2.738, 3.015, 3.365, 3.622], -[ 33, 0.853, 1.053, 1.308, 1.692, 2.035, 2.445, 2.733, 3.008, 3.356, 3.611], -[ 34, 0.852, 1.052, 1.307, 1.691, 2.032, 2.441, 2.728, 3.002, 3.348, 3.601], -[ 35, 0.852, 1.052, 1.306, 1.690, 2.030, 2.438, 2.724, 2.996, 3.340, 3.591], -[ 36, 0.852, 1.052, 1.306, 1.688, 2.028, 2.434, 2.719, 2.990, 3.333, 3.582], -[ 37, 0.851, 1.051, 1.305, 1.687, 2.026, 2.431, 2.715, 2.985, 3.326, 3.574], -[ 38, 0.851, 1.051, 1.304, 1.686, 2.024, 2.429, 2.712, 2.980, 3.319, 3.566], -[ 39, 0.851, 1.050, 1.304, 1.685, 2.023, 2.426, 2.708, 2.976, 3.313, 3.558], -[ 40, 0.851, 1.050, 1.303, 1.684, 2.021, 2.423, 2.704, 2.971, 3.307, 3.551], -[ 50, 0.849, 1.047, 1.299, 1.676, 2.009, 2.403, 2.678, 2.937, 3.261, 3.496], -[ 60, 0.848, 1.045, 1.296, 1.671, 2.000, 2.390, 2.660, 2.915, 3.232, 3.460], -[ 80, 0.846, 1.043, 1.292, 1.664, 1.990, 2.374, 2.639, 2.887, 3.195, 3.416], -[100, 0.845, 1.042, 1.290, 1.660, 1.984, 2.364, 2.626, 2.871, 3.174, 3.390], -[150, 0.844, 1.040, 1.287, 1.655, 1.976, 2.351, 2.609, 2.849, 3.145, 3.357] ] - - -# The functions use the following parameter name conventions: -# alpha - the alpha parameter -# degree - the degree-of-freedom parameter -# value - the t-distribution value for some alpha and degree -# deviations - a confidence interval radius, -# expressed as a multiple of the standard deviation of the sample -# ax - the alpha parameter index -# dx - the degree-of-freedom parameter index - -# The interface to this section of code is the last three functions, -# find_t_dist_value, find_t_dist_alpha, and find_t_dist_degree. - - -def t_dist_alpha_at_index(ax): - if ax == 0: - return .25 # effectively no confidence - else: - return t_dist_table[0][ax] - - -def t_dist_degree_at_index(dx): - return t_dist_table[dx][0] - - -def t_dist_value_at_index(ax, dx): - return t_dist_table[dx][ax] - - -def t_dist_index_of_degree(degree): - limit = len(t_dist_table) - 1 - dx = 0 - while dx < limit and t_dist_degree_at_index(dx+1) <= degree: - dx += 1 - return dx - - -def t_dist_index_of_alpha(alpha): - limit = len(t_dist_table[0]) - 1 - ax = 0 - while ax < limit and t_dist_alpha_at_index(ax+1) >= alpha: - ax += 1 - return ax - - -def t_dist_index_of_value(dx, value): - limit = len(t_dist_table[dx]) - 1 - ax = 0 - while ax < limit and t_dist_value_at_index(ax+1, dx) < value: - ax += 1 - return ax - - -def t_dist_value_within_deviations(dx, ax, deviations): - degree = t_dist_degree_at_index(dx) - count = degree + 1 - root = math.sqrt(count) - value = t_dist_value_at_index(ax, dx) - nominal = value / root - comparison = nominal <= deviations - return comparison - - -def t_dist_index_of_degree_for_deviations(ax, deviations): - limit = len(t_dist_table) - 1 - dx = 1 - while dx < limit and not t_dist_value_within_deviations(dx, ax, deviations): - dx += 1 - return dx - - -def find_t_dist_value(alpha, degree): - """ Return the t-distribution value. - The parameters are alpha and degree of freedom. - """ - dx = t_dist_index_of_degree(degree) - ax = t_dist_index_of_alpha(alpha) - return t_dist_value_at_index(ax, dx) - - -def find_t_dist_alpha(value, degree): - """ Return the alpha. - The parameters are the t-distribution value for a given degree of freedom. - """ - dx = t_dist_index_of_degree(degree) - ax = t_dist_index_of_value(dx, value) - return t_dist_alpha_at_index(ax) - - -def find_t_dist_degree(alpha, deviations): - """ Return the degree-of-freedom. - The parameters are the desired alpha and the number of standard deviations - away from the mean that the degree should handle. - """ - ax = t_dist_index_of_alpha(alpha) - dx = t_dist_index_of_degree_for_deviations(ax, deviations) - return t_dist_degree_at_index(dx) - - -############################################################## Core Statistical - - -# This section provides the core statistical classes and functions. - - -class Accumulator: - - """ An accumulator for statistical information using arithmetic mean. """ - - def __init__(self): - self.count = 0 - self.mean = 0 - self.sumsqdiff = 0 - - def insert(self, value): - self.count += 1 - diff = value - self.mean - self.mean += diff / self.count - self.sumsqdiff += (self.count - 1) * diff * diff / self.count - - -def fill_accumulator_from_values(values): - accumulator = Accumulator() - for value in values: - accumulator.insert(value) - return accumulator - - -def alpha_from_confidence(confidence): - scrubbed = min(99.99, max(confidence, 60)) - return (100.0 - scrubbed) / 200.0 - - -def confidence_from_alpha(alpha): - return 100 - 200 * alpha - - -class Sample: - - """ A description of a sample using an arithmetic mean. """ - - def __init__(self, accumulator, alpha): - if accumulator.count < 3: - sys.exit("Samples must contain three trials.") - self.count = accumulator.count - self.mean = accumulator.mean - variance = accumulator.sumsqdiff / (self.count - 1) - self.deviation = math.sqrt(variance) - self.error = self.deviation / math.sqrt(self.count) - self.alpha = alpha - self.radius = find_t_dist_value(alpha, self.count - 1) * self.error - - def alpha_for_radius(self, radius): - return find_t_dist_alpha(divide(radius, self.error), self.count) - - def degree_for_radius(self, radius): - return find_t_dist_degree(self.alpha, divide(radius, self.deviation)) - - def __str__(self): - text = "trial count is " + text_number(self.count) - text += ", mean is " + text_number(self.mean) - text += " (" + text_number(confidence_from_alpha(self.alpha)) +"%" - text += " confidence in " + text_number(self.mean - self.radius) - text += " to " + text_number(self.mean + self.radius) + ")" - text += ",\nstd.deviation is " + text_number(self.deviation) - text += ", std.error is " + text_number(self.error) - return text - - -def sample_from_values(values, alpha): - accumulator = fill_accumulator_from_values(values) - return Sample(accumulator, alpha) - - -class Comparison: - - """ A comparison of two samples using arithmetic means. """ - - def __init__(self, first, second, alpha): - if first.mean > second.mean: - self.upper = first - self.lower = second - self.larger = "first" - else: - self.upper = second - self.lower = first - self.larger = "second" - self.a_wanted = alpha - radius = self.upper.mean - self.lower.mean - rising = self.lower.alpha_for_radius(radius) - falling = self.upper.alpha_for_radius(radius) - self.a_actual = max(rising, falling) - rising = self.lower.degree_for_radius(radius) - falling = self.upper.degree_for_radius(radius) - self.count = max(rising, falling) + 1 - - def __str__(self): - message = "The " + self.larger + " sample appears to be " - change = divide(self.upper.mean, self.lower.mean) - 1 - message += text_percent(change) + " larger,\n" - confidence = confidence_from_alpha(self.a_actual) - if confidence >= 60: - message += "with " + text_number(confidence) + "% confidence" - message += " of being larger." - else: - message += "but with no confidence of actually being larger." - if self.a_actual > self.a_wanted: - confidence = confidence_from_alpha(self.a_wanted) - message += "\nTo reach " + text_number(confidence) + "% confidence," - if self.count < 100: - message += " you need roughly " + text_number(self.count) + " trials,\n" - message += "assuming the standard deviation is stable, which is iffy." - else: - message += "\nyou need to reduce the larger deviation" - message += " or increase the number of trials." - return message - - -############################################################ Single Value Files - - -# This section provides functions to compare two raw data files, -# each containing a whole sample consisting of single number per line. - - -# Should you repurpose this script, this code might help. -# -#def values_from_data_file(filename): -# for line in lines_in_file(filename): -# yield float(line) - - -# Should you repurpose this script, this code might help. -# -#def sample_from_data_file(filename, alpha): -# confidence = confidence_from_alpha(alpha) -# text = "\nArithmetic sample for data file\n\"" + filename + "\"" -# text += " with desired confidence " + text_number(confidence) + " is " -# print text -# values = values_from_data_file(filename) -# sample = sample_from_values(values, alpha) -# print sample -# return sample - - -# Should you repurpose this script, this code might help. -# -#def compare_two_data_files(filename1, filename2, confidence): -# alpha = alpha_from_confidence(confidence) -# sample1 = sample_from_data_file(filename1, alpha) -# sample2 = sample_from_data_file(filename2, alpha) -# print -# print Comparison(sample1, sample2, alpha) - - -# Should you repurpose this script, this code might help. -# -#def command_two_data_files(): -# argc = len(sys.argv) -# if argc < 2 or 4 < argc: -# message = "usage: " + sys.argv[0] -# message += " file-name file-name [confidence]" -# print message -# else: -# filename1 = sys.argv[1] -# filename2 = sys.argv[2] -# if len(sys.argv) >= 4: -# confidence = int(sys.argv[3]) -# else: -# confidence = 95 -# compare_two_data_files(filename1, filename2, confidence) - - -############################################### -ftime-report TimeVar Log Files - - -# This section provides functions to compare two sets of -ftime-report log -# files. Each set is a sample, where each data point is derived from the -# sum of values in a single log file. - - -label = r"^ *([^:]*[^: ]) *:" -number = r" *([0-9.]*) *" -percent = r"\( *[0-9]*\%\)" -numpct = number + percent -total_format = label + number + number + number + number + " kB\n" -total_parser = re.compile(total_format) -tmvar_format = label + numpct + " usr" + numpct + " sys" -tmvar_format += numpct + " wall" + number + " kB " + percent + " ggc\n" -tmvar_parser = re.compile(tmvar_format) -replace = r"\2\t\3\t\4\t\5\t\1" - - -def split_time_report(lines, pattern): - if pattern == "TOTAL": - parser = total_parser - else: - parser = tmvar_parser - for line in lines: - modified = parser.sub(replace, line) - if modified != line: - yield re.split("\t", modified) - - -def extract_cpu_time(tvtuples): - for tuple in tvtuples: - yield float(tuple[0]) + float(tuple[1]) - - -def sum_values(values): - sum = 0 - for value in values: - sum += value - return sum - - -def extract_time_for_timevar_log(filename, pattern): - lines = lines_in_file(filename) - tmvars = lines_containing_pattern(pattern, lines) - tuples = split_time_report(tmvars, pattern) - times = extract_cpu_time(tuples) - return sum_values(times) - - -def extract_times_for_timevar_logs(filelist, pattern): - for filename in filelist: - yield extract_time_for_timevar_log(filename, pattern) - - -def sample_from_timevar_logs(fileglob, pattern, alpha): - confidence = confidence_from_alpha(alpha) - text = "\nArithmetic sample for timevar log files\n\"" + fileglob + "\"" - text += "\nand selecting lines containing \"" + pattern + "\"" - text += " with desired confidence " + text_number(confidence) + " is " - print text - filelist = match_files(fileglob) - values = extract_times_for_timevar_logs(filelist, pattern) - sample = sample_from_values(values, alpha) - print sample - return sample - - -def compare_two_timevar_logs(fileglob1, fileglob2, pattern, confidence): - alpha = alpha_from_confidence(confidence) - sample1 = sample_from_timevar_logs(fileglob1, pattern, alpha) - sample2 = sample_from_timevar_logs(fileglob2, pattern, alpha) - print - print Comparison(sample1, sample2, alpha) - - -def command_two_timevar_logs(): - argc = len(sys.argv) - if argc < 3 or 5 < argc: - message = "usage: " + sys.argv[0] - message += " file-pattern file-pattern [line-pattern [confidence]]" - print message - else: - filepat1 = sys.argv[1] - filepat2 = sys.argv[2] - if len(sys.argv) >= 5: - confidence = int(sys.argv[4]) - else: - confidence = 95 - if len(sys.argv) >= 4: - linepat = sys.argv[3] - else: - linepat = "TOTAL" - compare_two_timevar_logs(filepat1, filepat2, linepat, confidence) - - -########################################################################## Main - - -# This section is the main code, implementing the command. - - -command_two_timevar_logs() |