1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
#!/usr/bin/env ruby
require 'fileutils'
def get_output_filenames
files = Dir.glob('*')
files.delete('Makefile')
files
end
def cleanup
get_output_filenames.each do |fname|
FileUtils.rm_rf fname
end
end
expected_failures = []
unexpected_passes = []
failures = []
passes = []
Dir.glob('testcase/*.mk').sort.each do |mk|
c = File.read(mk)
expected_failure = c =~ /\A# TODO/
name = mk[/([^\/]+)\.mk$/, 1]
dir = "out/#{name}"
FileUtils.mkdir_p(dir)
Dir.glob("#{dir}/*").each do |fname|
FileUtils.rm_rf(fname)
end
Dir.chdir(dir) do
File.open("Makefile", 'w') do |ofile|
ofile.print(c)
end
expected = ''
output = ''
testcases = c.scan(/^test\d*/).sort.uniq
if testcases.empty?
testcases = ['']
end
cleanup
testcases.each do |tc|
expected += "=== #{tc} ===\n" + `make #{tc} 2>&1`
expected_files = get_output_filenames
expected += "\n=== FILES ===\n#{expected_files * "\n"}\n"
end
cleanup
testcases.each do |tc|
res = IO.popen("../../kati -kati_log #{tc} 2>&1", 'r:binary', &:read)
output += "=== #{tc} ===\n" + res
output_files = get_output_filenames
output += "\n=== FILES ===\n#{output_files * "\n"}\n"
end
expected.gsub!(/^make\[\d+\]: (Entering|Leaving) directory.*\n/, '')
expected.gsub!(/^make\[\d+\]: /, '')
# Normalizations for old/new GNU make.
expected.gsub!(/[`'"]/, '"')
expected.gsub!(/ (?:commands|recipe) for target /,
' commands for target ')
expected.gsub!(/ (?:commands|recipe) commences /,
' commands commence ')
expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '')
expected.gsub!('Extraneous text after', 'extraneous text after')
# Not sure if this is useful.
expected.gsub!(/\s+Stop\.$/, '')
# GNU make 4.0 has this output.
expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '')
# We treat some warnings as errors.
expected.gsub!(/Nothing to be done for "test"\.\n/, '')
# kati specific log messages.
output.gsub!(/^\*kati\*.*\n/, '')
output.gsub!(/[`'"]/, '"')
File.open('out.make', 'w'){|ofile|ofile.print(expected)}
File.open('out.kati', 'w'){|ofile|ofile.print(output)}
if expected =~ /FAIL/
puts %Q(#{name} has a string "FAIL" in its expectation)
exit 1
end
if expected != output
if expected_failure
puts "#{name}: FAIL (expected)"
expected_failures << name
else
puts "#{name}: FAIL"
puts `diff -u out.make out.kati`
failures << name
end
else
if expected_failure
puts "#{name}: PASS (unexpected)"
unexpected_passes << name
else
puts "#{name}: PASS"
passes << name
end
end
end
end
puts
if !expected_failures.empty?
puts "=== Expected failures ==="
expected_failures.each do |n|
puts n
end
end
if !unexpected_passes.empty?
puts "=== Unexpected passes ==="
unexpected_passes.each do |n|
puts n
end
end
if !failures.empty?
puts "=== Failures ==="
failures.each do |n|
puts n
end
end
puts
if !unexpected_passes.empty? || !failures.empty?
puts 'FAIL!'
else
puts 'PASS!'
end
|
