aboutsummaryrefslogtreecommitdiffstats
path: root/run_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'run_test.go')
-rw-r--r--run_test.go512
1 files changed, 512 insertions, 0 deletions
diff --git a/run_test.go b/run_test.go
new file mode 100644
index 0000000..d295ddf
--- /dev/null
+++ b/run_test.go
@@ -0,0 +1,512 @@
+// Copyright 2020 Google Inc. All rights reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kati
+
+import (
+ "bytes"
+ "flag"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/sergi/go-diff/diffmatchpatch"
+)
+
+var ckati bool
+var ninja bool
+var genAllTargets bool
+
+func init() {
+ // suppress GNU make jobserver magic when calling "make"
+ os.Unsetenv("MAKEFLAGS")
+ os.Unsetenv("MAKELEVEL")
+ os.Setenv("NINJA_STATUS", "NINJACMD: ")
+
+ flag.BoolVar(&ckati, "ckati", false, "use ckati")
+ flag.BoolVar(&ninja, "ninja", false, "use ninja")
+ flag.BoolVar(&genAllTargets, "all", false, "use --gen_all_targets")
+}
+
+type normalization struct {
+ regexp *regexp.Regexp
+ replace string
+}
+
+var normalizeQuotes = normalization{
+ regexp.MustCompile("([`'\"]|\xe2\x80\x98|\xe2\x80\x99)"), `"`,
+}
+
+var normalizeMakeLog = []normalization{
+ normalizeQuotes,
+ {regexp.MustCompile(`make(?:\[\d+\])?: (Entering|Leaving) directory[^\n]*`), ""},
+ {regexp.MustCompile(`make(?:\[\d+\])?: `), ""},
+
+ // Normalizations for old/new GNU make.
+ {regexp.MustCompile(" recipe for target "), " commands for target "},
+ {regexp.MustCompile(" recipe commences "), " commands commence "},
+ {regexp.MustCompile("missing rule before recipe."), "missing rule before commands."},
+ {regexp.MustCompile(" (did you mean TAB instead of 8 spaces?)"), ""},
+ {regexp.MustCompile("Extraneous text after"), "extraneous text after"},
+ // Not sure if this is useful
+ {regexp.MustCompile(`\s+Stop\.`), ""},
+ // GNU make 4.0 has this output.
+ {regexp.MustCompile(`Makefile:\d+: commands for target ".*?" failed\n`), ""},
+ // We treat some warnings as errors.
+ {regexp.MustCompile(`/bin/(ba)?sh: line 0: `), ""},
+ // Normalization for "include foo" with C++ kati
+ {regexp.MustCompile(`(: \S+: No such file or directory)\n\*\*\* No rule to make target "[^"]+".`), "$1"},
+}
+
+var normalizeMakeNinja = normalization{
+ // We print out some ninja warnings in some tests to match what we expect
+ // ninja to produce. Remove them if we're not testing ninja
+ regexp.MustCompile("ninja: warning: [^\n]+"), "",
+}
+
+var normalizeKati = []normalization{
+ normalizeQuotes,
+
+ // kati specific log messages
+ {regexp.MustCompile(`\*kati\*[^\n]*`), ""},
+ {regexp.MustCompile(`c?kati: `), ""},
+ {regexp.MustCompile(`/bin/sh: line 0: `), ""},
+ {regexp.MustCompile(`/bin/sh: `), ""},
+ {regexp.MustCompile(`.*: warning for parse error in an unevaluated line: [^\n]*`), ""},
+ {regexp.MustCompile(`([^\n ]+: )?FindEmulator: `), ""},
+ // kati log ifles in find_command.mk
+ {regexp.MustCompile(` (\./+)+kati\.\S+`), ""},
+ // json files in find_command.mk
+ {regexp.MustCompile(` (\./+)+test\S+.json`), ""},
+ // Normalization for "include foo" with Go kati
+ {regexp.MustCompile(`(: )open (\S+): n(o such file or directory)\nNOTE:[^\n]*`), "${1}${2}: N${3}"},
+ // Bionic libc has different error messages than glibc
+ {regexp.MustCompile(`Too many symbolic links encountered`), "Too many levels of symbolic links"},
+}
+
+var normalizeNinja = []normalization{
+ {regexp.MustCompile(`NINJACMD: [^\n]*\n`), ""},
+ {regexp.MustCompile(`ninja: no work to do\.\n`), ""},
+ {regexp.MustCompile(`ninja: error: (.*, needed by .*),[^\n]*`),
+ "*** No rule to make target ${1}."},
+ {regexp.MustCompile(`ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct[^\n]*`),
+ "ninja: warning: multiple rules generate ${1}."},
+}
+
+var normalizeNinjaFail = []normalization{
+ {regexp.MustCompile(`FAILED: ([^\n]+\n/bin/bash)?[^\n]*\n`), "*** [test] Error 1\n"},
+ {regexp.MustCompile(`ninja: [^\n]+\n`), ""},
+}
+
+var normalizeNinjaIgnoreFail = []normalization{
+ {regexp.MustCompile(`FAILED: ([^\n]+\n/bin/bash)?[^\n]*\n`), ""},
+ {regexp.MustCompile(`ninja: [^\n]+\n`), ""},
+}
+
+var circularRE = regexp.MustCompile(`(Circular .* dropped\.\n)`)
+
+func normalize(log []byte, normalizations []normalization) []byte {
+ // We don't care when circular dependency detection happens.
+ ret := []byte{}
+ for _, circ := range circularRE.FindAllSubmatch(log, -1) {
+ ret = append(ret, circ[1]...)
+ }
+ ret = append(ret, circularRE.ReplaceAll(log, []byte{})...)
+
+ for _, n := range normalizations {
+ ret = n.regexp.ReplaceAll(ret, []byte(n.replace))
+ }
+ return ret
+}
+
+func runMake(t *testing.T, prefix []string, dir string, silent bool, tc string) string {
+ write := func(f string, data []byte) {
+ suffix := ""
+ if tc != "" {
+ suffix = "_" + tc
+ }
+ if err := ioutil.WriteFile(filepath.Join(dir, f+suffix), data, 0666); err != nil {
+ t.Error(err)
+ }
+ }
+
+ args := append(prefix, "make")
+ if silent {
+ args = append(args, "-s")
+ }
+ if tc != "" {
+ args = append(args, tc)
+ }
+ args = append(args, "SHELL=/bin/bash")
+
+ cmd := exec.Command(args[0], args[1:]...)
+ cmd.Dir = dir
+ output, _ := cmd.CombinedOutput()
+ write("stdout", output)
+
+ output = normalize(output, normalizeMakeLog)
+ if !ninja {
+ output = normalize(output, []normalization{normalizeMakeNinja})
+ }
+
+ write("stdout_normalized", output)
+ return string(output)
+}
+
+func runKati(t *testing.T, test, dir string, silent bool, tc string) string {
+ write := func(f string, data []byte) {
+ suffix := ""
+ if tc != "" {
+ suffix = "_" + tc
+ }
+ if err := ioutil.WriteFile(filepath.Join(dir, f+suffix), data, 0666); err != nil {
+ t.Error(err)
+ }
+ }
+
+ var cmd *exec.Cmd
+ if ckati {
+ cmd = exec.Command("../../../ckati", "--use_find_emulator")
+ } else {
+ json := tc
+ if json == "" {
+ json = "test"
+ }
+ cmd = exec.Command("../../../kati", "-save_json="+json+".json", "-log_dir=.", "--use_find_emulator")
+ }
+ if ninja {
+ cmd.Args = append(cmd.Args, "--ninja")
+ }
+ if genAllTargets {
+ cmd.Args = append(cmd.Args, "--gen_all_targets")
+ }
+ if silent {
+ cmd.Args = append(cmd.Args, "-s")
+ }
+ cmd.Args = append(cmd.Args, "SHELL=/bin/bash")
+ if tc != "" && (!genAllTargets || strings.Contains(test, "makecmdgoals")) {
+ cmd.Args = append(cmd.Args, tc)
+ }
+ cmd.Dir = dir
+ output, err := cmd.CombinedOutput()
+ write("stdout", output)
+ if err != nil {
+ output := normalize(output, normalizeKati)
+ write("stdout_normalized", output)
+ return string(output)
+ }
+
+ if ninja {
+ ninjaCmd := exec.Command("./ninja.sh", "-j1", "-v")
+ if genAllTargets && tc != "" {
+ ninjaCmd.Args = append(ninjaCmd.Args, tc)
+ }
+ ninjaCmd.Dir = dir
+ ninjaOutput, _ := ninjaCmd.CombinedOutput()
+ write("stdout_ninja", ninjaOutput)
+ ninjaOutput = normalize(ninjaOutput, normalizeNinja)
+ if test == "err_error_in_recipe.mk" {
+ ninjaOutput = normalize(ninjaOutput, normalizeNinjaIgnoreFail)
+ } else if strings.HasPrefix(test, "fail_") {
+ ninjaOutput = normalize(ninjaOutput, normalizeNinjaFail)
+ }
+ write("stdout_ninja_normalized", ninjaOutput)
+ output = append(output, ninjaOutput...)
+ }
+
+ output = normalize(output, normalizeKati)
+ write("stdout_normalized", output)
+ return string(output)
+}
+
+func runKatiInScript(t *testing.T, script, dir string, isNinjaTest bool) string {
+ write := func(f string, data []byte) {
+ if err := ioutil.WriteFile(filepath.Join(dir, f), data, 0666); err != nil {
+ t.Error(err)
+ }
+ }
+
+ args := []string{"bash", script}
+ if ckati {
+ args = append(args, "../../../ckati")
+ if isNinjaTest {
+ args = append(args, "--ninja", "--regen")
+ }
+ } else {
+ args = append(args, "../../../kati --use_cache -log_dir=.")
+ }
+ args = append(args, "SHELL=/bin/bash")
+
+ cmd := exec.Command(args[0], args[1:]...)
+ cmd.Dir = dir
+ output, _ := cmd.Output()
+ write("stdout", output)
+ if isNinjaTest {
+ output = normalize(output, normalizeNinja)
+ }
+ output = normalize(output, normalizeKati)
+ write("stdout_normalized", output)
+ return string(output)
+}
+
+func inList(list []string, item string) bool {
+ for _, i := range list {
+ if item == i {
+ return true
+ }
+ }
+ return false
+}
+
+func diffLists(a, b []string) (onlyA []string, onlyB []string) {
+ for _, i := range a {
+ if !inList(b, i) {
+ onlyA = append(onlyA, i)
+ }
+ }
+ for _, i := range b {
+ if !inList(a, i) {
+ onlyB = append(onlyB, i)
+ }
+ }
+ return
+}
+
+func outputFiles(t *testing.T, dir string) []string {
+ ret := []string{}
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ignoreFiles := []string{
+ ".", "..", "Makefile", "build.ninja", "env.sh", "ninja.sh", "gmon.out", "submake",
+ }
+ for _, fi := range files {
+ name := fi.Name()
+ if inList(ignoreFiles, name) ||
+ strings.HasPrefix(name, ".") ||
+ strings.HasSuffix(name, ".json") ||
+ strings.HasPrefix(name, "kati") ||
+ strings.HasPrefix(name, "stdout") {
+ continue
+ }
+ ret = append(ret, fi.Name())
+ }
+ return ret
+}
+
+var testcaseRE = regexp.MustCompile(`^test\d*`)
+
+func uniqueTestcases(c []byte) []string {
+ seen := map[string]bool{}
+ ret := []string{}
+ for _, line := range bytes.Split(c, []byte("\n")) {
+ line := string(line)
+ s := testcaseRE.FindString(line)
+ if s == "" {
+ continue
+ }
+ if _, ok := seen[s]; ok {
+ continue
+ }
+ seen[s] = true
+ ret = append(ret, s)
+ }
+ sort.Strings(ret)
+ if len(ret) == 0 {
+ return []string{""}
+ }
+ return ret
+}
+
+var todoRE = regexp.MustCompile(`^# TODO(?:\(([-a-z|]+)(?:/([-a-z0-9|]+))?\))?`)
+
+func isExpectedFailure(c []byte, tc string) bool {
+ for _, line := range bytes.Split(c, []byte("\n")) {
+ line := string(line)
+ if !strings.HasPrefix(line, "#!") && !strings.HasPrefix(line, "# TODO") {
+ break
+ }
+
+ todo := todoRE.FindStringSubmatch(line)
+ if todo == nil {
+ continue
+ }
+
+ if todo[1] == "" {
+ return true
+ }
+
+ todos := strings.Split(todo[1], "|")
+ if (inList(todos, "go") && !ckati) ||
+ (inList(todos, "c") && ckati) ||
+ (inList(todos, "go-ninja") && !ckati && ninja) ||
+ (inList(todos, "c-ninja") && ckati && ninja) ||
+ (inList(todos, "c-exec") && ckati && !ninja) ||
+ (inList(todos, "ninja") && ninja) ||
+ (inList(todos, "ninja-genall") && ninja && genAllTargets) ||
+ (inList(todos, "all")) {
+
+ if todo[2] == "" {
+ return true
+ }
+ tcs := strings.Split(todo[2], "|")
+ if inList(tcs, tc) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func TestKati(t *testing.T) {
+ if ckati {
+ os.Setenv("KATI_VARIANT", "c")
+ if _, err := os.Stat("ckati"); err != nil {
+ t.Fatalf("ckati must be built before testing: %s", err)
+ }
+ } else {
+ if _, err := os.Stat("kati"); err != nil {
+ t.Fatalf("kati must be built before testing: %s", err)
+ }
+ }
+ if ninja {
+ if _, err := exec.LookPath("ninja"); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ out, _ := filepath.Abs("out")
+ files, err := ioutil.ReadDir("testcase")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, fi := range files {
+ name := fi.Name()
+
+ isMkTest := strings.HasSuffix(name, ".mk")
+ isShTest := strings.HasSuffix(name, ".sh")
+ if strings.HasPrefix(name, ".") || !(isMkTest || isShTest) {
+ continue
+ }
+
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+
+ c, err := ioutil.ReadFile(filepath.Join("testcase", name))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out := filepath.Join(out, name)
+ if err := os.RemoveAll(out); err != nil {
+ t.Fatal(err)
+ }
+ outMake := filepath.Join(out, "make")
+ outKati := filepath.Join(out, "kati")
+ if err := os.MkdirAll(outMake, 0777); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(outKati, 0777); err != nil {
+ t.Fatal(err)
+ }
+
+ testcases := []string{""}
+ expected := map[string]string{}
+ expectedFiles := map[string][]string{}
+ expectedFailures := map[string]bool{}
+ got := map[string]string{}
+ gotFiles := map[string][]string{}
+
+ if isMkTest {
+ setup := func(dir string) {
+ if err = ioutil.WriteFile(filepath.Join(dir, "Makefile"), c, 0666); err != nil {
+ t.Fatal(err)
+ }
+ os.Symlink("../../../testcase/submake", filepath.Join(dir, "submake"))
+ }
+ setup(outMake)
+ setup(outKati)
+
+ testcases = uniqueTestcases(c)
+
+ isSilent := strings.HasPrefix(name, "submake_")
+
+ for _, tc := range testcases {
+ expected[tc] = runMake(t, nil, outMake, ninja || isSilent, tc)
+ expectedFiles[tc] = outputFiles(t, outMake)
+ expectedFailures[tc] = isExpectedFailure(c, tc)
+ }
+
+ for _, tc := range testcases {
+ got[tc] = runKati(t, name, outKati, isSilent, tc)
+ gotFiles[tc] = outputFiles(t, outKati)
+ }
+ } else if isShTest {
+ isNinjaTest := strings.HasPrefix(name, "ninja_")
+ if isNinjaTest && (!ckati || !ninja) {
+ t.SkipNow()
+ }
+
+ scriptName := "../../../testcase/" + name
+
+ expected[""] = runMake(t, []string{"bash", scriptName}, outMake, isNinjaTest, "")
+ expectedFailures[""] = isExpectedFailure(c, "")
+
+ got[""] = runKatiInScript(t, scriptName, outKati, isNinjaTest)
+ }
+
+ check := func(t *testing.T, m, k string, mFiles, kFiles []string, expectFail bool) {
+ if strings.Contains(m, "FAIL") {
+ t.Fatalf("Make returned 'FAIL':\n%q", m)
+ }
+
+ if !expectFail && m != k {
+ dmp := diffmatchpatch.New()
+ diffs := dmp.DiffMain(k, m, true)
+ diffs = dmp.DiffCleanupSemantic(diffs)
+ t.Errorf("Different output from kati (red) to the expected value from make (green):\n%s",
+ dmp.DiffPrettyText(diffs))
+ } else if expectFail && m == k {
+ t.Errorf("Expected failure, but output is the same")
+ }
+
+ if !expectFail {
+ onlyMake, onlyKati := diffLists(mFiles, kFiles)
+ if len(onlyMake) > 0 {
+ t.Errorf("Files only created by Make:\n%q", onlyMake)
+ }
+ if len(onlyKati) > 0 {
+ t.Errorf("Files only created by Kati:\n%q", onlyKati)
+ }
+ }
+ }
+
+ for _, tc := range testcases {
+ if tc == "" || len(testcases) == 1 {
+ check(t, expected[tc], got[tc], expectedFiles[tc], gotFiles[tc], expectedFailures[tc])
+ } else {
+ t.Run(tc, func(t *testing.T) {
+ check(t, expected[tc], got[tc], expectedFiles[tc], gotFiles[tc], expectedFailures[tc])
+ })
+ }
+ }
+ })
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-04 11:22:55 +0000