Switch to a golang-based test runner

This adds a few features:
* Parallel execution (and all other golang test runner features)
* Colored diffs of output
* Make and kati run in subdirs, all output is preserved
* Full logs saved of raw and normalized output per-testcase
* TODO(c/<testcase>) support to mark individual testcases as expected
  failures, instead of the full test.

Removed support for the [go]kati serialization tests

Change-Id: Id6be80a3168ccff3fa3a970fe089986f1aa066dd

30 files changed, 571 insertions, 486 deletions

diff --git a/.github/workflows/cpp-ci.yml b/.github/workflows/cpp-ci.yml
index c076cf1..022deeb 100644
--- a/.github/workflows/cpp-ci.yml
+++ b/.github/workflows/cpp-ci.yml
@@ -29,11 +29,11 @@ jobs:
     - name: clang format
       run: ./clang-format-check
     - name: run standalone tests
-      run: ruby runtest.rb -c
+      run: go test --ckati
     - name: run ninja tests
-      run: ruby runtest.rb -c -n
+      run: go test --ckati --ninja
     - name: run ninja all targets tests
-      run: ruby runtest.rb -c -n -a
+      run: go test --ckati --ninja --all
     - name: run ninja unit tests
       run: ./ninja_test
     - name: run stringpiece unit tests
diff --git a/Makefile b/Makefile
index 9a2aefc..15d8c1e 100644
--- a/Makefile
+++ b/Makefile
@@ -17,14 +17,8 @@ all: ckati ckati_tests
 include Makefile.kati
 include Makefile.ckati
 
-test: run_tests
-
-test_quietly: run_tests
-test_quietly: RUN_TESTS_QUIETLY := -q
-
-run_tests: all ckati_tests
-	ruby runtest.rb -c -n $(RUN_TESTS_QUIETLY)
-
+test: all ckati_tests
+	go test --ckati --ninja
 
 clean: ckati_clean
 
diff --git a/go.mod b/go.mod
index 67c9b11..3ce630e 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,7 @@ module github.com/google/kati
 
 go 1.14
 
-require github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+require (
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/sergi/go-diff v1.1.0
+)
diff --git a/go.sum b/go.sum
index e323037..d08941b 100644
--- a/go.sum
+++ b/go.sum
@@ -1,2 +1,16 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/run_test.go b/run_test.go
new file mode 100644
index 0000000..d295ddf
--- /dev/null
+++ b/run_test.go
@@ -0,0 +1,512 @@
+// Copyright 2020 Google Inc. All rights reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kati
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+var ckati bool
+var ninja bool
+var genAllTargets bool
+
+func init() {
+	// suppress GNU make jobserver magic when calling "make"
+	os.Unsetenv("MAKEFLAGS")
+	os.Unsetenv("MAKELEVEL")
+	os.Setenv("NINJA_STATUS", "NINJACMD: ")
+
+	flag.BoolVar(&ckati, "ckati", false, "use ckati")
+	flag.BoolVar(&ninja, "ninja", false, "use ninja")
+	flag.BoolVar(&genAllTargets, "all", false, "use --gen_all_targets")
+}
+
+type normalization struct {
+	regexp  *regexp.Regexp
+	replace string
+}
+
+var normalizeQuotes = normalization{
+	regexp.MustCompile("([`'\"]|\xe2\x80\x98|\xe2\x80\x99)"), `"`,
+}
+
+var normalizeMakeLog = []normalization{
+	normalizeQuotes,
+	{regexp.MustCompile(`make(?:\[\d+\])?: (Entering|Leaving) directory[^\n]*`), ""},
+	{regexp.MustCompile(`make(?:\[\d+\])?: `), ""},
+
+	// Normalizations for old/new GNU make.
+	{regexp.MustCompile(" recipe for target "), " commands for target "},
+	{regexp.MustCompile(" recipe commences "), " commands commence "},
+	{regexp.MustCompile("missing rule before recipe."), "missing rule before commands."},
+	{regexp.MustCompile(" (did you mean TAB instead of 8 spaces?)"), ""},
+	{regexp.MustCompile("Extraneous text after"), "extraneous text after"},
+	// Not sure if this is useful
+	{regexp.MustCompile(`\s+Stop\.`), ""},
+	// GNU make 4.0 has this output.
+	{regexp.MustCompile(`Makefile:\d+: commands for target ".*?" failed\n`), ""},
+	// We treat some warnings as errors.
+	{regexp.MustCompile(`/bin/(ba)?sh: line 0: `), ""},
+	// Normalization for "include foo" with C++ kati
+	{regexp.MustCompile(`(: \S+: No such file or directory)\n\*\*\* No rule to make target "[^"]+".`), "$1"},
+}
+
+var normalizeMakeNinja = normalization{
+	// We print out some ninja warnings in some tests to match what we expect
+	// ninja to produce. Remove them if we're not testing ninja
+	regexp.MustCompile("ninja: warning: [^\n]+"), "",
+}
+
+var normalizeKati = []normalization{
+	normalizeQuotes,
+
+	// kati specific log messages
+	{regexp.MustCompile(`\*kati\*[^\n]*`), ""},
+	{regexp.MustCompile(`c?kati: `), ""},
+	{regexp.MustCompile(`/bin/sh: line 0: `), ""},
+	{regexp.MustCompile(`/bin/sh: `), ""},
+	{regexp.MustCompile(`.*: warning for parse error in an unevaluated line: [^\n]*`), ""},
+	{regexp.MustCompile(`([^\n ]+: )?FindEmulator: `), ""},
+	// kati log ifles in find_command.mk
+	{regexp.MustCompile(` (\./+)+kati\.\S+`), ""},
+	// json files in find_command.mk
+	{regexp.MustCompile(` (\./+)+test\S+.json`), ""},
+	// Normalization for "include foo" with Go kati
+	{regexp.MustCompile(`(: )open (\S+): n(o such file or directory)\nNOTE:[^\n]*`), "${1}${2}: N${3}"},
+	// Bionic libc has different error messages than glibc
+	{regexp.MustCompile(`Too many symbolic links encountered`), "Too many levels of symbolic links"},
+}
+
+var normalizeNinja = []normalization{
+	{regexp.MustCompile(`NINJACMD: [^\n]*\n`), ""},
+	{regexp.MustCompile(`ninja: no work to do\.\n`), ""},
+	{regexp.MustCompile(`ninja: error: (.*, needed by .*),[^\n]*`),
+		"*** No rule to make target ${1}."},
+	{regexp.MustCompile(`ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct[^\n]*`),
+		"ninja: warning: multiple rules generate ${1}."},
+}
+
+var normalizeNinjaFail = []normalization{
+	{regexp.MustCompile(`FAILED: ([^\n]+\n/bin/bash)?[^\n]*\n`), "*** [test] Error 1\n"},
+	{regexp.MustCompile(`ninja: [^\n]+\n`), ""},
+}
+
+var normalizeNinjaIgnoreFail = []normalization{
+	{regexp.MustCompile(`FAILED: ([^\n]+\n/bin/bash)?[^\n]*\n`), ""},
+	{regexp.MustCompile(`ninja: [^\n]+\n`), ""},
+}
+
+var circularRE = regexp.MustCompile(`(Circular .* dropped\.\n)`)
+
+func normalize(log []byte, normalizations []normalization) []byte {
+	// We don't care when circular dependency detection happens.
+	ret := []byte{}
+	for _, circ := range circularRE.FindAllSubmatch(log, -1) {
+		ret = append(ret, circ[1]...)
+	}
+	ret = append(ret, circularRE.ReplaceAll(log, []byte{})...)
+
+	for _, n := range normalizations {
+		ret = n.regexp.ReplaceAll(ret, []byte(n.replace))
+	}
+	return ret
+}
+
+func runMake(t *testing.T, prefix []string, dir string, silent bool, tc string) string {
+	write := func(f string, data []byte) {
+		suffix := ""
+		if tc != "" {
+			suffix = "_" + tc
+		}
+		if err := ioutil.WriteFile(filepath.Join(dir, f+suffix), data, 0666); err != nil {
+			t.Error(err)
+		}
+	}
+
+	args := append(prefix, "make")
+	if silent {
+		args = append(args, "-s")
+	}
+	if tc != "" {
+		args = append(args, tc)
+	}
+	args = append(args, "SHELL=/bin/bash")
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = dir
+	output, _ := cmd.CombinedOutput()
+	write("stdout", output)
+
+	output = normalize(output, normalizeMakeLog)
+	if !ninja {
+		output = normalize(output, []normalization{normalizeMakeNinja})
+	}
+
+	write("stdout_normalized", output)
+	return string(output)
+}
+
+func runKati(t *testing.T, test, dir string, silent bool, tc string) string {
+	write := func(f string, data []byte) {
+		suffix := ""
+		if tc != "" {
+			suffix = "_" + tc
+		}
+		if err := ioutil.WriteFile(filepath.Join(dir, f+suffix), data, 0666); err != nil {
+			t.Error(err)
+		}
+	}
+
+	var cmd *exec.Cmd
+	if ckati {
+		cmd = exec.Command("../../../ckati", "--use_find_emulator")
+	} else {
+		json := tc
+		if json == "" {
+			json = "test"
+		}
+		cmd = exec.Command("../../../kati", "-save_json="+json+".json", "-log_dir=.", "--use_find_emulator")
+	}
+	if ninja {
+		cmd.Args = append(cmd.Args, "--ninja")
+	}
+	if genAllTargets {
+		cmd.Args = append(cmd.Args, "--gen_all_targets")
+	}
+	if silent {
+		cmd.Args = append(cmd.Args, "-s")
+	}
+	cmd.Args = append(cmd.Args, "SHELL=/bin/bash")
+	if tc != "" && (!genAllTargets || strings.Contains(test, "makecmdgoals")) {
+		cmd.Args = append(cmd.Args, tc)
+	}
+	cmd.Dir = dir
+	output, err := cmd.CombinedOutput()
+	write("stdout", output)
+	if err != nil {
+		output := normalize(output, normalizeKati)
+		write("stdout_normalized", output)
+		return string(output)
+	}
+
+	if ninja {
+		ninjaCmd := exec.Command("./ninja.sh", "-j1", "-v")
+		if genAllTargets && tc != "" {
+			ninjaCmd.Args = append(ninjaCmd.Args, tc)
+		}
+		ninjaCmd.Dir = dir
+		ninjaOutput, _ := ninjaCmd.CombinedOutput()
+		write("stdout_ninja", ninjaOutput)
+		ninjaOutput = normalize(ninjaOutput, normalizeNinja)
+		if test == "err_error_in_recipe.mk" {
+			ninjaOutput = normalize(ninjaOutput, normalizeNinjaIgnoreFail)
+		} else if strings.HasPrefix(test, "fail_") {
+			ninjaOutput = normalize(ninjaOutput, normalizeNinjaFail)
+		}
+		write("stdout_ninja_normalized", ninjaOutput)
+		output = append(output, ninjaOutput...)
+	}
+
+	output = normalize(output, normalizeKati)
+	write("stdout_normalized", output)
+	return string(output)
+}
+
+func runKatiInScript(t *testing.T, script, dir string, isNinjaTest bool) string {
+	write := func(f string, data []byte) {
+		if err := ioutil.WriteFile(filepath.Join(dir, f), data, 0666); err != nil {
+			t.Error(err)
+		}
+	}
+
+	args := []string{"bash", script}
+	if ckati {
+		args = append(args, "../../../ckati")
+		if isNinjaTest {
+			args = append(args, "--ninja", "--regen")
+		}
+	} else {
+		args = append(args, "../../../kati --use_cache -log_dir=.")
+	}
+	args = append(args, "SHELL=/bin/bash")
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = dir
+	output, _ := cmd.Output()
+	write("stdout", output)
+	if isNinjaTest {
+		output = normalize(output, normalizeNinja)
+	}
+	output = normalize(output, normalizeKati)
+	write("stdout_normalized", output)
+	return string(output)
+}
+
+func inList(list []string, item string) bool {
+	for _, i := range list {
+		if item == i {
+			return true
+		}
+	}
+	return false
+}
+
+func diffLists(a, b []string) (onlyA []string, onlyB []string) {
+	for _, i := range a {
+		if !inList(b, i) {
+			onlyA = append(onlyA, i)
+		}
+	}
+	for _, i := range b {
+		if !inList(a, i) {
+			onlyB = append(onlyB, i)
+		}
+	}
+	return
+}
+
+func outputFiles(t *testing.T, dir string) []string {
+	ret := []string{}
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ignoreFiles := []string{
+		".", "..", "Makefile", "build.ninja", "env.sh", "ninja.sh", "gmon.out", "submake",
+	}
+	for _, fi := range files {
+		name := fi.Name()
+		if inList(ignoreFiles, name) ||
+			strings.HasPrefix(name, ".") ||
+			strings.HasSuffix(name, ".json") ||
+			strings.HasPrefix(name, "kati") ||
+			strings.HasPrefix(name, "stdout") {
+			continue
+		}
+		ret = append(ret, fi.Name())
+	}
+	return ret
+}
+
+var testcaseRE = regexp.MustCompile(`^test\d*`)
+
+func uniqueTestcases(c []byte) []string {
+	seen := map[string]bool{}
+	ret := []string{}
+	for _, line := range bytes.Split(c, []byte("\n")) {
+		line := string(line)
+		s := testcaseRE.FindString(line)
+		if s == "" {
+			continue
+		}
+		if _, ok := seen[s]; ok {
+			continue
+		}
+		seen[s] = true
+		ret = append(ret, s)
+	}
+	sort.Strings(ret)
+	if len(ret) == 0 {
+		return []string{""}
+	}
+	return ret
+}
+
+var todoRE = regexp.MustCompile(`^# TODO(?:\(([-a-z|]+)(?:/([-a-z0-9|]+))?\))?`)
+
+func isExpectedFailure(c []byte, tc string) bool {
+	for _, line := range bytes.Split(c, []byte("\n")) {
+		line := string(line)
+		if !strings.HasPrefix(line, "#!") && !strings.HasPrefix(line, "# TODO") {
+			break
+		}
+
+		todo := todoRE.FindStringSubmatch(line)
+		if todo == nil {
+			continue
+		}
+
+		if todo[1] == "" {
+			return true
+		}
+
+		todos := strings.Split(todo[1], "|")
+		if (inList(todos, "go") && !ckati) ||
+			(inList(todos, "c") && ckati) ||
+			(inList(todos, "go-ninja") && !ckati && ninja) ||
+			(inList(todos, "c-ninja") && ckati && ninja) ||
+			(inList(todos, "c-exec") && ckati && !ninja) ||
+			(inList(todos, "ninja") && ninja) ||
+			(inList(todos, "ninja-genall") && ninja && genAllTargets) ||
+			(inList(todos, "all")) {
+
+			if todo[2] == "" {
+				return true
+			}
+			tcs := strings.Split(todo[2], "|")
+			if inList(tcs, tc) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func TestKati(t *testing.T) {
+	if ckati {
+		os.Setenv("KATI_VARIANT", "c")
+		if _, err := os.Stat("ckati"); err != nil {
+			t.Fatalf("ckati must be built before testing: %s", err)
+		}
+	} else {
+		if _, err := os.Stat("kati"); err != nil {
+			t.Fatalf("kati must be built before testing: %s", err)
+		}
+	}
+	if ninja {
+		if _, err := exec.LookPath("ninja"); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	out, _ := filepath.Abs("out")
+	files, err := ioutil.ReadDir("testcase")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, fi := range files {
+		name := fi.Name()
+
+		isMkTest := strings.HasSuffix(name, ".mk")
+		isShTest := strings.HasSuffix(name, ".sh")
+		if strings.HasPrefix(name, ".") || !(isMkTest || isShTest) {
+			continue
+		}
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			c, err := ioutil.ReadFile(filepath.Join("testcase", name))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			out := filepath.Join(out, name)
+			if err := os.RemoveAll(out); err != nil {
+				t.Fatal(err)
+			}
+			outMake := filepath.Join(out, "make")
+			outKati := filepath.Join(out, "kati")
+			if err := os.MkdirAll(outMake, 0777); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.MkdirAll(outKati, 0777); err != nil {
+				t.Fatal(err)
+			}
+
+			testcases := []string{""}
+			expected := map[string]string{}
+			expectedFiles := map[string][]string{}
+			expectedFailures := map[string]bool{}
+			got := map[string]string{}
+			gotFiles := map[string][]string{}
+
+			if isMkTest {
+				setup := func(dir string) {
+					if err = ioutil.WriteFile(filepath.Join(dir, "Makefile"), c, 0666); err != nil {
+						t.Fatal(err)
+					}
+					os.Symlink("../../../testcase/submake", filepath.Join(dir, "submake"))
+				}
+				setup(outMake)
+				setup(outKati)
+
+				testcases = uniqueTestcases(c)
+
+				isSilent := strings.HasPrefix(name, "submake_")
+
+				for _, tc := range testcases {
+					expected[tc] = runMake(t, nil, outMake, ninja || isSilent, tc)
+					expectedFiles[tc] = outputFiles(t, outMake)
+					expectedFailures[tc] = isExpectedFailure(c, tc)
+				}
+
+				for _, tc := range testcases {
+					got[tc] = runKati(t, name, outKati, isSilent, tc)
+					gotFiles[tc] = outputFiles(t, outKati)
+				}
+			} else if isShTest {
+				isNinjaTest := strings.HasPrefix(name, "ninja_")
+				if isNinjaTest && (!ckati || !ninja) {
+					t.SkipNow()
+				}
+
+				scriptName := "../../../testcase/" + name
+
+				expected[""] = runMake(t, []string{"bash", scriptName}, outMake, isNinjaTest, "")
+				expectedFailures[""] = isExpectedFailure(c, "")
+
+				got[""] = runKatiInScript(t, scriptName, outKati, isNinjaTest)
+			}
+
+			check := func(t *testing.T, m, k string, mFiles, kFiles []string, expectFail bool) {
+				if strings.Contains(m, "FAIL") {
+					t.Fatalf("Make returned 'FAIL':\n%q", m)
+				}
+
+				if !expectFail && m != k {
+					dmp := diffmatchpatch.New()
+					diffs := dmp.DiffMain(k, m, true)
+					diffs = dmp.DiffCleanupSemantic(diffs)
+					t.Errorf("Different output from kati (red) to the expected value from make (green):\n%s",
+						dmp.DiffPrettyText(diffs))
+				} else if expectFail && m == k {
+					t.Errorf("Expected failure, but output is the same")
+				}
+
+				if !expectFail {
+					onlyMake, onlyKati := diffLists(mFiles, kFiles)
+					if len(onlyMake) > 0 {
+						t.Errorf("Files only created by Make:\n%q", onlyMake)
+					}
+					if len(onlyKati) > 0 {
+						t.Errorf("Files only created by Kati:\n%q", onlyKati)
+					}
+				}
+			}
+
+			for _, tc := range testcases {
+				if tc == "" || len(testcases) == 1 {
+					check(t, expected[tc], got[tc], expectedFiles[tc], gotFiles[tc], expectedFailures[tc])
+				} else {
+					t.Run(tc, func(t *testing.T) {
+						check(t, expected[tc], got[tc], expectedFiles[tc], gotFiles[tc], expectedFailures[tc])
+					})
+				}
+			}
+		})
+	}
+}
diff --git a/runtest.rb b/runtest.rb
deleted file mode 100755
index 173c66d..0000000
--- a/runtest.rb
+++ /dev/null
@@ -1,440 +0,0 @@
-#!/usr/bin/env ruby
-# coding: binary
-#
-# Copyright 2015 Google Inc. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-require 'fileutils'
-
-# suppress GNU make jobserver magic when calling "make"
-ENV.delete('MAKEFLAGS')
-ENV.delete('MAKELEVEL')
-
-while true
-  if ARGV[0] == '-s'
-    test_serialization = true
-    ARGV.shift
-  elsif ARGV[0] == '-c'
-    $ckati = true
-    ARGV.shift
-    ENV['KATI_VARIANT'] = 'c'
-  elsif ARGV[0] == '-n'
-    $via_ninja = true
-    ARGV.shift
-    ENV['NINJA_STATUS'] = 'NINJACMD: '
-  elsif ARGV[0] == '-a'
-    gen_all_targets = true
-    ARGV.shift
-  elsif ARGV[0] == '-v'
-    $show_failing = true
-    ARGV.shift
-  elsif ARGV[0] == "-q"
-    $hide_passing = true
-    ARGV.shift
-  else
-    break
-  end
-end
-
-def get_output_filenames
-  files = Dir.glob('*')
-  files.delete('Makefile')
-  files.delete('build.ninja')
-  files.delete('env.sh')
-  files.delete('ninja.sh')
-  files.delete('gmon.out')
-  files.delete('submake')
-  files.reject!{|f|f =~ /\.json$/}
-  files.reject!{|f|f =~ /^kati\.*/}
-  files
-end
-
-def cleanup
-  (get_output_filenames + Dir.glob('.*')).each do |fname|
-    next if fname == '.' || fname == '..'
-    FileUtils.rm_rf fname
-  end
-end
-
-def move_circular_dep(l)
-  # We don't care when circular dependency detection happens.
-  circ = ''
-  while l.sub!(/Circular .* dropped\.\n/, '') do
-    circ += $&
-  end
-  circ + l
-end
-
-$expected_failures = []
-$unexpected_passes = []
-$failures = []
-$passes = []
-
-if !ARGV.empty?
-  test_files = ARGV.map do |test|
-    "testcase/#{File.basename(test)}"
-  end
-else
-  test_files = Dir.glob('testcase/*.mk').sort
-  test_files += Dir.glob('testcase/*.sh').sort
-end
-
-def run_in_testdir(test_filename)
-  c = File.read(test_filename)
-  name = File.basename(test_filename)
-  dir = "out/#{name}"
-
-  FileUtils.mkdir_p(dir)
-  Dir.glob("#{dir}/*").each do |fname|
-    FileUtils.rm_rf(fname)
-  end
-
-  Dir.chdir(dir) do
-    yield name
-  end
-end
-
-def normalize_ninja_log(log, mk)
-  log.gsub!(/^NINJACMD: .*\n/, '')
-  log.gsub!(/^ninja: no work to do\.\n/, '')
-  log.gsub!(/^ninja: error: (.*, needed by .*),.*/,
-            '*** No rule to make target \\1.')
-  log.gsub!(/^ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct.*$/,
-            'ninja: warning: multiple rules generate \\1.')
-
-  if mk =~ /err_error_in_recipe.mk/
-    # This test expects ninja fails. Strip ninja specific error logs.
-    ninja_failed_subst = ''
-  elsif mk =~ /\/fail_/
-    # Recipes in these tests fail.
-    ninja_failed_subst = "*** [test] Error 1\n"
-  end
-  if ninja_failed_subst
-    log.gsub!(/^FAILED: (.*\n\/bin\/bash)?.*\n/, ninja_failed_subst)
-    log.gsub!(/^ninja: .*\n/, '')
-  end
-  log
-end
-
-def normalize_quotes(log)
-  log.gsub!(/[`'"]/, '"')
-  # For recent GNU find, which uses Unicode characters.
-  log.gsub!(/(\xe2\x80\x98|\xe2\x80\x99)/, '"')
-  log
-end
-
-def normalize_make_log(expected, mk, via_ninja)
-  expected = normalize_quotes(expected)
-  expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '')
-  expected.gsub!(/^make(?:\[\d+\])?: /, '')
-  expected = move_circular_dep(expected)
-
-  # Normalizations for old/new GNU make.
-  expected.gsub!(' recipe for target ', ' commands for target ')
-  expected.gsub!(' recipe commences ', ' commands commence ')
-  expected.gsub!('missing rule before recipe.', 'missing rule before commands.')
-  expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '')
-  expected.gsub!('Extraneous text after', 'extraneous text after')
-  # Not sure if this is useful.
-  expected.gsub!(/\s+Stop\.$/, '')
-  # GNU make 4.0 has this output.
-  expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '')
-  # We treat some warnings as errors.
-  expected.gsub!(/^\/bin\/(ba)?sh: line 0: /, '')
-  # We print out some ninja warnings in some tests to match what we expect
-  # ninja to produce. Remove them if we're not testing ninja.
-  if !via_ninja
-    expected.gsub!(/^ninja: warning: .*\n/, '')
-  end
-  # Normalization for "include foo" with C++ kati.
-  expected.gsub!(/(: )(\S+): (No such file or directory)\n\*\*\* No rule to make target "\2"./, '\1\2: \3')
-
-  expected
-end
-
-def normalize_kati_log(output)
-  output = normalize_quotes(output)
-  output = move_circular_dep(output)
-
-  # kati specific log messages.
-  output.gsub!(/^\*kati\*.*\n/, '')
-  output.gsub!(/^c?kati: /, '')
-  output.gsub!(/\/bin\/sh: ([^:]*): command not found/,
-               "\\1: Command not found")
-  output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '')
-  output.gsub!(/^([^ ]+: )?FindEmulator: /, '')
-  output.gsub!(/^\/bin\/sh: line 0: /, '')
-  output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk
-  output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk
-  # Normalization for "include foo" with Go kati.
-  output.gsub!(/(: )open (\S+): n(o such file or directory)\nNOTE:.*/,
-               "\\1\\2: N\\3")
-  # Bionic libc has different error messages than glibc
-  output.gsub!(/Too many symbolic links encountered/, 'Too many levels of symbolic links')
-  output
-end
-
-bash_var = ' SHELL=/bin/bash'
-
-def is_expected_failure(f)
-  c = File.read(f)
-  expected_failure = false
-  if c =~ /\A(#!\/bin\/bash\n)?# TODO(?:\(([-a-z|]+)\))?/
-    if $2
-      todos = $2.split('|')
-      if todos.include?('go') && !$ckati
-        expected_failure = true
-      end
-      if todos.include?('c') && $ckati
-        expected_failure = true
-      end
-      if todos.include?('go-ninja') && !$ckati && $via_ninja
-        expected_failure = true
-      end
-      if todos.include?('c-ninja') && $ckati && $via_ninja
-        expected_failure = true
-      end
-      if todos.include?('c-exec') && $ckati && !$via_ninja
-        expected_failure = true
-      end
-      if todos.include?('ninja') && $via_ninja
-        expected_failure = true
-      end
-    else
-      expected_failure = true
-    end
-  end
-  return expected_failure
-end
-
-report_result = proc do |name, expected, output, expected_failure|
-  if expected != output
-    if expected_failure
-      if !$hide_passing
-        puts "#{name}: FAIL (expected)"
-      end
-      $expected_failures << name
-    else
-      puts "#{name}: FAIL"
-      $failures << name
-    end
-    if !expected_failure || $show_failing
-      puts `diff -u out.make out.kati`
-    end
-  else
-    if expected_failure
-      puts "#{name}: PASS (unexpected)"
-      $unexpected_passes << name
-    else
-      if !$hide_passing
-        puts "#{name}: PASS"
-      end
-      $passes << name
-    end
-  end
-end
-
-run_make_test = proc do |mk|
-  expected_failure = is_expected_failure(mk)
-  c = File.read(mk)
-  run_in_testdir(mk) do |name|
-    File.open("Makefile", 'w') do |ofile|
-      ofile.print(c)
-    end
-    File.symlink('../../testcase/submake', 'submake')
-
-    expected = ''
-    output = ''
-
-    testcases = c.scan(/^test\d*/).sort.uniq
-    if testcases.empty?
-      testcases = ['']
-    end
-
-    is_silent_test = mk =~ /\/submake_/
-
-    cleanup
-    testcases.each do |tc|
-      cmd = 'make'
-      if $via_ninja || is_silent_test
-        cmd += ' -s'
-      end
-      cmd += bash_var
-      cmd += " #{tc} 2>&1"
-      res = IO.popen(cmd, 'r:binary', &:read)
-      res = normalize_make_log(res, mk, $via_ninja)
-      expected += "=== #{tc} ===\n" + res
-      expected_files = get_output_filenames
-      expected += "\n=== FILES ===\n#{expected_files * "\n"}\n"
-    end
-
-    cleanup
-    testcases.each do |tc|
-      json = "#{tc.empty? ? 'test' : tc}"
-      cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator"
-      if $ckati
-        cmd = "../../ckati --use_find_emulator"
-      end
-      if $via_ninja
-        cmd += ' --ninja'
-      end
-      if gen_all_targets
-        if !$ckati || !$via_ninja
-          raise "-a should be used with -c -n"
-        end
-        cmd += ' --gen_all_targets'
-      end
-      if is_silent_test
-        cmd += ' -s'
-      end
-      cmd += bash_var
-      if !gen_all_targets || mk =~ /makecmdgoals/
-        cmd += " #{tc}"
-      end
-      cmd += " 2>&1"
-      res = IO.popen(cmd, 'r:binary', &:read)
-      if $via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh')
-        cmd = './ninja.sh -j1 -v'
-        if gen_all_targets
-          cmd += " #{tc}"
-        end
-        cmd += ' 2>&1'
-        log = IO.popen(cmd, 'r:binary', &:read)
-        res += normalize_ninja_log(log, mk)
-      end
-      res = normalize_kati_log(res)
-      output += "=== #{tc} ===\n" + res
-      output_files = get_output_filenames
-      output += "\n=== FILES ===\n#{output_files * "\n"}\n"
-    end
-
-    File.open('out.make', 'w'){|ofile|ofile.print(expected)}
-    File.open('out.kati', 'w'){|ofile|ofile.print(output)}
-
-    if expected =~ /FAIL/
-      puts %Q(#{name} has a string "FAIL" in its expectation)
-      exit 1
-    end
-
-    report_result.call(name, expected, output, expected_failure)
-
-    if name !~ /^err_/ && test_serialization && !expected_failure
-      testcases.each do |tc|
-        json = "#{tc.empty? ? 'test' : tc}"
-        cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1"
-        res = IO.popen(cmd, 'r:binary', &:read)
-        if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json")
-          puts "#{name}##{json}: Serialize failure (not exist)"
-          puts res
-        else
-          json1 = File.read("#{json}.json")
-          json2 = File.read("#{json}_2.json")
-          if json1 != json2
-            puts "#{name}##{json}: Serialize failure"
-            puts res
-          end
-        end
-      end
-    end
-  end
-end
-
-run_shell_test = proc do |sh|
-  is_ninja_test = sh =~ /\/ninja_/
-  if is_ninja_test && (!$ckati || !$via_ninja)
-    next
-  end
-
-  expected_failure = is_expected_failure(sh)
-
-  run_in_testdir(sh) do |name|
-    cleanup
-    cmd = "bash ../../#{sh} make"
-    if is_ninja_test
-      cmd += ' -s'
-    end
-    cmd += bash_var
-    expected = IO.popen(cmd, 'r:binary', &:read)
-    cleanup
-
-    if is_ninja_test
-      if $ckati
-        cmd = "bash ../../#{sh} ../../ckati --ninja --regen"
-      else
-        next
-      end
-    else
-      if $ckati
-        cmd = "bash ../../#{sh} ../../ckati"
-      else
-        cmd = "bash ../../#{sh} ../../kati --use_cache -log_dir=."
-      end
-    end
-    cmd += bash_var
-
-    output = IO.popen(cmd, 'r:binary', &:read)
-
-    expected = normalize_make_log(expected, sh, is_ninja_test)
-    output = normalize_kati_log(output)
-    if is_ninja_test
-      output = normalize_ninja_log(output, sh)
-    end
-    File.open('out.make', 'w'){|ofile|ofile.print(expected)}
-    File.open('out.kati', 'w'){|ofile|ofile.print(output)}
-
-    report_result.call(name, expected, output, expected_failure)
-  end
-end
-
-test_files.each do |test|
-  if /\.mk$/ =~ test
-    run_make_test.call(test)
-  elsif /\.sh$/ =~ test
-    run_shell_test.call(test)
-  else
-    raise "Unknown test type: #{test}"
-  end
-end
-
-puts
-
-if !$expected_failures.empty? && !$hide_passing
-  puts "=== Expected failures ==="
-  $expected_failures.each do |n|
-    puts n
-  end
-end
-
-if !$unexpected_passes.empty?
-  puts "=== Unexpected passes ==="
-  $unexpected_passes.each do |n|
-    puts n
-  end
-end
-
-if !$failures.empty?
-  puts "=== Failures ==="
-  $failures.each do |n|
-    puts n
-  end
-end
-
-puts
-
-if !$unexpected_passes.empty? || !$failures.empty?
-  puts "FAIL! (#{$failures.size + $unexpected_passes.size} fails #{$passes.size} passes)"
-  exit 1
-else
-  puts 'PASS!'
-end
diff --git a/testcase/abspath.mk b/testcase/abspath.mk
index 9522dd3..89d96f9 100644
--- a/testcase/abspath.mk
+++ b/testcase/abspath.mk
@@ -1,5 +1,5 @@
-foo = $(abspath ./foo bar/../foo bar//..//foo / /usr)
-bar = $(abspath .. ./. ./ /aa/.. a///)
+foo = $(subst /kati,,$(subst /make,,$(abspath ./foo bar/../foo bar//..//foo / /usr)))
+bar = $(subst /kati,,$(subst /make,,$(abspath .. ./. ./ /aa/.. a///)))
 
 test:
 	echo $(foo)
diff --git a/testcase/comment_in_command.mk b/testcase/comment_in_command.mk
index 881a241..a734bc2 100644
--- a/testcase/comment_in_command.mk
+++ b/testcase/comment_in_command.mk
@@ -1,4 +1,4 @@
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 test1:
 	# foo
diff --git a/testcase/curdir_var.mk b/testcase/curdir_var.mk
index 391f09f..6d35f77 100644
--- a/testcase/curdir_var.mk
+++ b/testcase/curdir_var.mk
@@ -1,2 +1,3 @@
 test:
-	echo $(CURDIR)
+	# Make and kati will be executed in different subdirs
+	echo $(dir $(CURDIR))
diff --git a/testcase/err_export_override.mk b/testcase/err_export_override.mk
index 6bb9f75..fc901f5 100644
--- a/testcase/err_export_override.mk
+++ b/testcase/err_export_override.mk
@@ -2,7 +2,7 @@
 
 # GNU make 4 accepts this syntax. Note kati doesn't agree with make 4
 # either.
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 ifeq ($(MAKE)$(MAKEVER),make4)
 $(error test skipped)
 endif
diff --git a/testcase/err_override_export.mk b/testcase/err_override_export.mk
index 2d72aab..b08f5e1 100644
--- a/testcase/err_override_export.mk
+++ b/testcase/err_override_export.mk
@@ -2,7 +2,7 @@
 
 # GNU make 4 accepts this syntax. Note kati doesn't agree with make 4
 # either.
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 ifeq ($(MAKE)$(MAKEVER),make4)
 $(error test skipped)
 endif
diff --git a/testcase/err_suffixes.mk b/testcase/err_suffixes.mk
index 2ac6d09..0e566be 100644
--- a/testcase/err_suffixes.mk
+++ b/testcase/err_suffixes.mk
@@ -1,4 +1,4 @@
-# TODO: Fix
+# TODO(all/test2): Fix
 
 test1:
 	touch a.src
diff --git a/testcase/find_command.mk b/testcase/find_command.mk
index b714879..941b7d0 100644
--- a/testcase/find_command.mk
+++ b/testcase/find_command.mk
@@ -1,4 +1,5 @@
-# TODO(go|ninja): This test is only for ckati. ninja: multiple problems
+# TODO(go): This test is only for ckati.
+# TODO(ninja/test2|test4|test5) ninja: multiple problems
 # go: symlink support isn't enough.
 # ninja: find . finds ninja temporary files
 # ninja: escaping ! doesn't seem to be working
@@ -31,7 +32,7 @@ test1:
 	ln -s ../../testdir/dir1 testdir/dir2/link2
 	ln -s broken testdir/dir2/link3
 	mkdir -p build/tools
-	cp ../../testcase/tools/findleaves.py build/tools
+	cp ../../../testcase/tools/findleaves.py build/tools
 
 	mkdir -p testdir3/b/c/d
 	ln -s b testdir3/a
diff --git a/testcase/implicit_pattern_rule_chain2.mk b/testcase/implicit_pattern_rule_chain2.mk
index 65e914b..a995a9f 100644
--- a/testcase/implicit_pattern_rule_chain2.mk
+++ b/testcase/implicit_pattern_rule_chain2.mk
@@ -1,4 +1,4 @@
-# TODO: Fix. We probably need to assume foo.y exists as there's a rule
+# TODO(all/test2): Fix. We probably need to assume foo.y exists as there's a rule
 # to generate it.
 
 test1:
diff --git a/testcase/implicit_pattern_rule_prefix.mk b/testcase/implicit_pattern_rule_prefix.mk
index 6df648c..8972e2c 100644
--- a/testcase/implicit_pattern_rule_prefix.mk
+++ b/testcase/implicit_pattern_rule_prefix.mk
@@ -1,4 +1,4 @@
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 test: abcd
 
diff --git a/testcase/include_glob_order.mk b/testcase/include_glob_order.mk
index 8b337f9..bceae5f 100644
--- a/testcase/include_glob_order.mk
+++ b/testcase/include_glob_order.mk
@@ -1,4 +1,4 @@
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 # GNU make 4 doesn't sort glob results.
 ifeq ($(MAKEVER,4))
diff --git a/testcase/multi_implicit_output_patterns.mk b/testcase/multi_implicit_output_patterns.mk
index 8b53b49..4ff3d62 100644
--- a/testcase/multi_implicit_output_patterns.mk
+++ b/testcase/multi_implicit_output_patterns.mk
@@ -1,6 +1,6 @@
 # TODO(go): Fix
 
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 all: a.h.x a.c.x a.h.z a.c.z b.h.x b.c.x b.h.z b.c.z
 
diff --git a/testcase/multiline_recipe.mk b/testcase/multiline_recipe.mk
index 74b4d27..69e37fb 100644
--- a/testcase/multiline_recipe.mk
+++ b/testcase/multiline_recipe.mk
@@ -1,5 +1,5 @@
-# TODO(c): fix test6 - \t$${empty} should be empty.
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+# TODO(c/test6): fix test6 - \t$${empty} should be empty.
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 ifeq ($(MAKEVER),4)
 AT=@
 endif
diff --git a/testcase/ninja_normalized_path.mk b/testcase/ninja_normalized_path.mk
index f46f603..183237f 100644
--- a/testcase/ninja_normalized_path.mk
+++ b/testcase/ninja_normalized_path.mk
@@ -1,4 +1,5 @@
-# TODO(ninja): This is tough to fix with ninja. Ninja normalizes
+# TODO(ninja-genall)
+# TODO(ninja/test2): This is tough to fix with ninja. Ninja normalizes
 # target names while make does not.
 
 test1:
diff --git a/testcase/order_only2.mk b/testcase/order_only2.mk
index 6baf49c..96dc795 100644
--- a/testcase/order_only2.mk
+++ b/testcase/order_only2.mk
@@ -1,4 +1,4 @@
-# TODO(ninja): Ninja does not believe the timestamp so this test is invalid.
+# TODO(ninja/test2): Ninja does not believe the timestamp so this test is invalid.
 
 test1:
 	touch -t 197101010000 old1
diff --git a/testcase/posix_var.mk b/testcase/posix_var.mk
index 5574f09..8065d3a 100644
--- a/testcase/posix_var.mk
+++ b/testcase/posix_var.mk
@@ -1,6 +1,6 @@
 # TODO(go): Fix
 
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 # GNU make 3.82 has this feature though.
 ifeq ($(MAKEVER),3)
diff --git a/testcase/realpath.mk b/testcase/realpath.mk
index 0d57e3b..456230d 100644
--- a/testcase/realpath.mk
+++ b/testcase/realpath.mk
@@ -1,13 +1,12 @@
-foo = $(realpath ./foo)
-bar = $(realpath ./bar)
-foofoo = $(realpath ./foo ./foo)
-foobar = $(realpath ./foo ./bar)
+$(shell touch ../foo)
 
-test: foo
+foo = $(realpath ../foo)
+bar = $(realpath ../bar)
+foofoo = $(realpath ../foo ../foo)
+foobar = $(realpath ../foo ../bar)
+
+test:
 	echo $(foo)
 	echo $(bar)
 	echo $(foofoo)
 	echo $(foobar)
-
-foo:
-	touch foo
diff --git a/testcase/shell.mk b/testcase/shell.mk
index 5698d1a..d149ff7 100644
--- a/testcase/shell.mk
+++ b/testcase/shell.mk
@@ -1,5 +1,5 @@
 test:
-	echo $(shell pwd)
+	echo $(shell dirname $$(pwd))
 	echo $(shell false)
 	echo $(shell  /bin/echo -e "\na \n  b 	\n " )
 	echo $(shell  /bin/echo -e "\na \n  b 	\n " )X
diff --git a/testcase/shell_var_with_args.mk b/testcase/shell_var_with_args.mk
index 4256eb0..b163106 100644
--- a/testcase/shell_var_with_args.mk
+++ b/testcase/shell_var_with_args.mk
@@ -1,6 +1,6 @@
 # TODO(go): Fix
 
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 ifeq ($(MAKEVER),4)
 
diff --git a/testcase/strip_and_shell.mk b/testcase/strip_and_shell.mk
index cb550f3..fccc37f 100644
--- a/testcase/strip_and_shell.mk
+++ b/testcase/strip_and_shell.mk
@@ -1,4 +1,4 @@
 # TODO(c-ninja): $(shell) in another make expression is not supported.
 
 test:
-	echo $(strip $(shell pwd))
+	echo $(strip $(shell dirname $$(pwd)))
diff --git a/testcase/var_with_space.mk b/testcase/var_with_space.mk
index 6f74816..1bfb782 100644
--- a/testcase/var_with_space.mk
+++ b/testcase/var_with_space.mk
@@ -1,4 +1,4 @@
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 ifeq ($(MAKEVER),4)
 # A variable name with space is invalid on GNU make 4.
diff --git a/testcase/vpath.mk b/testcase/vpath.mk
index 3a86668..5e37264 100644
--- a/testcase/vpath.mk
+++ b/testcase/vpath.mk
@@ -1,4 +1,4 @@
-# TODO(c): bar is built even if foo doesn't exist.
+# TODO(c/test2): bar is built even if foo doesn't exist.
 
 VPATH=dir
 
diff --git a/testcase/wildcard.mk b/testcase/wildcard.mk
index 33adfd6..5bfe18c 100644
--- a/testcase/wildcard.mk
+++ b/testcase/wildcard.mk
@@ -1,6 +1,6 @@
 # TODO(go): Fix
 
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 
 files = $(wildcard M*)
 
diff --git a/testcase/wildcard_cache.mk b/testcase/wildcard_cache.mk
index 83ba3db..957e4c2 100644
--- a/testcase/wildcard_cache.mk
+++ b/testcase/wildcard_cache.mk
@@ -1,7 +1,7 @@
 # TODO(c): Fix this. Maybe $(wildcard) always runs at eval-phase.
 
 # GNU make 4 agrees with ckati.
-MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+MAKEVER:=$(shell make --version | grep "Make [0-9]" | sed -E 's/.*Make ([0-9]).*/\1/')
 ifeq ($(MAKE)$(MAKEVER),make4)
 $(error test skipped)
 endif
diff --git a/testcase/wildcard_target.mk b/testcase/wildcard_target.mk
index 8cfbe1c..2b8f9af 100644
--- a/testcase/wildcard_target.mk
+++ b/testcase/wildcard_target.mk
@@ -1,4 +1,4 @@
-# TODO(c): Implement wildcard expansion in prerequisites.
+# TODO(c/test2): Implement wildcard expansion in prerequisites.
 
 test1:
 	touch foo.x