[go] cleanup wildcard cache

remove UseWildcardCache flag.
fix wildcard_cache.mk

6 files changed, 104 insertions, 95 deletions

diff --git a/cmd/kati/main.go b/cmd/kati/main.go
index 86a4503..2e273b0 100644
--- a/cmd/kati/main.go
+++ b/cmd/kati/main.go
@@ -102,7 +102,6 @@ func init() {
 
 	// TODO: Make this default.
 	flag.BoolVar(&kati.UseFindCache, "use_find_cache", false, "Use find cache.")
-	flag.BoolVar(&kati.UseWildcardCache, "use_wildcard_cache", true, "Use wildcard cache.")
 	flag.BoolVar(&kati.UseShellBuiltins, "use_shell_builtins", true, "Use shell builtins")
 	flag.StringVar(&kati.IgnoreOptionalInclude, "ignore_optional_include", "", "If specified, skip reading -include directives start with the specified path.")
 }
diff --git a/dep.go b/dep.go
index 2cb46ff..0b27578 100644
--- a/dep.go
+++ b/dep.go
@@ -574,10 +574,13 @@ func (db *depBuilder) Eval(targets []string) ([]*DepNode, error) {
 		targets = append(targets, phonys...)
 	}
 
-	logStats("%d variables", len(db.vars))
-	logStats("%d explicit rules", len(db.rules))
-	logStats("%d implicit rules", db.implicitRules.size())
-	logStats("%d suffix rules", len(db.suffixRules))
+	if StatsFlag {
+		logStats("%d variables", len(db.vars))
+		logStats("%d explicit rules", len(db.rules))
+		logStats("%d implicit rules", db.implicitRules.size())
+		logStats("%d suffix rules", len(db.suffixRules))
+		logStats("%d dirs %d files", wildcardCache.dirs(), wildcardCache.files())
+	}
 
 	var nodes []*DepNode
 	for _, target := range targets {
diff --git a/flags.go b/flags.go
index aa119eb..21535cd 100644
--- a/flags.go
+++ b/flags.go
@@ -23,7 +23,6 @@ var (
 	DryRunFlag bool
 
 	UseFindCache     bool
-	UseWildcardCache bool
 	UseShellBuiltins bool
 
 	IgnoreOptionalInclude string
diff --git a/func.go b/func.go
index 9fdb877..8ce4ef1 100644
--- a/func.go
+++ b/func.go
@@ -574,15 +574,7 @@ func (f *funcWildcard) Eval(w evalWriter, ev *Evaluator) error {
 		return err
 	}
 	te := traceEvent.begin("wildcard", tmpval(wb.Bytes()), traceEventMain)
-	if ev.avoidIO && !UseWildcardCache {
-		ev.hasIO = true
-		io.WriteString(w, "$(/bin/ls -d ")
-		w.Write(wb.Bytes())
-		io.WriteString(w, " 2> /dev/null)")
-		wb.release()
-		traceEvent.end(te)
-		return nil
-	}
+	// need to handle avoidIO here?
 	t := time.Now()
 	for _, word := range wb.words {
 		pat := string(word)
diff --git a/pathutil.go b/pathutil.go
index 1653b3e..fe1bba2 100644
--- a/pathutil.go
+++ b/pathutil.go
@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -31,12 +30,32 @@ import (
 )
 
 type wildcardCacheT struct {
-	mu sync.Mutex
-	m  map[string][]string
+	mu     sync.Mutex
+	dirent map[string][]string
 }
 
 var wildcardCache = &wildcardCacheT{
-	m: make(map[string][]string),
+	dirent: make(map[string][]string),
+}
+
+func (w *wildcardCacheT) dirs() int {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return len(w.dirent)
+}
+
+func (w *wildcardCacheT) files() int {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	n := 0
+	for _, names := range w.dirent {
+		n += len(names)
+	}
+	return n
+}
+
+func hasWildcardMeta(pat string) bool {
+	return strings.IndexAny(pat, "*?[") >= 0
 }
 
 func wildcardUnescape(pat string) string {
@@ -45,99 +64,97 @@ func wildcardUnescape(pat string) string {
 		if pat[i] == '\\' && i+1 < len(pat) {
 			switch pat[i+1] {
 			case '*', '?', '[', '\\':
-				writeByte(&buf, pat[i])
-			default:
-				i++
+				buf.WriteByte(pat[i])
 			}
+			continue
 		}
-		writeByte(&buf, pat[i])
+		buf.WriteByte(pat[i])
 	}
 	return buf.String()
 }
 
-func wildcardGlob(pat string) ([]string, error) {
-	// TODO(ukai): use find cache for glob if exists.
-	pat = wildcardUnescape(pat)
-	pattern := filepath.Clean(pat)
-	if pattern != pat {
-		// For some reason, go's Glob normalizes
-		// foo/../bar to bar.
-		i := strings.IndexAny(pattern, "*?[")
-		if i < 0 {
-			// no wildcard. if any files matched with pattern,
-			// return pat.
-			_, err := os.Stat(pat)
-			if err != nil {
-				return nil, nil
-			}
-			return []string{pat}, nil
-		}
-		if strings.Contains(pattern[i+1:], "..") {
-			// We ask shell to expand a glob to avoid this.
-			cmdline := []string{"/bin/sh", "-c", "/bin/ls -d " + pat}
-			cmd := exec.Cmd{
-				Path: cmdline[0],
-				Args: cmdline,
-			}
-			// Ignore errors.
-			out, _ := cmd.Output()
-			ws := newWordScanner(out)
-			var files []string
-			for ws.Scan() {
-				files = append(files, string(ws.Bytes()))
-			}
-			return files, nil
-		}
-		// prefix + meta + suffix, and suffix doesn't have '..'
-		prefix := pattern[:i]
-		i = strings.IndexAny(pat, "*?[")
-		if i < 0 {
-			return nil, fmt.Errorf("wildcard metachar mismatch? pattern=%q pat=%q", pattern, pat)
-		}
-		oprefix := pat[:i]
-		matched, err := filepath.Glob(pattern)
+func (w *wildcardCacheT) readdirnames(dir string) []string {
+	if dir == "" {
+		dir = "."
+	}
+	w.mu.Lock()
+	names, ok := w.dirent[dir]
+	w.mu.Unlock()
+	if ok {
+		return names
+	}
+	d, err := os.Open(dir)
+	if err != nil {
+		w.mu.Lock()
+		w.dirent[dir] = nil
+		w.mu.Unlock()
+		return nil
+	}
+	defer d.Close()
+	names, _ = d.Readdirnames(-1)
+	sort.Strings(names)
+	w.mu.Lock()
+	w.dirent[dir] = names
+	w.mu.Unlock()
+	return names
+}
+
+// glob searches for files matching pattern in the directory dir
+// and appends them to matches. ignore I/O errors.
+func (w *wildcardCacheT) glob(dir, pattern string, matches []string) ([]string, error) {
+	names := w.readdirnames(dir)
+	if dir != "" {
+		dir += string(filepath.Separator)
+	}
+	for _, n := range names {
+		matched, err := filepath.Match(pattern, n)
 		if err != nil {
 			return nil, err
 		}
-		var files []string
-		for _, m := range matched {
-			file := oprefix + strings.TrimPrefix(m, prefix)
-			_, err := os.Stat(file)
-			if err != nil {
-				continue
-			}
-			files = append(files, file)
+		if matched {
+			matches = append(matches, dir+n)
 		}
-		return files, nil
 	}
-	return filepath.Glob(pat)
+	return matches, nil
 }
 
-func wildcard(w evalWriter, pat string) error {
-	if UseWildcardCache {
-		// TODO(ukai): make sure it didn't chdir?
-		wildcardCache.mu.Lock()
-		files, ok := wildcardCache.m[pat]
-		wildcardCache.mu.Unlock()
-		if ok {
-			for _, file := range files {
-				w.writeWordString(file)
-			}
-			return nil
+func (w *wildcardCacheT) Glob(pat string) ([]string, error) {
+	// TODO(ukai): use find cache for glob if exists
+	// or use wildcardCache for find cache.
+	pat = wildcardUnescape(pat)
+	dir, file := filepath.Split(pat)
+	switch dir {
+	case "", string(filepath.Separator):
+		// nothing
+	default:
+		dir = dir[0 : len(dir)-1] // chop off trailing separator
+	}
+	if !hasWildcardMeta(dir) {
+		return w.glob(dir, file, nil)
+	}
+
+	m, err := w.Glob(dir)
+	if err != nil {
+		return nil, err
+	}
+	var matches []string
+	for _, d := range m {
+		matches, err = w.glob(d, file, matches)
+		if err != nil {
+			return nil, err
 		}
 	}
-	files, err := wildcardGlob(pat)
+	return matches, nil
+}
+
+func wildcard(w evalWriter, pat string) error {
+	files, err := wildcardCache.Glob(pat)
 	if err != nil {
 		return err
 	}
 	for _, file := range files {
 		w.writeWordString(file)
 	}
-	if UseWildcardCache {
-		wildcardCache.mu.Lock()
-		wildcardCache.m[pat] = files
-		wildcardCache.mu.Unlock()
-	}
 	return nil
 }
 
diff --git a/testcase/wildcard_cache.mk b/testcase/wildcard_cache.mk
index 70309c8..e8fb12d 100644
--- a/testcase/wildcard_cache.mk
+++ b/testcase/wildcard_cache.mk
@@ -1,11 +1,10 @@
-# maybe, make has wildcard cache at startup time?
+# TODO(c): Fix this. Maybe $(wildcard) always runs at eval-phase.
 files = $(wildcard *,*)
 
-# if make starts without foo,bar, expect foo,bar, but it will be empty.
+# if make starts without foo,bar, it will be empty, although expect foo,bar.
 test: foo,bar
 	echo $(files)
-# TODO: Fix this. Maybe $(wildcard) always runs at eval-phase.
-#	echo $(wildcard foo*)
+	echo $(wildcard foo*)
 
 # first $(files) will be empty since no foo,bar exists.
 # second $(files) expects foo, but empty.