diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Makefile | 17 | ||||
| -rw-r--r-- | ast.cc | 2 | ||||
| -rw-r--r-- | ast.go | 6 | ||||
| -rw-r--r-- | bootstrap.go | 56 | ||||
| -rw-r--r-- | cmd/kati/main.go | 289 | ||||
| -rw-r--r-- | cmdline.go | 48 | ||||
| -rw-r--r-- | dep.cc | 2 | ||||
| -rw-r--r-- | dep.go | 6 | ||||
| -rw-r--r-- | depgraph.go | 115 | ||||
| -rw-r--r-- | doc.go | 8 | ||||
| -rw-r--r-- | eval.cc | 2 | ||||
| -rw-r--r-- | eval.go | 31 | ||||
| -rw-r--r-- | exec.cc | 2 | ||||
| -rw-r--r-- | exec.go | 25 | ||||
| -rw-r--r-- | expr.go | 4 | ||||
| -rw-r--r-- | expr_test.go | 2 | ||||
| -rw-r--r-- | file.cc | 2 | ||||
| -rw-r--r-- | file_cache.cc | 2 | ||||
| -rw-r--r-- | fileutil.cc | 2 | ||||
| -rw-r--r-- | fileutil.go | 2 | ||||
| -rw-r--r-- | flags.go | 30 | ||||
| -rw-r--r-- | func.cc | 2 | ||||
| -rw-r--r-- | func.go | 20 | ||||
| -rw-r--r-- | func_test.go | 8 | ||||
| -rw-r--r-- | ioutil.go | 2 | ||||
| -rw-r--r-- | log.go | 20 | ||||
| -rw-r--r-- | main.cc | 2 | ||||
| -rw-r--r-- | main.go | 412 | ||||
| -rw-r--r-- | ninja.go | 24 | ||||
| -rw-r--r-- | ninja_test.go | 2 | ||||
| -rw-r--r-- | para.cc | 2 | ||||
| -rw-r--r-- | para.go | 8 | ||||
| -rw-r--r-- | para_test.go | 6 | ||||
| -rw-r--r-- | parser.cc | 2 | ||||
| -rw-r--r-- | parser.go | 8 | ||||
| -rw-r--r-- | pathutil.go | 20 | ||||
| -rw-r--r-- | query.go | 2 | ||||
| -rw-r--r-- | rule.cc | 2 | ||||
| -rw-r--r-- | rule_parser.go | 2 | ||||
| -rw-r--r-- | rule_parser_test.go | 2 | ||||
| -rw-r--r-- | serialize.go | 6 | ||||
| -rw-r--r-- | shellutil.go | 24 | ||||
| -rw-r--r-- | shellutil_test.go | 8 | ||||
| -rw-r--r-- | stats.go | 30 | ||||
| -rw-r--r-- | string_piece.cc | 2 | ||||
| -rw-r--r-- | string_piece_test.cc | 2 | ||||
| -rw-r--r-- | string_pool.cc | 2 | ||||
| -rw-r--r-- | stringprintf.cc | 2 | ||||
| -rw-r--r-- | strutil.cc | 2 | ||||
| -rw-r--r-- | strutil.go | 2 | ||||
| -rw-r--r-- | strutil_test.cc | 2 | ||||
| -rw-r--r-- | strutil_test.go | 2 | ||||
| -rw-r--r-- | symtab.go | 2 | ||||
| -rw-r--r-- | value.cc | 2 | ||||
| -rw-r--r-- | var.cc | 2 | ||||
| -rw-r--r-- | var.go | 2 | ||||
| -rw-r--r-- | worker.go | 38 |
58 files changed, 779 insertions, 552 deletions
@@ -1,4 +1,5 @@ kati +!cmd/kati/ ckati para *.o @@ -12,3 +13,4 @@ bench-old.out bench-new.out string_piece_test strutil_test +go_src_stamp @@ -41,8 +41,18 @@ CXXFLAGS:=-g -W -Wall -MMD # -O all: kati para ckati $(CXX_TEST_EXES) -kati: $(GO_SRCS) - env $(shell go env) go build -o $@ *.go +kati: go_src_stamp + GOPATH=$$(pwd)/out go install github.com/google/kati/cmd/kati + cp out/bin/kati $@ + +go_src_stamp: $(GO_SRCS) cmd/*/*.go + -rm -rf out/src/github.com/google/kati + mkdir -p out/src/github.com/google/kati + cp -a $(GO_SRCS) cmd out/src/github.com/google/kati + touch $@ + +go_test: $(GO_SRCS) para + go test *.go ckati: $(CXX_OBJS) $(CXX) -std=c++11 $(CXXFLAGS) -o $@ $(CXX_OBJS) @@ -54,9 +64,6 @@ $(CXX_TEST_EXES): $(filter-out main.o,$(CXX_OBJS)) $(CXX_TEST_EXES): %: %.o $(CXX) $^ -o $@ -go_test: $(GO_SRCS) para - env $(shell go env) go test *.go - para: para.cc $(CXX) -std=c++11 -g -O -W -Wall -MMD -o $@ $< @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "ast.h" #include "eval.h" @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -20,8 +20,6 @@ import ( "strings" ) -const BootstrapMakefile = "*bootstrap*" - type AST interface { eval(*Evaluator) show() @@ -46,7 +44,7 @@ func (ast *AssignAST) eval(ev *Evaluator) { func (ast *AssignAST) evalRHS(ev *Evaluator, lhs string) Var { origin := "file" - if ast.filename == BootstrapMakefile { + if ast.filename == bootstrapMakefileName { origin = "default" } if ast.opt == "override" { diff --git a/bootstrap.go b/bootstrap.go new file mode 100644 index 0000000..de13b2b --- /dev/null +++ b/bootstrap.go @@ -0,0 +1,56 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kati + +import ( + "fmt" + "path/filepath" + "strings" +) + +const bootstrapMakefileName = "*bootstrap*" + +func BootstrapMakefile(targets []string) Makefile { + bootstrap := ` +CC:=cc +CXX:=g++ +AR:=ar +MAKE:=kati +# Pretend to be GNU make 3.81, for compatibility. +MAKE_VERSION:=3.81 +SHELL:=/bin/sh +# TODO: Add more builtin vars. + +# http://www.gnu.org/software/make/manual/make.html#Catalogue-of-Rules +# The document above is actually not correct. See default.c: +# http://git.savannah.gnu.org/cgit/make.git/tree/default.c?id=4.1 +.c.o: + $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -o $@ $< +.cc.o: + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -o $@ $< +# TODO: Add more builtin rules. +` + bootstrap += fmt.Sprintf("MAKECMDGOALS:=%s\n", strings.Join(targets, " ")) + cwd, err := filepath.Abs(".") + if err != nil { + panic(err) + } + bootstrap += fmt.Sprintf("CURDIR:=%s\n", cwd) + mk, err := ParseMakefileString(bootstrap, bootstrapMakefileName, 0) + if err != nil { + panic(err) + } + return mk +} diff --git a/cmd/kati/main.go b/cmd/kati/main.go new file mode 100644 index 0000000..2128a07 --- /dev/null +++ b/cmd/kati/main.go @@ -0,0 +1,289 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "strings" + "text/template" + "time" + + "github.com/google/kati" +) + +const shellDateTimeformat = time.RFC3339 + +var ( + makefileFlag string + jobsFlag int + + loadJSON string + saveJSON string + loadGOB string + saveGOB string + useCache bool + + cpuprofile string + heapprofile string + memstats string + traceEventFile string + syntaxCheckOnlyFlag bool + queryFlag string + eagerCmdEvalFlag bool + generateNinja bool + gomaDir string + usePara bool + findCachePrunes string + findCacheLeafNames string + shellDate string +) + +func init() { + // TODO: Make this default and replace this by -d flag. + flag.StringVar(&makefileFlag, "f", "", "Use it as a makefile") + flag.IntVar(&jobsFlag, "j", 1, "Allow N jobs at once.") + + flag.StringVar(&loadGOB, "load", "", "") + flag.StringVar(&saveGOB, "save", "", "") + flag.StringVar(&loadJSON, "load_json", "", "") + flag.StringVar(&saveJSON, "save_json", "", "") + flag.BoolVar(&useCache, "use_cache", false, "Use cache.") + + flag.StringVar(&cpuprofile, "kati_cpuprofile", "", "write cpu profile to `file`") + flag.StringVar(&heapprofile, "kati_heapprofile", "", "write heap profile to `file`") + flag.StringVar(&memstats, "kati_memstats", "", "Show memstats with given templates") + flag.StringVar(&traceEventFile, "kati_trace_event", "", "write trace event to `file`") + flag.BoolVar(&syntaxCheckOnlyFlag, "c", false, "Syntax check only.") + flag.StringVar(&queryFlag, "query", "", "Show the target info") + flag.BoolVar(&eagerCmdEvalFlag, "eager_cmd_eval", false, "Eval commands first.") + flag.BoolVar(&generateNinja, "ninja", false, "Generate build.ninja.") + flag.StringVar(&gomaDir, "goma_dir", "", "If specified, use goma to build C/C++ files.") + flag.BoolVar(&usePara, "use_para", false, "Use para.") + + flag.StringVar(&findCachePrunes, "find_cache_prunes", "", + "space separated prune directories for find cache.") + flag.StringVar(&findCacheLeafNames, "find_cache_leaf_names", "", + "space separated leaf names for find cache.") + flag.StringVar(&shellDate, "shell_date", "", "specify $(shell date) time as "+shellDateTimeformat) + + flag.BoolVar(&kati.LogFlag, "kati_log", false, "Verbose kati specific log") + flag.BoolVar(&kati.StatsFlag, "kati_stats", false, "Show a bunch of statistics") + flag.BoolVar(&kati.PeriodicStatsFlag, "kati_periodic_stats", false, "Show a bunch of periodic statistics") + flag.BoolVar(&kati.EvalStatsFlag, "kati_eval_stats", false, "Show eval statistics") + + flag.BoolVar(&kati.DryRunFlag, "n", false, "Only print the commands that would be executed") + + // TODO: Make this default. + flag.BoolVar(&kati.UseFindCache, "use_find_cache", false, "Use find cache.") + flag.BoolVar(&kati.UseWildcardCache, "use_wildcard_cache", true, "Use wildcard cache.") + flag.BoolVar(&kati.UseShellBuiltins, "use_shell_builtins", true, "Use shell builtins") + flag.StringVar(&kati.IgnoreOptionalInclude, "ignore_optional_include", "", "If specified, skip reading -include directives start with the specified path.") +} + +func writeHeapProfile() { + f, err := os.Create(heapprofile) + if err != nil { + panic(err) + } + pprof.WriteHeapProfile(f) + f.Close() +} + +type memStatsDumper struct { + *template.Template +} + +func (t memStatsDumper) dump() { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + var buf bytes.Buffer + err := t.Template.Execute(&buf, ms) + fmt.Println(buf.String()) + if err != nil { + panic(err) + } +} + +func findPara() string { + switch runtime.GOOS { + case "linux": + katicmd, err := os.Readlink("/proc/self/exe") + if err != nil { + panic(err) + } + return filepath.Join(filepath.Dir(katicmd), "para") + default: + panic(fmt.Sprintf("unknown OS: %s", runtime.GOOS)) + } +} + +func load(makefile string, opt kati.LoadOpt) (*kati.DepGraph, error) { + startTime := time.Now() + + if loadGOB != "" { + g := kati.LoadDepGraph(loadGOB) + kati.LogStats("deserialize time: %q", time.Since(startTime)) + return g, nil + } + if loadJSON != "" { + g := kati.LoadDepGraphFromJSON(loadJSON) + kati.LogStats("deserialize time: %q", time.Since(startTime)) + return g, nil + } + return kati.Load(makefile, opt) +} + +func save(g *kati.DepGraph, targets []string) { + startTime := time.Now() + if saveGOB != "" { + kati.DumpDepGraph(g, saveGOB, targets) + kati.LogStats("serialize time: %q", time.Since(startTime)) + } + if saveJSON != "" { + kati.DumpDepGraphAsJSON(g, saveJSON, targets) + kati.LogStats("serialize time: %q", time.Since(startTime)) + } + + if useCache && !g.IsCached() { + kati.DumpDepGraphCache(g, targets) + kati.LogStats("serialize time: %q", time.Since(startTime)) + } +} + +func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + flag.Parse() + if cpuprofile != "" { + f, err := os.Create(cpuprofile) + if err != nil { + panic(err) + } + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + kati.AtError(pprof.StopCPUProfile) + } + if heapprofile != "" { + defer writeHeapProfile() + kati.AtError(writeHeapProfile) + } + defer kati.DumpStats() + kati.AtError(kati.DumpStats) + if memstats != "" { + ms := memStatsDumper{ + Template: template.Must(template.New("memstats").Parse(memstats)), + } + ms.dump() + defer ms.dump() + kati.AtError(ms.dump) + } + if traceEventFile != "" { + f, err := os.Create(traceEventFile) + if err != nil { + panic(err) + } + kati.TraceEventStart(f) + defer kati.TraceEventStop() + kati.AtError(kati.TraceEventStop) + } + + if shellDate != "" { + if shellDate == "ref" { + shellDate = shellDateTimeformat[:20] // until Z, drop 07:00 + } + t, err := time.Parse(shellDateTimeformat, shellDate) + if err != nil { + panic(err) + } + kati.ShellDateTimestamp = t + } + + var leafNames []string + if findCacheLeafNames != "" { + leafNames = strings.Fields(findCacheLeafNames) + } + if findCachePrunes != "" { + kati.UseFindCache = true + kati.AndroidFindCacheInit(strings.Fields(findCachePrunes), leafNames) + } + + clvars, targets := kati.ParseCommandLine(flag.Args()) + + g, err := load(makefileFlag, kati.LoadOpt{ + Targets: targets, + CommandLineVars: clvars, + EnvironmentVars: os.Environ(), + UseCache: useCache, + }) + if err != nil { + panic(err) + } + nodes := g.Nodes() + vars := g.Vars() + + if eagerCmdEvalFlag { + startTime := time.Now() + kati.EvalCommands(nodes, vars) + kati.LogStats("eager eval command time: %q", time.Since(startTime)) + } + + save(g, targets) + + if generateNinja { + startTime := time.Now() + kati.GenerateNinja(g, gomaDir) + kati.LogStats("generate ninja time: %q", time.Since(startTime)) + return + } + + if syntaxCheckOnlyFlag { + return + } + + if queryFlag != "" { + kati.HandleQuery(queryFlag, g) + return + } + + // TODO: Handle target specific variables. + ev := kati.NewEvaluator(vars) + for name, export := range g.Exports() { + if export { + os.Setenv(name, ev.EvaluateVar(name)) + } else { + os.Unsetenv(name) + } + } + + startTime := time.Now() + execOpt := &kati.ExecutorOpt{ + NumJobs: jobsFlag, + } + if usePara { + execOpt.ParaPath = findPara() + } + ex := kati.NewExecutor(vars, execOpt) + err = ex.Exec(nodes) + if err != nil { + panic(err) + } + kati.LogStats("exec time: %q", time.Since(startTime)) + +} diff --git a/cmdline.go b/cmdline.go new file mode 100644 index 0000000..fa75451 --- /dev/null +++ b/cmdline.go @@ -0,0 +1,48 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kati + +import ( + "fmt" + "strings" +) + +func ParseCommandLine(cmdline []string) ([]string, []string) { + var vars []string + var targets []string + for _, arg := range cmdline { + if strings.IndexByte(arg, '=') >= 0 { + vars = append(vars, arg) + continue + } + targets = append(targets, arg) + } + return vars, targets +} + +func InitVars(vars Vars, kvlist []string, origin string) error { + for _, v := range kvlist { + kv := strings.SplitN(v, "=", 2) + Logf("%s var %q", origin, v) + if len(kv) < 2 { + return fmt.Errorf("A weird %s variable %q", origin, kv) + } + vars.Assign(kv[0], &RecursiveVar{ + expr: literal(kv[1]), + origin: origin, + }) + } + return nil +} @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "dep.h" #include <algorithm> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "fmt" @@ -272,7 +272,7 @@ func (db *DepBuilder) buildPlan(output string, neededBy string, tsvs Vars) (*Dep if !present || oldVar.String() == "" { db.vars[name] = tsv } else { - v = oldVar.AppendVar(newEvaluator(db.vars), tsv) + v = oldVar.AppendVar(NewEvaluator(db.vars), tsv) db.vars[name] = v } tsvs[name] = v @@ -489,7 +489,7 @@ func (s bySuffix) Less(i, j int) bool { } func (db *DepBuilder) reportStats() { - if !katiLogFlag && !katiPeriodicStatsFlag { + if !LogFlag && !PeriodicStatsFlag { return } diff --git a/depgraph.go b/depgraph.go new file mode 100644 index 0000000..a349980 --- /dev/null +++ b/depgraph.go @@ -0,0 +1,115 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kati + +import ( + "crypto/sha1" + "io/ioutil" + "time" +) + +type DepGraph struct { + nodes []*DepNode + vars Vars + readMks []*ReadMakefile + exports map[string]bool + isCached bool +} + +func (g *DepGraph) Nodes() []*DepNode { return g.nodes } +func (g *DepGraph) Vars() Vars { return g.vars } +func (g *DepGraph) Exports() map[string]bool { return g.exports } +func (g *DepGraph) IsCached() bool { return g.isCached } + +type LoadOpt struct { + Targets []string + CommandLineVars []string + EnvironmentVars []string + UseCache bool +} + +func Load(makefile string, opt LoadOpt) (*DepGraph, error) { + startTime := time.Now() + if makefile == "" { + makefile = DefaultMakefile() + } + + if opt.UseCache { + g := LoadDepGraphCache(makefile, opt.Targets) + if g != nil { + return g, nil + } + } + + bmk := BootstrapMakefile(opt.Targets) + + content, err := ioutil.ReadFile(makefile) + if err != nil { + return nil, err + } + mk, err := ParseMakefile(content, makefile) + if err != nil { + return nil, err + } + + for _, stmt := range mk.stmts { + stmt.show() + } + + mk.stmts = append(bmk.stmts, mk.stmts...) + + vars := make(Vars) + err = InitVars(vars, opt.EnvironmentVars, "environment") + if err != nil { + return nil, err + } + err = InitVars(vars, opt.CommandLineVars, "command line") + if err != nil { + return nil, err + } + er, err := Eval(mk, vars, opt.UseCache) + if err != nil { + return nil, err + } + vars.Merge(er.vars) + + LogStats("eval time: %q", time.Since(startTime)) + LogStats("shell func time: %q %d", shellStats.Duration(), shellStats.Count()) + + startTime = time.Now() + db := NewDepBuilder(er, vars) + LogStats("dep build prepare time: %q", time.Since(startTime)) + + startTime = time.Now() + nodes, err := db.Eval(opt.Targets) + if err != nil { + return nil, err + } + LogStats("dep build time: %q", time.Since(startTime)) + var readMks []*ReadMakefile + // Always put the root Makefile as the first element. + readMks = append(readMks, &ReadMakefile{ + Filename: makefile, + Hash: sha1.Sum(content), + State: FileExists, + }) + readMks = append(readMks, er.readMks...) + return &DepGraph{ + nodes: nodes, + vars: vars, + readMks: readMks, + exports: er.exports, + }, nil +} @@ -0,0 +1,8 @@ +/* +Package kati provides GNU make compatible functions, especially +to speed up the continuous build of Android. + +*/ +package kati + +// TODO(ukai): cleanup API. make more symbol unexported. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "eval.h" #include <errno.h> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -53,6 +53,7 @@ type Evaluator struct { vars Vars lastRule *Rule currentScope Vars + useCache bool avoidIO bool hasIO bool readMks map[string]*ReadMakefile @@ -62,7 +63,7 @@ type Evaluator struct { lineno int } -func newEvaluator(vars map[string]Var) *Evaluator { +func NewEvaluator(vars map[string]Var) *Evaluator { return &Evaluator{ outVars: make(Vars), vars: vars, @@ -91,7 +92,7 @@ func (ev *Evaluator) args(buf *buffer, args ...Value) [][]byte { func (ev *Evaluator) evalAssign(ast *AssignAST) { ev.lastRule = nil lhs, rhs := ev.evalAssignAST(ast) - if katiLogFlag { + if LogFlag { Logf("ASSIGN: %s=%q (flavor:%q)", lhs, rhs, rhs.Flavor()) } if lhs == "" { @@ -128,7 +129,7 @@ func (ev *Evaluator) setTargetSpecificVar(assign *AssignAST, output string) { } ev.currentScope = vars lhs, rhs := ev.evalAssignAST(assign) - if katiLogFlag { + if LogFlag { Logf("rule outputs:%q assign:%q=%q (flavor:%q)", output, lhs, rhs, rhs.Flavor()) } vars.Assign(lhs, &TargetSpecificVar{v: rhs, op: assign.op}) @@ -147,7 +148,7 @@ func (ev *Evaluator) evalMaybeRule(ast *MaybeRuleAST) { if ast.term == '=' { line = append(line, ast.afterTerm...) } - if katiLogFlag { + if LogFlag { Logf("rule? %q=>%q", ast.expr, line) } @@ -166,7 +167,7 @@ func (ev *Evaluator) evalMaybeRule(ast *MaybeRuleAST) { Error(ast.filename, ast.lineno, "%v", err.Error()) } freeBuf(buf) - if katiLogFlag { + if LogFlag { Logf("rule %q => outputs:%q, inputs:%q", line, rule.outputs, rule.inputs) } @@ -201,7 +202,7 @@ func (ev *Evaluator) evalMaybeRule(ast *MaybeRuleAST) { if ast.term == ';' { rule.cmds = append(rule.cmds, string(ast.afterTerm[1:])) } - if katiLogFlag { + if LogFlag { Logf("rule outputs:%q cmds:%q", rule.outputs, rule.cmds) } ev.lastRule = rule @@ -288,7 +289,7 @@ func (ev *Evaluator) evalIncludeFile(fname string, mk Makefile) error { } func (ev *Evaluator) updateReadMakefile(fn string, hash [sha1.Size]byte, st FileState) { - if !useCache { + if !ev.useCache { return } @@ -349,7 +350,7 @@ func (ev *Evaluator) evalInclude(ast *IncludeAST) { } for _, fn := range files { - if ignoreOptionalInclude != "" && ast.op == "-include" && matchPattern(fn, ignoreOptionalInclude) { + if IgnoreOptionalInclude != "" && ast.op == "-include" && matchPattern(fn, IgnoreOptionalInclude) { continue } mk, hash, err := makefileCache.parse(fn) @@ -383,7 +384,7 @@ func (ev *Evaluator) evalIf(ast *IfAST) { val := buf.Len() freeBuf(buf) isTrue = (val > 0) == (ast.op == "ifdef") - if katiLogFlag { + if LogFlag { Logf("%s lhs=%q value=%q => %t", ast.op, ast.lhs, value, isTrue) } case "ifeq", "ifneq": @@ -395,7 +396,7 @@ func (ev *Evaluator) evalIf(ast *IfAST) { rhs := string(params[1]) freeBuf(buf) isTrue = (lhs == rhs) == (ast.op == "ifeq") - if katiLogFlag { + if LogFlag { Logf("%s lhs=%q %q rhs=%q %q => %t", ast.op, ast.lhs, lhs, ast.rhs, rhs, isTrue) } default: @@ -441,8 +442,9 @@ func createReadMakefileArray(mp map[string]*ReadMakefile) []*ReadMakefile { return r } -func Eval(mk Makefile, vars Vars) (er *EvalResult, err error) { - ev := newEvaluator(vars) +func Eval(mk Makefile, vars Vars, useCache bool) (er *EvalResult, err error) { + ev := NewEvaluator(vars) + ev.useCache = useCache defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic in eval %s: %v", mk.filename, r) @@ -450,6 +452,9 @@ func Eval(mk Makefile, vars Vars) (er *EvalResult, err error) { }() makefileList := vars.Lookup("MAKEFILE_LIST") + if !makefileList.IsDefined() { + makefileList = &SimpleVar{value: "", origin: "file"} + } makefileList = makefileList.Append(ev, mk.filename) ev.outVars.Assign("MAKEFILE_LIST", makefileList) @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "exec.h" #include <stdio.h> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -207,7 +207,7 @@ func (ex *Executor) makeJobs(n *DepNode, neededBy *Job) error { } func (ex *Executor) reportStats() { - if !katiLogFlag && !katiPeriodicStatsFlag { + if !LogFlag && !PeriodicStatsFlag { return } @@ -218,17 +218,28 @@ func (ex *Executor) reportStats() { } } -func NewExecutor(vars Vars) *Executor { +type ExecutorOpt struct { + NumJobs int + ParaPath string +} + +func NewExecutor(vars Vars, opt *ExecutorOpt) *Executor { + if opt == nil { + opt = &ExecutorOpt{NumJobs: 1} + } + if opt.NumJobs < 1 { + opt.NumJobs = 1 + } ex := &Executor{ rules: make(map[string]*Rule), suffixRules: make(map[string][]*Rule), done: make(map[string]*Job), vars: vars, - wm: NewWorkerManager(), + wm: NewWorkerManager(opt.NumJobs, opt.ParaPath), } // TODO: We should move this to somewhere around evalCmd so that // we can handle SHELL in target specific variables. - ev := newEvaluator(ex.vars) + ev := NewEvaluator(ex.vars) ex.shell = ev.EvaluateVar("SHELL") for k, v := range map[string]Var{ "@": AutoAtVar{AutoVar: AutoVar{ex: ex}}, @@ -275,7 +286,7 @@ func (ex *Executor) createRunners(n *DepNode, avoidIO bool) ([]runner, bool) { ex.vars[k] = v } - ev := newEvaluator(ex.vars) + ev := NewEvaluator(ex.vars) ev.avoidIO = avoidIO ev.filename = n.Filename ev.lineno = n.Lineno @@ -297,7 +308,7 @@ func (ex *Executor) createRunners(n *DepNode, avoidIO bool) ([]runner, bool) { func EvalCommands(nodes []*DepNode, vars Vars) { ioCnt := 0 - ex := NewExecutor(vars) + ex := NewExecutor(vars, nil) for i, n := range nodes { runners, hasIO := ex.createRunners(n, true) if hasIO { @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -591,7 +591,7 @@ func parseFunc(f Func, in []byte, s int, term []byte, funcName string, alloc boo if compactor, ok := f.(Compactor); ok { fv = compactor.Compact() } - if katiEvalStatsFlag || traceEvent.enabled() { + if EvalStatsFlag || traceEvent.enabled() { fv = funcstats{ Value: fv, str: fv.String(), diff --git a/expr_test.go b/expr_test.go index 9b776e0..a7f2fe3 100644 --- a/expr_test.go +++ b/expr_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "reflect" @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "file.h" #include <fcntl.h> diff --git a/file_cache.cc b/file_cache.cc index 0cd6ea5..b26d054 100644 --- a/file_cache.cc +++ b/file_cache.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "file_cache.h" #include <unordered_map> diff --git a/fileutil.cc b/fileutil.cc index 92726fa..ebf6fd4 100644 --- a/fileutil.cc +++ b/fileutil.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "fileutil.h" #include <errno.h> diff --git a/fileutil.go b/fileutil.go index a288b6d..cde353b 100644 --- a/fileutil.go +++ b/fileutil.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import "os" diff --git a/flags.go b/flags.go new file mode 100644 index 0000000..105df2d --- /dev/null +++ b/flags.go @@ -0,0 +1,30 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kati + +var ( + LogFlag bool + StatsFlag bool + PeriodicStatsFlag bool + EvalStatsFlag bool + + DryRunFlag bool + + UseFindCache bool + UseWildcardCache bool + UseShellBuiltins bool + + IgnoreOptionalInclude string +) @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "func.h" #include <glob.h> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -474,7 +474,7 @@ func (f *funcWildcard) Eval(w io.Writer, ev *Evaluator) { abuf := newBuf() f.args[1].Eval(abuf, ev) te := traceEvent.begin("wildcard", tmpval(abuf.Bytes()), traceEventMain) - if ev.avoidIO && !useWildcardCache { + if ev.avoidIO && !UseWildcardCache { ev.hasIO = true io.WriteString(w, "$(/bin/ls -d ") w.Write(abuf.Bytes()) @@ -774,7 +774,7 @@ func (f *funcShell) Eval(w io.Writer, ev *Evaluator) { shellVar := ev.LookupVar("SHELL") // TODO: Should be Eval, not String. cmdline := []string{shellVar.String(), "-c", arg} - if katiLogFlag { + if LogFlag { Logf("shell %q", cmdline) } cmd := exec.Cmd{ @@ -796,7 +796,7 @@ func (f *funcShell) Compact() Value { if len(f.args)-1 < 1 { return f } - if !useFindCache && !useShellBuiltins { + if !UseFindCache && !UseShellBuiltins { return f } @@ -807,7 +807,7 @@ func (f *funcShell) Compact() Value { default: expr = Expr{v} } - if useShellBuiltins { + if UseShellBuiltins { // hack for android for _, sb := range shBuiltins { if v, ok := matchExpr(expr, sb.pattern); ok { @@ -831,7 +831,7 @@ func (f *funcCall) Eval(w io.Writer, ev *Evaluator) { varname := fargs[0] variable := string(varname) te := traceEvent.begin("call", literal(variable), traceEventMain) - if katiLogFlag { + if LogFlag { Logf("call %q variable %q", f.args[1], variable) } v := ev.LookupVar(variable) @@ -846,7 +846,7 @@ func (f *funcCall) Eval(w io.Writer, ev *Evaluator) { for i, arg := range fargs[1:] { // f.args[2]=>args[1] will be $1. args = append(args, tmpval(arg)) - if katiLogFlag { + if LogFlag { Logf("call $%d: %q=>%q", i+1, arg, fargs[i+1]) } } @@ -854,13 +854,13 @@ func (f *funcCall) Eval(w io.Writer, ev *Evaluator) { ev.paramVars = args var buf bytes.Buffer - if katiLogFlag { + if LogFlag { w = io.MultiWriter(w, &buf) } v.Eval(w, ev) ev.paramVars = oldParams traceEvent.end(te) - if katiLogFlag { + if LogFlag { Logf("call %q variable %q return %q", f.args[1], variable, buf.Bytes()) } freeBuf(abuf) @@ -1040,7 +1040,7 @@ func (f *funcEvalAssign) Eval(w io.Writer, ev *Evaluator) { } rvalue = &RecursiveVar{expr: tmpval(rhs), origin: "file"} } - if katiLogFlag { + if LogFlag { Logf("Eval ASSIGN: %s=%q (flavor:%q)", f.lhs, rvalue, rvalue.Flavor()) } ev.outVars.Assign(f.lhs, rvalue) diff --git a/func_test.go b/func_test.go index 6758a9b..58d7570 100644 --- a/func_test.go +++ b/func_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -28,7 +28,7 @@ func BenchmarkFuncStrip(b *testing.B) { }, }, } - ev := newEvaluator(make(map[string]Var)) + ev := NewEvaluator(make(map[string]Var)) var buf bytes.Buffer b.ReportAllocs() b.ResetTimer() @@ -47,7 +47,7 @@ func BenchmarkFuncSort(b *testing.B) { }, }, } - ev := newEvaluator(make(map[string]Var)) + ev := NewEvaluator(make(map[string]Var)) var buf bytes.Buffer b.ReportAllocs() b.ResetTimer() @@ -68,7 +68,7 @@ func BenchmarkFuncPatsubst(b *testing.B) { }, }, } - ev := newEvaluator(make(map[string]Var)) + ev := NewEvaluator(make(map[string]Var)) var buf bytes.Buffer b.ReportAllocs() b.ResetTimer() @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import "io" @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" "fmt" "os" - "runtime/pprof" "sync" ) @@ -35,14 +34,14 @@ func LogAlways(f string, a ...interface{}) { } func LogStats(f string, a ...interface{}) { - if !katiLogFlag && !katiStatsFlag { + if !LogFlag && !StatsFlag { return } LogAlways(f, a...) } func Logf(f string, a ...interface{}) { - if !katiLogFlag { + if !LogFlag { return } LogAlways(f, a...) @@ -58,19 +57,22 @@ func WarnNoPrefix(filename string, lineno int, f string, a ...interface{}) { fmt.Printf(f, a...) } +var atErrors []func() + +func AtError(f func()) { + atErrors = append(atErrors, f) +} + func Error(filename string, lineno int, f string, a ...interface{}) { f = fmt.Sprintf("%s:%d: %s", filename, lineno, f) - maybeWriteHeapProfile() ErrorNoLocation(f, a...) } func ErrorNoLocation(f string, a ...interface{}) { f = fmt.Sprintf("%s\n", f) fmt.Printf(f, a...) - if cpuprofile != "" { - pprof.StopCPUProfile() + for i := len(atErrors) - 1; i >= 0; i-- { + atErrors[i]() } - maybeWriteHeapProfile() - dumpStats() os.Exit(2) } @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include <limits.h> #include <stdio.h> #include <string.h> diff --git a/main.go b/main.go deleted file mode 100644 index 15ad142..0000000 --- a/main.go +++ /dev/null @@ -1,412 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "crypto/sha1" - "flag" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "runtime/pprof" - "strings" - "text/template" - "time" -) - -const shellDateTimeformat = time.RFC3339 - -var ( - katiLogFlag bool - makefileFlag string - dryRunFlag bool - jobsFlag int - cpuprofile string - heapprofile string - memstats string - traceEventFile string - katiStatsFlag bool - katiPeriodicStatsFlag bool - katiEvalStatsFlag bool - loadJSON string - saveJSON string - loadGOB string - saveGOB string - syntaxCheckOnlyFlag bool - queryFlag string - eagerCmdEvalFlag bool - useParaFlag bool - useCache bool - useFindCache bool - findCachePrunes string - findCacheLeafNames string - useWildcardCache bool - useShellBuiltins bool - shellDate string - generateNinja bool - ignoreOptionalInclude string - gomaDir string - - katiDir string -) - -type DepGraph struct { - nodes []*DepNode - vars Vars - readMks []*ReadMakefile - exports map[string]bool - isCached bool -} - -func parseFlags() { - // TODO: Make this default and replace this by -d flag. - flag.BoolVar(&katiLogFlag, "kati_log", false, "Verbose kati specific log") - flag.StringVar(&makefileFlag, "f", "", "Use it as a makefile") - - flag.BoolVar(&dryRunFlag, "n", false, "Only print the commands that would be executed") - - flag.IntVar(&jobsFlag, "j", 1, "Allow N jobs at once.") - - flag.StringVar(&loadGOB, "load", "", "") - flag.StringVar(&saveGOB, "save", "", "") - flag.StringVar(&loadJSON, "load_json", "", "") - flag.StringVar(&saveJSON, "save_json", "", "") - - flag.StringVar(&cpuprofile, "kati_cpuprofile", "", "write cpu profile to `file`") - flag.StringVar(&heapprofile, "kati_heapprofile", "", "write heap profile to `file`") - flag.StringVar(&memstats, "kati_memstats", "", "Show memstats with given templates") - flag.StringVar(&traceEventFile, "kati_trace_event", "", "write trace event to `file`") - flag.BoolVar(&katiStatsFlag, "kati_stats", false, "Show a bunch of statistics") - flag.BoolVar(&katiPeriodicStatsFlag, "kati_periodic_stats", false, "Show a bunch of periodic statistics") - flag.BoolVar(&katiEvalStatsFlag, "kati_eval_stats", false, "Show eval statistics") - flag.BoolVar(&eagerCmdEvalFlag, "eager_cmd_eval", false, "Eval commands first.") - flag.BoolVar(&useParaFlag, "use_para", false, "Use para.") - flag.BoolVar(&syntaxCheckOnlyFlag, "c", false, "Syntax check only.") - flag.StringVar(&queryFlag, "query", "", "Show the target info") - // TODO: Make this default. - flag.BoolVar(&useCache, "use_cache", false, "Use cache.") - flag.BoolVar(&useFindCache, "use_find_cache", false, "Use find cache.") - flag.StringVar(&findCachePrunes, "find_cache_prunes", "", - "space separated prune directories for find cache.") - flag.StringVar(&findCacheLeafNames, "find_cache_leaf_names", "", - "space separated leaf names for find cache.") - flag.BoolVar(&useWildcardCache, "use_wildcard_cache", true, "Use wildcard cache.") - flag.BoolVar(&useShellBuiltins, "use_shell_builtins", true, "Use shell builtins") - flag.StringVar(&shellDate, "shell_date", "", "specify $(shell date) time as "+shellDateTimeformat) - flag.BoolVar(&generateNinja, "ninja", false, "Generate build.ninja.") - flag.StringVar(&ignoreOptionalInclude, "ignore_optional_include", "", "If specified, skip reading -include directives start with the specified path.") - flag.StringVar(&gomaDir, "goma_dir", "", "If specified, use goma to build C/C++ files.") - flag.Parse() -} - -func parseCommandLine() ([]string, []string) { - var vars []string - var targets []string - for _, arg := range flag.Args() { - if strings.IndexByte(arg, '=') >= 0 { - vars = append(vars, arg) - continue - } - targets = append(targets, arg) - } - return vars, targets -} - -func getBootstrapMakefile(targets []string) Makefile { - bootstrap := ` -CC:=cc -CXX:=g++ -AR:=ar -MAKE:=kati -# Pretend to be GNU make 3.81, for compatibility. -MAKE_VERSION:=3.81 -SHELL:=/bin/sh -# TODO: Add more builtin vars. - -# http://www.gnu.org/software/make/manual/make.html#Catalogue-of-Rules -# The document above is actually not correct. See default.c: -# http://git.savannah.gnu.org/cgit/make.git/tree/default.c?id=4.1 -.c.o: - $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -o $@ $< -.cc.o: - $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -o $@ $< -# TODO: Add more builtin rules. -` - bootstrap += fmt.Sprintf("MAKECMDGOALS:=%s\n", strings.Join(targets, " ")) - cwd, err := filepath.Abs(".") - if err != nil { - panic(err) - } - bootstrap += fmt.Sprintf("CURDIR:=%s\n", cwd) - mk, err := ParseMakefileString(bootstrap, BootstrapMakefile, 0) - if err != nil { - panic(err) - } - return mk -} - -func maybeWriteHeapProfile() { - if heapprofile != "" { - f, err := os.Create(heapprofile) - if err != nil { - panic(err) - } - pprof.WriteHeapProfile(f) - } -} - -func getDepGraph(clvars []string, targets []string) *DepGraph { - startTime := time.Now() - - if loadGOB != "" { - g := LoadDepGraph(loadGOB) - LogStats("deserialize time: %q", time.Since(startTime)) - return g - } - if loadJSON != "" { - g := LoadDepGraphFromJSON(loadJSON) - LogStats("deserialize time: %q", time.Since(startTime)) - return g - } - - makefile := makefileFlag - if makefile == "" { - makefile = GetDefaultMakefile() - } - - if useCache { - g := LoadDepGraphCache(makefile, targets) - if g != nil { - return g - } - } - - bmk := getBootstrapMakefile(targets) - - content, err := ioutil.ReadFile(makefile) - if err != nil { - panic(err) - } - mk, err := ParseMakefile(content, makefile) - if err != nil { - panic(err) - } - - for _, stmt := range mk.stmts { - stmt.show() - } - - mk.stmts = append(bmk.stmts, mk.stmts...) - - vars := make(Vars) - for _, env := range os.Environ() { - kv := strings.SplitN(env, "=", 2) - Logf("envvar %q", kv) - if len(kv) < 2 { - panic(fmt.Sprintf("A weird environ variable %q", kv)) - } - vars.Assign(kv[0], &RecursiveVar{ - expr: literal(kv[1]), - origin: "environment", - }) - } - vars.Assign("MAKEFILE_LIST", &SimpleVar{value: "", origin: "file"}) - for _, v := range clvars { - kv := strings.SplitN(v, "=", 2) - Logf("cmdlinevar %q", kv) - if len(kv) < 2 { - panic(fmt.Sprintf("unexpected command line var %q", kv)) - } - vars.Assign(kv[0], &RecursiveVar{ - expr: literal(kv[1]), - origin: "command line", - }) - } - - er, err := Eval(mk, vars) - if err != nil { - panic(err) - } - - vars.Merge(er.vars) - - LogStats("eval time: %q", time.Since(startTime)) - LogStats("shell func time: %q %d", shellStats.duration, shellStats.count) - - startTime = time.Now() - db := NewDepBuilder(er, vars) - LogStats("dep build prepare time: %q", time.Since(startTime)) - - startTime = time.Now() - nodes, err2 := db.Eval(targets) - if err2 != nil { - panic(err2) - } - LogStats("dep build time: %q", time.Since(startTime)) - var readMks []*ReadMakefile - // Always put the root Makefile as the first element. - readMks = append(readMks, &ReadMakefile{ - Filename: makefile, - Hash: sha1.Sum(content), - State: FileExists, - }) - readMks = append(readMks, er.readMks...) - return &DepGraph{ - nodes: nodes, - vars: vars, - readMks: readMks, - exports: er.exports, - } -} - -func findKatiDir() { - switch runtime.GOOS { - case "linux": - kati, err := os.Readlink("/proc/self/exe") - if err != nil { - panic(err) - } - katiDir = filepath.Dir(kati) - default: - panic(fmt.Sprintf("unknown OS: %s", runtime.GOOS)) - } -} - -func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - findKatiDir() - parseFlags() - if cpuprofile != "" { - f, err := os.Create(cpuprofile) - if err != nil { - panic(err) - } - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - defer maybeWriteHeapProfile() - defer dumpStats() - if memstats != "" { - t := template.Must(template.New("memstats").Parse(memstats)) - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - var buf bytes.Buffer - err := t.Execute(&buf, ms) - fmt.Println(buf.String()) - if err != nil { - panic(err) - } - defer func() { - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - var buf bytes.Buffer - t.Execute(&buf, ms) - fmt.Println(buf.String()) - }() - } - if traceEventFile != "" { - f, err := os.Create(traceEventFile) - if err != nil { - panic(err) - } - traceEvent.start(f) - defer traceEvent.stop() - } - - if shellDate != "" { - if shellDate == "ref" { - shellDate = shellDateTimeformat[:20] // until Z, drop 07:00 - } - t, err := time.Parse(shellDateTimeformat, shellDate) - if err != nil { - panic(err) - } - shellDateTimestamp = t - } - - if findCacheLeafNames != "" { - androidDefaultLeafNames = strings.Fields(findCacheLeafNames) - } - if findCachePrunes != "" { - useFindCache = true - androidFindCache.init(strings.Fields(findCachePrunes), androidDefaultLeafNames) - } - - clvars, targets := parseCommandLine() - - g := getDepGraph(clvars, targets) - nodes := g.nodes - vars := g.vars - - if eagerCmdEvalFlag { - startTime := time.Now() - EvalCommands(nodes, vars) - LogStats("eager eval command time: %q", time.Since(startTime)) - } - - if saveGOB != "" { - startTime := time.Now() - DumpDepGraph(g, saveGOB, targets) - LogStats("serialize time: %q", time.Since(startTime)) - } - if saveJSON != "" { - startTime := time.Now() - DumpDepGraphAsJSON(g, saveJSON, targets) - LogStats("serialize time: %q", time.Since(startTime)) - } - - if useCache && !g.isCached { - startTime := time.Now() - DumpDepGraphCache(g, targets) - LogStats("serialize time: %q", time.Since(startTime)) - } - - if generateNinja { - startTime := time.Now() - GenerateNinja(g) - LogStats("generate ninja time: %q", time.Since(startTime)) - return - } - - if syntaxCheckOnlyFlag { - return - } - - if queryFlag != "" { - HandleQuery(queryFlag, g) - return - } - - // TODO: Handle target specific variables. - ev := newEvaluator(vars) - for name, export := range g.exports { - if export { - os.Setenv(name, ev.EvaluateVar(name)) - } else { - os.Unsetenv(name) - } - } - - startTime := time.Now() - ex := NewExecutor(vars) - err := ex.Exec(nodes) - if err != nil { - panic(err) - } - LogStats("exec time: %q", time.Since(startTime)) -} @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -33,9 +33,10 @@ type NinjaGenerator struct { ruleID int done map[string]bool ccRe *regexp.Regexp + gomaDir string } -func NewNinjaGenerator(g *DepGraph) *NinjaGenerator { +func NewNinjaGenerator(g *DepGraph, gomaDir string) *NinjaGenerator { ccRe, err := regexp.Compile(`^prebuilts/(gcc|clang)/.*(gcc|g\+\+|clang|clang\+\+) .* -c `) if err != nil { panic(err) @@ -46,6 +47,7 @@ func NewNinjaGenerator(g *DepGraph) *NinjaGenerator { exports: g.exports, done: make(map[string]bool), ccRe: ccRe, + gomaDir: gomaDir, } } @@ -172,8 +174,8 @@ func (n *NinjaGenerator) genShellScript(runners []runner) (string, bool) { if cmd == "" { cmd = "true" } - if gomaDir != "" && n.ccRe.MatchString(cmd) { - cmd = fmt.Sprintf("%s/gomacc %s", gomaDir, cmd) + if n.gomaDir != "" && n.ccRe.MatchString(cmd) { + cmd = fmt.Sprintf("%s/gomacc %s", n.gomaDir, cmd) useGomacc = true } @@ -193,7 +195,7 @@ func (n *NinjaGenerator) genShellScript(runners []runner) (string, bool) { buf.WriteByte(')') } } - return buf.String(), gomaDir != "" && !useGomacc + return buf.String(), n.gomaDir != "" && !useGomacc } func (n *NinjaGenerator) genRuleName() string { @@ -283,7 +285,7 @@ func (n *NinjaGenerator) generateShell() { } defer f.Close() - ev := newEvaluator(n.vars) + ev := NewEvaluator(n.vars) shell := ev.EvaluateVar("SHELL") if shell == "" { shell = "/bin/sh" @@ -296,7 +298,7 @@ func (n *NinjaGenerator) generateShell() { fmt.Fprintf(f, "unset %s\n", name) } } - if gomaDir == "" { + if n.gomaDir == "" { fmt.Fprintln(f, `exec ninja "$@"`) } else { fmt.Fprintln(f, `exec ninja -j300 "$@"`) @@ -319,19 +321,19 @@ func (n *NinjaGenerator) generateNinja() { fmt.Fprintf(n.f, "# Generated by kati\n") fmt.Fprintf(n.f, "\n") - if gomaDir != "" { + if n.gomaDir != "" { fmt.Fprintf(n.f, "pool local_pool\n") fmt.Fprintf(n.f, " depth = %d\n", runtime.NumCPU()) } - n.ex = NewExecutor(n.vars) + n.ex = NewExecutor(n.vars, nil) for _, node := range n.nodes { n.emitNode(node) } } -func GenerateNinja(g *DepGraph) { - n := NewNinjaGenerator(g) +func GenerateNinja(g *DepGraph, gomaDir string) { + n := NewNinjaGenerator(g, gomaDir) n.generateShell() n.generateNinja() } diff --git a/ninja_test.go b/ninja_test.go index c757409..8b35206 100644 --- a/ninja_test.go +++ b/ninja_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import "testing" @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include <signal.h> #include <stdio.h> #include <stdlib.h> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bufio" @@ -20,7 +20,6 @@ import ( "fmt" "io" "os/exec" - "path/filepath" ) func btoi(b bool) int { @@ -133,9 +132,8 @@ type ParaWorker struct { doneChan chan bool } -func NewParaWorker(paraChan chan *ParaResult) *ParaWorker { - bin := filepath.Join(katiDir, "para") - para := exec.Command(bin, fmt.Sprintf("-j%d", jobsFlag), "--kati") +func newParaWorker(paraChan chan *ParaResult, numJobs int, paraPath string) *ParaWorker { + para := exec.Command(paraPath, fmt.Sprintf("-j%d", numJobs), "--kati") stdin, err := para.StdinPipe() if err != nil { panic(err) diff --git a/para_test.go b/para_test.go index 0cae106..26f2669 100644 --- a/para_test.go +++ b/para_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "fmt" @@ -25,8 +25,8 @@ func TestPara(t *testing.T) { if err != nil { panic(err) } - katiDir = cwd - jobsFlag = 4 + ParaPath = filepath.Join(cwd, "para") + JobsFlag = 4 paraChan := make(chan *ParaResult) para := NewParaWorker(paraChan) @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "parser.h" #include <stack> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati //go:generate go run testcase/gen_testcase_parse_benchmark.go // @@ -684,7 +684,7 @@ func ParseMakefileFd(filename string, f *os.File) (Makefile, error) { return parser.parse() } -func GetDefaultMakefile() string { +func DefaultMakefile() string { candidates := []string{"GNUmakefile", "makefile", "Makefile"} for _, filename := range candidates { if exists(filename) { @@ -746,12 +746,12 @@ func (mc *makefileCacheT) parse(filename string) (Makefile, [sha1.Size]byte, err Logf("parse Makefile %q", filename) mk, hash, ok, err := makefileCache.lookup(filename) if ok { - if katiLogFlag { + if LogFlag { Logf("makefile cache hit for %q", filename) } return mk, hash, err } - if katiLogFlag { + if LogFlag { Logf("reading makefile %q", filename) } c, err := ioutil.ReadFile(filename) diff --git a/pathutil.go b/pathutil.go index 38e613f..5c5649e 100644 --- a/pathutil.go +++ b/pathutil.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "errors" @@ -98,7 +98,7 @@ func wildcardGlob(pat string) []string { } func wildcard(sw *ssvWriter, pat string) { - if useWildcardCache { + if UseWildcardCache { // TODO(ukai): make sure it didn't chdir? wildcardCache.mu.Lock() files, ok := wildcardCache.m[pat] @@ -114,7 +114,7 @@ func wildcard(sw *ssvWriter, pat string) { for _, file := range files { sw.WriteString(file) } - if useWildcardCache { + if UseWildcardCache { wildcardCache.mu.Lock() wildcardCache.m[pat] = files wildcardCache.mu.Unlock() @@ -136,9 +136,17 @@ type androidFindCacheT struct { } var ( - androidFindCache androidFindCacheT + androidFindCache androidFindCacheT + androidDefaultLeafNames = []string{"CleanSpec.mk", "Android.mk"} ) +func AndroidFindCacheInit(prunes, leafNames []string) { + if leafNames != nil { + androidDefaultLeafNames = leafNames + } + androidFindCache.init(prunes) +} + func (c *androidFindCacheT) ready() bool { if c.files != nil { return true @@ -159,11 +167,11 @@ func (c *androidFindCacheT) leavesReady() bool { return c.leaves != nil } -func (c *androidFindCacheT) init(prunes, leaves []string) { +func (c *androidFindCacheT) init(prunes []string) { c.once.Do(func() { c.filesch = make(chan []fileInfo, 1) c.leavesch = make(chan []fileInfo, 1) - go c.start(prunes, leaves) + go c.start(prunes, androidDefaultLeafNames) }) } @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import "fmt" @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "rule.h" #include "log.h" diff --git a/rule_parser.go b/rule_parser.go index 2f575e7..ccc207d 100644 --- a/rule_parser.go +++ b/rule_parser.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" diff --git a/rule_parser_test.go b/rule_parser_test.go index 5dc8f24..b1db3ec 100644 --- a/rule_parser_test.go +++ b/rule_parser_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "reflect" diff --git a/serialize.go b/serialize.go index 8ed60d2..2b8c45c 100644 --- a/serialize.go +++ b/serialize.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -290,7 +290,7 @@ func DumpDepGraphCache(g *DepGraph, roots []string) { cacheFile := GetCacheFilename(g.readMks[0].Filename, roots) for _, mk := range g.readMks { // Inconsistent, do not dump this result. - if mk.State == 2 { + if mk.State == FileInconsistent { if exists(cacheFile) { os.Remove(cacheFile) } @@ -551,7 +551,7 @@ func showSerializedGraphStats(g SerializableGraph) { } func DeserializeGraph(g SerializableGraph) *DepGraph { - if katiLogFlag || katiStatsFlag { + if LogFlag || StatsFlag { showSerializedGraphStats(g) } nodes := DeserializeNodes(g) diff --git a/shellutil.go b/shellutil.go index c73711d..93a53df 100644 --- a/shellutil.go +++ b/shellutil.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "fmt" @@ -21,8 +21,6 @@ import ( "time" ) -var androidDefaultLeafNames = []string{"CleanSpec.mk", "Android.mk"} - var shBuiltins = []struct { name string pattern Expr @@ -59,7 +57,7 @@ var shBuiltins = []struct { if v[0] != v[1] { return sh } - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindFileInDir{ funcShell: sh, dir: v[0], @@ -78,7 +76,7 @@ var shBuiltins = []struct { literal(` -name "*.java" -and -not -name ".*"`), }, compact: func(sh *funcShell, v []Value) Value { - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindExtFilesUnder{ funcShell: sh, chdir: v[0], @@ -100,7 +98,7 @@ var shBuiltins = []struct { literal(" -name \"*.proto\" -and -not -name \".*\""), }, compact: func(sh *funcShell, v []Value) Value { - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindExtFilesUnder{ funcShell: sh, chdir: v[0], @@ -126,7 +124,7 @@ var shBuiltins = []struct { literal(` && find . -type d -a -name ".svn" -prune -o -type f -a \! -name "*.java" -a \! -name "package.html" -a \! -name "overview.html" -a \! -name ".*.swp" -a \! -name ".DS_Store" -a \! -name "*~" -print `), }, compact: func(sh *funcShell, v []Value) Value { - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindJavaResourceFileGroup{ funcShell: sh, dir: Expr(v), @@ -146,7 +144,7 @@ var shBuiltins = []struct { if !contains(androidDefaultLeafNames, "CleanSpec.mk") { return sh } - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindleaves{ funcShell: sh, prunes: []Value{ @@ -175,7 +173,7 @@ var shBuiltins = []struct { if !contains(androidDefaultLeafNames, "Android.mk") { return sh } - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindleaves{ funcShell: sh, prunes: []Value{ @@ -205,7 +203,7 @@ var shBuiltins = []struct { if !contains(androidDefaultLeafNames, "Android.mk") { return sh } - androidFindCache.init(nil, androidDefaultLeafNames) + androidFindCache.init(nil) return &funcShellAndroidFindleaves{ funcShell: sh, prunes: []Value{ @@ -408,7 +406,7 @@ func (f *funcShellAndroidFindleaves) Eval(w io.Writer, ev *Evaluator) { } var ( - shellDateTimestamp time.Time + ShellDateTimestamp time.Time shellDateFormatRef = map[string]string{ "%Y": "2006", "%m": "01", @@ -427,7 +425,7 @@ type funcShellDate struct { } func compactShellDate(sh *funcShell, v []Value) Value { - if shellDateTimestamp.IsZero() { + if ShellDateTimestamp.IsZero() { return sh } tf, ok := v[0].(literal) @@ -445,5 +443,5 @@ func compactShellDate(sh *funcShell, v []Value) Value { } func (f *funcShellDate) Eval(w io.Writer, ev *Evaluator) { - fmt.Fprint(w, shellDateTimestamp.Format(f.format)) + fmt.Fprint(w, ShellDateTimestamp.Format(f.format)) } diff --git a/shellutil_test.go b/shellutil_test.go index c54e761..e4c200d 100644 --- a/shellutil_test.go +++ b/shellutil_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "testing" @@ -42,10 +42,10 @@ func TestRot13(t *testing.T) { } func TestShellDate(t *testing.T) { - ts := shellDateTimestamp - shellDateTimestamp = time.Now() + ts := ShellDateTimestamp + ShellDateTimestamp = time.Now() defer func() { - shellDateTimestamp = ts + ShellDateTimestamp = ts }() for _, tc := range []struct { sharg literal @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "fmt" @@ -39,6 +39,14 @@ const ( var traceEvent traceEventT +func TraceEventStart(f io.WriteCloser) { + traceEvent.start(f) +} + +func TraceEventStop() { + traceEvent.stop() +} + func (t *traceEventT) start(f io.WriteCloser) { t.f = f t.t0 = time.Now() @@ -65,7 +73,7 @@ func (t *traceEventT) begin(name string, v Value, tid int) event { var e event e.tid = tid e.t = time.Now() - if t.f != nil || katiEvalStatsFlag { + if t.f != nil || EvalStatsFlag { e.name = name e.v = v.String() } @@ -123,7 +131,7 @@ var stats = &statsT{ } func (s *statsT) add(name, v string, t time.Time) { - if !katiEvalStatsFlag { + if !EvalStatsFlag { return } d := time.Since(t) @@ -139,8 +147,8 @@ func (s *statsT) add(name, v string, t time.Time) { s.mu.Unlock() } -func dumpStats() { - if !katiEvalStatsFlag { +func DumpStats() { + if !EvalStatsFlag { return } var sv byTotalTime @@ -177,3 +185,15 @@ func (s *shellStatsT) add(d time.Duration) { s.count++ s.mu.Unlock() } + +func (s *shellStatsT) Duration() time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return s.duration +} + +func (s *shellStatsT) Count() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.count +} diff --git a/string_piece.cc b/string_piece.cc index e5679b0..78de4ed 100644 --- a/string_piece.cc +++ b/string_piece.cc @@ -17,6 +17,8 @@ // found in the LICENSE file. // Copied from strings/stringpiece.cc with modifications +// +build ignore + #include <ctype.h> #include <limits.h> diff --git a/string_piece_test.cc b/string_piece_test.cc index 544c0e4..85675d8 100644 --- a/string_piece_test.cc +++ b/string_piece_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "string_piece.h" #include <assert.h> diff --git a/string_pool.cc b/string_pool.cc index 596b024..bd6a709 100644 --- a/string_pool.cc +++ b/string_pool.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "string_pool.h" #include <stdlib.h> diff --git a/stringprintf.cc b/stringprintf.cc index 63fd00b..4ca40d2 100644 --- a/stringprintf.cc +++ b/stringprintf.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "stringprintf.h" #include <assert.h> @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "strutil.h" #include <ctype.h> @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" diff --git a/strutil_test.cc b/strutil_test.cc index e8e773a..bcff29f 100644 --- a/strutil_test.cc +++ b/strutil_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "strutil.h" #include <assert.h> diff --git a/strutil_test.go b/strutil_test.go index 0cd77ed..362a09d 100644 --- a/strutil_test.go +++ b/strutil_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "fmt" @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import "sync" @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "value.h" #include <vector> @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build ignore + #include "var.h" #include "log.h" @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "bytes" @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package kati import ( "container/heap" @@ -87,7 +87,7 @@ func (jq *JobQueue) Pop() interface{} { return item } -func NewWorker(wm *WorkerManager) *Worker { +func newWorker(wm *WorkerManager) *Worker { w := &Worker{ wm: wm, jobChan: make(chan *Job), @@ -154,7 +154,7 @@ func newRunner(r runner, s string) runner { } switch s[0] { case '@': - if !dryRunFlag { + if !DryRunFlag { r.echo = false } s = s[1:] @@ -171,10 +171,10 @@ func newRunner(r runner, s string) runner { } func (r runner) run(output string) error { - if r.echo || dryRunFlag { + if r.echo || DryRunFlag { fmt.Printf("%s\n", r.cmd) } - if dryRunFlag { + if DryRunFlag { return nil } args := []string{r.shell, "-c", r.cmd} @@ -250,7 +250,7 @@ func (j *Job) build() { func (wm *WorkerManager) handleJobs() { for { - if !useParaFlag && len(wm.freeWorkers) == 0 { + if wm.para == nil && len(wm.freeWorkers) == 0 { return } if wm.readyQueue.Len() == 0 { @@ -259,7 +259,7 @@ func (wm *WorkerManager) handleJobs() { j := heap.Pop(&wm.readyQueue).(*Job) Logf("run: %s", j.n.Output) - if useParaFlag { + if wm.para != nil { j.runners = j.createRunners() if len(j.runners) == 0 { wm.updateParents(j) @@ -290,6 +290,7 @@ func (wm *WorkerManager) updateParents(j *Job) { } type WorkerManager struct { + maxJobs int jobs []*Job readyQueue JobQueue jobChan chan *Job @@ -307,8 +308,9 @@ type WorkerManager struct { finishCnt int } -func NewWorkerManager() *WorkerManager { +func NewWorkerManager(numJobs int, paraPath string) *WorkerManager { wm := &WorkerManager{ + maxJobs: numJobs, jobChan: make(chan *Job), resultChan: make(chan JobResult), newDepChan: make(chan NewDep), @@ -317,15 +319,15 @@ func NewWorkerManager() *WorkerManager { busyWorkers: make(map[*Worker]bool), } - if useParaFlag { + if paraPath != "" { wm.runnings = make(map[string]*Job) wm.paraChan = make(chan *ParaResult) - wm.para = NewParaWorker(wm.paraChan) + wm.para = newParaWorker(wm.paraChan, numJobs, paraPath) go wm.para.Run() } else { wm.busyWorkers = make(map[*Worker]bool) - for i := 0; i < jobsFlag; i++ { - w := NewWorker(wm) + for i := 0; i < numJobs; i++ { + w := newWorker(wm) wm.freeWorkers = append(wm.freeWorkers, w) go w.Run() } @@ -393,7 +395,7 @@ func (wm *WorkerManager) Run() { if pr.status < 0 && pr.signal < 0 { j := wm.runnings[pr.output] for _, r := range j.runners { - if r.echo || dryRunFlag { + if r.echo || DryRunFlag { fmt.Printf("%s\n", r.cmd) } } @@ -409,18 +411,18 @@ func (wm *WorkerManager) Run() { } wm.handleJobs() - if useParaFlag { + if wm.para != nil { numBusy := len(wm.runnings) - if numBusy > jobsFlag { - numBusy = jobsFlag + if numBusy > wm.maxJobs { + numBusy = wm.maxJobs } - Logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), jobsFlag-numBusy, numBusy) + Logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), wm.maxJobs-numBusy, numBusy) } else { Logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), len(wm.freeWorkers), len(wm.busyWorkers)) } } - if useParaFlag { + if wm.para != nil { Logf("Wait for para to finish") wm.para.Wait() } else { |