diff options
Diffstat (limited to 'golang/kati/ninja.go')
| -rw-r--r-- | golang/kati/ninja.go | 777 |
1 files changed, 777 insertions, 0 deletions
diff --git a/golang/kati/ninja.go b/golang/kati/ninja.go new file mode 100644 index 0000000..f90ff24 --- /dev/null +++ b/golang/kati/ninja.go @@ -0,0 +1,777 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kati + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "time" + + "github.com/golang/glog" +) + +type nodeState int + +const ( + nodeInit nodeState = iota // not visited + nodeVisit // visited + nodeFile // visited & file exists + nodeAlias // visited & alias for other target + nodeMissing // visited & no target for this output + nodeBuild // visited & build emitted +) + +func (s nodeState) String() string { + switch s { + case nodeInit: + return "node-init" + case nodeVisit: + return "node-visit" + case nodeFile: + return "node-file" + case nodeAlias: + return "node-alias" + case nodeMissing: + return "node-missing" + case nodeBuild: + return "node-build" + default: + return fmt.Sprintf("node-unknown[%d]", int(s)) + } +} + +// NinjaGenerator generates ninja build files from DepGraph. +type NinjaGenerator struct { + // Args is original arguments to generate the ninja file. + Args []string + // Suffix is suffix for generated files. + Suffix string + // GomaDir is goma directory. If empty, goma will not be used. + GomaDir string + // DetectAndroidEcho detects echo as description. + DetectAndroidEcho bool + + f *os.File + nodes []*DepNode + exports map[string]bool + + ctx *execContext + + ruleID int + done map[string]nodeState +} + +func (n *NinjaGenerator) init(g *DepGraph) { + g.resolveVPATH() + n.nodes = g.nodes + n.exports = g.exports + n.ctx = newExecContext(g.vars, g.vpaths, true) + n.done = make(map[string]nodeState) +} + +func getDepfileImpl(ss string) (string, error) { + tss := ss + " " + if (!strings.Contains(tss, " -MD ") && !strings.Contains(tss, " -MMD ")) || !strings.Contains(tss, " -c ") { + return "", nil + } + + mfIndex := strings.Index(ss, " -MF ") + if mfIndex >= 0 { + mf := trimLeftSpace(ss[mfIndex+4:]) + if strings.Index(mf, " -MF ") >= 0 { + return "", fmt.Errorf("Multiple output file candidates in %s", ss) + } + mfEndIndex := strings.IndexAny(mf, " \t\n") + if mfEndIndex >= 0 { + mf = mf[:mfEndIndex] + } + + return mf, nil + } + + outIndex := strings.Index(ss, " -o ") + if outIndex < 0 { + return "", fmt.Errorf("Cannot find the depfile in %s", ss) + } + out := trimLeftSpace(ss[outIndex+4:]) + if strings.Index(out, " -o ") >= 0 { + return "", fmt.Errorf("Multiple output file candidates in %s", ss) + } + outEndIndex := strings.IndexAny(out, " \t\n") + if outEndIndex >= 0 { + out = out[:outEndIndex] + } + return stripExt(out) + ".d", nil +} + +// getDepfile gets depfile from cmdline, and returns cmdline and depfile. +func getDepfile(cmdline string) (string, string, error) { + // A hack for Android - llvm-rs-cc seems not to emit a dep file. + if strings.Contains(cmdline, "bin/llvm-rs-cc ") { + return cmdline, "", nil + } + + depfile, err := getDepfileImpl(cmdline) + if depfile == "" || err != nil { + return cmdline, depfile, err + } + + // A hack for Makefiles generated by automake. + mvCmd := "(mv -f " + depfile + " " + if i := strings.LastIndex(cmdline, mvCmd); i >= 0 { + rest := cmdline[i+len(mvCmd):] + ei := strings.IndexByte(rest, ')') + if ei < 0 { + return cmdline, "", fmt.Errorf("unbalanced parenthes? %s", cmdline) + } + cmdline = cmdline[:i] + "(cp -f " + depfile + " " + rest + return cmdline, depfile, nil + } + + // A hack for Android to get .P files instead of .d. + p := stripExt(depfile) + ".P" + if strings.Contains(cmdline, p) { + rmfCmd := "; rm -f " + depfile + ncmdline := strings.Replace(cmdline, rmfCmd, "", 1) + if ncmdline == cmdline { + return cmdline, "", fmt.Errorf("cannot find removal of .d file: %s", cmdline) + } + return ncmdline, p, nil + } + + // A hack for Android. For .s files, GCC does not use + // C preprocessor, so it ignores -MF flag. + as := "/" + stripExt(filepath.Base(depfile)) + ".s" + if strings.Contains(cmdline, as) { + return cmdline, "", nil + } + + cmdline += fmt.Sprintf(" && cp %s %s.tmp", depfile, depfile) + depfile += ".tmp" + return cmdline, depfile, nil +} + +func trimTailingSlash(s string) string { + if s == "" { + return s + } + if s[len(s)-1] != '\\' { + return s + } + // drop single trailing slash - multiline_arg.mk + if len(s) > 2 && s[len(s)-2] != '\\' { + return s[:len(s)-1] + } + // preserve two trailing slash - escaped_backslash.mk + return s +} + +func stripShellComment(s string) string { + if strings.IndexByte(s, '#') < 0 { + // Fast path. + return s + } + // set space as an initial value so the leading comment will be + // stripped out. + lastch := rune(' ') + var escape bool + var quote rune + var skip rune + var cmdsubst []rune + var buf bytes.Buffer +Loop: + for _, c := range s { + if skip != 0 { + if skip != c { + continue Loop + } + if len(cmdsubst) > 0 && cmdsubst[len(cmdsubst)-1] == skip { + cmdsubst = cmdsubst[:len(cmdsubst)-1] + } + skip = 0 + } + if quote != 0 { + if quote == c && (quote == '\'' || !escape) { + quote = 0 + } + } else if !escape { + if c == '#' && isWhitespace(lastch) { + if len(cmdsubst) == 0 { + // strip comment until the end of line. + skip = '\n' + continue Loop + } + // strip comment until the end of command subst. + skip = cmdsubst[len(cmdsubst)-1] + continue Loop + } else if c == '\'' || c == '"' { + quote = c + } else if lastch == '$' && c == '(' { + cmdsubst = append(cmdsubst, ')') + } else if c == '`' { + cmdsubst = append(cmdsubst, '`') + } + } + if escape { + escape = false + } else if c == '\\' { + escape = true + } else { + escape = false + } + lastch = c + buf.WriteRune(c) + } + return buf.String() +} + +var ccRE = regexp.MustCompile(`^prebuilts/(gcc|clang)/.*(gcc|g\+\+|clang|clang\+\+) .* ?-c `) + +func gomaCmdForAndroidCompileCmd(cmd string) (string, bool) { + i := strings.Index(cmd, " ") + if i < 0 { + return cmd, false + } + driver := cmd[:i] + if strings.HasSuffix(driver, "ccache") { + return gomaCmdForAndroidCompileCmd(cmd[i+1:]) + } + return cmd, ccRE.MatchString(cmd) +} + +func descriptionFromCmd(cmd string) (string, bool) { + if !strings.HasPrefix(cmd, "echo") || !isWhitespace(rune(cmd[4])) { + return "", false + } + echoarg := cmd[5:] + + // strip outer quotes, and fail if it is not a single echo command. + var buf bytes.Buffer + var escape bool + var quote rune + for _, c := range echoarg { + if escape { + escape = false + buf.WriteRune(c) + continue + } + if c == '\\' { + escape = true + buf.WriteRune(c) + continue + } + if quote != 0 { + if c == quote { + quote = 0 + continue + } + buf.WriteRune(c) + continue + } + switch c { + case '\'', '"', '`': + quote = c + case '<', '>', '&', '|', ';': + return "", false + default: + buf.WriteRune(c) + } + } + return buf.String(), true +} + +func (n *NinjaGenerator) genShellScript(runners []runner) (cmd string, desc string, useLocalPool bool) { + const defaultDesc = "build $out" + var useGomacc bool + var buf bytes.Buffer + for i, r := range runners { + if i > 0 { + if runners[i-1].ignoreError { + buf.WriteString(" ; ") + } else { + buf.WriteString(" && ") + } + } + cmd := trimTailingSlash(r.cmd) + cmd = stripShellComment(cmd) + cmd = trimLeftSpace(cmd) + cmd = strings.Replace(cmd, "\\\n\t", "", -1) + cmd = strings.Replace(cmd, "\\\n", "", -1) + cmd = strings.TrimRight(cmd, " \t\n;") + cmd = escapeNinja(cmd) + if cmd == "" { + cmd = "true" + } + glog.V(2).Infof("cmd %q=>%q", r.cmd, cmd) + if n.GomaDir != "" { + rcmd, ok := gomaCmdForAndroidCompileCmd(cmd) + if ok { + cmd = fmt.Sprintf("%s/gomacc %s", n.GomaDir, rcmd) + useGomacc = true + } + } + if n.DetectAndroidEcho && desc == "" { + d, ok := descriptionFromCmd(cmd) + if ok { + desc = d + cmd = "true" + } + } + needsSubShell := i > 0 || len(runners) > 1 + if cmd[0] == '(' { + needsSubShell = false + } + + if needsSubShell { + buf.WriteByte('(') + } + buf.WriteString(cmd) + if i == len(runners)-1 && r.ignoreError { + buf.WriteString(" ; true") + } + if needsSubShell { + buf.WriteByte(')') + } + } + if desc == "" { + desc = defaultDesc + } + return buf.String(), desc, n.GomaDir != "" && !useGomacc +} + +func (n *NinjaGenerator) genRuleName() string { + ruleName := fmt.Sprintf("rule%d", n.ruleID) + n.ruleID++ + return ruleName +} + +func (n *NinjaGenerator) emitBuild(output, rule, inputs, orderOnlys string) { + fmt.Fprintf(n.f, "build %s: %s", escapeBuildTarget(output), rule) + if inputs != "" { + fmt.Fprintf(n.f, " %s", inputs) + } + if orderOnlys != "" { + fmt.Fprintf(n.f, " || %s", orderOnlys) + } +} + +func escapeBuildTarget(s string) string { + i := strings.IndexAny(s, "$: \\") + if i < 0 { + return s + } + // unescapeInput only "\ ", "\=" unescape as " ", "=". + // TODO(ukai): which char should unescape, which should not here? + var esc rune + var buf bytes.Buffer + for _, c := range s { + switch c { + case '\\': + esc = c + continue + case '$', ':', ' ': + esc = 0 + buf.WriteByte('$') + } + if esc != 0 { + buf.WriteRune(esc) + esc = 0 + } + buf.WriteRune(c) + } + if esc != 0 { + buf.WriteRune(esc) + } + return buf.String() +} + +func (n *NinjaGenerator) dependency(node *DepNode) (string, string) { + var deps []string + seen := make(map[string]bool) + for _, d := range node.Deps { + t := escapeBuildTarget(d.Output) + if seen[t] { + continue + } + deps = append(deps, t) + seen[t] = true + } + var orderOnlys []string + for _, d := range node.OrderOnlys { + t := escapeBuildTarget(d.Output) + if seen[t] { + continue + } + orderOnlys = append(orderOnlys, t) + seen[t] = true + } + return strings.Join(deps, " "), strings.Join(orderOnlys, " ") +} + +func escapeNinja(s string) string { + return strings.Replace(s, "$", "$$", -1) +} + +func escapeShell(s string) string { + i := strings.IndexAny(s, "$`!\\\"") + if i < 0 { + return s + } + var buf bytes.Buffer + var lastDollar bool + for _, c := range s { + switch c { + case '$': + if lastDollar { + buf.WriteRune(c) + lastDollar = false + continue + } + buf.WriteString(`\$`) + lastDollar = true + continue + case '`', '"', '!', '\\': + buf.WriteByte('\\') + } + buf.WriteRune(c) + lastDollar = false + } + return buf.String() +} + +func (n *NinjaGenerator) ninjaVars(s string, nv [][]string, esc func(string) string) string { + for _, v := range nv { + k, v := v[0], v[1] + if v == "" { + continue + } + if strings.Contains(v, "/./") || strings.Contains(v, "/../") || strings.Contains(v, "$") { + // ninja will normalize paths (/./, /../), so keep it as is + // ninja will emit quoted string for $ + continue + } + if esc != nil { + v = esc(v) + } + s = strings.Replace(s, v, k, -1) + } + return s +} + +func (n *NinjaGenerator) emitNode(node *DepNode) error { + output := node.Output + if _, found := n.done[output]; found { + return nil + } + n.done[output] = nodeVisit + + if len(node.Cmds) == 0 && len(node.Deps) == 0 && len(node.OrderOnlys) == 0 && !node.IsPhony { + if _, ok := n.ctx.vpaths.exists(output); ok { + n.done[output] = nodeFile + return nil + } + o := filepath.Clean(output) + if o != output { + // if normalized target has been done, it marks as alias. + if s, found := n.done[o]; found { + glog.V(1).Infof("node %s=%s => %s=alias", o, s, node.Output) + n.done[output] = nodeAlias + return nil + } + } + if node.Filename == "" { + n.done[output] = nodeMissing + } + return nil + } + + runners, _, err := createRunners(n.ctx, node) + if err != nil { + return err + } + ruleName := "phony" + useLocalPool := false + inputs, orderOnlys := n.dependency(node) + if len(runners) > 0 { + ruleName = n.genRuleName() + fmt.Fprintf(n.f, "\n# rule for %q\n", node.Output) + fmt.Fprintf(n.f, "rule %s\n", ruleName) + + ss, desc, ulp := n.genShellScript(runners) + if ulp { + useLocalPool = true + } + fmt.Fprintf(n.f, " description = %s\n", desc) + cmdline, depfile, err := getDepfile(ss) + if err != nil { + return err + } + if depfile != "" { + fmt.Fprintf(n.f, " depfile = %s\n", depfile) + fmt.Fprintf(n.f, " deps = gcc\n") + } + nv := [][]string{ + []string{"${in}", inputs}, + []string{"${out}", escapeNinja(output)}, + } + // It seems Linux is OK with ~130kB. + // TODO: Find this number automatically. + ArgLenLimit := 100 * 1000 + if len(cmdline) > ArgLenLimit { + fmt.Fprintf(n.f, " rspfile = $out.rsp\n") + cmdline = n.ninjaVars(cmdline, nv, nil) + fmt.Fprintf(n.f, " rspfile_content = %s\n", cmdline) + fmt.Fprintf(n.f, " command = %s $out.rsp\n", n.ctx.shell) + } else { + cmdline = escapeShell(cmdline) + cmdline = n.ninjaVars(cmdline, nv, escapeShell) + fmt.Fprintf(n.f, " command = %s -c \"%s\"\n", n.ctx.shell, cmdline) + } + } + n.emitBuild(output, ruleName, inputs, orderOnlys) + if useLocalPool { + fmt.Fprintf(n.f, " pool = local_pool\n") + } + fmt.Fprintf(n.f, "\n") + n.done[output] = nodeBuild + + for _, d := range node.Deps { + err := n.emitNode(d) + if err != nil { + return err + } + glog.V(1).Infof("node %s dep node %q %s", node.Output, d.Output, n.done[d.Output]) + } + for _, d := range node.OrderOnlys { + err := n.emitNode(d) + if err != nil { + return err + } + glog.V(1).Infof("node %s order node %q %s", node.Output, d.Output, n.done[d.Output]) + } + return nil +} + +func (n *NinjaGenerator) emitRegenRules() error { + if len(n.Args) == 0 { + return nil + } + mkfiles, err := n.ctx.ev.EvaluateVar("MAKEFILE_LIST") + if err != nil { + return err + } + fmt.Fprintf(n.f, ` +rule regen_ninja + description = Regenerate ninja files due to dependency + generator=1 + command=%s +`, strings.Join(n.Args, " ")) + fmt.Fprintf(n.f, "build %s: regen_ninja %s", n.ninjaName(), mkfiles) + // TODO: Add dependencies to directories read by $(wildcard) or + // $(shell find). + if len(usedEnvs) > 0 { + fmt.Fprintf(n.f, " %s", n.envlistName()) + } + fmt.Fprintf(n.f, "\n\n") + return nil +} + +func (n *NinjaGenerator) shName() string { + return fmt.Sprintf("ninja%s.sh", n.Suffix) +} + +func (n *NinjaGenerator) ninjaName() string { + return fmt.Sprintf("build%s.ninja", n.Suffix) +} + +func (n *NinjaGenerator) envlistName() string { + return fmt.Sprintf(".kati_env%s", n.Suffix) +} + +func (n *NinjaGenerator) generateEnvlist() (err error) { + f, err := os.Create(n.envlistName()) + if err != nil { + return err + } + defer func() { + cerr := f.Close() + if err == nil { + err = cerr + } + }() + for k := range usedEnvs { + v, err := n.ctx.ev.EvaluateVar(k) + if err != nil { + return err + } + fmt.Fprintf(f, "%q=%q\n", k, v) + } + return nil +} + +func (n *NinjaGenerator) generateShell() (err error) { + f, err := os.Create(n.shName()) + if err != nil { + return err + } + defer func() { + cerr := f.Close() + if err == nil { + err = cerr + } + }() + + fmt.Fprintf(f, "#!/bin/bash\n") + fmt.Fprintf(f, "# Generated by kati %s\n", gitVersion) + fmt.Fprintln(f) + fmt.Fprintln(f, `cd $(dirname "$0")`) + if n.Suffix != "" { + fmt.Fprintf(f, "if [ -f %s ]; then\n export $(cat %s)\nfi\n", n.envlistName(), n.envlistName()) + } + for name, export := range n.exports { + // export "a b"=c will error on bash + // bash: export `a b=c': not a valid identifier + if strings.ContainsAny(name, " \t\n\r") { + glog.V(1).Infof("ignore export %q (export:%t)", name, export) + continue + } + if export { + v, err := n.ctx.ev.EvaluateVar(name) + if err != nil { + return err + } + fmt.Fprintf(f, "export %q=%q\n", name, v) + } else { + fmt.Fprintf(f, "unset %q\n", name) + } + } + if n.GomaDir == "" { + fmt.Fprintf(f, `exec ninja -f %s "$@"`+"\n", n.ninjaName()) + } else { + fmt.Fprintf(f, `exec ninja -f %s -j500 "$@"`+"\n", n.ninjaName()) + } + + return f.Chmod(0755) +} + +func (n *NinjaGenerator) generateNinja(defaultTarget string) (err error) { + f, err := os.Create(n.ninjaName()) + if err != nil { + return err + } + defer func() { + cerr := f.Close() + if err == nil { + err = cerr + } + }() + + n.f = f + fmt.Fprintf(n.f, "# Generated by kati %s\n", gitVersion) + fmt.Fprintf(n.f, "\n") + + if len(usedEnvs) > 0 { + fmt.Fprintln(n.f, "# Environment variables used:") + var names []string + for name := range usedEnvs { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + v, err := n.ctx.ev.EvaluateVar(name) + if err != nil { + return err + } + fmt.Fprintf(n.f, "# %q=%q\n", name, v) + } + fmt.Fprintf(n.f, "\n") + } + + if n.GomaDir != "" { + fmt.Fprintf(n.f, "pool local_pool\n") + fmt.Fprintf(n.f, " depth = %d\n\n", runtime.NumCPU()) + } + + err = n.emitRegenRules() + if err != nil { + return err + } + + // defining $out for $@ and $in for $^ here doesn't work well, + // because these texts will be processed in escapeShell... + for _, node := range n.nodes { + err := n.emitNode(node) + if err != nil { + return err + } + glog.V(1).Infof("node %q %s", node.Output, n.done[node.Output]) + } + + // emit phony targets for visited nodes that are + // - not existing file + // - not alias for other targets. + var nodes []string + for node, state := range n.done { + if state != nodeVisit { + continue + } + nodes = append(nodes, node) + } + if len(nodes) > 0 { + fmt.Fprintln(n.f) + sort.Strings(nodes) + for _, node := range nodes { + n.emitBuild(node, "phony", "", "") + fmt.Fprintln(n.f) + n.done[node] = nodeBuild + } + } + + // emit default if the target was emitted. + if defaultTarget != "" && n.done[defaultTarget] == nodeBuild { + fmt.Fprintf(n.f, "\ndefault %s\n", escapeNinja(defaultTarget)) + } + return nil +} + +// Save generates build.ninja from DepGraph. +func (n *NinjaGenerator) Save(g *DepGraph, name string, targets []string) error { + startTime := time.Now() + n.init(g) + err := n.generateEnvlist() + if err != nil { + return err + } + err = n.generateShell() + if err != nil { + return err + } + var defaultTarget string + if len(targets) == 0 && len(g.nodes) > 0 { + defaultTarget = g.nodes[0].Output + } + err = n.generateNinja(defaultTarget) + if err != nil { + return err + } + logStats("generate ninja time: %q", time.Since(startTime)) + return nil +} |
