// Copyright 2015 Google Inc. All rights reserved // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kati import ( "bytes" "errors" "fmt" "io" "regexp" "strconv" "strings" "github.com/golang/glog" ) var ( errEndOfInput = errors.New("unexpected end of input") errNotLiteral = errors.New("valueNum: not literal") errUnterminatedVariableReference = errors.New("*** unterminated variable reference.") ) type evalWriter interface { io.Writer writeWord([]byte) writeWordString(string) resetSep() } // Value is an interface for value. type Value interface { String() string Eval(w evalWriter, ev *Evaluator) error serialize() serializableVar dump(d *dumpbuf) } // literal is literal value. type literal string func (s literal) String() string { return string(s) } func (s literal) Eval(w evalWriter, ev *Evaluator) error { io.WriteString(w, string(s)) return nil } func (s literal) serialize() serializableVar { return serializableVar{Type: "literal", V: string(s)} } func (s literal) dump(d *dumpbuf) { d.Byte(valueTypeLiteral) d.Bytes([]byte(s)) } // tmpval is temporary value. type tmpval []byte func (t tmpval) String() string { return string(t) } func (t tmpval) Eval(w evalWriter, ev *Evaluator) error { w.Write(t) return nil } func (t tmpval) Value() []byte { return []byte(t) } func (t tmpval) serialize() serializableVar { return serializableVar{Type: "tmpval", V: string(t)} } func (t tmpval) dump(d *dumpbuf) { d.Byte(valueTypeTmpval) d.Bytes(t) } // expr is a list of values. type expr []Value func (e expr) String() string { var s []string for _, v := range e { s = append(s, v.String()) } return strings.Join(s, "") } func (e expr) Eval(w evalWriter, ev *Evaluator) error { for _, v := range e { w.resetSep() err := v.Eval(w, ev) if err != nil { return err } } return nil } func (e expr) serialize() serializableVar { r := serializableVar{Type: "expr"} for _, v := range e { r.Children = append(r.Children, v.serialize()) } return r } func (e expr) dump(d *dumpbuf) { d.Byte(valueTypeExpr) d.Int(len(e)) for _, v := range e { v.dump(d) } } func compactExpr(e expr) Value { if len(e) == 1 { return e[0] } // TODO(ukai): concat literal return e } func toExpr(v Value) expr { if v == nil { return nil } if e, ok := v.(expr); ok { return e } return expr{v} } // varref is variable reference. e.g. ${foo}. type varref struct { varname Value paren byte } func (v *varref) String() string { varname := v.varname.String() if len(varname) == 1 && v.paren == 0 { return fmt.Sprintf("$%s", varname) } paren := v.paren if paren == 0 { paren = '{' } return fmt.Sprintf("$%c%s%c", paren, varname, closeParen(paren)) } func (v *varref) Eval(w evalWriter, ev *Evaluator) error { te := traceEvent.begin("var", v, traceEventMain) buf := newEbuf() err := v.varname.Eval(buf, ev) if err != nil { return err } vv := ev.LookupVar(buf.String()) buf.release() err = vv.Eval(w, ev) if err != nil { return err } traceEvent.end(te) return nil } func (v *varref) serialize() serializableVar { return serializableVar{ Type: "varref", V: string(v.paren), Children: []serializableVar{v.varname.serialize()}, } } func (v *varref) dump(d *dumpbuf) { d.Byte(valueTypeVarref) d.Byte(v.paren) v.varname.dump(d) } // paramref is parameter reference e.g. $1. type paramref int func (p paramref) String() string { return fmt.Sprintf("$%d", int(p)) } func (p paramref) Eval(w evalWriter, ev *Evaluator) error { te := traceEvent.begin("param", p, traceEventMain) n := int(p) if n < len(ev.paramVars) { err := ev.paramVars[n].Eval(w, ev) if err != nil { return err } } else { vv := ev.LookupVar(fmt.Sprintf("%d", n)) err := vv.Eval(w, ev) if err != nil { return err } } traceEvent.end(te) return nil } func (p paramref) serialize() serializableVar { return serializableVar{Type: "paramref", V: strconv.Itoa(int(p))} } func (p paramref) dump(d *dumpbuf) { d.Byte(valueTypeParamref) d.Int(int(p)) } // varsubst is variable substitutaion. e.g. ${var:pat=subst}. type varsubst struct { varname Value pat Value subst Value paren byte } func (v varsubst) String() string { paren := v.paren if paren == 0 { paren = '{' } return fmt.Sprintf("$%c%s:%s=%s%c", paren, v.varname, v.pat, v.subst, closeParen(paren)) } func (v varsubst) Eval(w evalWriter, ev *Evaluator) error { te := traceEvent.begin("varsubst", v, traceEventMain) buf := newEbuf() params, err := ev.args(buf, v.varname, v.pat, v.subst) if err != nil { return err } vname := string(params[0]) pat := string(params[1]) subst := string(params[2]) buf.Reset() vv := ev.LookupVar(vname) err = vv.Eval(buf, ev) if err != nil { return err } vals := splitSpaces(buf.String()) buf.release() space := false for _, val := range vals { if space { io.WriteString(w, " ") } io.WriteString(w, substRef(pat, subst, val)) space = true } traceEvent.end(te) return nil } func (v varsubst) serialize() serializableVar { return serializableVar{ Type: "varsubst", V: string(v.paren), Children: []serializableVar{ v.varname.serialize(), v.pat.serialize(), v.subst.serialize(), }, } } func (v varsubst) dump(d *dumpbuf) { d.Byte(valueTypeVarsubst) d.Byte(v.paren) v.varname.dump(d) v.pat.dump(d) v.subst.dump(d) } func str(buf []byte, alloc bool) Value { if alloc { return literal(string(buf)) } return tmpval(buf) } func appendStr(exp expr, buf []byte, alloc bool) expr { if len(buf) == 0 { return exp } if len(exp) == 0 { return append(exp, str(buf, alloc)) } switch v := exp[len(exp)-1].(type) { case literal: v += literal(string(buf)) exp[len(exp)-1] = v return exp case tmpval: v = append(v, buf...) exp[len(exp)-1] = v return exp } return append(exp, str(buf, alloc)) } func valueNum(v Value) (int, error) { switch v := v.(type) { case literal, tmpval: n, err := strconv.ParseInt(v.String(), 10, 64) return int(n), err } return 0, errNotLiteral } type parseOp struct { // alloc indicates text will be allocated as literal (string) alloc bool // matchParen matches parenthesis. // note: required for func arg matchParen bool } // parseExpr parses expression in `in` until it finds any byte in term. // if term is nil, it will parse to end of input. // if term is not nil, and it reaches to end of input, return error. // it returns parsed value, and parsed length `n`, so in[n-1] is any byte of // term, and in[n:] is next input. func parseExpr(in, term []byte, op parseOp) (Value, int, error) { var exp expr b := 0 i := 0 var saveParen byte parenDepth := 0 Loop: for i < len(in) { ch := in[i] if term != nil && bytes.IndexByte(term, ch) >= 0 { break Loop } switch ch { case '$': if i+1 >= len(in) { break Loop } if in[i+1] == '$' { exp = appendStr(exp, in[b:i+1], op.alloc) i += 2 b = i continue } if bytes.IndexByte(term, in[i+1]) >= 0 { exp = appendStr(exp, in[b:i], op.alloc) exp = append(exp, &varref{varname: literal("")}) i++ b = i break Loop } exp = appendStr(exp, in[b:i], op.alloc) v, n, err := parseDollar(in[i:], op.alloc) if err != nil { return nil, 0, err } i += n b = i exp = append(exp, v) continue case '(', '{': if !op.matchParen { break } cp := closeParen(ch) if i := bytes.IndexByte(term, cp); i >= 0 { parenDepth++ saveParen = cp term[i] = 0 } else if cp == saveParen { parenDepth++ } case saveParen: if !op.matchParen { break } parenDepth-- if parenDepth == 0 { i := bytes.IndexByte(term, 0) term[i] = saveParen saveParen = 0 } } i++ } exp = appendStr(exp, in[b:i], op.alloc) if i == len(in) && term != nil { glog.Warningf("parse: unexpected end of input: %q %d [%q]", in, i, term) return exp, i, errEndOfInput } return compactExpr(exp), i, nil } func closeParen(ch byte) byte { switch ch { case '(': return ')' case '{': return '}' } return 0 } // parseDollar parses // $(func expr[, expr...]) # func = literal SP // $(expr:expr=expr) // $(expr) // $x // it returns parsed value and parsed length. func parseDollar(in []byte, alloc bool) (Value, int, error) { if len(in) <= 1 { return nil, 0, errors.New("empty expr") } if in[0] != '$' { return nil, 0, errors.New("should starts with $") } if in[1] == '$' { return nil, 0, errors.New("should handle $$ as literal $") } oparen := in[1] paren := closeParen(oparen) if paren == 0 { // $x case. if in[1] >= '0' && in[1] <= '9' { return paramref(in[1] - '0'), 2, nil } return &varref{varname: str(in[1:2], alloc)}, 2, nil } term := []byte{paren, ':', ' '} var varname expr i := 2 op := parseOp{alloc: alloc} Again: for { e, n, err := parseExpr(in[i:], term, op) if err != nil { if err == errEndOfInput { // unmatched_paren2.mk varname = append(varname, toExpr(e)...) if len(varname) > 0 { for i, vn := range varname { if vr, ok := vn.(*varref); ok { if vr.paren == oparen { varname = varname[:i+1] varname[i] = expr{literal(fmt.Sprintf("$%c", oparen)), vr.varname} return &varref{varname: varname, paren: oparen}, i + 1 + n + 1, nil } } } } return nil, 0, errUnterminatedVariableReference } return nil, 0, err } varname = append(varname, toExpr(e)...) i += n switch in[i] { case paren: // ${expr} vname := compactExpr(varname) n, err := valueNum(vname) if err == nil { // ${n} return paramref(n), i + 1, nil } return &varref{varname: vname, paren: oparen}, i + 1, nil case ' ': // ${e ...} switch token := e.(type) { case literal, tmpval: funcName := intern(token.String()) if f, ok := funcMap[funcName]; ok { return parseFunc(f(), in, i+1, term[:1], funcName, op.alloc) } } term = term[:2] // drop ' ' continue Again case ':': // ${varname:...} colon := in[i : i+1] var vterm []byte vterm = append(vterm, term[:2]...) vterm[1] = '=' // term={paren, '='}. e, n, err := parseExpr(in[i+1:], vterm, op) if err != nil { return nil, 0, err } i += 1 + n if in[i] == paren { varname = appendStr(varname, colon, op.alloc) return &varref{varname: varname, paren: oparen}, i + 1, nil } // ${varname:xx=...} pat := e subst, n, err := parseExpr(in[i+1:], term[:1], op) if err != nil { return nil, 0, err } i += 1 + n // ${first:pat=e} return varsubst{ varname: compactExpr(varname), pat: pat, subst: subst, paren: oparen, }, i + 1, nil default: return nil, 0, fmt.Errorf("unexpected char %c at %d in %q", in[i], i, string(in)) } } } // skipSpaces skips spaces at front of `in` before any bytes in term. // in[n] will be the first non white space in in. func skipSpaces(in, term []byte) int { for i := 0; i < len(in); i++ { if bytes.IndexByte(term, in[i]) >= 0 { return i } switch in[i] { case ' ', '\t': default: return i } } return len(in) } // trimLiteralSpace trims literal space around v. func trimLiteralSpace(v Value) Value { switch v := v.(type) { case literal: return literal(strings.TrimSpace(string(v))) case tmpval: b := bytes.TrimSpace([]byte(v)) if len(b) == 0 { return literal("") } return tmpval(b) case expr: if len(v) == 0 { return v } switch s := v[0].(type) { case literal, tmpval: t := trimLiteralSpace(s) if t == literal("") { v = v[1:] } else { v[0] = t } } switch s := v[len(v)-1].(type) { case literal, tmpval: t := trimLiteralSpace(s) if t == literal("") { v = v[:len(v)-1] } else { v[len(v)-1] = t } } return compactExpr(v) } return v } // concatLine concatinates line with "\\\n" in function expression. // TODO(ukai): less alloc? func concatLine(v Value) Value { switch v := v.(type) { case literal: for { s := string(v) i := strings.Index(s, "\\\n") if i < 0 { return v } v = literal(s[:i] + strings.TrimLeft(s[i+2:], " \t")) } case tmpval: for { b := []byte(v) i := bytes.Index(b, []byte{'\\', '\n'}) if i < 0 { return v } var buf bytes.Buffer buf.Write(b[:i]) buf.Write(bytes.TrimLeft(b[i+2:], " \t")) v = tmpval(buf.Bytes()) } case expr: for i := range v { switch vv := v[i].(type) { case literal, tmpval: v[i] = concatLine(vv) } } return v } return v } // parseFunc parses function arguments from in[s:] for f. // in[0] is '$' and in[s] is space just after func name. // in[:n] will be "${func args...}" func parseFunc(f mkFunc, in []byte, s int, term []byte, funcName string, alloc bool) (Value, int, error) { f.AddArg(str(in[1:s-1], alloc)) arity := f.Arity() term = append(term, ',') i := skipSpaces(in[s:], term) i = s + i if i == len(in) { return f, i, nil } narg := 1 op := parseOp{alloc: alloc, matchParen: true} for { if arity != 0 && narg >= arity { // final arguments. term = term[:1] // drop ',' } v, n, err := parseExpr(in[i:], term, op) if err != nil { if err == errEndOfInput { return nil, 0, fmt.Errorf("*** unterminated call to function `%s': missing `)'.", funcName) } return nil, 0, err } v = concatLine(v) // TODO(ukai): do this in funcIf, funcAnd, or funcOr's compactor? if (narg == 1 && funcName == "if") || funcName == "and" || funcName == "or" { v = trimLiteralSpace(v) } f.AddArg(v) i += n narg++ if in[i] == term[0] { i++ break } i++ // should be ',' if i == len(in) { break } } var fv Value fv = f if compactor, ok := f.(compactor); ok { fv = compactor.Compact() } if EvalStatsFlag || traceEvent.enabled() { fv = funcstats{ Value: fv, str: fv.String(), } } return fv, i, nil } type compactor interface { Compact() Value } type funcstats struct { Value str string } func (f funcstats) Eval(w evalWriter, ev *Evaluator) error { te := traceEvent.begin("func", literal(f.str), traceEventMain) err := f.Value.Eval(w, ev) if err != nil { return err } // TODO(ukai): per functype? traceEvent.end(te) return nil } type matcherValue struct{} func (m matcherValue) Eval(w evalWriter, ev *Evaluator) error { return fmt.Errorf("couldn't eval matcher") } func (m matcherValue) serialize() serializableVar { return serializableVar{Type: ""} } func (m matcherValue) dump(d *dumpbuf) { d.err = fmt.Errorf("couldn't dump matcher") } type matchVarref struct{ matcherValue } func (m matchVarref) String() string { return "$(match-any)" } type literalRE struct { matcherValue *regexp.Regexp } func mustLiteralRE(s string) literalRE { return literalRE{ Regexp: regexp.MustCompile(s), } } func (r literalRE) String() string { return r.Regexp.String() } func matchValue(exp, pat Value) bool { switch pat := pat.(type) { case literal: return literal(exp.String()) == pat } // TODO: other type match? return false } func matchExpr(exp, pat expr) ([]Value, bool) { if len(exp) != len(pat) { return nil, false } var mv matchVarref var matches []Value for i := range exp { if pat[i] == mv { switch exp[i].(type) { case paramref, *varref: matches = append(matches, exp[i]) continue } return nil, false } if patre, ok := pat[i].(literalRE); ok { re := patre.Regexp m := re.FindStringSubmatch(exp[i].String()) if m == nil { return nil, false } for _, sm := range m[1:] { matches = append(matches, literal(sm)) } continue } if !matchValue(exp[i], pat[i]) { return nil, false } } return matches, true }