diff options
author | Colin Cross <ccross@android.com> | 2015-01-30 17:27:36 -0800 |
---|---|---|
committer | Colin Cross <ccross@android.com> | 2015-03-13 20:28:16 -0700 |
commit | 3f40fa460d85b10646d383a3b6b01ea6d569b01b (patch) | |
tree | 542d913a3f0f818042b503948869818a77e99ebc /androidmk | |
parent | e441b9df9a68595d0dd7b8ed184aecb27c86054b (diff) | |
download | build_soong-3f40fa460d85b10646d383a3b6b01ea6d569b01b.tar.gz build_soong-3f40fa460d85b10646d383a3b6b01ea6d569b01b.tar.bz2 build_soong-3f40fa460d85b10646d383a3b6b01ea6d569b01b.zip |
Add soong_build primary builder
Initial build logic for building android with soong. It can build
a variety of C and C++ files for arm/arm64 and host.
Change-Id: I10eb37c2c2a50be6af1bb5fd568c0962b9476bf0
Diffstat (limited to 'androidmk')
-rw-r--r-- | androidmk/cmd/androidmk/android.go | 115 | ||||
-rw-r--r-- | androidmk/cmd/androidmk/androidmk.go | 438 | ||||
-rw-r--r-- | androidmk/cmd/androidmk/values.go | 192 | ||||
-rw-r--r-- | androidmk/parser/make_strings.go | 170 | ||||
-rw-r--r-- | androidmk/parser/make_strings_test.go | 96 | ||||
-rw-r--r-- | androidmk/parser/makething.go | 142 | ||||
-rw-r--r-- | androidmk/parser/parser.go | 633 | ||||
-rw-r--r-- | androidmk/parser/scope.go | 88 |
8 files changed, 1874 insertions, 0 deletions
diff --git a/androidmk/cmd/androidmk/android.go b/androidmk/cmd/androidmk/android.go new file mode 100644 index 00000000..53ae6b32 --- /dev/null +++ b/androidmk/cmd/androidmk/android.go @@ -0,0 +1,115 @@ +package main + +import ( + "android/soong/androidmk/parser" +) + +const ( + clear_vars = "__android_mk_clear_vars" + build_shared_library = "cc_library_shared" + build_static_library = "cc_library_static" + build_host_static_library = "cc_library_host_static" + build_host_shared_library = "cc_library_host_shared" + build_executable = "cc_binary" + build_host_executable = "cc_binary_host" + build_native_test = "cc_test" + build_prebuilt = "prebuilt" +) + +var stringProperties = map[string]string{ + "LOCAL_MODULE": "name", + "LOCAL_MODULE_STEM": "stem", + "LOCAL_MODULE_CLASS": "class", + "LOCAL_CXX_STL": "cxx_stl", + "LOCAL_STRIP_MODULE": "strip", + "LOCAL_MULTILIB": "compile_multilib", +} + +var listProperties = map[string]string{ + "LOCAL_SRC_FILES": "srcs", + "LOCAL_SHARED_LIBRARIES": "shared_libs", + "LOCAL_STATIC_LIBRARIES": "static_libs", + "LOCAL_WHOLE_STATIC_LIBRARIES": "whole_static_libs", + "LOCAL_SYSTEM_SHARED_LIBRARIES": "system_shared_libs", + "LOCAL_C_INCLUDES": "include_dirs", + "LOCAL_EXPORT_C_INCLUDE_DIRS": "export_include_dirs", + "LOCAL_ASFLAGS": "asflags", + "LOCAL_CLANG_ASFLAGS": "clang_asflags", + "LOCAL_CFLAGS": "cflags", + "LOCAL_CONLYFLAGS": "conlyflags", + "LOCAL_CPPFLAGS": "cppflags", + "LOCAL_LDFLAGS": "ldflags", + "LOCAL_REQUIRED_MODULES": "required", + "LOCAL_MODULE_TAGS": "tags", + "LOCAL_LDLIBS": "host_ldlibs", + "LOCAL_CLANG_CFLAGS": "clang_cflags", +} + +var boolProperties = map[string]string{ + "LOCAL_IS_HOST_MODULE": "host", + "LOCAL_CLANG": "clang", + "LOCAL_FORCE_STATIC_EXECUTABLE": "static", + "LOCAL_ADDRESS_SANITIZER": "asan", + "LOCAL_NATIVE_COVERAGE": "native_coverage", + "LOCAL_NO_CRT": "nocrt", + "LOCAL_ALLOW_UNDEFINED_SYMBOLS": "allow_undefined_symbols", + "LOCAL_RTTI_FLAG": "rtti", +} + +var propertySuffixes = []struct { + suffix string + class string +}{ + {"arm", "arch"}, + {"arm64", "arch"}, + {"mips", "arch"}, + {"mips64", "arch"}, + {"x86", "arch"}, + {"x86_64", "arch"}, + {"32", "multilib"}, + {"64", "multilib"}, +} + +var propertySuffixTranslations = map[string]string{ + "32": "lib32", + "64": "lib64", +} + +var conditionalTranslations = map[string]struct { + class string + suffix string +}{ + "($(HOST_OS),darwin)": {"host_os", "darwin"}, + "($(HOST_OS), darwin)": {"host_os", "darwin"}, + "($(HOST_OS),windows)": {"host_os", "windows"}, + "($(HOST_OS), windows)": {"host_os", "windows"}, +} + +func mydir(args []string) string { + return "." +} + +func androidScope() parser.Scope { + globalScope := parser.NewScope(nil) + globalScope.Set("CLEAR_VARS", clear_vars) + globalScope.Set("BUILD_HOST_EXECUTABLE", build_host_executable) + globalScope.Set("BUILD_SHARED_LIBRARY", build_shared_library) + globalScope.Set("BUILD_STATIC_LIBRARY", build_static_library) + globalScope.Set("BUILD_HOST_STATIC_LIBRARY", build_host_static_library) + globalScope.Set("BUILD_HOST_SHARED_LIBRARY", build_host_shared_library) + globalScope.Set("BUILD_NATIVE_TEST", build_native_test) + globalScope.Set("BUILD_EXECUTABLE", build_executable) + globalScope.Set("BUILD_PREBUILT", build_prebuilt) + globalScope.SetFunc("my-dir", mydir) + + globalScope.Set("lib32", "lib32") + globalScope.Set("lib64", "lib64") + globalScope.Set("arm", "arm") + globalScope.Set("arm64", "arm64") + globalScope.Set("mips", "mips") + globalScope.Set("mips64", "mips64") + globalScope.Set("x86", "x86") + globalScope.Set("x86_64", "x86_64") + + return globalScope +} diff --git a/androidmk/cmd/androidmk/androidmk.go b/androidmk/cmd/androidmk/androidmk.go new file mode 100644 index 00000000..6695181b --- /dev/null +++ b/androidmk/cmd/androidmk/androidmk.go @@ -0,0 +1,438 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "strings" + "text/scanner" + + mkparser "android/soong/androidmk/parser" + + bpparser "blueprint/parser" +) + +// TODO: non-expanded variables with expressions + +type bpFile struct { + comments []bpparser.Comment + defs []bpparser.Definition + localAssignments map[string]*bpparser.Property + globalAssignments map[string]*bpparser.Value + scope mkparser.Scope + module *bpparser.Module + + pos scanner.Position + prevLine, line int +} + +func (f *bpFile) errorf(thing mkparser.MakeThing, s string, args ...interface{}) { + orig := thing.Dump() + s = fmt.Sprintf(s, args...) + f.comments = append(f.comments, bpparser.Comment{ + Comment: fmt.Sprintf("// ANDROIDMK TRANSLATION ERROR: %s", s), + Pos: f.pos, + }) + lines := strings.Split(orig, "\n") + for _, l := range lines { + f.incPos() + f.comments = append(f.comments, bpparser.Comment{ + Comment: "// " + l, + Pos: f.pos, + }) + } +} + +func (f *bpFile) setPos(pos, endPos scanner.Position) { + f.pos = pos + + f.line++ + if f.pos.Line > f.prevLine+1 { + f.line++ + } + + f.pos.Line = f.line + f.prevLine = endPos.Line +} + +func (f *bpFile) incPos() { + f.pos.Line++ + f.line++ + f.prevLine++ +} + +type conditional struct { + cond string + eq bool +} + +func main() { + b, err := ioutil.ReadFile(os.Args[1]) + if err != nil { + fmt.Println(err.Error()) + return + } + + p := mkparser.NewParser(os.Args[1], bytes.NewBuffer(b)) + + things, errs := p.Parse() + if len(errs) > 0 { + for _, err := range errs { + fmt.Println("ERROR: ", err) + } + return + } + + file := &bpFile{ + scope: androidScope(), + localAssignments: make(map[string]*bpparser.Property), + globalAssignments: make(map[string]*bpparser.Value), + } + + var conds []*conditional + var cond *conditional + + for _, t := range things { + file.setPos(t.Pos(), t.EndPos()) + + if comment, ok := t.AsComment(); ok { + file.comments = append(file.comments, bpparser.Comment{ + Pos: file.pos, + Comment: "//" + comment.Comment, + }) + } else if assignment, ok := t.AsAssignment(); ok { + handleAssignment(file, assignment, cond) + } else if directive, ok := t.AsDirective(); ok { + switch directive.Name { + case "include": + val := directive.Args.Value(file.scope) + switch val { + case build_shared_library, build_static_library, + build_executable, build_host_executable, + build_prebuilt, build_host_static_library, + build_host_shared_library, build_native_test: + + handleModuleConditionals(file, directive, cond) + makeModule(file, val) + case clear_vars: + resetModule(file) + default: + file.errorf(directive, "unsupported include") + continue + } + case "ifeq", "ifneq": + args := directive.Args.Dump() + eq := directive.Name == "ifeq" + switch args { + case "($(HOST_OS),windows)", "($(HOST_OS), windows)", + "($(HOST_OS),darwin)", "($(HOST_OS), darwin)": + newCond := conditional{args, eq} + conds = append(conds, &newCond) + if cond == nil { + cond = &newCond + } else { + file.errorf(directive, "unsupported nested conditional") + } + default: + file.errorf(directive, "unsupported conditional") + conds = append(conds, nil) + continue + } + case "else": + if len(conds) == 0 { + file.errorf(directive, "missing if before else") + continue + } else if conds[len(conds)-1] == nil { + file.errorf(directive, "else from unsupported contitional") + continue + } + cond.eq = !cond.eq + case "endif": + if len(conds) == 0 { + file.errorf(directive, "missing if before endif") + continue + } else if conds[len(conds)-1] == nil { + file.errorf(directive, "endif from unsupported contitional") + conds = conds[:len(conds)-1] + } else { + if cond == conds[len(conds)-1] { + cond = nil + } + conds = conds[:len(conds)-1] + } + default: + file.errorf(directive, "unsupported directive") + continue + } + } + } + + out, err := bpparser.Print(&bpparser.File{ + Defs: file.defs, + Comments: file.comments, + }) + if err != nil { + fmt.Println(err) + return + } + + fmt.Print(string(out)) +} + +func handleAssignment(file *bpFile, assignment mkparser.Assignment, c *conditional) { + if !assignment.Name.Const() { + file.errorf(assignment, "unsupported non-const variable name") + return + } + + if assignment.Target != nil { + file.errorf(assignment, "unsupported target assignment") + return + } + + name := assignment.Name.Value(nil) + suffix := "" + class := "" + + if strings.HasPrefix(name, "LOCAL_") { + for _, v := range propertySuffixes { + s, c := v.suffix, v.class + if strings.HasSuffix(name, "_"+s) { + name = strings.TrimSuffix(name, "_"+s) + suffix = s + if s, ok := propertySuffixTranslations[s]; ok { + suffix = s + } + class = c + break + } + } + + if c != nil { + if class != "" { + file.errorf(assignment, "suffix assignment inside conditional, skipping conditional") + } else { + if v, ok := conditionalTranslations[c.cond]; ok { + class = v.class + suffix = v.suffix + if !c.eq { + suffix = "not_" + suffix + } + } else { + panic("unknown conditional") + } + } + } + } else { + if c != nil { + eq := "eq" + if !c.eq { + eq = "neq" + } + file.errorf(assignment, "conditional %s %s on global assignment", eq, c.cond) + } + } + + var err error + if prop, ok := stringProperties[name]; ok { + err = setVariable(file, assignment.Value, assignment.Type == "+=", prop, bpparser.String, true, class, suffix) + } else if prop, ok := listProperties[name]; ok { + err = setVariable(file, assignment.Value, assignment.Type == "+=", prop, bpparser.List, true, class, suffix) + } else if prop, ok := boolProperties[name]; ok { + err = setVariable(file, assignment.Value, assignment.Type == "+=", prop, bpparser.Bool, true, class, suffix) + } else { + if name == "LOCAL_PATH" { + // Nothing to do, except maybe avoid the "./" in paths? + } else if strings.HasPrefix(name, "LOCAL_") { + //setVariable(file, assignment, name, bpparser.String, true) + switch name { + case "LOCAL_ADDITIONAL_DEPENDENCIES": + // TODO: check for only .mk files? + default: + file.errorf(assignment, "unsupported assignment to %s", name) + return + } + } else { + err = setVariable(file, assignment.Value, assignment.Type == "+=", name, bpparser.List, false, class, suffix) + } + } + if err != nil { + file.errorf(assignment, err.Error()) + } +} + +func handleModuleConditionals(file *bpFile, directive mkparser.Directive, c *conditional) { + if c == nil { + return + } + + if v, ok := conditionalTranslations[c.cond]; ok { + class := v.class + suffix := v.suffix + disabledSuffix := v.suffix + if !c.eq { + suffix = "not_" + suffix + } else { + disabledSuffix = "not_" + disabledSuffix + } + + // Hoist all properties inside the condtional up to the top level + file.module.Properties = file.localAssignments[class+"___"+suffix].Value.MapValue + file.module.Properties = append(file.module.Properties, file.localAssignments[class]) + file.localAssignments[class+"___"+suffix].Value.MapValue = nil + for i := range file.localAssignments[class].Value.MapValue { + if file.localAssignments[class].Value.MapValue[i].Name.Name == suffix { + file.localAssignments[class].Value.MapValue = + append(file.localAssignments[class].Value.MapValue[:i], + file.localAssignments[class].Value.MapValue[i+1:]...) + } + } + + // Create a fake assignment with enabled = false + err := setVariable(file, mkparser.SimpleMakeString("true", file.pos), false, + "disabled", bpparser.Bool, true, class, disabledSuffix) + if err != nil { + file.errorf(directive, err.Error()) + } + } else { + panic("unknown conditional") + } +} + +func makeModule(file *bpFile, t string) { + file.module.Type = bpparser.Ident{ + Name: t, + Pos: file.module.LbracePos, + } + file.module.RbracePos = file.pos + file.defs = append(file.defs, file.module) +} + +func resetModule(file *bpFile) { + file.module = &bpparser.Module{} + file.module.LbracePos = file.pos + file.localAssignments = make(map[string]*bpparser.Property) +} + +func setVariable(file *bpFile, val *mkparser.MakeString, plusequals bool, name string, + typ bpparser.ValueType, local bool, class string, suffix string) error { + + pos := file.pos + + var oldValue *bpparser.Value + if local { + var oldProp *bpparser.Property + if class != "" { + oldProp = file.localAssignments[name+"___"+class+"___"+suffix] + } else { + oldProp = file.localAssignments[name] + } + if oldProp != nil { + oldValue = &oldProp.Value + } + } else { + oldValue = file.globalAssignments[name] + } + + var exp *bpparser.Value + var err error + switch typ { + case bpparser.List: + exp, err = makeToListExpression(val) + case bpparser.String: + exp, err = makeToStringExpression(val) + case bpparser.Bool: + exp, err = makeToBoolExpression(val) + default: + panic("unknown type") + } + + if err != nil { + return err + } + + if local { + if oldValue != nil && plusequals { + val, err := addValues(oldValue, exp) + if err != nil { + return fmt.Errorf("unsupported addition: %s", err.Error()) + } + val.Expression.Pos = pos + *oldValue = *val + } else if class == "" { + prop := &bpparser.Property{ + Name: bpparser.Ident{Name: name, Pos: pos}, + Pos: pos, + Value: *exp, + } + file.localAssignments[name] = prop + file.module.Properties = append(file.module.Properties, prop) + } else { + classProp := file.localAssignments[class] + if classProp == nil { + classProp = &bpparser.Property{ + Name: bpparser.Ident{Name: class, Pos: pos}, + Pos: pos, + Value: bpparser.Value{ + Type: bpparser.Map, + MapValue: []*bpparser.Property{}, + }, + } + file.localAssignments[class] = classProp + file.module.Properties = append(file.module.Properties, classProp) + } + + suffixProp := file.localAssignments[class+"___"+suffix] + if suffixProp == nil { + suffixProp = &bpparser.Property{ + Name: bpparser.Ident{Name: suffix, Pos: pos}, + Pos: pos, + Value: bpparser.Value{ + Type: bpparser.Map, + MapValue: []*bpparser.Property{}, + }, + } + file.localAssignments[class+"___"+suffix] = suffixProp + classProp.Value.MapValue = append(classProp.Value.MapValue, suffixProp) + } + + prop := &bpparser.Property{ + Name: bpparser.Ident{Name: name, Pos: pos}, + Pos: pos, + Value: *exp, + } + file.localAssignments[class+"___"+suffix+"___"+name] = prop + suffixProp.Value.MapValue = append(suffixProp.Value.MapValue, prop) + } + } else { + if oldValue != nil && plusequals { + a := &bpparser.Assignment{ + Name: bpparser.Ident{ + Name: name, + Pos: pos, + }, + Value: *exp, + OrigValue: *exp, + Pos: pos, + Assigner: "+=", + } + file.defs = append(file.defs, a) + } else { + a := &bpparser.Assignment{ + Name: bpparser.Ident{ + Name: name, + Pos: pos, + }, + Value: *exp, + OrigValue: *exp, + Pos: pos, + Assigner: "=", + } + file.globalAssignments[name] = &a.Value + file.defs = append(file.defs, a) + } + } + + return nil +} diff --git a/androidmk/cmd/androidmk/values.go b/androidmk/cmd/androidmk/values.go new file mode 100644 index 00000000..2ba0829e --- /dev/null +++ b/androidmk/cmd/androidmk/values.go @@ -0,0 +1,192 @@ +package main + +import ( + "fmt" + "strings" + + mkparser "android/soong/androidmk/parser" + + bpparser "blueprint/parser" +) + +func stringToStringValue(s string) *bpparser.Value { + return &bpparser.Value{ + Type: bpparser.String, + StringValue: s, + } +} + +func addValues(val1, val2 *bpparser.Value) (*bpparser.Value, error) { + if val1.Type == bpparser.String && val2.Type == bpparser.List { + val1 = &bpparser.Value{ + Type: bpparser.List, + ListValue: []bpparser.Value{*val1}, + } + } else if val2.Type == bpparser.String && val1.Type == bpparser.List { + val2 = &bpparser.Value{ + Type: bpparser.List, + ListValue: []bpparser.Value{*val1}, + } + } else if val1.Type != val2.Type { + return nil, fmt.Errorf("cannot add mismatched types") + } + + return &bpparser.Value{ + Type: val1.Type, + Expression: &bpparser.Expression{ + Operator: '+', + Args: [2]bpparser.Value{*val1, *val2}, + }, + }, nil +} + +func makeToStringExpression(ms *mkparser.MakeString) (*bpparser.Value, error) { + var val *bpparser.Value + var err error + + if ms.Strings[0] != "" { + val = stringToStringValue(ms.Strings[0]) + } + + for i, s := range ms.Strings[1:] { + name := ms.Variables[i].Name + if !name.Const() { + return nil, fmt.Errorf("Unsupported non-const variable name %s", name.Dump()) + } + tmp := &bpparser.Value{ + Type: bpparser.String, + Variable: name.Value(nil), + } + + if val != nil { + val, err = addValues(val, tmp) + if err != nil { + return nil, err + } + } else { + val = tmp + } + + if s != "" { + tmp := stringToStringValue(s) + val, err = addValues(val, tmp) + if err != nil { + return nil, err + } + } + } + + return val, nil +} + +func stringToListValue(s string) *bpparser.Value { + list := strings.Fields(s) + valList := make([]bpparser.Value, len(list)) + for i, l := range list { + valList[i] = bpparser.Value{ + Type: bpparser.String, + StringValue: l, + } + } + return &bpparser.Value{ + Type: bpparser.List, + ListValue: valList, + } + +} + +func makeToListExpression(ms *mkparser.MakeString) (*bpparser.Value, error) { + fields := ms.Split(" \t") + + var listOfListValues []*bpparser.Value + + listValue := &bpparser.Value{ + Type: bpparser.List, + } + + for _, f := range fields { + if len(f.Variables) == 1 && f.Strings[0] == "" && f.Strings[1] == "" { + // Variable by itself, variable is probably a list + if !f.Variables[0].Name.Const() { + return nil, fmt.Errorf("unsupported non-const variable name") + } + if len(listValue.ListValue) > 0 { + listOfListValues = append(listOfListValues, listValue) + } + listOfListValues = append(listOfListValues, &bpparser.Value{ + Type: bpparser.List, + Variable: f.Variables[0].Name.Value(nil), + }) + listValue = &bpparser.Value{ + Type: bpparser.List, + } + } else { + s, err := makeToStringExpression(f) + if err != nil { + return nil, err + } + if s == nil { + continue + } + + listValue.ListValue = append(listValue.ListValue, *s) + } + } + + if len(listValue.ListValue) > 0 { + listOfListValues = append(listOfListValues, listValue) + } + + if len(listOfListValues) == 0 { + return listValue, nil + } + + val := listOfListValues[0] + for _, tmp := range listOfListValues[1:] { + var err error + val, err = addValues(val, tmp) + if err != nil { + return nil, err + } + } + + return val, nil +} + +func stringToBoolValue(s string) (*bpparser.Value, error) { + var b bool + s = strings.TrimSpace(s) + switch s { + case "true": + b = true + case "false", "": + b = false + case "-frtti": // HACK for LOCAL_RTTI_VALUE + b = true + default: + return nil, fmt.Errorf("unexpected bool value %s", s) + } + return &bpparser.Value{ + Type: bpparser.Bool, + BoolValue: b, + }, nil +} + +func makeToBoolExpression(ms *mkparser.MakeString) (*bpparser.Value, error) { + if !ms.Const() { + if len(ms.Variables) == 1 && ms.Strings[0] == "" && ms.Strings[1] == "" { + name := ms.Variables[0].Name + if !name.Const() { + return nil, fmt.Errorf("unsupported non-const variable name") + } + return &bpparser.Value{ + Type: bpparser.Bool, + Variable: name.Value(nil), + }, nil + } else { + return nil, fmt.Errorf("non-const bool expression %s", ms.Dump()) + } + } + + return stringToBoolValue(ms.Value(nil)) +} diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go new file mode 100644 index 00000000..558853f6 --- /dev/null +++ b/androidmk/parser/make_strings.go @@ -0,0 +1,170 @@ +package parser + +import ( + "strings" + "text/scanner" + "unicode" +) + +// A MakeString is a string that may contain variable substitutions in it. +// It can be considered as an alternating list of raw Strings and variable +// substitutions, where the first and last entries in the list must be raw +// Strings (possibly empty). A MakeString that starts with a variable +// will have an empty first raw string, and a MakeString that ends with a +// variable will have an empty last raw string. Two sequential Variables +// will have an empty raw string between them. +// +// The MakeString is stored as two lists, a list of raw Strings and a list +// of Variables. The raw string list is always one longer than the variable +// list. +type MakeString struct { + Pos scanner.Position + Strings []string + Variables []Variable +} + +func SimpleMakeString(s string, pos scanner.Position) *MakeString { + return &MakeString{ + Pos: pos, + Strings: []string{s}, + } +} + +func (ms *MakeString) appendString(s string) { + if len(ms.Strings) == 0 { + ms.Strings = []string{s} + return + } else { + ms.Strings[len(ms.Strings)-1] += s + } +} + +func (ms *MakeString) appendVariable(v Variable) { + if len(ms.Strings) == 0 { + ms.Strings = []string{"", ""} + ms.Variables = []Variable{v} + } else { + ms.Strings = append(ms.Strings, "") + ms.Variables = append(ms.Variables, v) + } +} + +func (ms *MakeString) appendMakeString(other *MakeString) { + last := len(ms.Strings) - 1 + ms.Strings[last] += other.Strings[0] + ms.Strings = append(ms.Strings, other.Strings[1:]...) + ms.Variables = append(ms.Variables, other.Variables...) +} + +func (ms *MakeString) Value(scope Scope) string { + if len(ms.Strings) == 0 { + return "" + } else { + ret := ms.Strings[0] + for i := range ms.Strings[1:] { + ret += ms.Variables[i].Value(scope) + ret += ms.Strings[i+1] + } + return ret + } +} + +func (ms *MakeString) Dump() string { + if len(ms.Strings) == 0 { + return "" + } else { + ret := ms.Strings[0] + for i := range ms.Strings[1:] { + ret += ms.Variables[i].Dump() + ret += ms.Strings[i+1] + } + return ret + } +} + +func (ms *MakeString) Const() bool { + return len(ms.Strings) <= 1 +} + +func (ms *MakeString) Empty() bool { + return len(ms.Strings) == 0 || (len(ms.Strings) == 1 && ms.Strings[0] == "") +} + +func (ms *MakeString) Split(sep string) []*MakeString { + return ms.SplitN(sep, -1) +} + +func (ms *MakeString) SplitN(sep string, n int) []*MakeString { + ret := []*MakeString{} + + curMs := SimpleMakeString("", ms.Pos) + + var i int + var s string + for i, s = range ms.Strings { + if n != 0 { + split := splitAnyN(s, sep, n) + if n != -1 { + if len(split) > n { + panic("oops!") + } else { + n -= len(split) + } + } + curMs.appendString(split[0]) + + for _, r := range split[1:] { + ret = append(ret, curMs) + curMs = SimpleMakeString(r, ms.Pos) + } + } else { + curMs.appendString(s) + } + + if i < len(ms.Strings)-1 { + curMs.appendVariable(ms.Variables[i]) + } + } + + ret = append(ret, curMs) + return ret +} + +func (ms *MakeString) TrimLeftSpaces() { + ms.Strings[0] = strings.TrimLeftFunc(ms.Strings[0], unicode.IsSpace) +} + +func (ms *MakeString) TrimRightSpaces() { + last := len(ms.Strings) - 1 + ms.Strings[last] = strings.TrimRightFunc(ms.Strings[last], unicode.IsSpace) +} + +func (ms *MakeString) TrimRightOne() { + last := len(ms.Strings) - 1 + if len(ms.Strings[last]) > 1 { + ms.Strings[last] = ms.Strings[last][0 : len(ms.Strings[last])-1] + } +} + +func (ms *MakeString) EndsWith(ch rune) bool { + s := ms.Strings[len(ms.Strings)-1] + return s[len(s)-1] == uint8(ch) +} + +func splitAnyN(s, sep string, n int) []string { + ret := []string{} + for n == -1 || n > 1 { + index := strings.IndexAny(s, sep) + if index >= 0 { + ret = append(ret, s[0:index]) + s = s[index+1:] + if n > 0 { + n-- + } + } else { + break + } + } + ret = append(ret, s) + return ret +} diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go new file mode 100644 index 00000000..db5840c6 --- /dev/null +++ b/androidmk/parser/make_strings_test.go @@ -0,0 +1,96 @@ +package parser + +import ( + "strings" + "testing" +) + +var splitNTestCases = []struct { + in *MakeString + expected []*MakeString + sep string + n int +}{ + { + in: &MakeString{ + strings: []string{ + "a b c", + "d e f", + " h i j", + }, + variables: []Variable{ + variable{name: SimpleMakeString("var1")}, + variable{name: SimpleMakeString("var2")}, + }, + }, + sep: " ", + n: -1, + expected: []*MakeString{ + SimpleMakeString("a"), + SimpleMakeString("b"), + &MakeString{ + strings: []string{"c", "d"}, + variables: []Variable{ + variable{name: SimpleMakeString("var1")}, + }, + }, + SimpleMakeString("e"), + &MakeString{ + strings: []string{"f", ""}, + variables: []Variable{ + variable{name: SimpleMakeString("var2")}, + }, + }, + SimpleMakeString("h"), + SimpleMakeString("i"), + SimpleMakeString("j"), + }, + }, + { + in: &MakeString{ + strings: []string{ + "a b c", + "d e f", + " h i j", + }, + variables: []Variable{ + variable{name: SimpleMakeString("var1")}, + variable{name: SimpleMakeString("var2")}, + }, + }, + sep: " ", + n: 3, + expected: []*MakeString{ + SimpleMakeString("a"), + SimpleMakeString("b"), + &MakeString{ + strings: []string{"c", "d e f", " h i j"}, + variables: []Variable{ + variable{name: SimpleMakeString("var1")}, + variable{name: SimpleMakeString("var2")}, + }, + }, + }, + }, +} + +func TestMakeStringSplitN(t *testing.T) { + for _, test := range splitNTestCases { + got := test.in.SplitN(test.sep, test.n) + gotString := dumpArray(got) + expectedString := dumpArray(test.expected) + if gotString != expectedString { + t.Errorf("expected:\n%s\ngot:\n%s", expectedString, gotString) + } + } +} + +func dumpArray(a []*MakeString) string { + ret := make([]string, len(a)) + + for i, s := range a { + ret[i] = s.Dump() + } + + return strings.Join(ret, "|||") +} diff --git a/androidmk/parser/makething.go b/androidmk/parser/makething.go new file mode 100644 index 00000000..7d60a779 --- /dev/null +++ b/androidmk/parser/makething.go @@ -0,0 +1,142 @@ +package parser + +import ( + "text/scanner" +) + +type MakeThing interface { + AsAssignment() (Assignment, bool) + AsComment() (Comment, bool) + AsDirective() (Directive, bool) + AsRule() (Rule, bool) + AsVariable() (Variable, bool) + Dump() string + Pos() scanner.Position + EndPos() scanner.Position +} + +type Assignment struct { + makeThing + Name *MakeString + Value *MakeString + Target *MakeString + Type string +} + +type Comment struct { + makeThing + Comment string +} + +type Directive struct { + makeThing + Name string + Args *MakeString +} + +type Rule struct { + makeThing + Target *MakeString + Prerequisites *MakeString + Recipe string +} + +type Variable struct { + makeThing + Name *MakeString +} + +type makeThing struct { + pos scanner.Position + endPos scanner.Position +} + +func (m makeThing) Pos() scanner.Position { + return m.pos +} + +func (m makeThing) EndPos() scanner.Position { + return m.endPos +} + +func (makeThing) AsAssignment() (a Assignment, ok bool) { + return +} + +func (a Assignment) AsAssignment() (Assignment, bool) { + return a, true +} + +func (a Assignment) Dump() string { + target := "" + if a.Target != nil { + target = a.Target.Dump() + ": " + } + return target + a.Name.Dump() + a.Type + a.Value.Dump() +} + +func (makeThing) AsComment() (c Comment, ok bool) { + return +} + +func (c Comment) AsComment() (Comment, bool) { + return c, true +} + +func (c Comment) Dump() string { + return "#" + c.Comment +} + +func (makeThing) AsDirective() (d Directive, ok bool) { + return +} + +func (d Directive) AsDirective() (Directive, bool) { + return d, true +} + +func (d Directive) Dump() string { + return d.Name + " " + d.Args.Dump() +} + +func (makeThing) AsRule() (r Rule, ok bool) { + return +} + +func (r Rule) AsRule() (Rule, bool) { + return r, true +} + +func (r Rule) Dump() string { + recipe := "" + if r.Recipe != "" { + recipe = "\n" + r.Recipe + } + return "rule: " + r.Target.Dump() + ": " + r.Prerequisites.Dump() + recipe +} + +func (makeThing) AsVariable() (v Variable, ok bool) { + return +} + +func (v Variable) AsVariable() (Variable, bool) { + return v, true +} + +func (v Variable) Dump() string { + return "$(" + v.Name.Dump() + ")" +} + +type byPosition []MakeThing + +func (s byPosition) Len() int { + return len(s) +} + +func (s byPosition) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s byPosition) Less(i, j int) bool { + return s[i].Pos().Offset < s[j].Pos().Offset +} diff --git a/androidmk/parser/parser.go b/androidmk/parser/parser.go new file mode 100644 index 00000000..58e612ea --- /dev/null +++ b/androidmk/parser/parser.go @@ -0,0 +1,633 @@ +package parser + +import ( + "errors" + "fmt" + "io" + "sort" + "text/scanner" +) + +var errTooManyErrors = errors.New("too many errors") + +const maxErrors = 100 + +type ParseError struct { + Err error + Pos scanner.Position +} + +func (e *ParseError) Error() string { + return fmt.Sprintf("%s: %s", e.Pos, e.Err) +} + +func (p *parser) Parse() ([]MakeThing, []error) { + defer func() { + if r := recover(); r != nil { + if r == errTooManyErrors { + return + } + panic(r) + } + }() + + p.parseLines() + p.accept(scanner.EOF) + p.things = append(p.things, p.comments...) + sort.Sort(byPosition(p.things)) + + return p.things, p.errors +} + +type parser struct { + scanner scanner.Scanner + tok rune + errors []error + comments []MakeThing + things []MakeThing +} + +func NewParser(filename string, r io.Reader) *parser { + p := &parser{} + p.scanner.Init(r) + p.scanner.Error = func(sc *scanner.Scanner, msg string) { + p.errorf(msg) + } + p.scanner.Whitespace = 0 + p.scanner.IsIdentRune = func(ch rune, i int) bool { + return ch > 0 && ch != ':' && ch != '#' && ch != '=' && ch != '+' && ch != '$' && + ch != '\\' && ch != '(' && ch != ')' && ch != '{' && ch != '}' && ch != ';' && + ch != '|' && ch != '?' && ch != '\r' && !isWhitespace(ch) + } + p.scanner.Mode = scanner.ScanIdents + p.scanner.Filename = filename + p.next() + return p +} + +func (p *parser) errorf(format string, args ...interface{}) { + pos := p.scanner.Position + if !pos.IsValid() { + pos = p.scanner.Pos() + } + err := &ParseError{ + Err: fmt.Errorf(format, args...), + Pos: pos, + } + p.errors = append(p.errors, err) + if len(p.errors) >= maxErrors { + panic(errTooManyErrors) + } +} + +func (p *parser) accept(toks ...rune) bool { + for _, tok := range toks { + if p.tok != tok { + p.errorf("expected %s, found %s", scanner.TokenString(tok), + scanner.TokenString(p.tok)) + return false + } + p.next() + } + return true +} + +func (p *parser) next() { + if p.tok != scanner.EOF { + p.tok = p.scanner.Scan() + for p.tok == '\r' { + p.tok = p.scanner.Scan() + } + } + return +} + +func (p *parser) parseLines() { + for { + p.ignoreWhitespace() + + if p.parseDirective() { + continue + } + + ident, _ := p.parseExpression('=', '?', ':', '#', '\n') + + p.ignoreSpaces() + + switch p.tok { + case '?': + p.accept('?') + if p.tok == '=' { + p.parseAssignment("?=", nil, ident) + } else { + p.errorf("expected = after ?") + } + case '+': + p.accept('+') + if p.tok == '=' { + p.parseAssignment("+=", nil, ident) + } else { + p.errorf("expected = after +") + } + case ':': + p.accept(':') + switch p.tok { + case '=': + p.parseAssignment(":=", nil, ident) + default: + p.parseRule(ident) + } + case '=': + p.parseAssignment("=", nil, ident) + case '#', '\n', scanner.EOF: + ident.TrimRightSpaces() + if v, ok := toVariable(ident); ok { + p.things = append(p.things, v) + } else if !ident.Empty() { + p.errorf("expected directive, rule, or assignment after ident " + ident.Dump()) + } + switch p.tok { + case scanner.EOF: + return + case '\n': + p.accept('\n') + case '#': + p.parseComment() + } + default: + p.errorf("expected assignment or rule definition, found %s\n", + p.scanner.TokenText()) + return + } + } +} + +func (p *parser) parseDirective() bool { + if p.tok != scanner.Ident || !isDirective(p.scanner.TokenText()) { + return false + } + + d := p.scanner.TokenText() + pos := p.scanner.Position + endPos := pos + p.accept(scanner.Ident) + + expression := SimpleMakeString("", pos) + + switch d { + case "endif", "endef", "else": + // Nothing + case "define": + expression = p.parseDefine() + default: + p.ignoreSpaces() + expression, endPos = p.parseExpression() + } + + p.things = append(p.things, Directive{ + makeThing: makeThing{ + pos: pos, + endPos: endPos, + }, + Name: d, + Args: expression, + }) + return true +} + +func (p *parser) parseDefine() *MakeString { + value := SimpleMakeString("", p.scanner.Position) + +loop: + for { + switch p.tok { + case scanner.Ident: + if p.scanner.TokenText() == "endef" { + p.accept(scanner.Ident) + break loop + } + value.appendString(p.scanner.TokenText()) + p.accept(scanner.Ident) + case '\\': + p.parseEscape() + switch p.tok { + case '\n': + value.appendString(" ") + case scanner.EOF: + p.errorf("expected escaped character, found %s", + scanner.TokenString(p.tok)) + break loop + default: + value.appendString(`\` + string(p.tok)) + } + p.accept(p.tok) + //TODO: handle variables inside defines? result depends if + //define is used in make or rule context + //case '$': + // variable := p.parseVariable() + // value.appendVariable(variable) + case scanner.EOF: + p.errorf("unexpected EOF while looking for endef") + break loop + default: + value.appendString(p.scanner.TokenText()) + p.accept(p.tok) + } + } + + return value +} + +func (p *parser) parseEscape() { + p.scanner.Mode = 0 + p.accept('\\') + p.scanner.Mode = scanner.ScanIdents +} + +func (p *parser) parseExpression(end ...rune) (*MakeString, scanner.Position) { + value := SimpleMakeString("", p.scanner.Position) + + endParen := false + for _, r := range end { + if r == ')' { + endParen = true + } + } + parens := 0 + + endPos := p.scanner.Position + +loop: + for { + if endParen && parens > 0 && p.tok == ')' { + parens-- + value.appendString(")") + endPos = p.scanner.Position + p.accept(')') + continue + } + + for _, r := range end { + if p.tok == r { + break loop + } + } + + switch p.tok { + case '\n': + break loop + case scanner.Ident: + value.appendString(p.scanner.TokenText()) + endPos = p.scanner.Position + p.accept(scanner.Ident) + case '\\': + p.parseEscape() + switch p.tok { + case '\n': + value.appendString(" ") + case scanner.EOF: + p.errorf("expected escaped character, found %s", + scanner.TokenString(p.tok)) + return value, endPos + default: + value.appendString(`\` + string(p.tok)) + } + endPos = p.scanner.Position + p.accept(p.tok) + case '#': + p.parseComment() + break loop + case '$': + var variable Variable + variable, endPos = p.parseVariable() + value.appendVariable(variable) + case scanner.EOF: + break loop + case '(': + if endParen { + parens++ + } + value.appendString("(") + endPos = p.scanner.Position + p.accept('(') + default: + value.appendString(p.scanner.TokenText()) + endPos = p.scanner.Position + p.accept(p.tok) + } + } + + if parens > 0 { + p.errorf("expected closing paren %s", value.Dump()) + } + return value, endPos +} + +func (p *parser) parseVariable() (Variable, scanner.Position) { + pos := p.scanner.Position + endPos := pos + p.accept('$') + var name *MakeString + switch p.tok { + case '(': + return p.parseBracketedVariable('(', ')', pos) + case '{': + return p.parseBracketedVariable('{', '}', pos) + case '$': + name = SimpleMakeString("__builtin_dollar", scanner.Position{}) + case scanner.EOF: + p.errorf("expected variable name, found %s", + scanner.TokenString(p.tok)) + default: + name, endPos = p.parseExpression(variableNameEndRunes...) + } + + return p.nameToVariable(name, pos, endPos), endPos +} + +func (p *parser) parseBracketedVariable(start, end rune, pos scanner.Position) (Variable, scanner.Position) { + p.accept(start) + name, endPos := p.parseExpression(end) + p.accept(end) + return p.nameToVariable(name, pos, endPos), endPos +} + +func (p *parser) nameToVariable(name *MakeString, pos, endPos scanner.Position) Variable { + return Variable{ + makeThing: makeThing{ + pos: pos, + endPos: endPos, + }, + Name: name, + } +} + +func (p *parser) parseRule(target *MakeString) { + prerequisites, newLine := p.parseRulePrerequisites(target) + + recipe := "" + endPos := p.scanner.Position +loop: + for { + if newLine { + if p.tok == '\t' { + endPos = p.scanner.Position + p.accept('\t') + newLine = false + continue loop + } else if p.parseDirective() { + newLine = false + continue + } else { + break loop + } + } + + newLine = false + switch p.tok { + case '\\': + p.parseEscape() + recipe += string(p.tok) + endPos = p.scanner.Position + p.accept(p.tok) + case '\n': + newLine = true + recipe += "\n" + endPos = p.scanner.Position + p.accept('\n') + case scanner.EOF: + break loop + default: + recipe += p.scanner.TokenText() + endPos = p.scanner.Position + p.accept(p.tok) + } + } + + if prerequisites != nil { + p.things = append(p.things, Rule{ + makeThing: makeThing{ + pos: target.Pos, + endPos: endPos, + }, + Target: target, + Prerequisites: prerequisites, + Recipe: recipe, + }) + } +} + +func (p *parser) parseRulePrerequisites(target *MakeString) (*MakeString, bool) { + newLine := false + + p.ignoreSpaces() + + prerequisites, _ := p.parseExpression('#', '\n', ';', ':', '=') + + switch p.tok { + case '\n': + p.accept('\n') + newLine = true + case '#': + p.parseComment() + newLine = true + case ';': + p.accept(';') + case ':': + p.accept(':') + if p.tok == '=' { + p.parseAssignment(":=", target, prerequisites) + return nil, true + } else { + more, _ := p.parseExpression('#', '\n', ';') + prerequisites.appendMakeString(more) + } + case '=': + p.parseAssignment("=", target, prerequisites) + return nil, true + default: + p.errorf("unexpected token %s after rule prerequisites", scanner.TokenString(p.tok)) + } + + return prerequisites, newLine +} + +func (p *parser) parseComment() { + pos := p.scanner.Position + p.accept('#') + comment := "" + endPos := pos +loop: + for { + switch p.tok { + case '\\': + p.parseEscape() + if p.tok == '\n' { + comment += "\n" + } else { + comment += "\\" + p.scanner.TokenText() + } + endPos = p.scanner.Position + p.accept(p.tok) + case '\n': + endPos = p.scanner.Position + p.accept('\n') + break loop + case scanner.EOF: + break loop + default: + comment += p.scanner.TokenText() + endPos = p.scanner.Position + p.accept(p.tok) + } + } + + p.comments = append(p.comments, Comment{ + makeThing: makeThing{ + pos: pos, + endPos: endPos, + }, + Comment: comment, + }) +} + +func (p *parser) parseAssignment(t string, target *MakeString, ident *MakeString) { + // The value of an assignment is everything including and after the first + // non-whitespace character after the = until the end of the logical line, + // which may included escaped newlines + p.accept('=') + value, endPos := p.parseExpression() + value.TrimLeftSpaces() + if ident.EndsWith('+') && t == "=" { + ident.TrimRightOne() + t = "+=" + } + + ident.TrimRightSpaces() + + p.things = append(p.things, Assignment{ + makeThing: makeThing{ + pos: ident.Pos, + endPos: endPos, + }, + Name: ident, + Value: value, + Target: target, + Type: t, + }) +} + +type androidMkModule struct { + assignments map[string]string +} + +type androidMkFile struct { + assignments map[string]string + modules []androidMkModule + includes []string +} + +var directives = [...]string{ + "define", + "else", + "endef", + "endif", + "ifdef", + "ifeq", + "ifndef", + "ifneq", + "include", + "-include", +} + +var functions = [...]string{ + "abspath", + "addprefix", + "addsuffix", + "basename", + "dir", + "notdir", + "subst", + "suffix", + "filter", + "filter-out", + "findstring", + "firstword", + "flavor", + "join", + "lastword", + "patsubst", + "realpath", + "shell", + "sort", + "strip", + "wildcard", + "word", + "wordlist", + "words", + "origin", + "foreach", + "call", + "info", + "error", + "warning", + "if", + "or", + "and", + "value", + "eval", + "file", +} + +func init() { + sort.Strings(directives[:]) + sort.Strings(functions[:]) +} + +func isDirective(s string) bool { + for _, d := range directives { + if s == d { + return true + } else if s < d { + return false + } + } + return false +} + +func isFunctionName(s string) bool { + for _, f := range functions { + if s == f { + return true + } else if s < f { + return false + } + } + return false +} + +func isWhitespace(ch rune) bool { + return ch == ' ' || ch == '\t' || ch == '\n' +} + +func isValidVariableRune(ch rune) bool { + return ch != scanner.Ident && ch != ':' && ch != '=' && ch != '#' +} + +var whitespaceRunes = []rune{' ', '\t', '\n'} +var variableNameEndRunes = append([]rune{':', '=', '#', ')', '}'}, whitespaceRunes...) + +func (p *parser) ignoreSpaces() int { + skipped := 0 + for p.tok == ' ' || p.tok == '\t' { + p.accept(p.tok) + skipped++ + } + return skipped +} + +func (p *parser) ignoreWhitespace() { + for isWhitespace(p.tok) { + p.accept(p.tok) + } +} diff --git a/androidmk/parser/scope.go b/androidmk/parser/scope.go new file mode 100644 index 00000000..742ad388 --- /dev/null +++ b/androidmk/parser/scope.go @@ -0,0 +1,88 @@ +package parser + +import "strings" + +type Scope interface { + Get(name string) string + Set(name, value string) + Call(name string, args []string) string + SetFunc(name string, f func([]string) string) +} + +type scope struct { + variables map[string]string + functions map[string]func([]string) string + parent Scope +} + +func (s *scope) Get(name string) string { + if val, ok := s.variables[name]; ok { + return val + } else if s.parent != nil { + return s.parent.Get(name) + } else if val, ok := builtinScope[name]; ok { + return val + } else { + return "<'" + name + "' unset>" + } +} + +func (s *scope) Set(name, value string) { + s.variables[name] = value +} + +func (s *scope) Call(name string, args []string) string { + if f, ok := s.functions[name]; ok { + return f(args) + } + + return "<func:'" + name + "' unset>" +} + +func (s *scope) SetFunc(name string, f func([]string) string) { + s.functions[name] = f +} + +func NewScope(parent Scope) Scope { + return &scope{ + variables: make(map[string]string), + functions: make(map[string]func([]string) string), + parent: parent, + } +} + +var builtinScope map[string]string + +func init() { + builtinScope := make(map[string]string) + builtinScope["__builtin_dollar"] = "$" +} + +func (v Variable) Value(scope Scope) string { + f := v.Name.SplitN(" \t", 2) + if len(f) > 1 && f[0].Const() { + fname := f[0].Value(nil) + if isFunctionName(fname) { + args := f[1].Split(",") + argVals := make([]string, len(args)) + for i, a := range args { + argVals[i] = a.Value(scope) + } + + if fname == "call" { + return scope.Call(argVals[0], argVals[1:]) + } else { + return "__builtin_func:" + fname + " " + strings.Join(argVals, " ") + } + } + } + + return scope.Get(v.Name.Value(scope)) +} + +func toVariable(ms *MakeString) (Variable, bool) { + if len(ms.Variables) == 1 && ms.Strings[0] == "" && ms.Strings[1] == "" { + return ms.Variables[0], true + } + return Variable{}, false +} |