diff options
82 files changed, 5112 insertions, 1067 deletions
@@ -38,12 +38,14 @@ bootstrap_go_package { "soong", "soong-env", "soong-shared", + "soong-lineage", ], srcs: [ "android/androidmk.go", "android/apex.go", "android/api_levels.go", "android/arch.go", + "android/bootjar.go", "android/config.go", "android/defaults.go", "android/defs.go", diff --git a/android/androidmk.go b/android/androidmk.go index bd49e4c6..4a968f5e 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -32,6 +32,8 @@ func init() { RegisterSingletonType("androidmk", AndroidMkSingleton) } +// Deprecated: consider using AndroidMkEntriesProvider instead, especially if you're not going to +// use the Custom function. type AndroidMkDataProvider interface { AndroidMk() AndroidMkData BaseModuleName() string @@ -55,6 +57,195 @@ type AndroidMkData struct { type AndroidMkExtraFunc func(w io.Writer, outputFile Path) +// Allows modules to customize their Android*.mk output. +type AndroidMkEntriesProvider interface { + AndroidMkEntries() AndroidMkEntries + BaseModuleName() string +} + +type AndroidMkEntries struct { + Class string + SubName string + DistFile OptionalPath + OutputFile OptionalPath + Disabled bool + Include string + Required []string + + header bytes.Buffer + footer bytes.Buffer + + ExtraEntries []AndroidMkExtraEntriesFunc + + EntryMap map[string][]string + entryOrder []string +} + +type AndroidMkExtraEntriesFunc func(entries *AndroidMkEntries) + +func (a *AndroidMkEntries) SetString(name, value string) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = []string{value} +} + +func (a *AndroidMkEntries) SetBoolIfTrue(name string, flag bool) { + if flag { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = []string{"true"} + } +} + +func (a *AndroidMkEntries) AddStrings(name string, value ...string) { + if len(value) == 0 { + return + } + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = append(a.EntryMap[name], value...) +} + +func (a *AndroidMkEntries) fillInEntries(config Config, bpPath string, mod blueprint.Module) { + a.EntryMap = make(map[string][]string) + amod := mod.(Module).base() + name := amod.BaseModuleName() + + if a.Include == "" { + a.Include = "$(BUILD_PREBUILT)" + } + a.Required = append(a.Required, amod.commonProperties.Required...) + + // Fill in the header part. + if len(amod.commonProperties.Dist.Targets) > 0 { + distFile := a.DistFile + if !distFile.Valid() { + distFile = a.OutputFile + } + if distFile.Valid() { + dest := filepath.Base(distFile.String()) + + if amod.commonProperties.Dist.Dest != nil { + var err error + if dest, err = validateSafePath(*amod.commonProperties.Dist.Dest); err != nil { + // This was checked in ModuleBase.GenerateBuildActions + panic(err) + } + } + + if amod.commonProperties.Dist.Suffix != nil { + ext := filepath.Ext(dest) + suffix := *amod.commonProperties.Dist.Suffix + dest = strings.TrimSuffix(dest, ext) + suffix + ext + } + + if amod.commonProperties.Dist.Dir != nil { + var err error + if dest, err = validateSafePath(*amod.commonProperties.Dist.Dir, dest); err != nil { + // This was checked in ModuleBase.GenerateBuildActions + panic(err) + } + } + + goals := strings.Join(amod.commonProperties.Dist.Targets, " ") + fmt.Fprintln(&a.header, ".PHONY:", goals) + fmt.Fprintf(&a.header, "$(call dist-for-goals,%s,%s:%s)\n", + goals, distFile.String(), dest) + } + } + + fmt.Fprintln(&a.header, "\ninclude $(CLEAR_VARS)") + + // Collect make variable assignment entries. + a.SetString("LOCAL_PATH", filepath.Dir(bpPath)) + a.SetString("LOCAL_MODULE", name+a.SubName) + a.SetString("LOCAL_MODULE_CLASS", a.Class) + a.SetString("LOCAL_PREBUILT_MODULE_FILE", a.OutputFile.String()) + a.AddStrings("LOCAL_REQUIRED_MODULES", a.Required...) + + archStr := amod.Arch().ArchType.String() + host := false + switch amod.Os().Class { + case Host: + // Make cannot identify LOCAL_MODULE_HOST_ARCH:= common. + if archStr != "common" { + a.SetString("LOCAL_MODULE_HOST_ARCH", archStr) + } + host = true + case HostCross: + // Make cannot identify LOCAL_MODULE_HOST_CROSS_ARCH:= common. + if archStr != "common" { + a.SetString("LOCAL_MODULE_HOST_CROSS_ARCH", archStr) + } + host = true + case Device: + // Make cannot identify LOCAL_MODULE_TARGET_ARCH:= common. + if archStr != "common" { + a.SetString("LOCAL_MODULE_TARGET_ARCH", archStr) + } + + a.AddStrings("LOCAL_INIT_RC", amod.commonProperties.Init_rc...) + a.AddStrings("LOCAL_VINTF_FRAGMENTS", amod.commonProperties.Vintf_fragments...) + a.SetBoolIfTrue("LOCAL_PROPRIETARY_MODULE", Bool(amod.commonProperties.Proprietary)) + if Bool(amod.commonProperties.Vendor) || Bool(amod.commonProperties.Soc_specific) { + a.SetString("LOCAL_VENDOR_MODULE", "true") + } + a.SetBoolIfTrue("LOCAL_ODM_MODULE", Bool(amod.commonProperties.Device_specific)) + a.SetBoolIfTrue("LOCAL_PRODUCT_MODULE", Bool(amod.commonProperties.Product_specific)) + a.SetBoolIfTrue("LOCAL_PRODUCT_SERVICES_MODULE", Bool(amod.commonProperties.Product_services_specific)) + if amod.commonProperties.Owner != nil { + a.SetString("LOCAL_MODULE_OWNER", *amod.commonProperties.Owner) + } + } + + if amod.noticeFile.Valid() { + a.SetString("LOCAL_NOTICE_FILE", amod.noticeFile.String()) + } + + if host { + makeOs := amod.Os().String() + if amod.Os() == Linux || amod.Os() == LinuxBionic { + makeOs = "linux" + } + a.SetString("LOCAL_MODULE_HOST_OS", makeOs) + a.SetString("LOCAL_IS_HOST_MODULE", "true") + } + + prefix := "" + if amod.ArchSpecific() { + switch amod.Os().Class { + case Host: + prefix = "HOST_" + case HostCross: + prefix = "HOST_CROSS_" + case Device: + prefix = "TARGET_" + + } + + if amod.Arch().ArchType != config.Targets[amod.Os()][0].Arch.ArchType { + prefix = "2ND_" + prefix + } + } + for _, extra := range a.ExtraEntries { + extra(a) + } + + // Write to footer. + fmt.Fprintln(&a.footer, "include "+a.Include) +} + +func (a *AndroidMkEntries) write(w io.Writer) { + w.Write(a.header.Bytes()) + for _, name := range a.entryOrder { + fmt.Fprintln(w, name+" := "+strings.Join(a.EntryMap[name], " ")) + } + w.Write(a.footer.Bytes()) +} + func AndroidMkSingleton() Singleton { return &androidMkSingleton{} } @@ -157,6 +348,8 @@ func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.M return translateAndroidModule(ctx, w, mod, x) case bootstrap.GoBinaryTool: return translateGoBinaryModule(ctx, w, mod, x) + case AndroidMkEntriesProvider: + return translateAndroidMkEntriesModule(ctx, w, mod, x) default: return nil } @@ -176,35 +369,30 @@ func translateGoBinaryModule(ctx SingletonContext, w io.Writer, mod blueprint.Mo func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, provider AndroidMkDataProvider) error { - name := provider.BaseModuleName() amod := mod.(Module).base() - - if !amod.Enabled() { - return nil - } - - if amod.commonProperties.SkipInstall { - return nil - } - - if !amod.commonProperties.NamespaceExportedToMake { - // TODO(jeffrygaston) do we want to validate that there are no modules being - // exported to Kati that depend on this module? + if shouldSkipAndroidMkProcessing(amod) { return nil } data := provider.AndroidMk() - if data.Include == "" { data.Include = "$(BUILD_PREBUILT)" } - data.Required = append(data.Required, amod.commonProperties.Required...) - - // Make does not understand LinuxBionic - if amod.Os() == LinuxBionic { - return nil + // Get the preamble content through AndroidMkEntries logic. + entries := AndroidMkEntries{ + Class: data.Class, + SubName: data.SubName, + DistFile: data.DistFile, + OutputFile: data.OutputFile, + Disabled: data.Disabled, + Include: data.Include, + Required: data.Required, } + entries.fillInEntries(ctx.Config(), ctx.BlueprintFile(mod), mod) + // preamble doesn't need the footer content. + entries.footer = bytes.Buffer{} + entries.write(&data.preamble) prefix := "" if amod.ArchSpecific() { @@ -223,115 +411,7 @@ func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Mod } } - if len(amod.commonProperties.Dist.Targets) > 0 { - distFile := data.DistFile - if !distFile.Valid() { - distFile = data.OutputFile - } - if distFile.Valid() { - dest := filepath.Base(distFile.String()) - - if amod.commonProperties.Dist.Dest != nil { - var err error - dest, err = validateSafePath(*amod.commonProperties.Dist.Dest) - if err != nil { - // This was checked in ModuleBase.GenerateBuildActions - panic(err) - } - } - - if amod.commonProperties.Dist.Suffix != nil { - ext := filepath.Ext(dest) - suffix := *amod.commonProperties.Dist.Suffix - dest = strings.TrimSuffix(dest, ext) + suffix + ext - } - - if amod.commonProperties.Dist.Dir != nil { - var err error - dest, err = validateSafePath(*amod.commonProperties.Dist.Dir, dest) - if err != nil { - // This was checked in ModuleBase.GenerateBuildActions - panic(err) - } - } - - goals := strings.Join(amod.commonProperties.Dist.Targets, " ") - fmt.Fprintln(&data.preamble, ".PHONY:", goals) - fmt.Fprintf(&data.preamble, "$(call dist-for-goals,%s,%s:%s)\n", - goals, distFile.String(), dest) - } - } - - fmt.Fprintln(&data.preamble, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(&data.preamble, "LOCAL_PATH :=", filepath.Dir(ctx.BlueprintFile(mod))) - fmt.Fprintln(&data.preamble, "LOCAL_MODULE :=", name+data.SubName) - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_CLASS :=", data.Class) - fmt.Fprintln(&data.preamble, "LOCAL_PREBUILT_MODULE_FILE :=", data.OutputFile.String()) - - if len(data.Required) > 0 { - fmt.Fprintln(&data.preamble, "LOCAL_REQUIRED_MODULES := "+strings.Join(data.Required, " ")) - } - - archStr := amod.Arch().ArchType.String() - host := false - switch amod.Os().Class { - case Host: - // Make cannot identify LOCAL_MODULE_HOST_ARCH:= common. - if archStr != "common" { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_ARCH :=", archStr) - } - host = true - case HostCross: - // Make cannot identify LOCAL_MODULE_HOST_CROSS_ARCH:= common. - if archStr != "common" { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr) - } - host = true - case Device: - // Make cannot identify LOCAL_MODULE_TARGET_ARCH:= common. - if archStr != "common" { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_TARGET_ARCH :=", archStr) - } - - if len(amod.commonProperties.Init_rc) > 0 { - fmt.Fprintln(&data.preamble, "LOCAL_INIT_RC := ", strings.Join(amod.commonProperties.Init_rc, " ")) - } - if len(amod.commonProperties.Vintf_fragments) > 0 { - fmt.Fprintln(&data.preamble, "LOCAL_VINTF_FRAGMENTS := ", strings.Join(amod.commonProperties.Vintf_fragments, " ")) - } - if Bool(amod.commonProperties.Proprietary) { - fmt.Fprintln(&data.preamble, "LOCAL_PROPRIETARY_MODULE := true") - } - if Bool(amod.commonProperties.Vendor) || Bool(amod.commonProperties.Soc_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_VENDOR_MODULE := true") - } - if Bool(amod.commonProperties.Device_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_ODM_MODULE := true") - } - if Bool(amod.commonProperties.Product_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_PRODUCT_MODULE := true") - } - if Bool(amod.commonProperties.Product_services_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_PRODUCT_SERVICES_MODULE := true") - } - if amod.commonProperties.Owner != nil { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_OWNER :=", *amod.commonProperties.Owner) - } - } - - if amod.noticeFile.Valid() { - fmt.Fprintln(&data.preamble, "LOCAL_NOTICE_FILE :=", amod.noticeFile.String()) - } - - if host { - makeOs := amod.Os().String() - if amod.Os() == Linux || amod.Os() == LinuxBionic { - makeOs = "linux" - } - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_OS :=", makeOs) - fmt.Fprintln(&data.preamble, "LOCAL_IS_HOST_MODULE := true") - } - + name := provider.BaseModuleName() blueprintDir := filepath.Dir(ctx.BlueprintFile(mod)) if data.Custom != nil { @@ -360,3 +440,30 @@ func WriteAndroidMkData(w io.Writer, data AndroidMkData) { fmt.Fprintln(w, "include "+data.Include) } + +func translateAndroidMkEntriesModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, + provider AndroidMkEntriesProvider) error { + if shouldSkipAndroidMkProcessing(mod.(Module).base()) { + return nil + } + + entries := provider.AndroidMkEntries() + entries.fillInEntries(ctx.Config(), ctx.BlueprintFile(mod), mod) + + entries.write(w) + + return nil +} + +func shouldSkipAndroidMkProcessing(module *ModuleBase) bool { + if !module.commonProperties.NamespaceExportedToMake { + // TODO(jeffrygaston) do we want to validate that there are no modules being + // exported to Kati that depend on this module? + return true + } + + return !module.Enabled() || + module.commonProperties.SkipInstall || + // Make does not understand LinuxBionic + module.Os() == LinuxBionic +} diff --git a/android/arch.go b/android/arch.go index 957a659c..6e8c4bdb 100644 --- a/android/arch.go +++ b/android/arch.go @@ -557,6 +557,10 @@ func newArch(name, multilib string) ArchType { return archType } +func ArchTypeList() []ArchType { + return append([]ArchType(nil), archTypeList...) +} + func (a ArchType) String() string { return a.Name } diff --git a/android/bootjar.go b/android/bootjar.go new file mode 100644 index 00000000..ea04dc8f --- /dev/null +++ b/android/bootjar.go @@ -0,0 +1,117 @@ +// Copyright (C) 2019 The LineageOS Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "strings" +) + +// Keys are bootjar name, value is whether or not +// we've marked a module as being a provider for it. +var jarMap map[string]bool + +func init() { + PreArchMutators(RegisterBootJarMutators) +} + +// Note: registration function is structured this way so that it can be included +// from soong module tests. +func RegisterBootJarMutators(ctx RegisterMutatorsContext) { + // Note: can't use Parallel() since we touch global jarMap + ctx.TopDown("bootjar_exportednamespace", bootJarMutatorExportedNamespace) + ctx.TopDown("bootjar_anynamespace", bootJarMutatorAnyNamespace) +} + +func mutatorInit(mctx TopDownMutatorContext) { + // Did we init already ? + if jarMap != nil { + return + } + + jarMap = make(map[string]bool) + for _, moduleName := range mctx.Config().BootJars() { + jarMap[moduleName] = false + } +} + +// Mark modules in soong exported namespace as providing a boot jar. +func bootJarMutatorExportedNamespace(mctx TopDownMutatorContext) { + bootJarMutator(mctx, true) +} + +// Mark modules in any namespace (incl root) as providing a boot jar. +func bootJarMutatorAnyNamespace(mctx TopDownMutatorContext) { + bootJarMutator(mctx, false) +} + +func bootJarMutator(mctx TopDownMutatorContext, requireExportedNamespace bool) { + mutatorInit(mctx) + + module, ok := mctx.Module().(Module) + if !ok { + // Not a proper module + return + } + + // Does this module produce a dex jar ? + if _, ok := module.(interface{ DexJar() Path }); !ok { + // No + return + } + + // If jarMap is empty we must be running in a test so + // set boot jar provide to true for all modules. + if len(jarMap) == 0 { + module.base().commonProperties.BootJarProvider = true + return + } + + name := mctx.ModuleName() + dir := mctx.ModuleDir() + + // Special treatment for hiddenapi modules - create extra + // jarMap entries if needed. + baseName := strings.TrimSuffix(name, "-hiddenapi") + if name != baseName { + _, baseIsBootJar := jarMap[baseName] + _, alreadyExists := jarMap[name] + if baseIsBootJar && !alreadyExists { + // This is a hidden api module whose base name exists in the boot jar list + // and we've not visited it before. Create a map entry for it. + jarMap[name] = false + } + } + + // Does this module match the name of a boot jar ? + if found, exists := jarMap[name]; !exists || found { + // No + return + } + + if requireExportedNamespace { + for _, n := range mctx.Config().ExportedNamespaces() { + if strings.HasPrefix(dir, n) { + jarMap[name] = true + module.base().commonProperties.BootJarProvider = true + break + } + } + } else { + jarMap[name] = true + module.base().commonProperties.BootJarProvider = true + } + + return +} diff --git a/android/config.go b/android/config.go index d1db87b2..7c604a42 100644 --- a/android/config.go +++ b/android/config.go @@ -584,7 +584,7 @@ func (c *config) DefaultAppCertificateDir(ctx PathContext) SourcePath { if defaultCert != "" { return PathForSource(ctx, filepath.Dir(defaultCert)) } else { - return PathForSource(ctx, "build/target/product/security") + return PathForSource(ctx, "build/make/target/product/security") } } @@ -601,7 +601,7 @@ func (c *config) DefaultAppCertificate(ctx PathContext) (pem, key SourcePath) { func (c *config) ApexKeyDir(ctx ModuleContext) SourcePath { // TODO(b/121224311): define another variable such as TARGET_APEX_KEY_OVERRIDE defaultCert := String(c.productVariables.DefaultAppCertificate) - if defaultCert == "" || filepath.Dir(defaultCert) == "build/target/product/security" { + if defaultCert == "" || filepath.Dir(defaultCert) == "build/make/target/product/security" { // When defaultCert is unset or is set to the testkeys path, use the APEX keys // that is under the module dir return pathForModuleSrc(ctx) @@ -882,6 +882,10 @@ func (c *deviceConfig) DeviceKernelHeaderDirs() []string { return c.config.productVariables.DeviceKernelHeaders } +func (c *deviceConfig) SpecificCameraParametersLibrary() string { + return String(c.config.productVariables.Lineage.Specific_camera_parameter_library) +} + func (c *deviceConfig) NativeCoverageEnabled() bool { return Bool(c.config.productVariables.NativeCoverage) } diff --git a/android/module.go b/android/module.go index 6a796227..17ddb050 100644 --- a/android/module.go +++ b/android/module.go @@ -129,6 +129,7 @@ type ModuleContext interface { AddMissingDependencies(deps []string) InstallInData() bool + InstallInTestcases() bool InstallInSanitizerDir() bool InstallInRecovery() bool @@ -185,6 +186,7 @@ type Module interface { Enabled() bool Target() Target InstallInData() bool + InstallInTestcases() bool InstallInSanitizerDir() bool InstallInRecovery() bool SkipInstall() @@ -302,6 +304,9 @@ type commonProperties struct { SkipInstall bool `blueprint:"mutated"` NamespaceExportedToMake bool `blueprint:"mutated"` + + // Whether this module provides a boot jar + BootJarProvider bool `blueprint:"mutated"` } type hostAndDeviceProperties struct { @@ -519,6 +524,10 @@ func (a *ModuleBase) Name() string { return String(a.nameProperties.Name) } +func (a *ModuleBase) BootJarProvider() bool { + return a.commonProperties.BootJarProvider +} + // BaseModuleName returns the name of the module as specified in the blueprints file. func (a *ModuleBase) BaseModuleName() string { return String(a.nameProperties.Name) @@ -656,6 +665,10 @@ func (p *ModuleBase) InstallInData() bool { return false } +func (p *ModuleBase) InstallInTestcases() bool { + return false +} + func (p *ModuleBase) InstallInSanitizerDir() bool { return false } @@ -1252,6 +1265,10 @@ func (a *androidModuleContext) InstallInData() bool { return a.module.InstallInData() } +func (a *androidModuleContext) InstallInTestcases() bool { + return a.module.InstallInTestcases() +} + func (a *androidModuleContext) InstallInSanitizerDir() bool { return a.module.InstallInSanitizerDir() } diff --git a/android/namespace.go b/android/namespace.go index 50bdcba7..a2bd0242 100644 --- a/android/namespace.go +++ b/android/namespace.go @@ -228,6 +228,11 @@ func (r *NameResolver) parseFullyQualifiedName(name string) (namespaceName strin } func (r *NameResolver) getNamespacesToSearchForModule(sourceNamespace *Namespace) (searchOrder []*Namespace) { + if sourceNamespace.visibleNamespaces == nil { + // When handling dependencies before namespaceMutator, assume they are non-Soong Blueprint modules and give + // access to all namespaces. + return r.sortedNamespaces.sortedItems() + } return sourceNamespace.visibleNamespaces } @@ -265,7 +270,12 @@ func (r *NameResolver) FindNamespaceImports(namespace *Namespace) (err error) { for _, name := range namespace.importedNamespaceNames { imp, ok := r.namespaceAt(name) if !ok { - return fmt.Errorf("namespace %v does not exist", name) + if (name != "all") { + return fmt.Errorf("namespace %v does not exist", name) + } else { + namespace.visibleNamespaces = make([]*Namespace, 0, 2+len(namespace.importedNamespaceNames)) + return nil + } } namespace.visibleNamespaces = append(namespace.visibleNamespaces, imp) } diff --git a/android/namespace_test.go b/android/namespace_test.go index 9a791a53..ec392dfc 100644 --- a/android/namespace_test.go +++ b/android/namespace_test.go @@ -93,6 +93,28 @@ func TestImplicitlyImportRootNamespace(t *testing.T) { // setupTest will report any errors } +func TestDependingOnBlueprintModuleInRootNamespace(t *testing.T) { + _ = setupTest(t, + map[string]string{ + ".": ` + blueprint_test_module { + name: "a", + } + `, + "dir1": ` + soong_namespace { + } + blueprint_test_module { + name: "b", + deps: ["a"], + } + `, + }, + ) + + // setupTest will report any errors +} + func TestDependingOnModuleInImportedNamespace(t *testing.T) { ctx := setupTest(t, map[string]string{ @@ -625,6 +647,7 @@ func setupTestFromFiles(bps map[string][]byte) (ctx *TestContext, errs []error) ctx.MockFileSystem(bps) ctx.RegisterModuleType("test_module", ModuleFactoryAdaptor(newTestModule)) ctx.RegisterModuleType("soong_namespace", ModuleFactoryAdaptor(NamespaceFactory)) + ctx.RegisterModuleType("blueprint_test_module", newBlueprintTestModule) ctx.PreArchMutators(RegisterNamespaceMutator) ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { ctx.BottomUp("rename", renameMutator) @@ -649,6 +672,7 @@ func setupTestExpectErrs(bps map[string]string) (ctx *TestContext, errs []error) } func setupTest(t *testing.T, bps map[string]string) (ctx *TestContext) { + t.Helper() ctx, errs := setupTestExpectErrs(bps) FailIfErrored(t, errs) return ctx @@ -726,3 +750,22 @@ func newTestModule() Module { InitAndroidModule(m) return m } + +type blueprintTestModule struct { + blueprint.SimpleName + properties struct { + Deps []string + } +} + +func (b *blueprintTestModule) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string { + return b.properties.Deps +} + +func (b *blueprintTestModule) GenerateBuildActions(blueprint.ModuleContext) { +} + +func newBlueprintTestModule() (blueprint.Module, []interface{}) { + m := &blueprintTestModule{} + return m, []interface{}{&m.properties, &m.SimpleName.Properties} +} diff --git a/android/paths.go b/android/paths.go index 0f20b844..edddce90 100644 --- a/android/paths.go +++ b/android/paths.go @@ -46,6 +46,7 @@ type ModuleInstallPathContext interface { androidBaseContext InstallInData() bool + InstallInTestcases() bool InstallInSanitizerDir() bool InstallInRecovery() bool } @@ -644,6 +645,31 @@ func pathForSource(ctx PathContext, pathComponents ...string) (SourcePath, error return ret, nil } +// pathForSourceRelaxed creates a SourcePath from pathComponents, but does not check that it exists. +// It differs from pathForSource in that the path is allowed to exist outside of the PathContext. +func pathForSourceRelaxed(ctx PathContext, pathComponents ...string) (SourcePath, error) { + p := filepath.Join(pathComponents...) + ret := SourcePath{basePath{p, ctx.Config(), ""}} + + abs, err := filepath.Abs(ret.String()) + if err != nil { + return ret, err + } + buildroot, err := filepath.Abs(ctx.Config().buildDir) + if err != nil { + return ret, err + } + if strings.HasPrefix(abs, buildroot) { + return ret, fmt.Errorf("source path %s is in output", abs) + } + + if pathtools.IsGlob(ret.String()) { + return ret, fmt.Errorf("path may not contain a glob: %s", ret.String()) + } + + return ret, nil +} + // existsWithDependencies returns true if the path exists, and adds appropriate dependencies to rerun if the // path does not exist. func existsWithDependencies(ctx PathContext, path SourcePath) (exists bool, err error) { @@ -697,6 +723,31 @@ func PathForSource(ctx PathContext, pathComponents ...string) SourcePath { return path } +// PathForSourceRelaxed joins the provided path components. Unlike PathForSource, +// the result is allowed to exist outside of the source dir. +// On error, it will return a usable, but invalid SourcePath, and report a ModuleError. +func PathForSourceRelaxed(ctx PathContext, pathComponents ...string) SourcePath { + path, err := pathForSourceRelaxed(ctx, pathComponents...) + if err != nil { + reportPathError(ctx, err) + } + + if modCtx, ok := ctx.(ModuleContext); ok && ctx.Config().AllowMissingDependencies() { + exists, err := existsWithDependencies(ctx, path) + if err != nil { + reportPathError(ctx, err) + } + if !exists { + modCtx.AddMissingDependencies([]string{path.String()}) + } + } else if exists, _, err := ctx.Fs().Exists(path.String()); err != nil { + reportPathErrorf(ctx, "%s: %s", path, err.Error()) + } else if !exists { + reportPathErrorf(ctx, "source path %s does not exist", path) + } + return path +} + // ExistentPathForSource returns an OptionalPath with the SourcePath if the // path exists, or an empty OptionalPath if it doesn't exist. Dependencies are added // so that the ninja file will be regenerated if the state of the path changes. @@ -1117,6 +1168,8 @@ func modulePartition(ctx ModuleInstallPathContext) string { var partition string if ctx.InstallInData() { partition = "data" + } else if ctx.InstallInTestcases() { + partition = "testcases" } else if ctx.InstallInRecovery() { // the layout of recovery partion is the same as that of system partition partition = "recovery/root/system" diff --git a/android/paths_test.go b/android/paths_test.go index b52d7133..c956a795 100644 --- a/android/paths_test.go +++ b/android/paths_test.go @@ -203,6 +203,7 @@ type moduleInstallPathContextImpl struct { androidBaseContextImpl inData bool + inTestcases bool inSanitizerDir bool inRecovery bool } @@ -221,6 +222,10 @@ func (m moduleInstallPathContextImpl) InstallInData() bool { return m.inData } +func (m moduleInstallPathContextImpl) InstallInTestcases() bool { + return m.inTestcases +} + func (m moduleInstallPathContextImpl) InstallInSanitizerDir() bool { return m.inSanitizerDir } diff --git a/android/prebuilt.go b/android/prebuilt.go index 3be10f72..48d3b68b 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -16,6 +16,7 @@ package android import ( "fmt" + "reflect" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -43,13 +44,21 @@ type Prebuilt struct { properties PrebuiltProperties module Module srcs *[]string - src *string + + // Metadata for single source Prebuilt modules. + srcProps reflect.Value + srcField reflect.StructField } func (p *Prebuilt) Name(name string) string { return "prebuilt_" + name } +// The below source-related functions and the srcs, src fields are based on an assumption that +// prebuilt modules have a static source property at the moment. Currently there is only one +// exception, android_app_import, which chooses a source file depending on the product's DPI +// preference configs. We'll want to add native support for dynamic source cases if we end up having +// more modules like this. func (p *Prebuilt) SingleSourcePath(ctx ModuleContext) Path { if p.srcs != nil { if len(*p.srcs) == 0 { @@ -66,11 +75,16 @@ func (p *Prebuilt) SingleSourcePath(ctx ModuleContext) Path { // sources. return PathForModuleSrc(ctx, (*p.srcs)[0]) } else { - if proptools.String(p.src) == "" { - ctx.PropertyErrorf("src", "missing prebuilt source file") + if !p.srcProps.IsValid() { + ctx.ModuleErrorf("prebuilt source was not set") + } + src := p.getSingleSourceFieldValue() + if src == "" { + ctx.PropertyErrorf(proptools.FieldNameForProperty(p.srcField.Name), + "missing prebuilt source file") return nil } - return PathForModuleSrc(ctx, *p.src) + return PathForModuleSrc(ctx, src) } } @@ -84,10 +98,12 @@ func InitPrebuiltModule(module PrebuiltInterface, srcs *[]string) { p.srcs = srcs } -func InitSingleSourcePrebuiltModule(module PrebuiltInterface, src *string) { +func InitSingleSourcePrebuiltModule(module PrebuiltInterface, srcProps interface{}, srcField string) { p := module.Prebuilt() module.AddProperties(&p.properties) - p.src = src + p.srcProps = reflect.ValueOf(srcProps).Elem() + p.srcField, _ = p.srcProps.Type().FieldByName(srcField) + p.checkSingleSourceProperties() } type PrebuiltInterface interface { @@ -124,7 +140,7 @@ func PrebuiltMutator(ctx BottomUpMutatorContext) { func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { if m, ok := ctx.Module().(PrebuiltInterface); ok && m.Prebuilt() != nil { p := m.Prebuilt() - if p.srcs == nil && p.src == nil { + if p.srcs == nil && !p.srcProps.IsValid() { panic(fmt.Errorf("prebuilt module did not have InitPrebuiltModule called on it")) } if !p.properties.SourceExists { @@ -167,7 +183,7 @@ func (p *Prebuilt) usePrebuilt(ctx TopDownMutatorContext, source Module) bool { return false } - if p.src != nil && *p.src == "" { + if p.srcProps.IsValid() && p.getSingleSourceFieldValue() == "" { return false } @@ -182,3 +198,24 @@ func (p *Prebuilt) usePrebuilt(ctx TopDownMutatorContext, source Module) bool { func (p *Prebuilt) SourceExists() bool { return p.properties.SourceExists } + +func (p *Prebuilt) checkSingleSourceProperties() { + if !p.srcProps.IsValid() || p.srcField.Name == "" { + panic(fmt.Errorf("invalid single source prebuilt %+v", p)) + } + + if p.srcProps.Kind() != reflect.Struct && p.srcProps.Kind() != reflect.Interface { + panic(fmt.Errorf("invalid single source prebuilt %+v", p.srcProps)) + } +} + +func (p *Prebuilt) getSingleSourceFieldValue() string { + value := p.srcProps.FieldByIndex(p.srcField.Index) + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + if value.Kind() != reflect.String { + panic(fmt.Errorf("prebuilt src field %q should be a string or a pointer to one", p.srcField.Name)) + } + return value.String() +} diff --git a/android/prebuilt_etc.go b/android/prebuilt_etc.go index 8f23d78e..f35e3c76 100644 --- a/android/prebuilt_etc.go +++ b/android/prebuilt_etc.go @@ -14,11 +14,7 @@ package android -import ( - "fmt" - "io" - "strings" -) +import "strconv" // TODO(jungw): Now that it handles more than the ones in etc/, consider renaming this file. @@ -135,38 +131,27 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx ModuleContext) { }) } -func (p *PrebuiltEtc) AndroidMk() AndroidMkData { - return AndroidMkData{ - Custom: func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData) { - nameSuffix := "" - if p.inRecovery() && !p.onlyInRecovery() { - nameSuffix = ".recovery" - } - fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) - fmt.Fprintln(w, "LOCAL_MODULE :=", name+nameSuffix) - fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC") - if p.commonProperties.Owner != nil { - fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", *p.commonProperties.Owner) - } - fmt.Fprintln(w, "LOCAL_MODULE_TAGS := optional") - if p.Host() { - fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") - } - fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", p.outputFilePath.String()) - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", "$(OUT_DIR)/"+p.installDirPath.RelPathString()) - fmt.Fprintln(w, "LOCAL_INSTALLED_MODULE_STEM :=", p.outputFilePath.Base()) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !p.Installable()) - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES :=", strings.Join(data.Required, " ")) - fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", p.Arch().ArchType.String()) - if p.additionalDependencies != nil { - fmt.Fprint(w, "LOCAL_ADDITIONAL_DEPENDENCIES :=") - for _, path := range *p.additionalDependencies { - fmt.Fprint(w, " "+path.String()) +func (p *PrebuiltEtc) AndroidMkEntries() AndroidMkEntries { + nameSuffix := "" + if p.inRecovery() && !p.onlyInRecovery() { + nameSuffix = ".recovery" + } + return AndroidMkEntries{ + Class: "ETC", + SubName: nameSuffix, + OutputFile: OptionalPathForPath(p.outputFilePath), + ExtraEntries: []AndroidMkExtraEntriesFunc{ + func(entries *AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_TAGS", "optional") + entries.SetString("LOCAL_MODULE_PATH", "$(OUT_DIR)/"+p.installDirPath.RelPathString()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.outputFilePath.Base()) + entries.SetString("LOCAL_UNINSTALLABLE_MODULE", strconv.FormatBool(!p.Installable())) + if p.additionalDependencies != nil { + for _, path := range *p.additionalDependencies { + entries.SetString("LOCAL_ADDITIONAL_DEPENDENCIES", path.String()) + } } - fmt.Fprintln(w, "") - } - fmt.Fprintln(w, "include $(BUILD_PREBUILT)") + }, }, } } diff --git a/android/prebuilt_etc_test.go b/android/prebuilt_etc_test.go index e0ade7e5..ee6e990e 100644 --- a/android/prebuilt_etc_test.go +++ b/android/prebuilt_etc_test.go @@ -15,12 +15,10 @@ package android import ( - "bufio" - "bytes" "io/ioutil" "os" "path/filepath" - "strings" + "reflect" "testing" ) @@ -139,45 +137,33 @@ func TestPrebuiltEtcGlob(t *testing.T) { } func TestPrebuiltEtcAndroidMk(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` + ctx, config := testPrebuiltEtc(t, ` prebuilt_etc { name: "foo", src: "foo.conf", owner: "abc", filename_from_src: true, + required: ["modA", "moduleB"], } `) - data := AndroidMkData{} - data.Required = append(data.Required, "modA", "moduleB") - - expected := map[string]string{ - "LOCAL_MODULE": "foo", - "LOCAL_MODULE_CLASS": "ETC", - "LOCAL_MODULE_OWNER": "abc", - "LOCAL_INSTALLED_MODULE_STEM": "foo.conf", - "LOCAL_REQUIRED_MODULES": "modA moduleB", + expected := map[string][]string{ + "LOCAL_MODULE": {"foo"}, + "LOCAL_MODULE_CLASS": {"ETC"}, + "LOCAL_MODULE_OWNER": {"abc"}, + "LOCAL_INSTALLED_MODULE_STEM": {"foo.conf"}, + "LOCAL_REQUIRED_MODULES": {"modA", "moduleB"}, } mod := ctx.ModuleForTests("foo", "android_arm64_armv8-a_core").Module().(*PrebuiltEtc) - buf := &bytes.Buffer{} - mod.AndroidMk().Custom(buf, "foo", "", "", data) - for k, expected := range expected { - found := false - scanner := bufio.NewScanner(bytes.NewReader(buf.Bytes())) - for scanner.Scan() { - line := scanner.Text() - tok := strings.Split(line, " := ") - if tok[0] == k { - found = true - if tok[1] != expected { - t.Errorf("Incorrect %s '%s', expected '%s'", k, tok[1], expected) - } + entries := AndroidMkEntriesForTest(t, config, "", mod) + for k, expectedValue := range expected { + if value, ok := entries.EntryMap[k]; ok { + if !reflect.DeepEqual(value, expectedValue) { + t.Errorf("Incorrect %s '%s', expected '%s'", k, value, expectedValue) } - } - - if !found { - t.Errorf("No %s defined, saw %s", k, buf.String()) + } else { + t.Errorf("No %s defined, saw %q", k, entries.EntryMap) } } } diff --git a/android/sh_binary.go b/android/sh_binary.go index cf415c56..ba0c8be7 100644 --- a/android/sh_binary.go +++ b/android/sh_binary.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "io" "strings" ) @@ -30,6 +29,7 @@ func init() { RegisterModuleType("sh_binary", ShBinaryFactory) RegisterModuleType("sh_binary_host", ShBinaryHostFactory) RegisterModuleType("sh_test", ShTestFactory) + RegisterModuleType("sh_test_host", ShTestHostFactory) } type shBinaryProperties struct { @@ -58,6 +58,10 @@ type TestProperties struct { // the name of the test configuration (for example "AndroidTest.xml") that should be // installed with the module. Test_config *string `android:"arch_variant"` + + // list of files or filegroup modules that provide data that should be installed alongside + // the test. + Data []string `android:"path,arch_variant"` } type ShBinary struct { @@ -73,6 +77,8 @@ type ShTest struct { ShBinary testProperties TestProperties + + data Paths } func (s *ShBinary) DepsMutator(ctx BottomUpMutatorContext) { @@ -122,30 +128,54 @@ func (s *ShBinary) GenerateAndroidBuildActions(ctx ModuleContext) { }) } -func (s *ShBinary) AndroidMk() AndroidMkData { - return AndroidMkData{ +func (s *ShBinary) AndroidMkEntries() AndroidMkEntries { + return AndroidMkEntries{ Class: "EXECUTABLES", OutputFile: OptionalPathForPath(s.outputFilePath), Include: "$(BUILD_SYSTEM)/soong_cc_prebuilt.mk", - Extra: []AndroidMkExtraFunc{ - func(w io.Writer, outputFile Path) { - fmt.Fprintln(w, "LOCAL_MODULE_RELATIVE_PATH :=", String(s.properties.Sub_dir)) - fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX :=") - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", s.outputFilePath.Rel()) + ExtraEntries: []AndroidMkExtraEntriesFunc{ + func(entries *AndroidMkEntries) { + s.customAndroidMkEntries(entries) }, }, } } -func (s *ShTest) AndroidMk() AndroidMkData { - data := s.ShBinary.AndroidMk() - data.Class = "NATIVE_TESTS" - data.Extra = append(data.Extra, func(w io.Writer, outputFile Path) { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", - strings.Join(s.testProperties.Test_suites, " ")) - fmt.Fprintln(w, "LOCAL_TEST_CONFIG :=", String(s.testProperties.Test_config)) - }) - return data +func (s *ShBinary) customAndroidMkEntries(entries *AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_RELATIVE_PATH", String(s.properties.Sub_dir)) + entries.SetString("LOCAL_MODULE_SUFFIX", "") + entries.SetString("LOCAL_MODULE_STEM", s.outputFilePath.Rel()) +} + +func (s *ShTest) GenerateAndroidBuildActions(ctx ModuleContext) { + s.ShBinary.GenerateAndroidBuildActions(ctx) + + s.data = PathsForModuleSrc(ctx, s.testProperties.Data) +} + +func (s *ShTest) AndroidMkEntries() AndroidMkEntries { + return AndroidMkEntries{ + Class: "NATIVE_TESTS", + OutputFile: OptionalPathForPath(s.outputFilePath), + Include: "$(BUILD_SYSTEM)/soong_cc_prebuilt.mk", + ExtraEntries: []AndroidMkExtraEntriesFunc{ + func(entries *AndroidMkEntries) { + s.customAndroidMkEntries(entries) + + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", s.testProperties.Test_suites...) + entries.SetString("LOCAL_TEST_CONFIG", String(s.testProperties.Test_config)) + for _, d := range s.data { + rel := d.Rel() + path := d.String() + if !strings.HasSuffix(path, rel) { + panic(fmt.Errorf("path %q does not end with %q", path, rel)) + } + path = strings.TrimSuffix(path, rel) + entries.AddStrings("LOCAL_TEST_DATA", path+":"+rel) + } + }, + }, + } } func InitShBinaryModule(s *ShBinary) { @@ -170,6 +200,7 @@ func ShBinaryHostFactory() Module { return module } +// sh_test defines a shell script based test module. func ShTestFactory() Module { module := &ShTest{} InitShBinaryModule(&module.ShBinary) @@ -178,3 +209,13 @@ func ShTestFactory() Module { InitAndroidArchModule(module, HostAndDeviceSupported, MultilibFirst) return module } + +// sh_test_host defines a shell script based test module that runs on a host. +func ShTestHostFactory() Module { + module := &ShTest{} + InitShBinaryModule(&module.ShBinary) + module.AddProperties(&module.testProperties) + + InitAndroidArchModule(module, HostSupported, MultilibFirst) + return module +} diff --git a/android/sh_binary_test.go b/android/sh_binary_test.go new file mode 100644 index 00000000..67360830 --- /dev/null +++ b/android/sh_binary_test.go @@ -0,0 +1,79 @@ +package android + +import ( + "io/ioutil" + "os" + "reflect" + "testing" +) + +func testShBinary(t *testing.T, bp string) (*TestContext, Config) { + buildDir, err := ioutil.TempDir("", "soong_sh_binary_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(buildDir) + + config := TestArchConfig(buildDir, nil) + + ctx := NewTestArchContext() + ctx.RegisterModuleType("sh_test", ModuleFactoryAdaptor(ShTestFactory)) + ctx.RegisterModuleType("sh_test_host", ModuleFactoryAdaptor(ShTestHostFactory)) + ctx.Register() + mockFiles := map[string][]byte{ + "Android.bp": []byte(bp), + "test.sh": nil, + "testdata/data1": nil, + "testdata/sub/data2": nil, + } + ctx.MockFileSystem(mockFiles) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + return ctx, config +} + +func TestShTestTestData(t *testing.T) { + ctx, config := testShBinary(t, ` + sh_test { + name: "foo", + src: "test.sh", + filename: "test.sh", + data: [ + "testdata/data1", + "testdata/sub/data2", + ], + } + `) + + mod := ctx.ModuleForTests("foo", "android_arm64_armv8-a").Module().(*ShTest) + + entries := AndroidMkEntriesForTest(t, config, "", mod) + expected := []string{":testdata/data1", ":testdata/sub/data2"} + actual := entries.EntryMap["LOCAL_TEST_DATA"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected test data expected: %q, actual: %q", expected, actual) + } +} + +func TestShTestHost(t *testing.T) { + ctx, _ := testShBinary(t, ` + sh_test_host { + name: "foo", + src: "test.sh", + filename: "test.sh", + data: [ + "testdata/data1", + "testdata/sub/data2", + ], + } + `) + + buildOS := BuildOs.String() + mod := ctx.ModuleForTests("foo", buildOS+"_x86_64").Module().(*ShTest) + if !mod.Host() { + t.Errorf("host bit is not set for a sh_test_host module.") + } +} diff --git a/android/testing.go b/android/testing.go index 0ec5af58..ab3c69cf 100644 --- a/android/testing.go +++ b/android/testing.go @@ -369,3 +369,14 @@ func FailIfNoMatchingErrors(t *testing.T, pattern string, errs []error) { } } } + +func AndroidMkEntriesForTest(t *testing.T, config Config, bpPath string, mod blueprint.Module) AndroidMkEntries { + var p AndroidMkEntriesProvider + var ok bool + if p, ok = mod.(AndroidMkEntriesProvider); !ok { + t.Errorf("module does not implmement AndroidMkEntriesProvider: " + mod.Name()) + } + entries := p.AndroidMkEntries() + entries.fillInEntries(config, bpPath, mod) + return entries +} diff --git a/android/variable.go b/android/variable.go index e643c0eb..56fdcc18 100644 --- a/android/variable.go +++ b/android/variable.go @@ -20,6 +20,8 @@ import ( "runtime" "strings" + "lineage/soong/android" + "github.com/google/blueprint/proptools" ) @@ -124,6 +126,9 @@ type variableProperties struct { Static_libs []string Srcs []string } + + // include Lineage variables + Lineage android.Product_variables } `android:"arch_variant"` } @@ -287,6 +292,9 @@ type productVariables struct { ProductHiddenAPIStubsTest []string `json:",omitempty"` TargetFSConfigGen []string `json:",omitempty"` + + // include Lineage variables + Lineage android.ProductVariables } func boolPtr(v bool) *bool { @@ -346,7 +354,13 @@ func variableMutator(mctx BottomUpMutatorContext) { a := module.base() variableValues := reflect.ValueOf(&a.variableProperties.Product_variables).Elem() zeroValues := reflect.ValueOf(zeroProductVariables.Product_variables) + valStruct := reflect.ValueOf(mctx.Config().productVariables) + doVariableMutation(mctx, a, variableValues, zeroValues, valStruct) +} + +func doVariableMutation(mctx BottomUpMutatorContext, a *ModuleBase, variableValues reflect.Value, zeroValues reflect.Value, + valStruct reflect.Value) { for i := 0; i < variableValues.NumField(); i++ { variableValue := variableValues.Field(i) zeroValue := zeroValues.Field(i) @@ -354,8 +368,11 @@ func variableMutator(mctx BottomUpMutatorContext) { property := "product_variables." + proptools.PropertyNameForField(name) // Check that the variable was set for the product - val := reflect.ValueOf(mctx.Config().productVariables).FieldByName(name) - if !val.IsValid() || val.Kind() != reflect.Ptr || val.IsNil() { + val := valStruct.FieldByName(name) + if val.IsValid() && val.Kind() == reflect.Struct { + doVariableMutation(mctx, a, variableValue, zeroValue, val) + continue + } else if !val.IsValid() || val.Kind() != reflect.Ptr || val.IsNil() { continue } diff --git a/androidmk/cmd/androidmk/android.go b/androidmk/cmd/androidmk/android.go index 52bcf9c9..0c048083 100644 --- a/androidmk/cmd/androidmk/android.go +++ b/androidmk/cmd/androidmk/android.go @@ -169,6 +169,8 @@ func init() { // Jacoco filters: "LOCAL_JACK_COVERAGE_INCLUDE_FILTER": "jacoco.include_filter", "LOCAL_JACK_COVERAGE_EXCLUDE_FILTER": "jacoco.exclude_filter", + + "LOCAL_FULL_LIBS_MANIFEST_FILES": "additional_manifests", }) addStandardProperties(bpparser.BoolType, @@ -933,6 +935,7 @@ var prebuiltTypes = map[string]string{ "STATIC_LIBRARIES": "cc_prebuilt_library_static", "EXECUTABLES": "cc_prebuilt_binary", "JAVA_LIBRARIES": "java_import", + "APPS": "android_app_import", "ETC": "prebuilt_etc", } diff --git a/androidmk/cmd/androidmk/androidmk_test.go b/androidmk/cmd/androidmk/androidmk_test.go index f2dc6ff2..9570d364 100644 --- a/androidmk/cmd/androidmk/androidmk_test.go +++ b/androidmk/cmd/androidmk/androidmk_test.go @@ -1112,6 +1112,32 @@ android_app { } `, }, + { + desc: "android_app_import", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_SRC_FILES := foo.apk +LOCAL_PRIVILEGED_MODULE := true +LOCAL_MODULE_CLASS := APPS +LOCAL_MODULE_TAGS := optional +LOCAL_DEX_PREOPT := false +include $(BUILD_PREBUILT) +`, + expected: ` +android_app_import { + name: "foo", + + privileged: true, + + dex_preopt: { + enabled: false, + }, + apk: "foo.apk", + +} +`, + }, } func TestEndToEnd(t *testing.T) { diff --git a/apex/apex.go b/apex/apex.go index 81160a9c..1e1c33a7 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1358,6 +1358,13 @@ type PrebuiltProperties struct { // Optional name for the installed apex. If unspecified, name of the // module is used as the file name Filename *string + + // Names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string } func (p *Prebuilt) installable() bool { @@ -1452,16 +1459,17 @@ func (p *Prebuilt) Name() string { return p.prebuilt.Name(p.ModuleBase.Name()) } -func (p *Prebuilt) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (p *Prebuilt) AndroidMkEntries() android.AndroidMkEntries { + return android.AndroidMkEntries{ Class: "ETC", OutputFile: android.OptionalPathForPath(p.inputApex), Include: "$(BUILD_PREBUILT)", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", filepath.Join("$(OUT_DIR)", p.installDir.RelPathString())) - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", p.installFilename) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !p.installable()) + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", filepath.Join("$(OUT_DIR)", p.installDir.RelPathString())) + entries.SetString("LOCAL_MODULE_STEM", p.installFilename) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !p.installable()) + entries.AddStrings("LOCAL_OVERRIDES_PACKAGES", p.properties.Overrides...) }, }, } @@ -1471,7 +1479,7 @@ func (p *Prebuilt) AndroidMk() android.AndroidMkData { func PrebuiltFactory() android.Module { module := &Prebuilt{} module.AddProperties(&module.properties) - android.InitSingleSourcePrebuiltModule(module, &module.properties.Source) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Source") android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) return module } diff --git a/apex/apex_test.go b/apex/apex_test.go index c671e7c3..1ae1073c 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -29,7 +29,7 @@ import ( var buildDir string -func testApex(t *testing.T, bp string) *android.TestContext { +func testApex(t *testing.T, bp string) (*android.TestContext, android.Config) { var config android.Config config, buildDir = setup(t) defer teardown(buildDir) @@ -158,7 +158,7 @@ func testApex(t *testing.T, bp string) *android.TestContext { ctx.MockFileSystem(map[string][]byte{ "Android.bp": []byte(bp), - "build/target/product/security": nil, + "build/make/target/product/security": nil, "apex_manifest.json": nil, "AndroidManifest.xml": nil, "system/sepolicy/apex/myapex-file_contexts": nil, @@ -188,7 +188,7 @@ func testApex(t *testing.T, bp string) *android.TestContext { _, errs = ctx.PrepareBuildActions(config) android.FailIfErrored(t, errs) - return ctx + return ctx, config } func setup(t *testing.T) (config android.Config, buildDir string) { @@ -238,7 +238,7 @@ func ensureListNotContains(t *testing.T, result []string, notExpected string) { // Minimal test func TestBasicApex(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex_defaults { name: "myapex-defaults", manifest: ":myapex.manifest", @@ -362,7 +362,7 @@ func TestBasicApex(t *testing.T) { } func TestBasicZipApex(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -410,7 +410,7 @@ func TestBasicZipApex(t *testing.T) { } func TestApexWithStubs(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -494,7 +494,7 @@ func TestApexWithStubs(t *testing.T) { } func TestApexWithExplicitStubsDependency(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -561,7 +561,7 @@ func TestApexWithExplicitStubsDependency(t *testing.T) { } func TestApexWithSystemLibsStubs(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -682,7 +682,7 @@ func TestApexWithSystemLibsStubs(t *testing.T) { } func TestFilesInSubDir(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -742,7 +742,7 @@ func TestFilesInSubDir(t *testing.T) { } func TestUseVendor(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -792,7 +792,7 @@ func TestUseVendor(t *testing.T) { } func TestStaticLinking(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -832,7 +832,7 @@ func TestStaticLinking(t *testing.T) { } func TestKeys(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex_keytest", key: "myapex.key", @@ -886,7 +886,7 @@ func TestKeys(t *testing.T) { } func TestMacro(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -930,7 +930,7 @@ func TestMacro(t *testing.T) { } func TestHeaderLibsDependency(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -978,7 +978,7 @@ func TestHeaderLibsDependency(t *testing.T) { } func TestNonTestApex(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1029,7 +1029,7 @@ func TestTestApex(t *testing.T) { if android.InAnyApex("mylib_common_test") { t.Fatal("mylib_common_test must not be used in any other tests since this checks that global state is not updated in an illegal way!") } - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex_test { name: "myapex", key: "myapex.key", @@ -1077,7 +1077,7 @@ func TestTestApex(t *testing.T) { } func TestApexWithTarget(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1157,7 +1157,7 @@ func TestApexWithTarget(t *testing.T) { } func TestApexWithShBinary(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1185,7 +1185,7 @@ func TestApexWithShBinary(t *testing.T) { } func TestApexInProductPartition(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1217,7 +1217,7 @@ func TestApexInProductPartition(t *testing.T) { } func TestApexKeyFromOtherModule(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex_key { name: "myapex.key", public_key: ":my.avbpubkey", @@ -1250,7 +1250,7 @@ func TestApexKeyFromOtherModule(t *testing.T) { } func TestPrebuilt(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` prebuilt_apex { name: "myapex", arch: { @@ -1273,7 +1273,7 @@ func TestPrebuilt(t *testing.T) { } func TestPrebuiltFilenameOverride(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` prebuilt_apex { name: "myapex", src: "myapex-arm.apex", diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go index 706c0ec2..80e7b4a6 100644 --- a/bpfix/bpfix/bpfix.go +++ b/bpfix/bpfix/bpfix.go @@ -102,6 +102,10 @@ var fixSteps = []fixStep{ name: "rewriteAndroidTest", fix: rewriteAndroidTest, }, + { + name: "rewriteAndroidAppImport", + fix: rewriteAndroidAppImport, + }, } func NewFixRequest() FixRequest { @@ -497,27 +501,8 @@ func rewriteAndroidmkPrebuiltEtc(f *Fixer) error { continue } - // The rewriter converts LOCAL_SRC_FILES to `srcs` attribute. Convert - // it to 'src' attribute (which is where the file is installed). If the - // value 'srcs' is a list, we can convert it only if it contains a single - // element. - if srcs, ok := mod.GetProperty("srcs"); ok { - if srcList, ok := srcs.Value.(*parser.List); ok { - removeProperty(mod, "srcs") - if len(srcList.Values) == 1 { - mod.Properties = append(mod.Properties, - &parser.Property{Name: "src", NamePos: srcs.NamePos, ColonPos: srcs.ColonPos, Value: resolveLocalModule(mod, srcList.Values[0])}) - } else if len(srcList.Values) > 1 { - indicateAttributeError(mod, "src", "LOCAL_SRC_FILES should contain at most one item") - } - } else if _, ok = srcs.Value.(*parser.Variable); ok { - removeProperty(mod, "srcs") - mod.Properties = append(mod.Properties, - &parser.Property{Name: "src", NamePos: srcs.NamePos, ColonPos: srcs.ColonPos, Value: resolveLocalModule(mod, srcs.Value)}) - } else { - renameProperty(mod, "srcs", "src") - } - } + // 'srcs' --> 'src' conversion + convertToSingleSource(mod, "src") // The rewriter converts LOCAL_MODULE_PATH attribute into a struct attribute // 'local_module_path'. Analyze its contents and create the correct sub_dir:, @@ -589,6 +574,62 @@ func rewriteAndroidTest(f *Fixer) error { return nil } +func rewriteAndroidAppImport(f *Fixer) error { + for _, def := range f.tree.Defs { + mod, ok := def.(*parser.Module) + if !(ok && mod.Type == "android_app_import") { + continue + } + // 'srcs' --> 'apk' conversion + convertToSingleSource(mod, "apk") + // Handle special certificate value, "PRESIGNED". + if cert, ok := mod.GetProperty("certificate"); ok { + if certStr, ok := cert.Value.(*parser.String); ok { + if certStr.Value == "PRESIGNED" { + removeProperty(mod, "certificate") + prop := &parser.Property{ + Name: "presigned", + Value: &parser.Bool{ + Value: true, + }, + } + mod.Properties = append(mod.Properties, prop) + } + } + } + } + return nil +} + +// Converts the default source list property, 'srcs', to a single source property with a given name. +// "LOCAL_MODULE" reference is also resolved during the conversion process. +func convertToSingleSource(mod *parser.Module, srcPropertyName string) { + if srcs, ok := mod.GetProperty("srcs"); ok { + if srcList, ok := srcs.Value.(*parser.List); ok { + removeProperty(mod, "srcs") + if len(srcList.Values) == 1 { + mod.Properties = append(mod.Properties, + &parser.Property{ + Name: srcPropertyName, + NamePos: srcs.NamePos, + ColonPos: srcs.ColonPos, + Value: resolveLocalModule(mod, srcList.Values[0])}) + } else if len(srcList.Values) > 1 { + indicateAttributeError(mod, srcPropertyName, "LOCAL_SRC_FILES should contain at most one item") + } + } else if _, ok = srcs.Value.(*parser.Variable); ok { + removeProperty(mod, "srcs") + mod.Properties = append(mod.Properties, + &parser.Property{Name: srcPropertyName, + NamePos: srcs.NamePos, + ColonPos: srcs.ColonPos, + Value: resolveLocalModule(mod, srcs.Value)}) + } else { + renameProperty(mod, "srcs", "apk") + } + } +} + func runPatchListMod(modFunc func(mod *parser.Module, buf []byte, patchlist *parser.PatchList) error) func(*Fixer) error { return func(f *Fixer) error { // Make sure all the offsets are accurate diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index 459cd36b..5e0b8175 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -784,3 +784,52 @@ func TestRewriteAndroidTest(t *testing.T) { }) } } + +func TestRewriteAndroidAppImport(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "android_app_import apk", + in: ` + android_app_import { + name: "foo", + srcs: ["package.apk"], + } + `, + out: ` + android_app_import { + name: "foo", + apk: "package.apk", + } + `, + }, + { + name: "android_app_import presigned", + in: ` + android_app_import { + name: "foo", + apk: "package.apk", + certificate: "PRESIGNED", + } + `, + out: ` + android_app_import { + name: "foo", + apk: "package.apk", + presigned: true, + + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPass(t, test.in, test.out, func(fixer *Fixer) error { + return rewriteAndroidAppImport(fixer) + }) + }) + } +} diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index cd7c4107..d37e4862 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -236,6 +236,7 @@ var ( "cortex-a72": "${config.ArmClangCortexA53Cflags}", "cortex-a73": "${config.ArmClangCortexA53Cflags}", "cortex-a75": "${config.ArmClangCortexA55Cflags}", + "cortex-a76": "${config.ArmClangCortexA55Cflags}", "krait": "${config.ArmClangKraitCflags}", "kryo": "${config.ArmClangKryoCflags}", "kryo385": "${config.ArmClangCortexA53Cflags}", diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 330c5dd2..4c6e243a 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -37,18 +37,7 @@ import ( "android/soong/zip" ) -// We default to number of cpus / 4, which seems to be the sweet spot for my -// system. I suspect this is mostly due to memory or disk bandwidth though, and -// may depend on the size ofthe source tree, so this probably isn't a great -// default. -func detectNumJobs() int { - if runtime.NumCPU() < 4 { - return 1 - } - return runtime.NumCPU() / 4 -} - -var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") +var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]") var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts") var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)") @@ -156,10 +145,12 @@ type mpContext struct { } func main() { - writer := terminal.NewWriter(terminal.StdioImpl{}) - defer writer.Finish() + stdio := terminal.StdioImpl{} + + output := terminal.NewStatusOutput(stdio.Stdout(), "", false, + build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")) - log := logger.New(writer) + log := logger.New(output) defer log.Cleanup() flag.Parse() @@ -172,8 +163,7 @@ func main() { stat := &status.Status{} defer stat.Finish() - stat.AddOutput(terminal.NewStatusOutput(writer, "", - build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))) + stat.AddOutput(output) var failures failureCount stat.AddOutput(&failures) @@ -188,7 +178,7 @@ func main() { Context: ctx, Logger: log, Tracer: trace, - Writer: writer, + Writer: output, Status: stat, }} @@ -229,6 +219,21 @@ func main() { trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) } + var jobs = *numJobs + if jobs < 1 { + jobs = runtime.NumCPU() / 4 + + ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024) + if ramJobs := ramGb / 20; ramGb > 0 && jobs > ramJobs { + jobs = ramJobs + } + + if jobs < 1 { + jobs = 1 + } + } + log.Verbosef("Using %d parallel jobs", jobs) + setMaxFiles(log) finder := build.NewSourceFinder(buildCtx, config) @@ -303,7 +308,7 @@ func main() { }() var wg sync.WaitGroup - for i := 0; i < *numJobs; i++ { + for i := 0; i < jobs; i++ { wg.Add(1) go func() { defer wg.Done() @@ -341,7 +346,7 @@ func main() { } else if failures > 1 { log.Fatalf("%d failures", failures) } else { - writer.Print("Success") + fmt.Fprintln(output, "Success") } } @@ -386,11 +391,11 @@ func buildProduct(mpctx *mpContext, product string) { Context: mpctx.Context, Logger: log, Tracer: mpctx.Tracer, - Writer: terminal.NewWriter(terminal.NewCustomStdio(nil, f, f)), + Writer: f, Thread: mpctx.Tracer.NewThread(product), Status: &status.Status{}, }} - ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", + ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", false, build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))) config := build.NewConfig(ctx, flag.Args()...) @@ -466,3 +471,8 @@ func (f *failureCount) Message(level status.MsgLevel, message string) { } func (f *failureCount) Flush() {} + +func (f *failureCount) Write(p []byte) (int, error) { + // discard writes + return len(p), nil +} diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go index 7057b33f..151e8bbf 100644 --- a/cmd/sbox/sbox.go +++ b/cmd/sbox/sbox.go @@ -35,6 +35,7 @@ var ( rawCommand string outputRoot string keepOutDir bool + copyAllOutput bool depfileOut string ) @@ -47,6 +48,8 @@ func init() { "root of directory to copy outputs into") flag.BoolVar(&keepOutDir, "keep-out-dir", false, "whether to keep the sandbox directory when done") + flag.BoolVar(©AllOutput, "copy-all-output", false, + "whether to copy all output files") flag.StringVar(&depfileOut, "depfile-out", "", "file path of the depfile to generate. This value will replace '__SBOX_DEPFILE__' in the command and will be treated as an output but won't be added to __SBOX_OUT_FILES__") @@ -117,7 +120,7 @@ func run() error { // the contents of the __SBOX_OUT_FILES__ variable outputsVarEntries := flag.Args() - if len(outputsVarEntries) == 0 { + if !copyAllOutput && len(outputsVarEntries) == 0 { usageViolation("at least one output file must be given") } @@ -223,7 +226,7 @@ func run() error { missingOutputErrors = append(missingOutputErrors, fmt.Sprintf("%s: not a file", filePath)) } } - if len(missingOutputErrors) > 0 { + if !copyAllOutput && len(missingOutputErrors) > 0 { // find all created files for making a more informative error message createdFiles := findAllFilesUnder(tempDir) @@ -255,8 +258,14 @@ func run() error { keepOutDir = true return errors.New(errorMessage) } + var filePathList []string + if copyAllOutput { + filePathList = findAllFilesUnder(tempDir) + } else { + filePathList = allOutputs + } // the created files match the declared files; now move them - for _, filePath := range allOutputs { + for _, filePath := range filePathList { tempPath := filepath.Join(tempDir, filePath) destPath := filePath if len(outputRoot) != 0 { diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 9f40e33e..974c644a 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -32,42 +32,103 @@ import ( "android/soong/ui/tracer" ) +// A command represents an operation to be executed in the soong build +// system. +type command struct { + // the flag name (must have double dashes) + flag string + + // description for the flag (to display when running help) + description string + + // Forces the status output into dumb terminal mode. + forceDumbOutput bool + + // Sets a prefix string to use for filenames of log files. + logsPrefix string + + // Creates the build configuration based on the args and build context. + config func(ctx build.Context, args ...string) build.Config + + // Returns what type of IO redirection this Command requires. + stdio func() terminal.StdioInterface + + // run the command + run func(ctx build.Context, config build.Config, args []string, logsDir string) +} + +const makeModeFlagName = "--make-mode" + +// list of supported commands (flags) supported by soong ui +var commands []command = []command{ + { + flag: makeModeFlagName, + description: "build the modules by the target name (i.e. soong_docs)", + config: func(ctx build.Context, args ...string) build.Config { + return build.NewConfig(ctx, args...) + }, + stdio: stdio, + run: make, + }, { + flag: "--dumpvar-mode", + description: "print the value of the legacy make variable VAR to stdout", + forceDumbOutput: true, + logsPrefix: "dumpvars-", + config: dumpVarConfig, + stdio: customStdio, + run: dumpVar, + }, { + flag: "--dumpvars-mode", + description: "dump the values of one or more legacy make variables, in shell syntax", + forceDumbOutput: true, + logsPrefix: "dumpvars-", + config: dumpVarConfig, + stdio: customStdio, + run: dumpVars, + }, { + flag: "--build-mode", + description: "build modules based on the specified build action", + config: buildActionConfig, + stdio: stdio, + run: make, + }, +} + +// indexList returns the index of first found s. -1 is return if s is not +// found. func indexList(s string, list []string) int { for i, l := range list { if l == s { return i } } - return -1 } +// inList returns true if one or more of s is in the list. func inList(s string, list []string) bool { return indexList(s, list) != -1 } +// Main execution of soong_ui. The command format is as follows: +// +// soong_ui <command> [<arg 1> <arg 2> ... <arg n>] +// +// Command is the type of soong_ui execution. Only one type of +// execution is specified. The args are specific to the command. func main() { - var stdio terminal.StdioInterface - stdio = terminal.StdioImpl{} - - // dumpvar uses stdout, everything else should be in stderr - if os.Args[1] == "--dumpvar-mode" || os.Args[1] == "--dumpvars-mode" { - stdio = terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) + c, args := getCommand(os.Args) + if c == nil { + fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n") + os.Exit(1) } - writer := terminal.NewWriter(stdio) - defer writer.Finish() + output := terminal.NewStatusOutput(c.stdio().Stdout(), os.Getenv("NINJA_STATUS"), c.forceDumbOutput, + build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")) - log := logger.New(writer) + log := logger.New(output) defer log.Cleanup() - if len(os.Args) < 2 || !(inList("--make-mode", os.Args) || - os.Args[1] == "--dumpvars-mode" || - os.Args[1] == "--dumpvar-mode") { - - log.Fatalln("The `soong` native UI is not yet available.") - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -78,8 +139,7 @@ func main() { stat := &status.Status{} defer stat.Finish() - stat.AddOutput(terminal.NewStatusOutput(writer, os.Getenv("NINJA_STATUS"), - build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))) + stat.AddOutput(output) stat.AddOutput(trace.StatusTracer()) build.SetupSignals(log, cancel, func() { @@ -93,15 +153,11 @@ func main() { Logger: log, Metrics: met, Tracer: trace, - Writer: writer, + Writer: output, Status: stat, }} - var config build.Config - if os.Args[1] == "--dumpvars-mode" || os.Args[1] == "--dumpvar-mode" { - config = build.NewConfig(buildCtx) - } else { - config = build.NewConfig(buildCtx, os.Args[1:]...) - } + + config := c.config(buildCtx, args...) build.SetupOutDir(buildCtx, config) @@ -111,12 +167,14 @@ func main() { } os.MkdirAll(logsDir, 0777) - log.SetOutput(filepath.Join(logsDir, "soong.log")) - trace.SetOutput(filepath.Join(logsDir, "build.trace")) - stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log"))) - stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) + log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log")) + trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace")) + stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log"))) + stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log"))) + stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error"))) + stat.AddOutput(status.NewCriticalPath(log)) - defer met.Dump(filepath.Join(logsDir, "build_metrics")) + defer met.Dump(filepath.Join(logsDir, c.logsPrefix+"soong_metrics")) if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { if !strings.HasSuffix(start, "N") { @@ -141,28 +199,7 @@ func main() { defer f.Shutdown() build.FindSources(buildCtx, config, f) - if os.Args[1] == "--dumpvar-mode" { - dumpVar(buildCtx, config, os.Args[2:]) - } else if os.Args[1] == "--dumpvars-mode" { - dumpVars(buildCtx, config, os.Args[2:]) - } else { - if config.IsVerbose() { - writer.Print("! The argument `showcommands` is no longer supported.") - writer.Print("! Instead, the verbose log is always written to a compressed file in the output dir:") - writer.Print("!") - writer.Print(fmt.Sprintf("! gzip -cd %s/verbose.log.gz | less -R", logsDir)) - writer.Print("!") - writer.Print("! Older versions are saved in verbose.log.#.gz files") - writer.Print("") - time.Sleep(5 * time.Second) - } - - toBuild := build.BuildAll - if config.Checkbuild() { - toBuild |= build.RunBuildTests - } - build.Build(buildCtx, config, toBuild) - } + c.run(buildCtx, config, args, logsDir) } func fixBadDanglingLink(ctx build.Context, name string) { @@ -179,16 +216,16 @@ func fixBadDanglingLink(ctx build.Context, name string) { } } -func dumpVar(ctx build.Context, config build.Config, args []string) { +func dumpVar(ctx build.Context, config build.Config, args []string, _ string) { flags := flag.NewFlagSet("dumpvar", flag.ExitOnError) flags.Usage = func() { - fmt.Fprintf(os.Stderr, "usage: %s --dumpvar-mode [--abs] <VAR>\n\n", os.Args[0]) - fmt.Fprintln(os.Stderr, "In dumpvar mode, print the value of the legacy make variable VAR to stdout") - fmt.Fprintln(os.Stderr, "") + fmt.Fprintf(ctx.Writer, "usage: %s --dumpvar-mode [--abs] <VAR>\n\n", os.Args[0]) + fmt.Fprintln(ctx.Writer, "In dumpvar mode, print the value of the legacy make variable VAR to stdout") + fmt.Fprintln(ctx.Writer, "") - fmt.Fprintln(os.Stderr, "'report_config' is a special case that prints the human-readable config banner") - fmt.Fprintln(os.Stderr, "from the beginning of the build.") - fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(ctx.Writer, "'report_config' is a special case that prints the human-readable config banner") + fmt.Fprintln(ctx.Writer, "from the beginning of the build.") + fmt.Fprintln(ctx.Writer, "") flags.PrintDefaults() } abs := flags.Bool("abs", false, "Print the absolute path of the value") @@ -229,18 +266,18 @@ func dumpVar(ctx build.Context, config build.Config, args []string) { } } -func dumpVars(ctx build.Context, config build.Config, args []string) { +func dumpVars(ctx build.Context, config build.Config, args []string, _ string) { flags := flag.NewFlagSet("dumpvars", flag.ExitOnError) flags.Usage = func() { - fmt.Fprintf(os.Stderr, "usage: %s --dumpvars-mode [--vars=\"VAR VAR ...\"]\n\n", os.Args[0]) - fmt.Fprintln(os.Stderr, "In dumpvars mode, dump the values of one or more legacy make variables, in") - fmt.Fprintln(os.Stderr, "shell syntax. The resulting output may be sourced directly into a shell to") - fmt.Fprintln(os.Stderr, "set corresponding shell variables.") - fmt.Fprintln(os.Stderr, "") - - fmt.Fprintln(os.Stderr, "'report_config' is a special case that dumps a variable containing the") - fmt.Fprintln(os.Stderr, "human-readable config banner from the beginning of the build.") - fmt.Fprintln(os.Stderr, "") + fmt.Fprintf(ctx.Writer, "usage: %s --dumpvars-mode [--vars=\"VAR VAR ...\"]\n\n", os.Args[0]) + fmt.Fprintln(ctx.Writer, "In dumpvars mode, dump the values of one or more legacy make variables, in") + fmt.Fprintln(ctx.Writer, "shell syntax. The resulting output may be sourced directly into a shell to") + fmt.Fprintln(ctx.Writer, "set corresponding shell variables.") + fmt.Fprintln(ctx.Writer, "") + + fmt.Fprintln(ctx.Writer, "'report_config' is a special case that dumps a variable containing the") + fmt.Fprintln(ctx.Writer, "human-readable config banner from the beginning of the build.") + fmt.Fprintln(ctx.Writer, "") flags.PrintDefaults() } @@ -296,3 +333,158 @@ func dumpVars(ctx build.Context, config build.Config, args []string) { fmt.Printf("%s%s='%s'\n", *absVarPrefix, name, strings.Join(res, " ")) } } + +func stdio() terminal.StdioInterface { + return terminal.StdioImpl{} +} + +func customStdio() terminal.StdioInterface { + return terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) +} + +// dumpVarConfig does not require any arguments to be parsed by the NewConfig. +func dumpVarConfig(ctx build.Context, args ...string) build.Config { + return build.NewConfig(ctx) +} + +func buildActionConfig(ctx build.Context, args ...string) build.Config { + flags := flag.NewFlagSet("build-mode", flag.ContinueOnError) + flags.Usage = func() { + fmt.Fprintf(ctx.Writer, "usage: %s --build-mode --dir=<path> <build action> [<build arg 1> <build arg 2> ...]\n\n", os.Args[0]) + fmt.Fprintln(ctx.Writer, "In build mode, build the set of modules based on the specified build") + fmt.Fprintln(ctx.Writer, "action. The --dir flag is required to determine what is needed to") + fmt.Fprintln(ctx.Writer, "build in the source tree based on the build action. See below for") + fmt.Fprintln(ctx.Writer, "the list of acceptable build action flags.") + fmt.Fprintln(ctx.Writer, "") + flags.PrintDefaults() + } + + buildActionFlags := []struct { + name string + description string + action build.BuildAction + set bool + }{{ + name: "all-modules", + description: "Build action: build from the top of the source tree.", + action: build.BUILD_MODULES, + }, { + // This is redirecting to mma build command behaviour. Once it has soaked for a + // while, the build command is deleted from here once it has been removed from the + // envsetup.sh. + name: "modules-in-a-dir-no-deps", + description: "Build action: builds all of the modules in the current directory without their dependencies.", + action: build.BUILD_MODULES_IN_A_DIRECTORY, + }, { + // This is redirecting to mmma build command behaviour. Once it has soaked for a + // while, the build command is deleted from here once it has been removed from the + // envsetup.sh. + name: "modules-in-dirs-no-deps", + description: "Build action: builds all of the modules in the supplied directories without their dependencies.", + action: build.BUILD_MODULES_IN_DIRECTORIES, + }, { + name: "modules-in-a-dir", + description: "Build action: builds all of the modules in the current directory and their dependencies.", + action: build.BUILD_MODULES_IN_A_DIRECTORY, + }, { + name: "modules-in-dirs", + description: "Build action: builds all of the modules in the supplied directories and their dependencies.", + action: build.BUILD_MODULES_IN_DIRECTORIES, + }} + for i, flag := range buildActionFlags { + flags.BoolVar(&buildActionFlags[i].set, flag.name, false, flag.description) + } + dir := flags.String("dir", "", "Directory of the executed build command.") + + // Only interested in the first two args which defines the build action and the directory. + // The remaining arguments are passed down to the config. + const numBuildActionFlags = 2 + if len(args) < numBuildActionFlags { + flags.Usage() + ctx.Fatalln("Improper build action arguments.") + } + flags.Parse(args[0:numBuildActionFlags]) + + // The next block of code is to validate that exactly one build action is set and the dir flag + // is specified. + buildActionCount := 0 + var buildAction build.BuildAction + for _, flag := range buildActionFlags { + if flag.set { + buildActionCount++ + buildAction = flag.action + } + } + if buildActionCount != 1 { + ctx.Fatalln("Build action not defined.") + } + if *dir == "" { + ctx.Fatalln("-dir not specified.") + } + + // Remove the build action flags from the args as they are not recognized by the config. + args = args[numBuildActionFlags:] + return build.NewBuildActionConfig(buildAction, *dir, ctx, args...) +} + +func make(ctx build.Context, config build.Config, _ []string, logsDir string) { + if config.IsVerbose() { + writer := ctx.Writer + fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.") + fmt.Fprintln(writer, "! Instead, the verbose log is always written to a compressed file in the output dir:") + fmt.Fprintln(writer, "!") + fmt.Fprintf(writer, "! gzip -cd %s/verbose.log.gz | less -R\n", logsDir) + fmt.Fprintln(writer, "!") + fmt.Fprintln(writer, "! Older versions are saved in verbose.log.#.gz files") + fmt.Fprintln(writer, "") + select { + case <-time.After(5 * time.Second): + case <-ctx.Done(): + return + } + } + + if _, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok { + writer := ctx.Writer + fmt.Fprintln(writer, "! The variable `ONE_SHOT_MAKEFILE` is obsolete.") + fmt.Fprintln(writer, "!") + fmt.Fprintln(writer, "! If you're using `mm`, you'll need to run `source build/envsetup.sh` to update.") + fmt.Fprintln(writer, "!") + fmt.Fprintln(writer, "! Otherwise, either specify a module name with m, or use mma / MODULES-IN-...") + fmt.Fprintln(writer, "") + ctx.Fatal("done") + } + + toBuild := build.BuildAll + if config.Checkbuild() { + toBuild |= build.RunBuildTests + } + build.Build(ctx, config, toBuild) +} + +// getCommand finds the appropriate command based on args[1] flag. args[0] +// is the soong_ui filename. +func getCommand(args []string) (*command, []string) { + if len(args) < 2 { + return nil, args + } + + for _, c := range commands { + if c.flag == args[1] { + return &c, args[2:] + } + + // special case for --make-mode: if soong_ui was called from + // build/make/core/main.mk, the makeparallel with --ninja + // option specified puts the -j<num> before --make-mode. + // TODO: Remove this hack once it has been fixed. + if c.flag == makeModeFlagName { + if inList(makeModeFlagName, args) { + return &c, args[1:] + } + } + } + + // command not found + return nil, args +} diff --git a/java/aar.go b/java/aar.go index 6273a9b5..53d946f6 100644 --- a/java/aar.go +++ b/java/aar.go @@ -30,7 +30,7 @@ type AndroidLibraryDependency interface { ExportedProguardFlagFiles() android.Paths ExportedRRODirs() []rroDir ExportedStaticPackages() android.Paths - ExportedManifest() android.Path + ExportedManifests() android.Paths } func init() { @@ -69,21 +69,26 @@ type aaptProperties struct { // path to AndroidManifest.xml. If unset, defaults to "AndroidManifest.xml". Manifest *string `android:"path"` + + // paths to additional manifest files to merge with main manifest. + Additional_manifests []string `android:"path"` } type aapt struct { - aaptSrcJar android.Path - exportPackage android.Path - manifestPath android.Path - proguardOptionsFile android.Path - rroDirs []rroDir - rTxt android.Path - extraAaptPackagesFile android.Path - noticeFile android.OptionalPath - isLibrary bool - uncompressedJNI bool - useEmbeddedDex bool - usesNonSdkApis bool + aaptSrcJar android.Path + exportPackage android.Path + manifestPath android.Path + transitiveManifestPaths android.Paths + proguardOptionsFile android.Path + rroDirs []rroDir + rTxt android.Path + extraAaptPackagesFile android.Path + mergedManifestFile android.Path + noticeFile android.OptionalPath + isLibrary bool + uncompressedJNI bool + useEmbeddedDex bool + usesNonSdkApis bool splitNames []string splits []split @@ -105,8 +110,8 @@ func (a *aapt) ExportedRRODirs() []rroDir { return a.rroDirs } -func (a *aapt) ExportedManifest() android.Path { - return a.manifestPath +func (a *aapt) ExportedManifests() android.Paths { + return a.transitiveManifestPaths } func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, manifestPath android.Path) (flags []string, @@ -177,7 +182,7 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, mani if !hasVersionName { var versionName string - if ctx.ModuleName() == "framework-res" { + if ctx.ModuleName() == "framework-res" || ctx.ModuleName() == "org.lineageos.platform-res" { // Some builds set AppsDefaultVersionName() to include the build number ("O-123456"). aapt2 copies the // version name of framework-res into app manifests as compileSdkVersionCodename, which confuses things // if it contains the build number. Use the PlatformVersionName instead. @@ -197,17 +202,37 @@ func (a *aapt) deps(ctx android.BottomUpMutatorContext, sdkContext sdkContext) { if sdkDep.frameworkResModule != "" { ctx.AddVariationDependencies(nil, frameworkResTag, sdkDep.frameworkResModule) } + if sdkDep.lineageResModule != "" { + ctx.AddDependency(ctx.Module(), lineageResTag, sdkDep.lineageResModule) + } } func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, extraLinkFlags ...string) { - transitiveStaticLibs, staticLibManifests, staticRRODirs, libDeps, libFlags := aaptLibs(ctx, sdkContext) + transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, libDeps, libFlags := aaptLibs(ctx, sdkContext) // App manifest file manifestFile := proptools.StringDefault(a.aaptProperties.Manifest, "AndroidManifest.xml") manifestSrcPath := android.PathForModuleSrc(ctx, manifestFile) - manifestPath := manifestMerger(ctx, manifestSrcPath, sdkContext, staticLibManifests, a.isLibrary, - a.uncompressedJNI, a.useEmbeddedDex, a.usesNonSdkApis) + manifestPath := manifestFixer(ctx, manifestSrcPath, sdkContext, + a.isLibrary, a.uncompressedJNI, a.usesNonSdkApis, a.useEmbeddedDex) + + // Add additional manifest files to transitive manifests. + additionalManifests := android.PathsForModuleSrc(ctx, a.aaptProperties.Additional_manifests) + a.transitiveManifestPaths = append(android.Paths{manifestPath}, additionalManifests...) + a.transitiveManifestPaths = append(a.transitiveManifestPaths, transitiveStaticLibManifests...) + + if len(a.transitiveManifestPaths) > 1 { + a.mergedManifestFile = manifestMerger(ctx, a.transitiveManifestPaths[0], a.transitiveManifestPaths[1:], a.isLibrary) + if !a.isLibrary { + // Only use the merged manifest for applications. For libraries, the transitive closure of manifests + // will be propagated to the final application and merged there. The merged manifest for libraries is + // only passed to Make, which can't handle transitive dependencies. + manifestPath = a.mergedManifestFile + } + } else { + a.mergedManifestFile = manifestPath + } linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resZips := a.aapt2Flags(ctx, sdkContext, manifestPath) @@ -294,7 +319,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, ex } // aaptLibs collects libraries from dependencies and sdk_version and converts them into paths -func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStaticLibs, staticLibManifests android.Paths, +func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStaticLibs, transitiveStaticLibManifests android.Paths, staticRRODirs []rroDir, deps android.Paths, flags []string) { var sharedLibs android.Paths @@ -314,7 +339,7 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStati switch ctx.OtherModuleDependencyTag(module) { case instrumentationForTag: // Nothing, instrumentationForTag is treated as libTag for javac but not for aapt2. - case libTag, frameworkResTag: + case libTag, frameworkResTag, lineageResTag: if exportPackage != nil { sharedLibs = append(sharedLibs, exportPackage) } @@ -322,7 +347,7 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStati if exportPackage != nil { transitiveStaticLibs = append(transitiveStaticLibs, aarDep.ExportedStaticPackages()...) transitiveStaticLibs = append(transitiveStaticLibs, exportPackage) - staticLibManifests = append(staticLibManifests, aarDep.ExportedManifest()) + transitiveStaticLibManifests = append(transitiveStaticLibManifests, aarDep.ExportedManifests()...) outer: for _, d := range aarDep.ExportedRRODirs() { @@ -349,8 +374,9 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStati } transitiveStaticLibs = android.FirstUniquePaths(transitiveStaticLibs) + transitiveStaticLibManifests = android.FirstUniquePaths(transitiveStaticLibManifests) - return transitiveStaticLibs, staticLibManifests, staticRRODirs, deps, flags + return transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, deps, flags } type AndroidLibrary struct { @@ -506,8 +532,8 @@ func (a *AARImport) ExportedStaticPackages() android.Paths { return a.exportedStaticPackages } -func (a *AARImport) ExportedManifest() android.Path { - return a.manifest +func (a *AARImport) ExportedManifests() android.Paths { + return android.Paths{a.manifest} } func (a *AARImport) Prebuilt() *android.Prebuilt { @@ -524,6 +550,9 @@ func (a *AARImport) DepsMutator(ctx android.BottomUpMutatorContext) { if sdkDep.useModule && sdkDep.frameworkResModule != "" { ctx.AddVariationDependencies(nil, frameworkResTag, sdkDep.frameworkResModule) } + if sdkDep.useModule && sdkDep.lineageResModule != "" { + ctx.AddDependency(ctx.Module(), lineageResTag, sdkDep.lineageResModule) + } } ctx.AddVariationDependencies(nil, libTag, a.properties.Libs...) diff --git a/java/android_manifest.go b/java/android_manifest.go index 8dc3b475..ea7c2dd4 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -36,13 +36,14 @@ var manifestFixerRule = pctx.AndroidStaticRule("manifestFixer", var manifestMergerRule = pctx.AndroidStaticRule("manifestMerger", blueprint.RuleParams{ - Command: `${config.ManifestMergerCmd} --main $in $libs --out $out`, + Command: `${config.ManifestMergerCmd} $args --main $in $libs --out $out`, CommandDeps: []string{"${config.ManifestMergerCmd}"}, }, - "libs") + "args", "libs") -func manifestMerger(ctx android.ModuleContext, manifest android.Path, sdkContext sdkContext, - staticLibManifests android.Paths, isLibrary, uncompressedJNI, useEmbeddedDex, usesNonSdkApis bool) android.Path { +// Uses manifest_fixer.py to inject minSdkVersion, etc. into an AndroidManifest.xml +func manifestFixer(ctx android.ModuleContext, manifest android.Path, sdkContext sdkContext, + isLibrary, uncompressedJNI, usesNonSdkApis, useEmbeddedDex bool) android.Path { var args []string if isLibrary { @@ -79,35 +80,44 @@ func manifestMerger(ctx android.ModuleContext, manifest android.Path, sdkContext deps = append(deps, apiFingerprint) } - // Inject minSdkVersion into the manifest fixedManifest := android.PathForModuleOut(ctx, "manifest_fixer", "AndroidManifest.xml") ctx.Build(pctx, android.BuildParams{ - Rule: manifestFixerRule, - Input: manifest, - Implicits: deps, - Output: fixedManifest, + Rule: manifestFixerRule, + Description: "fix manifest", + Input: manifest, + Implicits: deps, + Output: fixedManifest, Args: map[string]string{ "minSdkVersion": sdkVersionOrDefault(ctx, sdkContext.minSdkVersion()), "targetSdkVersion": targetSdkVersion, "args": strings.Join(args, " "), }, }) - manifest = fixedManifest - - // Merge static aar dependency manifests if necessary - if len(staticLibManifests) > 0 { - mergedManifest := android.PathForModuleOut(ctx, "manifest_merger", "AndroidManifest.xml") - ctx.Build(pctx, android.BuildParams{ - Rule: manifestMergerRule, - Input: manifest, - Implicits: staticLibManifests, - Output: mergedManifest, - Args: map[string]string{ - "libs": android.JoinWithPrefix(staticLibManifests.Strings(), "--libs "), - }, - }) - manifest = mergedManifest + + return fixedManifest +} + +func manifestMerger(ctx android.ModuleContext, manifest android.Path, staticLibManifests android.Paths, + isLibrary bool) android.Path { + + var args string + if !isLibrary { + // Follow Gradle's behavior, only pass --remove-tools-declarations when merging app manifests. + args = "--remove-tools-declarations" } - return manifest + mergedManifest := android.PathForModuleOut(ctx, "manifest_merger", "AndroidManifest.xml") + ctx.Build(pctx, android.BuildParams{ + Rule: manifestMergerRule, + Description: "merge manifest", + Input: manifest, + Implicits: staticLibManifests, + Output: mergedManifest, + Args: map[string]string{ + "libs": android.JoinWithPrefix(staticLibManifests.Strings(), "--libs "), + "args": args, + }, + }) + + return mergedManifest } diff --git a/java/androidmk.go b/java/androidmk.go index 43b9f488..9d238c5e 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -39,8 +39,8 @@ func (library *Library) AndroidMkHostDex(w io.Writer, name string, data android. fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", library.headerJarFile.String()) fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", library.implementationAndResourcesJar.String()) fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(data.Required, " ")) - if r := library.deviceProperties.Target.Hostdex.Required; len(r) > 0 { - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(r, " ")) + if len(data.Required) > 0 { + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES :=", strings.Join(data.Required, " ")) } fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_java_prebuilt.mk") } @@ -271,7 +271,7 @@ func (app *AndroidApp) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_SOONG_PROGUARD_DICT :=", app.proguardDictionary.String()) } - if app.Name() == "framework-res" { + if app.Name() == "framework-res" || app.Name() == "org.lineageos.platform-res" { fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(TARGET_OUT_JAVA_LIBRARIES)") // Make base_rules.mk not put framework-res in a subdirectory called // framework_res. @@ -320,7 +320,8 @@ func (app *AndroidApp) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED :=", app.dexpreopter.builtInstalled) } for _, split := range app.aapt.splits { - install := "$(LOCAL_MODULE_PATH)/" + strings.TrimSuffix(app.installApkName, ".apk") + split.suffix + ".apk" + install := app.onDeviceDir + "/" + + strings.TrimSuffix(app.installApkName, ".apk") + "_" + split.suffix + ".apk" fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED +=", split.path.String()+":"+install) } }, @@ -372,7 +373,7 @@ func (a *AndroidLibrary) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_SOONG_PROGUARD_DICT :=", a.proguardDictionary.String()) } - if a.Name() == "framework-res" { + if a.Name() == "framework-res" || a.Name() == "org.lineageos.platform-res" { fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(TARGET_OUT_JAVA_LIBRARIES)") // Make base_rules.mk not put framework-res in a subdirectory called // framework_res. @@ -381,7 +382,7 @@ func (a *AndroidLibrary) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE :=", a.exportPackage.String()) fmt.Fprintln(w, "LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES :=", a.extraAaptPackagesFile.String()) - fmt.Fprintln(w, "LOCAL_FULL_MANIFEST_FILE :=", a.manifestPath.String()) + fmt.Fprintln(w, "LOCAL_FULL_MANIFEST_FILE :=", a.mergedManifestFile.String()) fmt.Fprintln(w, "LOCAL_SOONG_EXPORT_PROGUARD_FLAGS :=", strings.Join(a.exportedProguardFlagFiles.Strings(), " ")) fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") @@ -584,6 +585,29 @@ func (dstubs *Droidstubs) AndroidMk() android.AndroidMkData { } } +func (a *AndroidAppImport) AndroidMkEntries() android.AndroidMkEntries { + return android.AndroidMkEntries{ + Class: "APPS", + OutputFile: android.OptionalPathForPath(a.outputFile), + Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBoolIfTrue("LOCAL_PRIVILEGED_MODULE", Bool(a.properties.Privileged)) + if a.certificate != nil { + entries.SetString("LOCAL_CERTIFICATE", a.certificate.Pem.String()) + } else { + entries.SetString("LOCAL_CERTIFICATE", "PRESIGNED") + } + entries.AddStrings("LOCAL_OVERRIDES_PACKAGES", a.properties.Overrides...) + if len(a.dexpreopter.builtInstalled) > 0 { + entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", a.dexpreopter.builtInstalled) + } + entries.AddStrings("LOCAL_INSTALLED_MODULE_STEM", a.installPath.Rel()) + }, + }, + } +} + func androidMkWriteTestData(data android.Paths, ret *android.AndroidMkData) { var testFiles []string for _, d := range data { diff --git a/java/app.go b/java/app.go index ad672ead..fdea4350 100644 --- a/java/app.go +++ b/java/app.go @@ -17,24 +17,29 @@ package java // This file contains the module types for compiling Android apps. import ( + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" "path/filepath" + "reflect" "sort" "strings" - "github.com/google/blueprint" - "github.com/google/blueprint/proptools" - "android/soong/android" "android/soong/cc" "android/soong/tradefed" ) +var supportedDpis = []string{"ldpi", "mdpi", "hdpi", "xhdpi", "xxhdpi", "xxxhdpi"} + func init() { android.RegisterModuleType("android_app", AndroidAppFactory) android.RegisterModuleType("android_test", AndroidTestFactory) android.RegisterModuleType("android_test_helper_app", AndroidTestHelperAppFactory) android.RegisterModuleType("android_app_certificate", AndroidAppCertificateFactory) android.RegisterModuleType("override_android_app", OverrideAndroidAppModuleFactory) + android.RegisterModuleType("android_app_import", AndroidAppImportFactory) + + initAndroidAppImportVariantGroupTypes() } // AndroidManifest.xml merging @@ -114,6 +119,10 @@ type AndroidApp struct { // the install APK name is normally the same as the module name, but can be overridden with PRODUCT_PACKAGE_NAME_OVERRIDES. installApkName string + installDir android.OutputPath + + onDeviceDir string + additionalAaptFlags []string } @@ -188,11 +197,9 @@ func (a *AndroidApp) shouldUncompressDex(ctx android.ModuleContext) bool { return true } - // Uncompress dex in APKs of privileged apps, and modules used by privileged apps - // (even for unbundled builds, they may be preinstalled as prebuilts). - if ctx.Config().UncompressPrivAppDex() && - (Bool(a.appProperties.Privileged) || - inList(ctx.ModuleName(), ctx.Config().ModulesLoadedByPrivilegedModules())) { + // Uncompress dex in APKs of privileged apps (even for unbundled builds, they may + // be preinstalled as prebuilts). + if ctx.Config().UncompressPrivAppDex() && Bool(a.appProperties.Privileged) { return true } @@ -200,12 +207,7 @@ func (a *AndroidApp) shouldUncompressDex(ctx android.ModuleContext) bool { return false } - // Uncompress if the dex files is preopted on /system. - if !a.dexpreopter.dexpreoptDisabled(ctx) && (ctx.Host() || !odexOnSystemOther(ctx, a.dexpreopter.installPath)) { - return true - } - - return false + return shouldUncompressDex(ctx, &a.dexpreopter) } func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { @@ -286,7 +288,7 @@ func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { a.dexpreopter.uncompressedDex = a.shouldUncompressDex(ctx) a.deviceProperties.UncompressDex = a.dexpreopter.uncompressedDex - if ctx.ModuleName() != "framework-res" { + if ctx.ModuleName() != "framework-res" && ctx.ModuleName() != "org.lineageos.platform-res" { a.Module.compile(ctx, a.aaptSrcJar) } @@ -308,40 +310,41 @@ func (a *AndroidApp) jniBuildActions(jniLibs []jniLib, ctx android.ModuleContext return jniJarFile } -func (a *AndroidApp) certificateBuildActions(certificateDeps []Certificate, ctx android.ModuleContext) []Certificate { - cert := a.getCertString(ctx) - certModule := android.SrcIsModule(cert) - if certModule != "" { - a.certificate = certificateDeps[0] - certificateDeps = certificateDeps[1:] - } else if cert != "" { - defaultDir := ctx.Config().DefaultAppCertificateDir(ctx) - a.certificate = Certificate{ - defaultDir.Join(ctx, cert+".x509.pem"), - defaultDir.Join(ctx, cert+".pk8"), +// Reads and prepends a main cert from the default cert dir if it hasn't been set already, i.e. it +// isn't a cert module reference. Also checks and enforces system cert restriction if applicable. +func processMainCert(m android.ModuleBase, certPropValue string, certificates []Certificate, ctx android.ModuleContext) []Certificate { + if android.SrcIsModule(certPropValue) == "" { + var mainCert Certificate + if certPropValue != "" { + defaultDir := ctx.Config().DefaultAppCertificateDir(ctx) + mainCert = Certificate{ + defaultDir.Join(ctx, certPropValue+".x509.pem"), + defaultDir.Join(ctx, certPropValue+".pk8"), + } + } else { + pem, key := ctx.Config().DefaultAppCertificate(ctx) + mainCert = Certificate{pem, key} } - } else { - pem, key := ctx.Config().DefaultAppCertificate(ctx) - a.certificate = Certificate{pem, key} + certificates = append([]Certificate{mainCert}, certificates...) } - if !a.Module.Platform() { - certPath := a.certificate.Pem.String() + if !m.Platform() { + certPath := certificates[0].Pem.String() systemCertPath := ctx.Config().DefaultAppCertificateDir(ctx).String() if strings.HasPrefix(certPath, systemCertPath) { enforceSystemCert := ctx.Config().EnforceSystemCertificate() whitelist := ctx.Config().EnforceSystemCertificateWhitelist() - if enforceSystemCert && !inList(a.Module.Name(), whitelist) { + if enforceSystemCert && !inList(m.Name(), whitelist) { ctx.PropertyErrorf("certificate", "The module in product partition cannot be signed with certificate in system.") } } } - return append([]Certificate{a.certificate}, certificateDeps...) + return certificates } -func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext, installDir android.OutputPath) android.OptionalPath { +func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext) android.OptionalPath { if !Bool(a.appProperties.Embed_notices) && !ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { return android.OptionalPath{} } @@ -388,7 +391,7 @@ func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext, installDir an sort.Slice(noticePaths, func(i, j int) bool { return noticePaths[i].String() < noticePaths[j].String() }) - noticeFile := android.BuildNoticeOutput(ctx, installDir, a.installApkName+".apk", noticePaths) + noticeFile := android.BuildNoticeOutput(ctx, a.installDir, a.installApkName+".apk", noticePaths) return android.OptionalPathForPath(noticeFile) } @@ -397,17 +400,22 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Check if the install APK name needs to be overridden. a.installApkName = ctx.DeviceConfig().OverridePackageNameFor(a.Name()) - var installDir android.OutputPath if ctx.ModuleName() == "framework-res" { // framework-res.apk is installed as system/framework/framework-res.apk - installDir = android.PathForModuleInstall(ctx, "framework") + a.installDir = android.PathForModuleInstall(ctx, "framework") + } else if ctx.ModuleName() == "org.lineageos.platform-res" { + // org.lineageos.platform-res.apk needs to be in system/framework + a.installDir = android.PathForModuleInstall(ctx, "framework") } else if Bool(a.appProperties.Privileged) { - installDir = android.PathForModuleInstall(ctx, "priv-app", a.installApkName) + a.installDir = android.PathForModuleInstall(ctx, "priv-app", a.installApkName) + } else if ctx.InstallInTestcases() { + a.installDir = android.PathForModuleInstall(ctx, a.installApkName) } else { - installDir = android.PathForModuleInstall(ctx, "app", a.installApkName) + a.installDir = android.PathForModuleInstall(ctx, "app", a.installApkName) } + a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir) - a.aapt.noticeFile = a.noticeBuildActions(ctx, installDir) + a.aapt.noticeFile = a.noticeBuildActions(ctx) // Process all building blocks, from AAPT to certificates. a.aaptBuildActions(ctx) @@ -416,25 +424,26 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { dexJarFile := a.dexBuildActions(ctx) - jniLibs, certificateDeps := a.collectAppDeps(ctx) + jniLibs, certificateDeps := collectAppDeps(ctx) jniJarFile := a.jniBuildActions(jniLibs, ctx) if ctx.Failed() { return } - certificates := a.certificateBuildActions(certificateDeps, ctx) + certificates := processMainCert(a.ModuleBase, a.getCertString(ctx), certificateDeps, ctx) + a.certificate = certificates[0] // Build a final signed app package. // TODO(jungjw): Consider changing this to installApkName. packageFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".apk") - CreateAppPackage(ctx, packageFile, a.exportPackage, jniJarFile, dexJarFile, certificates) + CreateAndSignAppPackage(ctx, packageFile, a.exportPackage, jniJarFile, dexJarFile, certificates) a.outputFile = packageFile for _, split := range a.aapt.splits { // Sign the split APKs packageFile := android.PathForModuleOut(ctx, ctx.ModuleName()+"_"+split.suffix+".apk") - CreateAppPackage(ctx, packageFile, split.path, nil, nil, certificates) + CreateAndSignAppPackage(ctx, packageFile, split.path, nil, nil, certificates) a.extraOutputFiles = append(a.extraOutputFiles, packageFile) } @@ -444,13 +453,13 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.bundleFile = bundleFile // Install the app package. - ctx.InstallFile(installDir, a.installApkName+".apk", a.outputFile) + ctx.InstallFile(a.installDir, a.installApkName+".apk", a.outputFile) for _, split := range a.aapt.splits { - ctx.InstallFile(installDir, a.installApkName+"_"+split.suffix+".apk", split.path) + ctx.InstallFile(a.installDir, a.installApkName+"_"+split.suffix+".apk", split.path) } } -func (a *AndroidApp) collectAppDeps(ctx android.ModuleContext) ([]jniLib, []Certificate) { +func collectAppDeps(ctx android.ModuleContext) ([]jniLib, []Certificate) { var jniLibs []jniLib var certificates []Certificate @@ -472,7 +481,6 @@ func (a *AndroidApp) collectAppDeps(ctx android.ModuleContext) ([]jniLib, []Cert } } else { ctx.ModuleErrorf("jni_libs dependency %q must be a cc library", otherName) - } } else if tag == certificateTag { if dep, ok := module.(*AndroidAppCertificate); ok { @@ -539,6 +547,10 @@ type AndroidTest struct { data android.Paths } +func (a *AndroidTest) InstallInTestcases() bool { + return true +} + func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Check if the instrumentation target package is overridden before generating build actions. if a.appTestProperties.Instrumentation_for != nil { @@ -680,3 +692,284 @@ func OverrideAndroidAppModuleFactory() android.Module { android.InitOverrideModule(m) return m } + +type AndroidAppImport struct { + android.ModuleBase + android.DefaultableModuleBase + prebuilt android.Prebuilt + + properties AndroidAppImportProperties + dpiVariants interface{} + archVariants interface{} + + outputFile android.Path + certificate *Certificate + + dexpreopter + + installPath android.OutputPath +} + +type AndroidAppImportProperties struct { + // A prebuilt apk to import + Apk *string + + // The name of a certificate in the default certificate directory or an android_app_certificate + // module name in the form ":module". Should be empty if presigned or default_dev_cert is set. + Certificate *string + + // Set this flag to true if the prebuilt apk is already signed. The certificate property must not + // be set for presigned modules. + Presigned *bool + + // Sign with the default system dev certificate. Must be used judiciously. Most imported apps + // need to either specify a specific certificate or be presigned. + Default_dev_cert *bool + + // Specifies that this app should be installed to the priv-app directory, + // where the system will grant it additional privileges not available to + // normal apps. + Privileged *bool + + // Names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string + + // Optional name for the installed app. If unspecified, it is derived from the module name. + Filename *string +} + +// Updates properties with variant-specific values. +func (a *AndroidAppImport) processVariants(ctx android.LoadHookContext) { + config := ctx.Config() + + dpiProps := reflect.ValueOf(a.dpiVariants).Elem().FieldByName("Dpi_variants") + // Try DPI variant matches in the reverse-priority order so that the highest priority match + // overwrites everything else. + // TODO(jungjw): Can we optimize this by making it priority order? + for i := len(config.ProductAAPTPrebuiltDPI()) - 1; i >= 0; i-- { + MergePropertiesFromVariant(ctx, &a.properties, dpiProps, config.ProductAAPTPrebuiltDPI()[i]) + } + if config.ProductAAPTPreferredConfig() != "" { + MergePropertiesFromVariant(ctx, &a.properties, dpiProps, config.ProductAAPTPreferredConfig()) + } + + archProps := reflect.ValueOf(a.archVariants).Elem().FieldByName("Arch") + archType := ctx.Config().Targets[android.Android][0].Arch.ArchType + MergePropertiesFromVariant(ctx, &a.properties, archProps, archType.Name) +} + +func MergePropertiesFromVariant(ctx android.BaseModuleContext, + dst interface{}, variantGroup reflect.Value, variant string) { + src := variantGroup.FieldByName(proptools.FieldNameForProperty(variant)) + if !src.IsValid() { + return + } + + err := proptools.ExtendMatchingProperties([]interface{}{dst}, src.Interface(), nil, proptools.OrderAppend) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) + } + } +} + +func (a *AndroidAppImport) DepsMutator(ctx android.BottomUpMutatorContext) { + cert := android.SrcIsModule(String(a.properties.Certificate)) + if cert != "" { + ctx.AddDependency(ctx.Module(), certificateTag, cert) + } +} + +func (a *AndroidAppImport) uncompressEmbeddedJniLibs( + ctx android.ModuleContext, inputPath android.Path, outputPath android.OutputPath) { + rule := android.NewRuleBuilder() + rule.Command(). + Textf(`if (zipinfo %s 'lib/*.so' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then`, inputPath). + Tool(ctx.Config().HostToolPath(ctx, "zip2zip")). + FlagWithInput("-i ", inputPath). + FlagWithOutput("-o ", outputPath). + FlagWithArg("-0 ", "'lib/**/*.so'"). + Textf(`; else cp -f %s %s; fi`, inputPath, outputPath) + rule.Build(pctx, ctx, "uncompress-embedded-jni-libs", "Uncompress embedded JIN libs") +} + +// Returns whether this module should have the dex file stored uncompressed in the APK. +func (a *AndroidAppImport) shouldUncompressDex(ctx android.ModuleContext) bool { + if ctx.Config().UnbundledBuild() { + return false + } + + // Uncompress dex in APKs of privileged apps + if ctx.Config().UncompressPrivAppDex() && Bool(a.properties.Privileged) { + return true + } + + return shouldUncompressDex(ctx, &a.dexpreopter) +} + +func (a *AndroidAppImport) uncompressDex( + ctx android.ModuleContext, inputPath android.Path, outputPath android.OutputPath) { + rule := android.NewRuleBuilder() + rule.Command(). + Textf(`if (zipinfo %s '*.dex' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then`, inputPath). + Tool(ctx.Config().HostToolPath(ctx, "zip2zip")). + FlagWithInput("-i ", inputPath). + FlagWithOutput("-o ", outputPath). + FlagWithArg("-0 ", "'classes*.dex'"). + Textf(`; else cp -f %s %s; fi`, inputPath, outputPath) + rule.Build(pctx, ctx, "uncompress-dex", "Uncompress dex files") +} + +func (a *AndroidAppImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { + numCertPropsSet := 0 + if String(a.properties.Certificate) != "" { + numCertPropsSet++ + } + if Bool(a.properties.Presigned) { + numCertPropsSet++ + } + if Bool(a.properties.Default_dev_cert) { + numCertPropsSet++ + } + if numCertPropsSet != 1 { + ctx.ModuleErrorf("One and only one of certficate, presigned, and default_dev_cert properties must be set") + } + + _, certificates := collectAppDeps(ctx) + + // TODO: LOCAL_EXTRACT_APK/LOCAL_EXTRACT_DPI_APK + // TODO: LOCAL_PACKAGE_SPLITS + + srcApk := a.prebuilt.SingleSourcePath(ctx) + + // TODO: Install or embed JNI libraries + + // Uncompress JNI libraries in the apk + jnisUncompressed := android.PathForModuleOut(ctx, "jnis-uncompressed", ctx.ModuleName()+".apk") + a.uncompressEmbeddedJniLibs(ctx, srcApk, jnisUncompressed.OutputPath) + + installDir := android.PathForModuleInstall(ctx, "app", a.BaseModuleName()) + a.dexpreopter.installPath = installDir.Join(ctx, a.BaseModuleName()+".apk") + a.dexpreopter.isInstallable = true + a.dexpreopter.isPresignedPrebuilt = Bool(a.properties.Presigned) + a.dexpreopter.uncompressedDex = a.shouldUncompressDex(ctx) + dexOutput := a.dexpreopter.dexpreopt(ctx, jnisUncompressed) + if a.dexpreopter.uncompressedDex { + dexUncompressed := android.PathForModuleOut(ctx, "dex-uncompressed", ctx.ModuleName()+".apk") + a.uncompressDex(ctx, dexOutput, dexUncompressed.OutputPath) + dexOutput = dexUncompressed + } + + // Sign or align the package + // TODO: Handle EXTERNAL + if !Bool(a.properties.Presigned) { + // If the certificate property is empty at this point, default_dev_cert must be set to true. + // Which makes processMainCert's behavior for the empty cert string WAI. + certificates = processMainCert(a.ModuleBase, String(a.properties.Certificate), certificates, ctx) + if len(certificates) != 1 { + ctx.ModuleErrorf("Unexpected number of certificates were extracted: %q", certificates) + } + a.certificate = &certificates[0] + signed := android.PathForModuleOut(ctx, "signed", ctx.ModuleName()+".apk") + SignAppPackage(ctx, signed, dexOutput, certificates) + a.outputFile = signed + } else { + alignedApk := android.PathForModuleOut(ctx, "zip-aligned", ctx.ModuleName()+".apk") + TransformZipAlign(ctx, alignedApk, dexOutput) + a.outputFile = alignedApk + } + + // TODO: Optionally compress the output apk. + + a.installPath = ctx.InstallFile(installDir, + proptools.StringDefault(a.properties.Filename, a.BaseModuleName()+".apk"), a.outputFile) + + // TODO: androidmk converter jni libs +} + +func (a *AndroidAppImport) Prebuilt() *android.Prebuilt { + return &a.prebuilt +} + +func (a *AndroidAppImport) Name() string { + return a.prebuilt.Name(a.ModuleBase.Name()) +} + +var dpiVariantGroupType reflect.Type +var archVariantGroupType reflect.Type + +func initAndroidAppImportVariantGroupTypes() { + dpiVariantGroupType = createVariantGroupType(supportedDpis, "Dpi_variants") + + archNames := make([]string, len(android.ArchTypeList())) + for i, archType := range android.ArchTypeList() { + archNames[i] = archType.Name + } + archVariantGroupType = createVariantGroupType(archNames, "Arch") +} + +// Populates all variant struct properties at creation time. +func (a *AndroidAppImport) populateAllVariantStructs() { + a.dpiVariants = reflect.New(dpiVariantGroupType).Interface() + a.AddProperties(a.dpiVariants) + + a.archVariants = reflect.New(archVariantGroupType).Interface() + a.AddProperties(a.archVariants) +} + +func createVariantGroupType(variants []string, variantGroupName string) reflect.Type { + props := reflect.TypeOf((*AndroidAppImportProperties)(nil)) + + variantFields := make([]reflect.StructField, len(variants)) + for i, variant := range variants { + variantFields[i] = reflect.StructField{ + Name: proptools.FieldNameForProperty(variant), + Type: props, + } + } + + variantGroupStruct := reflect.StructOf(variantFields) + return reflect.StructOf([]reflect.StructField{ + { + Name: variantGroupName, + Type: variantGroupStruct, + }, + }) +} + +// android_app_import imports a prebuilt apk with additional processing specified in the module. +// DPI-specific apk source files can be specified using dpi_variants. Example: +// +// android_app_import { +// name: "example_import", +// apk: "prebuilts/example.apk", +// dpi_variants: { +// mdpi: { +// apk: "prebuilts/example_mdpi.apk", +// }, +// xhdpi: { +// apk: "prebuilts/example_xhdpi.apk", +// }, +// }, +// certificate: "PRESIGNED", +// } +func AndroidAppImportFactory() android.Module { + module := &AndroidAppImport{} + module.AddProperties(&module.properties) + module.AddProperties(&module.dexpreoptProperties) + module.populateAllVariantStructs() + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + module.processVariants(ctx) + }) + + InitJavaModule(module, android.DeviceSupported) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Apk") + + return module +} diff --git a/java/app_builder.go b/java/app_builder.go index 5bacb677..82a390f6 100644 --- a/java/app_builder.go +++ b/java/app_builder.go @@ -62,7 +62,7 @@ var combineApk = pctx.AndroidStaticRule("combineApk", CommandDeps: []string{"${config.MergeZipsCmd}"}, }) -func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath, +func CreateAndSignAppPackage(ctx android.ModuleContext, outputFile android.WritablePath, packageFile, jniJarFile, dexJarFile android.Path, certificates []Certificate) { unsignedApkName := strings.TrimSuffix(outputFile.Base(), ".apk") + "-unsigned.apk" @@ -83,6 +83,11 @@ func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath Output: unsignedApk, }) + SignAppPackage(ctx, outputFile, unsignedApk, certificates) +} + +func SignAppPackage(ctx android.ModuleContext, signedApk android.WritablePath, unsignedApk android.Path, certificates []Certificate) { + var certificateArgs []string var deps android.Paths for _, c := range certificates { @@ -93,7 +98,7 @@ func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath ctx.Build(pctx, android.BuildParams{ Rule: Signapk, Description: "signapk", - Output: outputFile, + Output: signedApk, Input: unsignedApk, Implicits: deps, Args: map[string]string{ diff --git a/java/app_test.go b/java/app_test.go index 2bd44ad4..a0beaca8 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -15,17 +15,22 @@ package java import ( - "android/soong/android" - "android/soong/cc" - "fmt" "path/filepath" "reflect" +/* + "regexp" +*/ "sort" "strings" "testing" +/* "github.com/google/blueprint/proptools" +*/ + + "android/soong/android" + "android/soong/cc" ) var ( @@ -82,8 +87,11 @@ func TestApp(t *testing.T) { expectedLinkImplicits = append(expectedLinkImplicits, manifestFixer.Output.String()) frameworkRes := ctx.ModuleForTests("framework-res", "android_common") + lineageRes := ctx.ModuleForTests("org.lineageos.platform-res", "android_common") expectedLinkImplicits = append(expectedLinkImplicits, frameworkRes.Output("package-res.apk").Output.String()) + expectedLinkImplicits = append(expectedLinkImplicits, + lineageRes.Output("package-res.apk").Output.String()) // Test the mapping from input files to compiled output file names compile := foo.Output(compiledResourceFiles[0]) @@ -714,7 +722,7 @@ func TestCertificates(t *testing.T) { } `, certificateOverride: "", - expected: "build/target/product/security/testkey.x509.pem build/target/product/security/testkey.pk8", + expected: "build/make/target/product/security/testkey.x509.pem build/make/target/product/security/testkey.pk8", }, { name: "module certificate property", @@ -743,7 +751,7 @@ func TestCertificates(t *testing.T) { } `, certificateOverride: "", - expected: "build/target/product/security/expiredkey.x509.pem build/target/product/security/expiredkey.pk8", + expected: "build/make/target/product/security/expiredkey.x509.pem build/make/target/product/security/expiredkey.pk8", }, { name: "certificate overrides", @@ -912,7 +920,7 @@ func TestOverrideAndroidApp(t *testing.T) { { variantName: "android_common", apkPath: "/target/product/test_device/system/app/foo/foo.apk", - signFlag: "build/target/product/security/expiredkey.x509.pem build/target/product/security/expiredkey.pk8", + signFlag: "build/make/target/product/security/expiredkey.x509.pem build/make/target/product/security/expiredkey.pk8", overrides: []string{"baz"}, aaptFlag: "", }, @@ -926,7 +934,7 @@ func TestOverrideAndroidApp(t *testing.T) { { variantName: "baz_android_common", apkPath: "/target/product/test_device/system/app/baz/baz.apk", - signFlag: "build/target/product/security/expiredkey.x509.pem build/target/product/security/expiredkey.pk8", + signFlag: "build/make/target/product/security/expiredkey.x509.pem build/make/target/product/security/expiredkey.pk8", overrides: []string{"baz", "foo"}, aaptFlag: "--rename-manifest-package org.dandroid.bp", }, @@ -1066,6 +1074,7 @@ func TestEmbedNotice(t *testing.T) { } } +/* func TestUncompressDex(t *testing.T) { testCases := []struct { name string @@ -1148,3 +1157,310 @@ func TestUncompressDex(t *testing.T) { }) } } +*/ + +func TestAndroidAppImport(t *testing.T) { + ctx := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + dex_preopt: { + enabled: true, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + t.Errorf("can't find dexpreopt outputs") + } + + // Check cert signing flag. + signedApk := variant.Output("signed/foo.apk") + signingFlag := signedApk.Args["certificates"] + expected := "build/make/target/product/security/platform.x509.pem build/make/target/product/security/platform.pk8" + if expected != signingFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected, signingFlag) + } +} + +func TestAndroidAppImport_NoDexPreopt(t *testing.T) { + ctx := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + dex_preopt: { + enabled: false, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. They shouldn't exist. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule != nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule != nil { + t.Errorf("dexpreopt shouldn't have run.") + } +} + +func TestAndroidAppImport_Presigned(t *testing.T) { + ctx := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + t.Errorf("can't find dexpreopt outputs") + } + // Make sure stripping wasn't done. + stripRule := variant.Output("dexpreopt/foo.apk") + if !strings.HasPrefix(stripRule.RuleParams.Command, "cp -f") { + t.Errorf("unexpected, non-skipping strip command: %q", stripRule.RuleParams.Command) + } + + // Make sure signing was skipped and aligning was done instead. + if variant.MaybeOutput("signed/foo.apk").Rule != nil { + t.Errorf("signing rule shouldn't be included.") + } + if variant.MaybeOutput("zip-aligned/foo.apk").Rule == nil { + t.Errorf("can't find aligning rule") + } +} + +func TestAndroidAppImport_DefaultDevCert(t *testing.T) { + ctx := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + default_dev_cert: true, + dex_preopt: { + enabled: true, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + t.Errorf("can't find dexpreopt outputs") + } + + // Check cert signing flag. + signedApk := variant.Output("signed/foo.apk") + signingFlag := signedApk.Args["certificates"] + expected := "build/make/target/product/security/testkey.x509.pem build/make/target/product/security/testkey.pk8" + if expected != signingFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected, signingFlag) + } +} + +/* +func TestAndroidAppImport_DpiVariants(t *testing.T) { + bp := ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + dpi_variants: { + xhdpi: { + apk: "prebuilts/apk/app_xhdpi.apk", + }, + xxhdpi: { + apk: "prebuilts/apk/app_xxhdpi.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + ` + testCases := []struct { + name string + aaptPreferredConfig *string + aaptPrebuiltDPI []string + expected string + }{ + { + name: "no preferred", + aaptPreferredConfig: nil, + aaptPrebuiltDPI: []string{}, + expected: "prebuilts/apk/app.apk", + }, + { + name: "AAPTPreferredConfig matches", + aaptPreferredConfig: proptools.StringPtr("xhdpi"), + aaptPrebuiltDPI: []string{"xxhdpi", "ldpi"}, + expected: "prebuilts/apk/app_xhdpi.apk", + }, + { + name: "AAPTPrebuiltDPI matches", + aaptPreferredConfig: proptools.StringPtr("mdpi"), + aaptPrebuiltDPI: []string{"xxhdpi", "xhdpi"}, + expected: "prebuilts/apk/app_xxhdpi.apk", + }, + { + name: "non-first AAPTPrebuiltDPI matches", + aaptPreferredConfig: proptools.StringPtr("mdpi"), + aaptPrebuiltDPI: []string{"ldpi", "xhdpi"}, + expected: "prebuilts/apk/app_xhdpi.apk", + }, + { + name: "no matches", + aaptPreferredConfig: proptools.StringPtr("mdpi"), + aaptPrebuiltDPI: []string{"ldpi", "xxxhdpi"}, + expected: "prebuilts/apk/app.apk", + }, + } + + jniRuleRe := regexp.MustCompile("^if \\(zipinfo (\\S+)") + for _, test := range testCases { + config := testConfig(nil) + config.TestProductVariables.AAPTPreferredConfig = test.aaptPreferredConfig + config.TestProductVariables.AAPTPrebuiltDPI = test.aaptPrebuiltDPI + ctx := testAppContext(config, bp, nil) + + run(t, ctx, config) + + variant := ctx.ModuleForTests("foo", "android_common") + jniRuleCommand := variant.Output("jnis-uncompressed/foo.apk").RuleParams.Command + matches := jniRuleRe.FindStringSubmatch(jniRuleCommand) + if len(matches) != 2 { + t.Errorf("failed to extract the src apk path from %q", jniRuleCommand) + } + if test.expected != matches[1] { + t.Errorf("wrong src apk, expected: %q got: %q", test.expected, matches[1]) + } + } +} +*/ + +func TestAndroidAppImport_Filename(t *testing.T) { + config := testConfig(nil) + ctx := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + } + + android_app_import { + name: "bar", + apk: "prebuilts/apk/app.apk", + presigned: true, + filename: "bar_sample.apk" + } + `) + + testCases := []struct { + name string + expected string + }{ + { + name: "foo", + expected: "foo.apk", + }, + { + name: "bar", + expected: "bar_sample.apk", + }, + } + + for _, test := range testCases { + variant := ctx.ModuleForTests(test.name, "android_common") + if variant.MaybeOutput(test.expected).Rule == nil { + t.Errorf("can't find output named %q - all outputs: %v", test.expected, variant.AllOutputs()) + } + + a := variant.Module().(*AndroidAppImport) + expectedValues := []string{test.expected} + actualValues := android.AndroidMkEntriesForTest( + t, config, "", a).EntryMap["LOCAL_INSTALLED_MODULE_STEM"] + if !reflect.DeepEqual(actualValues, expectedValues) { + t.Errorf("Incorrect LOCAL_INSTALLED_MODULE_STEM value '%s', expected '%s'", + actualValues, expectedValues) + } + } +} + +/* +func TestAndroidAppImport_ArchVariants(t *testing.T) { + // The test config's target arch is ARM64. + testCases := []struct { + name string + bp string + expected string + }{ + { + name: "matching arch", + bp: ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + arch: { + arm64: { + apk: "prebuilts/apk/app_arm64.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `, + expected: "prebuilts/apk/app_arm64.apk", + }, + { + name: "no matching arch", + bp: ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + arch: { + arm: { + apk: "prebuilts/apk/app_arm.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `, + expected: "prebuilts/apk/app.apk", + }, + } + + jniRuleRe := regexp.MustCompile("^if \\(zipinfo (\\S+)") + for _, test := range testCases { + ctx := testJava(t, test.bp) + + variant := ctx.ModuleForTests("foo", "android_common") + jniRuleCommand := variant.Output("jnis-uncompressed/foo.apk").RuleParams.Command + matches := jniRuleRe.FindStringSubmatch(jniRuleCommand) + if len(matches) != 2 { + t.Errorf("failed to extract the src apk path from %q", jniRuleCommand) + } + if test.expected != matches[1] { + t.Errorf("wrong src apk, expected: %q got: %q", test.expected, matches[1]) + } + } +} +*/ diff --git a/java/config/config.go b/java/config/config.go index 7f968bcc..7263205b 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -53,7 +53,7 @@ func init() { pctx.StaticVariable("JavacHeapSize", "2048M") pctx.StaticVariable("JavacHeapFlags", "-J-Xmx${JavacHeapSize}") - pctx.StaticVariable("DexFlags", "-JXX:+TieredCompilation -JXX:TieredStopAtLevel=1") + pctx.StaticVariable("DexFlags", "-JXX:OnError='cat hs_err_pid%p.log' -JXX:CICompilerCount=6 -JXX:+UseDynamicNumberOfGCThreads") pctx.StaticVariable("CommonJdkFlags", strings.Join([]string{ `-Xmaxerrs 9999999`, diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 9141f9ef..088c12df 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -22,11 +22,12 @@ import ( type dexpreopter struct { dexpreoptProperties DexpreoptProperties - installPath android.OutputPath - uncompressedDex bool - isSDKLibrary bool - isTest bool - isInstallable bool + installPath android.OutputPath + uncompressedDex bool + isSDKLibrary bool + isTest bool + isInstallable bool + isPresignedPrebuilt bool builtInstalled string } @@ -177,6 +178,8 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Mo NoCreateAppImage: !BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, true), ForceCreateAppImage: BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, false), + PresignedPrebuilt: d.isPresignedPrebuilt, + NoStripping: Bool(d.dexpreoptProperties.Dex_preopt.No_stripping), StripInputPath: dexJarFile, StripOutputPath: strippedDexJarFile.OutputPath, diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 4d87b2f7..87d8428d 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -151,6 +151,10 @@ func buildBootImage(ctx android.SingletonContext, config bootImageConfig) *bootI bootDexJars := make(android.Paths, len(image.modules)) ctx.VisitAllModules(func(module android.Module) { + if m, ok := module.(interface{ BootJarProvider() bool }); !ok || + !m.BootJarProvider() { + return + } // Collect dex jar paths for the modules listed above. if j, ok := module.(interface{ DexJar() android.Path }); ok { name := ctx.ModuleName(module) diff --git a/java/dexpreopt_bootjars_test.go b/java/dexpreopt_bootjars_test.go index cbb52f15..d4f84914 100644 --- a/java/dexpreopt_bootjars_test.go +++ b/java/dexpreopt_bootjars_test.go @@ -53,6 +53,8 @@ func TestDexpreoptBootJars(t *testing.T) { ctx := testContext(config, bp, nil) + ctx.PreArchMutators(android.RegisterBootJarMutators) + ctx.RegisterSingletonType("dex_bootjars", android.SingletonFactoryAdaptor(dexpreoptBootJarsFactory)) run(t, ctx, config) diff --git a/java/droiddoc.go b/java/droiddoc.go index f56cae82..003811ac 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -429,8 +429,10 @@ func InitDroiddocModule(module android.DefaultableModule, hod android.HostOrDevi android.InitDefaultableModule(module) } -func apiCheckEnabled(apiToCheck ApiToCheck, apiVersionTag string) bool { - if String(apiToCheck.Api_file) != "" && String(apiToCheck.Removed_api_file) != "" { +func apiCheckEnabled(ctx android.ModuleContext, apiToCheck ApiToCheck, apiVersionTag string) bool { + if ctx.Config().IsEnvTrue("WITHOUT_CHECK_API") { + return false + } else if String(apiToCheck.Api_file) != "" && String(apiToCheck.Removed_api_file) != "" { return true } else if String(apiToCheck.Api_file) != "" { panic("for " + apiVersionTag + " removed_api_file has to be non-empty!") @@ -993,8 +995,8 @@ func (d *Droiddoc) collectDoclavaDocsFlags(ctx android.ModuleContext, implicits func (d *Droiddoc) collectStubsFlags(ctx android.ModuleContext, implicitOutputs *android.WritablePaths) string { var doclavaFlags string - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Api_filename) != "" { d.apiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_api.txt") doclavaFlags += " -api " + d.apiFile.String() @@ -1002,8 +1004,8 @@ func (d *Droiddoc) collectStubsFlags(ctx android.ModuleContext, d.apiFilePath = d.apiFile } - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Removed_api_filename) != "" { d.removedApiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_removed.txt") doclavaFlags += " -removedApi " + d.removedApiFile.String() @@ -1183,7 +1185,7 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { flags.postDoclavaCmds) } - if apiCheckEnabled(d.properties.Check_api.Current, "current") && + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") && !ctx.Config().IsPdkBuild() { apiFile := ctx.ExpandSource(String(d.properties.Check_api.Current.Api_file), "check_api.current.api_file") @@ -1209,7 +1211,7 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { d.updateCurrentApiTimestamp) } - if apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") && + if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") && !ctx.Config().IsPdkBuild() { apiFile := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Api_file), "check_api.last_released.api_file") @@ -1336,8 +1338,8 @@ func (d *Droidstubs) initBuilderFlags(ctx android.ModuleContext, implicits *andr func (d *Droidstubs) collectStubsFlags(ctx android.ModuleContext, implicitOutputs *android.WritablePaths) string { var metalavaFlags string - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Api_filename) != "" { d.apiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_api.txt") metalavaFlags = metalavaFlags + " --api " + d.apiFile.String() @@ -1345,8 +1347,8 @@ func (d *Droidstubs) collectStubsFlags(ctx android.ModuleContext, d.apiFilePath = d.apiFile } - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Removed_api_filename) != "" { d.removedApiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_removed.txt") metalavaFlags = metalavaFlags + " --removed-api " + d.removedApiFile.String() @@ -1683,7 +1685,7 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { flags.metalavaStubsFlags+flags.metalavaAnnotationsFlags+flags.metalavaInclusionAnnotationsFlags+ flags.metalavaApiLevelsAnnotationsFlags+flags.metalavaApiToXmlFlags+" "+d.Javadoc.args) - if apiCheckEnabled(d.properties.Check_api.Current, "current") && + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") && !ctx.Config().IsPdkBuild() { apiFile := ctx.ExpandSource(String(d.properties.Check_api.Current.Api_file), "check_api.current.api_file") @@ -1720,7 +1722,7 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { d.updateCurrentApiTimestamp) } - if apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") && + if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") && !ctx.Config().IsPdkBuild() { apiFile := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Api_file), "check_api.last_released.api_file") diff --git a/java/hiddenapi.go b/java/hiddenapi.go index 6020aba6..28724f2a 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -73,7 +73,14 @@ func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOu // to the hidden API for the bootclassloader. If information is gathered for modules // not on the list then that will cause failures in the CtsHiddenApiBlacklist... // tests. - if inList(bootJarName, ctx.Config().BootJars()) { + isBootJarProvider := false + ctx.VisitAllModuleVariants(func(module android.Module) { + if m, ok := module.(interface{ BootJarProvider() bool }); ok && + m.BootJarProvider() { + isBootJarProvider = true + } + }) + if isBootJarProvider && inList(bootJarName, ctx.Config().BootJars()) { // Derive the greylist from classes jar. flagsCSV := android.PathForModuleOut(ctx, "hiddenapi", "flags.csv") metadataCSV := android.PathForModuleOut(ctx, "hiddenapi", "metadata.csv") diff --git a/java/java.go b/java/java.go index 9ac38c92..7fedf6d6 100644 --- a/java/java.go +++ b/java/java.go @@ -406,6 +406,7 @@ var ( bootClasspathTag = dependencyTag{name: "bootclasspath"} systemModulesTag = dependencyTag{name: "system modules"} frameworkResTag = dependencyTag{name: "framework-res"} + lineageResTag = dependencyTag{name: "org.lineageos.platform-res"} frameworkApkTag = dependencyTag{name: "framework-apk"} kotlinStdlibTag = dependencyTag{name: "kotlin-stdlib"} kotlinAnnotationsTag = dependencyTag{name: "kotlin-annotations"} @@ -421,6 +422,7 @@ type sdkDep struct { systemModules string frameworkResModule string + lineageResModule string jars android.Paths aidl android.OptionalPath @@ -486,12 +488,21 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { } if (ctx.ModuleName() == "framework") || (ctx.ModuleName() == "framework-annotation-proc") { ctx.AddVariationDependencies(nil, frameworkResTag, "framework-res") + ctx.AddDependency(ctx.Module(), lineageResTag, "org.lineageos.platform-res") } if ctx.ModuleName() == "android_stubs_current" || ctx.ModuleName() == "android_system_stubs_current" || ctx.ModuleName() == "android_test_stubs_current" { ctx.AddVariationDependencies(nil, frameworkApkTag, "framework-res") } + if ctx.ModuleName() == "org.lineageos.platform-res" { + ctx.AddDependency(ctx.Module(), frameworkResTag, "framework-res") + } + if ctx.ModuleName() == "org.lineageos.platform" || + ctx.ModuleName() == "org.lineageos.platform.internal" || + ctx.ModuleName() == "org.lineageos.platform.sdk" { + ctx.AddDependency(ctx.Module(), lineageResTag, "org.lineageos.platform-res") + } } ctx.AddVariationDependencies(nil, libTag, j.properties.Libs...) @@ -760,6 +771,19 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { // generated by framework-res.apk deps.srcJars = append(deps.srcJars, dep.(*AndroidApp).aaptSrcJar) } + case lineageResTag: + if ctx.ModuleName() == "org.lineageos.platform" || + ctx.ModuleName() == "org.lineageos.platform.internal" || + ctx.ModuleName() == "org.lineageos.platform.sdk" { + // org.lineageos.platform.jar has a one-off dependency on the R.java and Manifest.java files + // generated by org.lineageos.platform-res.apk + deps.srcJars = append(deps.srcJars, dep.(*AndroidApp).aaptSrcJar) + } + if ctx.ModuleName() == "framework" { + // framework.jar has a one-off dependency on the R.java and Manifest.java files + // generated by org.lineageos.platform-res.apk + deps.srcJars = append(deps.srcJars, dep.(*AndroidApp).aaptSrcJar) + } case frameworkApkTag: if ctx.ModuleName() == "android_stubs_current" || ctx.ModuleName() == "android_system_stubs_current" || diff --git a/java/java_test.go b/java/java_test.go index 31b23e76..bafb466c 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -63,6 +63,7 @@ func testContext(config android.Config, bp string, ctx := android.NewTestArchContext() ctx.RegisterModuleType("android_app", android.ModuleFactoryAdaptor(AndroidAppFactory)) ctx.RegisterModuleType("android_app_certificate", android.ModuleFactoryAdaptor(AndroidAppCertificateFactory)) + ctx.RegisterModuleType("android_app_import", android.ModuleFactoryAdaptor(AndroidAppImportFactory)) ctx.RegisterModuleType("android_library", android.ModuleFactoryAdaptor(AndroidLibraryFactory)) ctx.RegisterModuleType("android_test", android.ModuleFactoryAdaptor(AndroidTestFactory)) ctx.RegisterModuleType("android_test_helper_app", android.ModuleFactoryAdaptor(AndroidTestHelperAppFactory)) @@ -167,9 +168,15 @@ func testContext(config android.Config, bp string, "prebuilts/sdk/tools/core-lambda-stubs.jar": nil, "prebuilts/sdk/Android.bp": []byte(`prebuilt_apis { name: "sdk", api_dirs: ["14", "28", "current"],}`), + "prebuilts/apk/app.apk": nil, + "prebuilts/apk/app_arm.apk": nil, + "prebuilts/apk/app_arm64.apk": nil, + "prebuilts/apk/app_xhdpi.apk": nil, + "prebuilts/apk/app_xxhdpi.apk": nil, + // For framework-res, which is an implicit dependency for framework - "AndroidManifest.xml": nil, - "build/target/product/security/testkey": nil, + "AndroidManifest.xml": nil, + "build/make/target/product/security/testkey": nil, "build/soong/scripts/jar-wrapper.sh": nil, diff --git a/java/sdk.go b/java/sdk.go index e93f8fb6..dab04e71 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -182,6 +182,7 @@ func decodeSdkDep(ctx android.BaseContext, sdkContext sdkContext) sdkDep { return sdkDep{ useDefaultLibs: true, frameworkResModule: "framework-res", + lineageResModule: "org.lineageos.platform-res", } case "current": return toModule("android_stubs_current", "framework-res", sdkFrameworkAidlPath(ctx)) diff --git a/java/testing.go b/java/testing.go index 6b35bd04..c805e0ef 100644 --- a/java/testing.go +++ b/java/testing.go @@ -76,6 +76,11 @@ func GatherRequiredDepsForTest() string { name: "framework-res", no_framework_libs: true, } + + android_app { + name: "org.lineageos.platform-res", + no_framework_libs: true, + } ` systemModules := []string{ diff --git a/scripts/microfactory.bash b/scripts/microfactory.bash index 4bb6058a..b8bb81db 100644 --- a/scripts/microfactory.bash +++ b/scripts/microfactory.bash @@ -59,7 +59,7 @@ function soong_build_go BUILDDIR=$(getoutdir) \ SRCDIR=${TOP} \ BLUEPRINTDIR=${TOP}/build/blueprint \ - EXTRA_ARGS="-pkg-path android/soong=${TOP}/build/soong -pkg-path github.com/golang/protobuf=${TOP}/external/golang-protobuf" \ + EXTRA_ARGS="-pkg-path android/soong=${TOP}/build/soong -pkg-path lineage/soong=${TOP}/vendor/lineage/build/soong -pkg-path github.com/golang/protobuf=${TOP}/external/golang-protobuf" \ build_go $@ } diff --git a/sysprop/sysprop_test.go b/sysprop/sysprop_test.go index 07406b3c..09076820 100644 --- a/sysprop/sysprop_test.go +++ b/sysprop/sysprop_test.go @@ -136,8 +136,8 @@ func testContext(config android.Config, bp string, "prebuilts/sdk/Android.bp": []byte(`prebuilt_apis { name: "sdk", api_dirs: ["28", "current"],}`), // For framework-res, which is an implicit dependency for framework - "AndroidManifest.xml": nil, - "build/target/product/security/testkey": nil, + "AndroidManifest.xml": nil, + "build/make/target/product/security/testkey": nil, "build/soong/scripts/jar-wrapper.sh": nil, diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 1ddaf68a..b9e7f8d4 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -65,11 +65,13 @@ bootstrap_go_package { ], darwin: { srcs: [ + "config_darwin.go", "sandbox_darwin.go", ], }, linux: { srcs: [ + "config_linux.go", "sandbox_linux.go", ], }, diff --git a/ui/build/build.go b/ui/build/build.go index 0ae06d61..6adccc69 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -128,6 +128,7 @@ func help(ctx Context, config Config, what int) { func Build(ctx Context, config Config, what int) { ctx.Verboseln("Starting build with args:", config.Arguments()) ctx.Verboseln("Environment:", config.Environment().Environ()) + ctx.Verbosef("Total RAM: %dGB", config.TotalRAM()/1024/1024/1024) if config.SkipMake() { ctx.Verboseln("Skipping Make/Kati as requested") diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go index c47f614d..9c0c76ff 100644 --- a/ui/build/cleanbuild.go +++ b/ui/build/cleanbuild.go @@ -98,6 +98,7 @@ func installClean(ctx Context, config Config, what int) { hostOut("vts"), productOut("*.img"), productOut("*.zip"), + productOut("*.zip.md5sum"), productOut("android-info.txt"), productOut("kernel"), productOut("data"), diff --git a/ui/build/config.go b/ui/build/config.go index 7eb3a725..0dbd8165 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -16,7 +16,6 @@ package build import ( "io/ioutil" - "log" "os" "path/filepath" "runtime" @@ -51,6 +50,9 @@ type configImpl struct { targetDevice string targetDeviceDir string + // Autodetected + totalRAM uint64 + pdkBuild bool brokenDupRules bool @@ -62,6 +64,33 @@ type configImpl struct { const srcDirFileCheck = "build/soong/root.bp" +var buildFiles = []string{"Android.mk", "Android.bp"} + +type BuildAction uint + +const ( + // Builds all of the modules and their dependencies of a specified directory, relative to the root + // directory of the source tree. + BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota + + // Builds all of the modules and their dependencies of a list of specified directories. All specified + // directories are relative to the root directory of the source tree. + BUILD_MODULES_IN_DIRECTORIES + + // Build a list of specified modules. If none was specified, simply build the whole source tree. + BUILD_MODULES +) + +// checkTopDir validates that the current directory is at the root directory of the source tree. +func checkTopDir(ctx Context) { + if _, err := os.Stat(srcDirFileCheck); err != nil { + if os.IsNotExist(err) { + ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) + } + ctx.Fatalln("Error verifying tree state:", err) + } +} + func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ environ: OsEnvironment(), @@ -71,6 +100,8 @@ func NewConfig(ctx Context, args ...string) Config { ret.parallel = runtime.NumCPU() + 2 ret.keepGoing = 1 + ret.totalRAM = detectTotalRAM(ctx) + ret.parseArgs(ctx, args) // Make sure OUT_DIR is set appropriately @@ -155,35 +186,30 @@ func NewConfig(ctx Context, args ...string) Config { ret.environ.Set("TMPDIR", absPath(ctx, ret.TempDir())) // Precondition: the current directory is the top of the source tree - if _, err := os.Stat(srcDirFileCheck); err != nil { - if os.IsNotExist(err) { - log.Fatalf("Current working directory must be the source tree. %q not found", srcDirFileCheck) - } - log.Fatalln("Error verifying tree state:", err) - } + checkTopDir(ctx) if srcDir := absPath(ctx, "."); strings.ContainsRune(srcDir, ' ') { - log.Println("You are building in a directory whose absolute path contains a space character:") - log.Println() - log.Printf("%q\n", srcDir) - log.Println() - log.Fatalln("Directory names containing spaces are not supported") + ctx.Println("You are building in a directory whose absolute path contains a space character:") + ctx.Println() + ctx.Printf("%q\n", srcDir) + ctx.Println() + ctx.Fatalln("Directory names containing spaces are not supported") } if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { - log.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") - log.Println() - log.Printf("%q\n", outDir) - log.Println() - log.Fatalln("Directory names containing spaces are not supported") + ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") + ctx.Println() + ctx.Printf("%q\n", outDir) + ctx.Println() + ctx.Fatalln("Directory names containing spaces are not supported") } if distDir := ret.DistDir(); strings.ContainsRune(distDir, ' ') { - log.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") - log.Println() - log.Printf("%q\n", distDir) - log.Println() - log.Fatalln("Directory names containing spaces are not supported") + ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") + ctx.Println() + ctx.Printf("%q\n", distDir) + ctx.Println() + ctx.Fatalln("Directory names containing spaces are not supported") } // Configure Java-related variables, including adding it to $PATH @@ -230,6 +256,225 @@ func NewConfig(ctx Context, args ...string) Config { return Config{ret} } +// NewBuildActionConfig returns a build configuration based on the build action. The arguments are +// processed based on the build action and extracts any arguments that belongs to the build action. +func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { + return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) +} + +// getConfigArgs processes the command arguments based on the build action and creates a set of new +// arguments to be accepted by Config. +func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { + // The next block of code verifies that the current directory is the root directory of the source + // tree. It then finds the relative path of dir based on the root directory of the source tree + // and verify that dir is inside of the source tree. + checkTopDir(ctx) + topDir, err := os.Getwd() + if err != nil { + ctx.Fatalf("Error retrieving top directory: %v", err) + } + dir, err = filepath.EvalSymlinks(dir) + if err != nil { + ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) + } + dir, err = filepath.Abs(dir) + if err != nil { + ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) + } + relDir, err := filepath.Rel(topDir, dir) + if err != nil { + ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) + } + // If there are ".." in the path, it's not in the source tree. + if strings.Contains(relDir, "..") { + ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) + } + + configArgs := args[:] + + // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to + // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. + targetNamePrefix := "MODULES-IN-" + if inList("GET-INSTALL-PATH", configArgs) { + targetNamePrefix = "GET-INSTALL-PATH-IN-" + configArgs = removeFromList("GET-INSTALL-PATH", configArgs) + } + + var targets []string + + switch action { + case BUILD_MODULES: + // No additional processing is required when building a list of specific modules or all modules. + case BUILD_MODULES_IN_A_DIRECTORY: + // If dir is the root source tree, all the modules are built of the source tree are built so + // no need to find the build file. + if topDir == dir { + break + } + + buildFile := findBuildFile(ctx, relDir) + if buildFile == "" { + ctx.Fatalf("Build file not found for %s directory", relDir) + } + targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} + case BUILD_MODULES_IN_DIRECTORIES: + newConfigArgs, dirs := splitArgs(configArgs) + configArgs = newConfigArgs + targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) + } + + // Tidy only override all other specified targets. + tidyOnly := os.Getenv("WITH_TIDY_ONLY") + if tidyOnly == "true" || tidyOnly == "1" { + configArgs = append(configArgs, "tidy_only") + } else { + configArgs = append(configArgs, targets...) + } + + return configArgs +} + +// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. +func convertToTarget(dir string, targetNamePrefix string) string { + return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") +} + +// hasBuildFile returns true if dir contains an Android build file. +func hasBuildFile(ctx Context, dir string) bool { + for _, buildFile := range buildFiles { + _, err := os.Stat(filepath.Join(dir, buildFile)) + if err == nil { + return true + } + if !os.IsNotExist(err) { + ctx.Fatalf("Error retrieving the build file stats: %v", err) + } + } + return false +} + +// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file +// in the current and any sub directory of dir. If a build file is not found, traverse the path +// up by one directory and repeat again until either a build file is found or reached to the root +// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank +// string is returned. +func findBuildFile(ctx Context, dir string) string { + // If the string is empty or ".", assume it is top directory of the source tree. + if dir == "" || dir == "." { + return "" + } + + found := false + for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { + err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if found { + return filepath.SkipDir + } + if info.IsDir() { + return nil + } + for _, buildFile := range buildFiles { + if info.Name() == buildFile { + found = true + return filepath.SkipDir + } + } + return nil + }) + if err != nil { + ctx.Fatalf("Error finding Android build file: %v", err) + } + + if found { + return filepath.Join(buildDir, "Android.mk") + } + } + + return "" +} + +// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. +func splitArgs(args []string) (newArgs []string, dirs []string) { + specialArgs := map[string]bool{ + "showcommands": true, + "snod": true, + "dist": true, + "checkbuild": true, + } + + newArgs = []string{} + dirs = []string{} + + for _, arg := range args { + // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. + if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { + newArgs = append(newArgs, arg) + continue + } + + if _, ok := specialArgs[arg]; ok { + newArgs = append(newArgs, arg) + continue + } + + dirs = append(dirs, arg) + } + + return newArgs, dirs +} + +// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a +// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the +// source root tree where the build action command was invoked. Each directory is validated if the +// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. +func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { + for _, dir := range dirs { + // The directory may have specified specific modules to build. ":" is the separator to separate + // the directory and the list of modules. + s := strings.Split(dir, ":") + l := len(s) + if l > 2 { // more than one ":" was specified. + ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) + } + + dir = filepath.Join(relDir, s[0]) + if _, err := os.Stat(dir); err != nil { + ctx.Fatalf("couldn't find directory %s", dir) + } + + // Verify that if there are any targets specified after ":". Each target is separated by ",". + var newTargets []string + if l == 2 && s[1] != "" { + newTargets = strings.Split(s[1], ",") + if inList("", newTargets) { + ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) + } + } + + // If there are specified targets to build in dir, an android build file must exist for the one + // shot build. For the non-targets case, find the appropriate build file and build all the + // modules in dir (or the closest one in the dir path). + if len(newTargets) > 0 { + if !hasBuildFile(ctx, dir) { + ctx.Fatalf("Couldn't locate a build file from %s directory", dir) + } + } else { + buildFile := findBuildFile(ctx, dir) + if buildFile == "" { + ctx.Fatalf("Build file not found for %s directory", dir) + } + newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} + } + + targets = append(targets, newTargets...) + } + + return targets +} + func (c *configImpl) parseArgs(ctx Context, args []string) { for i := 0; i < len(args); i++ { arg := strings.TrimSpace(args[i]) @@ -386,7 +631,7 @@ func (c *configImpl) Arguments() []string { func (c *configImpl) OutDir() string { if outDir, ok := c.environ.Get("OUT_DIR"); ok { - return filepath.Clean(outDir) + return outDir } return "out" } @@ -469,6 +714,10 @@ func (c *configImpl) Parallel() int { return c.parallel } +func (c *configImpl) TotalRAM() uint64 { + return c.totalRAM +} + func (c *configImpl) UseGoma() bool { if v, ok := c.environ.Get("USE_GOMA"); ok { v = strings.TrimSpace(v) diff --git a/ui/build/config_darwin.go b/ui/build/config_darwin.go new file mode 100644 index 00000000..480d8d1f --- /dev/null +++ b/ui/build/config_darwin.go @@ -0,0 +1,40 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "encoding/binary" + "syscall" +) + +func detectTotalRAM(ctx Context) uint64 { + s, err := syscall.Sysctl("hw.memsize") + if err != nil { + ctx.Printf("Failed to get system memory size: %s") + return 0 + } + + // syscall.Sysctl assumes that the return value is a string and trims the last byte if it is 0. + if len(s) == 7 { + s += "\x00" + } + + if len(s) != 8 { + ctx.Printf("Failed to get system memory size, returned %d bytes, 8", len(s)) + return 0 + } + + return binary.LittleEndian.Uint64([]byte(s)) +} diff --git a/ui/build/config_linux.go b/ui/build/config_linux.go new file mode 100644 index 00000000..9e1bdc7f --- /dev/null +++ b/ui/build/config_linux.go @@ -0,0 +1,28 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import "syscall" + +func detectTotalRAM(ctx Context) uint64 { + var info syscall.Sysinfo_t + err := syscall.Sysinfo(&info) + if err != nil { + ctx.Printf("Failed to get system memory size: %s") + return 0 + } + memBytes := uint64(info.Totalram) * uint64(info.Unit) + return memBytes +} diff --git a/ui/build/config_test.go b/ui/build/config_test.go index 242e3afb..df618c4e 100644 --- a/ui/build/config_test.go +++ b/ui/build/config_test.go @@ -17,19 +17,22 @@ package build import ( "bytes" "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" "reflect" "strings" "testing" "android/soong/ui/logger" - "android/soong/ui/terminal" ) func testContext() Context { return Context{&ContextImpl{ Context: context.Background(), Logger: logger.New(&bytes.Buffer{}), - Writer: terminal.NewWriter(terminal.NewCustomStdio(&bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{})), + Writer: &bytes.Buffer{}, }} } @@ -173,3 +176,820 @@ func TestConfigParseArgsVars(t *testing.T) { }) } } + +func TestConfigCheckTopDir(t *testing.T) { + ctx := testContext() + buildRootDir := filepath.Dir(srcDirFileCheck) + expectedErrStr := fmt.Sprintf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) + + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // ********* Action ********* + // If set to true, the build root file is created. + rootBuildFile bool + + // The current path where Soong is being executed. + path string + + // ********* Validation ********* + // Expecting error and validate the error string against expectedErrStr. + wantErr bool + }{{ + description: "current directory is the root source tree", + rootBuildFile: true, + path: ".", + wantErr: false, + }, { + description: "one level deep in the source tree", + rootBuildFile: true, + path: "1", + wantErr: true, + }, { + description: "very deep in the source tree", + rootBuildFile: true, + path: "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/7", + wantErr: true, + }, { + description: "outside of source tree", + rootBuildFile: false, + path: "1/2/3/4/5", + wantErr: true, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + if !tt.wantErr { + t.Fatalf("Got unexpected error: %v", err) + } + if expectedErrStr != err.Error() { + t.Fatalf("expected %s, got %s", expectedErrStr, err.Error()) + } + }) + + // Create the root source tree. + rootDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(rootDir) + + // Create the build root file. This is to test if topDir returns an error if the build root + // file does not exist. + if tt.rootBuildFile { + dir := filepath.Join(rootDir, buildRootDir) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Errorf("failed to create %s directory: %v", dir, err) + } + f := filepath.Join(rootDir, srcDirFileCheck) + if err := ioutil.WriteFile(f, []byte{}, 0644); err != nil { + t.Errorf("failed to create file %s: %v", f, err) + } + } + + // Next block of code is to set the current directory. + dir := rootDir + if tt.path != "" { + dir = filepath.Join(dir, tt.path) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Errorf("failed to create %s directory: %v", dir, err) + } + } + curDir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get the current directory: %v", err) + } + defer func() { os.Chdir(curDir) }() + + if err := os.Chdir(dir); err != nil { + t.Fatalf("failed to change directory to %s: %v", dir, err) + } + + checkTopDir(ctx) + }) + } +} + +func TestConfigConvertToTarget(t *testing.T) { + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // ********* Action ********* + // The current directory where Soong is being executed. + dir string + + // The current prefix string to be pre-appended to the target. + prefix string + + // ********* Validation ********* + // The expected target to be invoked in ninja. + expectedTarget string + }{{ + description: "one level directory in source tree", + dir: "test1", + prefix: "MODULES-IN-", + expectedTarget: "MODULES-IN-test1", + }, { + description: "multiple level directories in source tree", + dir: "test1/test2/test3/test4", + prefix: "GET-INSTALL-PATH-IN-", + expectedTarget: "GET-INSTALL-PATH-IN-test1-test2-test3-test4", + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + target := convertToTarget(tt.dir, tt.prefix) + if target != tt.expectedTarget { + t.Errorf("expected %s, got %s for target", tt.expectedTarget, target) + } + }) + } +} + +func setTop(t *testing.T, dir string) func() { + curDir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get current directory: %v", err) + } + if err := os.Chdir(dir); err != nil { + t.Fatalf("failed to change directory to top dir %s: %v", dir, err) + } + return func() { os.Chdir(curDir) } +} + +func createBuildFiles(t *testing.T, topDir string, buildFiles []string) { + for _, buildFile := range buildFiles { + buildFile = filepath.Join(topDir, buildFile) + if err := ioutil.WriteFile(buildFile, []byte{}, 0644); err != nil { + t.Errorf("failed to create file %s: %v", buildFile, err) + } + } +} + +func createDirectories(t *testing.T, topDir string, dirs []string) { + for _, dir := range dirs { + dir = filepath.Join(topDir, dir) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Errorf("failed to create %s directory: %v", dir, err) + } + } +} + +func TestConfigGetTargets(t *testing.T) { + ctx := testContext() + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // Directories that exist in the source tree. + dirsInTrees []string + + // Build files that exists in the source tree. + buildFiles []string + + // ********* Action ********* + // Directories passed in to soong_ui. + dirs []string + + // Current directory that the user executed the build action command. + curDir string + + // ********* Validation ********* + // Expected targets from the function. + expectedTargets []string + + // Expecting error from running test case. + errStr string + }{{ + description: "one target dir specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3"}, + curDir: "0", + expectedTargets: []string{"MODULES-IN-0-1-2-3"}, + }, { + description: "one target dir specified, build file does not exist", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{}, + dirs: []string{"1/2/3"}, + curDir: "0", + errStr: "Build file not found for 0/1/2/3 directory", + }, { + description: "one target dir specified, invalid targets specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{}, + dirs: []string{"1/2/3:t1:t2"}, + curDir: "0", + errStr: "1/2/3:t1:t2 not in proper directory:target1,target2,... format (\":\" was specified more than once)", + }, { + description: "one target dir specified, no targets specified but has colon", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:"}, + curDir: "0", + expectedTargets: []string{"MODULES-IN-0-1-2-3"}, + }, { + description: "one target dir specified, two targets specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:t1,t2"}, + curDir: "0", + expectedTargets: []string{"t1", "t2"}, + }, { + description: "one target dir specified, no targets and has a comma", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:,"}, + curDir: "0", + errStr: "0/1/2/3 not in proper directory:target1,target2,... format", + }, { + description: "one target dir specified, improper targets defined", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:,t1"}, + curDir: "0", + errStr: "0/1/2/3 not in proper directory:target1,target2,... format", + }, { + description: "one target dir specified, blank target", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:t1,"}, + curDir: "0", + errStr: "0/1/2/3 not in proper directory:target1,target2,... format", + }, { + description: "one target dir specified, many targets specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:t1,t2,t3,t4,t5,t6,t7,t8,t9,t10"}, + curDir: "0", + expectedTargets: []string{"t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", "t10"}, + }, { + description: "one target dir specified, one target specified, build file does not exist", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{}, + dirs: []string{"1/2/3:t1"}, + curDir: "0", + errStr: "Couldn't locate a build file from 0/1/2/3 directory", + }, { + description: "one target dir specified, one target specified, build file not in target dir", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/Android.mk"}, + dirs: []string{"1/2/3:t1"}, + curDir: "0", + errStr: "Couldn't locate a build file from 0/1/2/3 directory", + }, { + description: "one target dir specified, build file not in target dir", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/Android.mk"}, + dirs: []string{"1/2/3"}, + curDir: "0", + expectedTargets: []string{"MODULES-IN-0-1-2"}, + }, { + description: "multiple targets dir specified, targets specified", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{"1/2/3:t1,t2", "3/4:t3,t4,t5"}, + curDir: "0", + expectedTargets: []string{"t1", "t2", "t3", "t4", "t5"}, + }, { + description: "multiple targets dir specified, one directory has targets specified", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{"1/2/3:t1,t2", "3/4"}, + curDir: "0", + expectedTargets: []string{"t1", "t2", "MODULES-IN-0-3-4"}, + }, { + description: "two dirs specified, only one dir exist", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.mk"}, + dirs: []string{"1/2/3:t1", "3/4"}, + curDir: "0", + errStr: "couldn't find directory 0/3/4", + }, { + description: "multiple targets dirs specified at root source tree", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{"0/1/2/3:t1,t2", "0/3/4"}, + curDir: ".", + expectedTargets: []string{"t1", "t2", "MODULES-IN-0-3-4"}, + }, { + description: "no directories specified", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{}, + curDir: ".", + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + if tt.errStr == "" { + t.Fatalf("Got unexpected error: %v", err) + } + if tt.errStr != err.Error() { + t.Errorf("expected %s, got %s", tt.errStr, err.Error()) + } + }) + + // Create the root source tree. + topDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(topDir) + + createDirectories(t, topDir, tt.dirsInTrees) + createBuildFiles(t, topDir, tt.buildFiles) + r := setTop(t, topDir) + defer r() + + targets := getTargetsFromDirs(ctx, tt.curDir, tt.dirs, "MODULES-IN-") + if !reflect.DeepEqual(targets, tt.expectedTargets) { + t.Errorf("expected %v, got %v for targets", tt.expectedTargets, targets) + } + + // If the execution reached here and there was an expected error code, the unit test case failed. + if tt.errStr != "" { + t.Errorf("expecting error %s", tt.errStr) + } + }) + } +} + +func TestConfigFindBuildFile(t *testing.T) { + ctx := testContext() + + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // Array of build files to create in dir. + buildFiles []string + + // Directories that exist in the source tree. + dirsInTrees []string + + // ********* Action ********* + // The base directory is where findBuildFile is invoked. + dir string + + // ********* Validation ********* + // Expected build file path to find. + expectedBuildFile string + }{{ + description: "build file exists at leaf directory", + buildFiles: []string{"1/2/3/Android.bp"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/2/3/Android.mk", + }, { + description: "build file exists in all directory paths", + buildFiles: []string{"1/Android.mk", "1/2/Android.mk", "1/2/3/Android.mk"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/2/3/Android.mk", + }, { + description: "build file does not exist in all directory paths", + buildFiles: []string{}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "", + }, { + description: "build file exists only at top directory", + buildFiles: []string{"Android.bp"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "", + }, { + description: "build file exist in a subdirectory", + buildFiles: []string{"1/2/Android.bp"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/2/Android.mk", + }, { + description: "build file exists in a subdirectory", + buildFiles: []string{"1/Android.mk"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/Android.mk", + }, { + description: "top directory", + buildFiles: []string{"Android.bp"}, + dirsInTrees: []string{}, + dir: ".", + expectedBuildFile: "", + }, { + description: "build file exists in subdirectory", + buildFiles: []string{"1/2/3/Android.bp", "1/2/4/Android.bp"}, + dirsInTrees: []string{"1/2/3", "1/2/4"}, + dir: "1/2", + expectedBuildFile: "1/2/Android.mk", + }, { + description: "build file exists in parent subdirectory", + buildFiles: []string{"1/5/Android.bp"}, + dirsInTrees: []string{"1/2/3", "1/2/4", "1/5"}, + dir: "1/2", + expectedBuildFile: "1/Android.mk", + }, { + description: "build file exists in deep parent's subdirectory.", + buildFiles: []string{"1/5/6/Android.bp"}, + dirsInTrees: []string{"1/2/3", "1/2/4", "1/5/6", "1/5/7"}, + dir: "1/2", + expectedBuildFile: "1/Android.mk", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + t.Fatalf("Got unexpected error: %v", err) + }) + + topDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(topDir) + + createDirectories(t, topDir, tt.dirsInTrees) + createBuildFiles(t, topDir, tt.buildFiles) + + curDir, err := os.Getwd() + if err != nil { + t.Fatalf("Could not get working directory: %v", err) + } + defer func() { os.Chdir(curDir) }() + if err := os.Chdir(topDir); err != nil { + t.Fatalf("Could not change top dir to %s: %v", topDir, err) + } + + buildFile := findBuildFile(ctx, tt.dir) + if buildFile != tt.expectedBuildFile { + t.Errorf("expected %q, got %q for build file", tt.expectedBuildFile, buildFile) + } + }) + } +} + +func TestConfigSplitArgs(t *testing.T) { + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // ********* Action ********* + // Arguments passed in to soong_ui. + args []string + + // ********* Validation ********* + // Expected newArgs list after extracting the directories. + expectedNewArgs []string + + // Expected directories + expectedDirs []string + }{{ + description: "flags but no directories specified", + args: []string{"showcommands", "-j", "-k"}, + expectedNewArgs: []string{"showcommands", "-j", "-k"}, + expectedDirs: []string{}, + }, { + description: "flags and one directory specified", + args: []string{"snod", "-j", "dir:target1,target2"}, + expectedNewArgs: []string{"snod", "-j"}, + expectedDirs: []string{"dir:target1,target2"}, + }, { + description: "flags and directories specified", + args: []string{"dist", "-k", "dir1", "dir2:target1,target2"}, + expectedNewArgs: []string{"dist", "-k"}, + expectedDirs: []string{"dir1", "dir2:target1,target2"}, + }, { + description: "only directories specified", + args: []string{"dir1", "dir2", "dir3:target1,target2"}, + expectedNewArgs: []string{}, + expectedDirs: []string{"dir1", "dir2", "dir3:target1,target2"}, + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + args, dirs := splitArgs(tt.args) + if !reflect.DeepEqual(tt.expectedNewArgs, args) { + t.Errorf("expected %v, got %v for arguments", tt.expectedNewArgs, args) + } + if !reflect.DeepEqual(tt.expectedDirs, dirs) { + t.Errorf("expected %v, got %v for directories", tt.expectedDirs, dirs) + } + }) + } +} + +type envVar struct { + name string + value string +} + +type buildActionTestCase struct { + // ********* Setup ********* + // Test description. + description string + + // Directories that exist in the source tree. + dirsInTrees []string + + // Build files that exists in the source tree. + buildFiles []string + + // Create root symlink that points to topDir. + rootSymlink bool + + // ********* Action ********* + // Arguments passed in to soong_ui. + args []string + + // Directory where the build action was invoked. + curDir string + + // WITH_TIDY_ONLY environment variable specified. + tidyOnly string + + // ********* Validation ********* + // Expected arguments to be in Config instance. + expectedArgs []string + + // Expecting error from running test case. + expectedErrStr string +} + +func testGetConfigArgs(t *testing.T, tt buildActionTestCase, action BuildAction) { + ctx := testContext() + + defer logger.Recover(func(err error) { + if tt.expectedErrStr == "" { + t.Fatalf("Got unexpected error: %v", err) + } + if tt.expectedErrStr != err.Error() { + t.Errorf("expected %s, got %s", tt.expectedErrStr, err.Error()) + } + }) + + // Environment variables to set it to blank on every test case run. + resetEnvVars := []string{ + "WITH_TIDY_ONLY", + } + + for _, name := range resetEnvVars { + if err := os.Unsetenv(name); err != nil { + t.Fatalf("failed to unset environment variable %s: %v", name, err) + } + } + if tt.tidyOnly != "" { + if err := os.Setenv("WITH_TIDY_ONLY", tt.tidyOnly); err != nil { + t.Errorf("failed to set WITH_TIDY_ONLY to %s: %v", tt.tidyOnly, err) + } + } + + // Create the root source tree. + topDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(topDir) + + createDirectories(t, topDir, tt.dirsInTrees) + createBuildFiles(t, topDir, tt.buildFiles) + + if tt.rootSymlink { + // Create a secondary root source tree which points to the true root source tree. + symlinkTopDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create symlink temp dir: %v", err) + } + defer os.RemoveAll(symlinkTopDir) + + symlinkTopDir = filepath.Join(symlinkTopDir, "root") + err = os.Symlink(topDir, symlinkTopDir) + if err != nil { + t.Fatalf("failed to create symlink: %v", err) + } + topDir = symlinkTopDir + } + + r := setTop(t, topDir) + defer r() + + // The next block is to create the root build file. + rootBuildFileDir := filepath.Dir(srcDirFileCheck) + if err := os.MkdirAll(rootBuildFileDir, 0755); err != nil { + t.Fatalf("Failed to create %s directory: %v", rootBuildFileDir, err) + } + + if err := ioutil.WriteFile(srcDirFileCheck, []byte{}, 0644); err != nil { + t.Fatalf("failed to create %s file: %v", srcDirFileCheck, err) + } + + args := getConfigArgs(action, tt.curDir, ctx, tt.args) + if !reflect.DeepEqual(tt.expectedArgs, args) { + t.Fatalf("expected %v, got %v for config arguments", tt.expectedArgs, args) + } + + // If the execution reached here and there was an expected error code, the unit test case failed. + if tt.expectedErrStr != "" { + t.Errorf("expecting error %s", tt.expectedErrStr) + } +} + +func TestGetConfigArgsBuildModules(t *testing.T) { + tests := []buildActionTestCase{{ + description: "normal execution from the root source tree directory", + dirsInTrees: []string{"0/1/2", "0/2", "0/3"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp", "0/3/Android.mk"}, + args: []string{"-j", "fake_module", "fake_module2"}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{"-j", "fake_module", "fake_module2"}, + }, { + description: "normal execution in deep directory", + dirsInTrees: []string{"0/1/2", "0/2", "0/3", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/Android.mk"}, + args: []string{"-j", "fake_module", "fake_module2", "-k"}, + curDir: "1/2/3/4/5/6/7/8/9", + tidyOnly: "", + expectedArgs: []string{"-j", "fake_module", "fake_module2", "-k"}, + }, { + description: "normal execution in deep directory, no targets", + dirsInTrees: []string{"0/1/2", "0/2", "0/3", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/Android.mk"}, + args: []string{"-j", "-k"}, + curDir: "1/2/3/4/5/6/7/8/9", + tidyOnly: "", + expectedArgs: []string{"-j", "-k"}, + }, { + description: "normal execution in root source tree, no args", + dirsInTrees: []string{"0/1/2", "0/2", "0/3"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp"}, + args: []string{}, + curDir: "0/2", + tidyOnly: "", + expectedArgs: []string{}, + }, { + description: "normal execution in symlink root source tree, no args", + dirsInTrees: []string{"0/1/2", "0/2", "0/3"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp"}, + rootSymlink: true, + args: []string{}, + curDir: "0/2", + tidyOnly: "", + expectedArgs: []string{}, + }} + for _, tt := range tests { + t.Run("build action BUILD_MODULES with dependencies, "+tt.description, func(t *testing.T) { + testGetConfigArgs(t, tt, BUILD_MODULES) + }) + } +} + +func TestGetConfigArgsBuildModulesInDirectory(t *testing.T) { + tests := []buildActionTestCase{{ + description: "normal execution in a directory", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/2/Android.mk"}, + args: []string{"fake-module"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"fake-module", "MODULES-IN-0-1-2"}, + }, { + description: "build file in parent directory", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1"}, + }, + { + description: "build file in parent directory, multiple module names passed in", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{"fake-module1", "fake-module2", "fake-module3"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"fake-module1", "fake-module2", "fake-module3", "MODULES-IN-0-1"}, + }, { + description: "build file in 2nd level parent directory", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/Android.bp"}, + args: []string{}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0"}, + }, { + description: "build action executed at root directory", + dirsInTrees: []string{}, + buildFiles: []string{}, + rootSymlink: false, + args: []string{}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{}, + }, { + description: "build action executed at root directory in symlink", + dirsInTrees: []string{}, + buildFiles: []string{}, + rootSymlink: true, + args: []string{}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{}, + }, { + description: "build file not found", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{}, + args: []string{}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2"}, + expectedErrStr: "Build file not found for 0/1/2 directory", + }, { + description: "GET-INSTALL-PATH specified,", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{"GET-INSTALL-PATH", "-j", "-k", "GET-INSTALL-PATH"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"-j", "-k", "GET-INSTALL-PATH-IN-0-1"}, + }, { + description: "tidy only environment variable specified,", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{"GET-INSTALL-PATH"}, + curDir: "0/1/2", + tidyOnly: "true", + expectedArgs: []string{"tidy_only"}, + }, { + description: "normal execution in root directory with args", + dirsInTrees: []string{}, + buildFiles: []string{}, + args: []string{"-j", "-k", "fake_module"}, + curDir: "", + tidyOnly: "", + expectedArgs: []string{"-j", "-k", "fake_module"}, + }} + for _, tt := range tests { + t.Run("build action BUILD_MODULES_IN_DIR, "+tt.description, func(t *testing.T) { + testGetConfigArgs(t, tt, BUILD_MODULES_IN_A_DIRECTORY) + }) + } +} + +func TestGetConfigArgsBuildModulesInDirectories(t *testing.T) { + tests := []buildActionTestCase{{ + description: "normal execution in a directory", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/2/3.3"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/2/3.3/Android.bp"}, + args: []string{"3.1/", "3.2/", "3.3/"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2-3.1", "MODULES-IN-0-1-2-3.2", "MODULES-IN-0-1-2-3.3"}, + }, { + description: "GET-INSTALL-PATH specified", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/Android.bp"}, + args: []string{"GET-INSTALL-PATH", "2/3.1/", "2/3.2", "3"}, + curDir: "0/1", + tidyOnly: "", + expectedArgs: []string{"GET-INSTALL-PATH-IN-0-1-2-3.1", "GET-INSTALL-PATH-IN-0-1-2-3.2", "GET-INSTALL-PATH-IN-0-1"}, + }, { + description: "tidy only environment variable specified", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/2/3.3"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/2/3.3/Android.bp"}, + args: []string{"GET-INSTALL-PATH", "3.1/", "3.2/", "3.3"}, + curDir: "0/1/2", + tidyOnly: "1", + expectedArgs: []string{"tidy_only"}, + }, { + description: "normal execution from top dir directory", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/3/Android.bp", "0/2/Android.bp"}, + rootSymlink: false, + args: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2-3.1", "MODULES-IN-0-1-2-3.2", "MODULES-IN-0-1-3", "MODULES-IN-0-2"}, + }, { + description: "normal execution from top dir directory in symlink", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/3/Android.bp", "0/2/Android.bp"}, + rootSymlink: true, + args: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2-3.1", "MODULES-IN-0-1-2-3.2", "MODULES-IN-0-1-3", "MODULES-IN-0-2"}, + }} + for _, tt := range tests { + t.Run("build action BUILD_MODULES_IN_DIRS, "+tt.description, func(t *testing.T) { + testGetConfigArgs(t, tt, BUILD_MODULES_IN_DIRECTORIES) + }) + } +} diff --git a/ui/build/context.go b/ui/build/context.go index 249e8982..3945ce00 100644 --- a/ui/build/context.go +++ b/ui/build/context.go @@ -16,12 +16,12 @@ package build import ( "context" + "io" "android/soong/ui/logger" "android/soong/ui/metrics" "android/soong/ui/metrics/metrics_proto" "android/soong/ui/status" - "android/soong/ui/terminal" "android/soong/ui/tracer" ) @@ -35,7 +35,7 @@ type ContextImpl struct { Metrics *metrics.Metrics - Writer terminal.Writer + Writer io.Writer Status *status.Status Thread tracer.Thread @@ -70,7 +70,7 @@ func (c ContextImpl) CompleteTrace(name, desc string, begin, end uint64) { if c.Metrics != nil { realTime := end - begin c.Metrics.SetTimeMetrics( - metrics_proto.PerfInfo{ + soong_metrics_proto.PerfInfo{ Desc: &desc, Name: &name, StartTime: &begin, diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index 3e387c1e..fd572a9b 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -125,6 +125,7 @@ func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_ var BannerVars = []string{ "PLATFORM_VERSION_CODENAME", "PLATFORM_VERSION", + "LINEAGE_VERSION", "TARGET_PRODUCT", "TARGET_BUILD_VARIANT", "TARGET_BUILD_TYPE", @@ -149,6 +150,8 @@ var BannerVars = []string{ "TARGET_BUILD_PDK", "PDK_FUSION_PLATFORM_ZIP", "PRODUCT_SOONG_NAMESPACES", + "WITH_SU", + "WITH_GMS", } func Banner(make_vars map[string]string) string { @@ -221,7 +224,7 @@ func runMakeProductConfig(ctx Context, config Config) { env := config.Environment() // Print the banner like make does if !env.IsEnvTrue("ANDROID_QUIET_BUILD") { - ctx.Writer.Print(Banner(make_vars)) + fmt.Fprintln(ctx.Writer, Banner(make_vars)) } // Populate the environment diff --git a/ui/build/kati.go b/ui/build/kati.go index 959d0bdf..90414055 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -42,9 +42,6 @@ func genKatiSuffix(ctx Context, config Config) { if args := config.KatiArgs(); len(args) > 0 { katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_")) } - if oneShot, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok { - katiSuffix += "-" + spaceSlashReplacer.Replace(oneShot) - } // If the suffix is too long, replace it with a md5 hash and write a // file that contains the original suffix. diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go index f4bb89fb..d17b9f77 100644 --- a/ui/build/paths/config.go +++ b/ui/build/paths/config.go @@ -94,6 +94,7 @@ var Configuration = map[string]PathConfig{ "javap": Allowed, "lsof": Allowed, "m4": Allowed, + "nproc": Allowed, "openssl": Allowed, "patch": Allowed, "pstree": Allowed, diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go index b94db744..37c09aa1 100644 --- a/ui/build/sandbox_linux.go +++ b/ui/build/sandbox_linux.go @@ -90,10 +90,7 @@ func (c *Cmd) sandboxSupported() bool { return } - c.ctx.Println("Build sandboxing disabled due to nsjail error. This may become fatal in the future.") - c.ctx.Println("Please let us know why nsjail doesn't work in your environment at:") - c.ctx.Println(" https://groups.google.com/forum/#!forum/android-building") - c.ctx.Println(" https://issuetracker.google.com/issues/new?component=381517") + c.ctx.Println("Build sandboxing disabled due to nsjail error.") for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { c.ctx.Verboseln(line) diff --git a/ui/build/util.go b/ui/build/util.go index 0676a860..75e6753b 100644 --- a/ui/build/util.go +++ b/ui/build/util.go @@ -44,6 +44,17 @@ func inList(s string, list []string) bool { return indexList(s, list) != -1 } +// removeFromlist removes all occurrences of the string in list. +func removeFromList(s string, list []string) []string { + filteredList := make([]string, 0, len(list)) + for _, ls := range list { + if s != ls { + filteredList = append(filteredList, ls) + } + } + return filteredList +} + // ensureDirectoriesExist is a shortcut to os.MkdirAll, sending errors to the ctx logger. func ensureDirectoriesExist(ctx Context, dirs ...string) { for _, dir := range dirs { diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 790b67ad..bc86f0a8 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -33,19 +33,19 @@ const ( ) type Metrics struct { - metrics metrics_proto.MetricsBase + metrics soong_metrics_proto.MetricsBase TimeTracer TimeTracer } func New() (metrics *Metrics) { m := &Metrics{ - metrics: metrics_proto.MetricsBase{}, + metrics: soong_metrics_proto.MetricsBase{}, TimeTracer: &timeTracerImpl{}, } return m } -func (m *Metrics) SetTimeMetrics(perf metrics_proto.PerfInfo) { +func (m *Metrics) SetTimeMetrics(perf soong_metrics_proto.PerfInfo) { switch perf.GetName() { case RunKati: m.metrics.KatiRuns = append(m.metrics.KatiRuns, &perf) @@ -76,11 +76,11 @@ func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { case "TARGET_BUILD_VARIANT": switch v { case "user": - m.metrics.TargetBuildVariant = metrics_proto.MetricsBase_USER.Enum() + m.metrics.TargetBuildVariant = soong_metrics_proto.MetricsBase_USER.Enum() case "userdebug": - m.metrics.TargetBuildVariant = metrics_proto.MetricsBase_USERDEBUG.Enum() + m.metrics.TargetBuildVariant = soong_metrics_proto.MetricsBase_USERDEBUG.Enum() case "eng": - m.metrics.TargetBuildVariant = metrics_proto.MetricsBase_ENG.Enum() + m.metrics.TargetBuildVariant = soong_metrics_proto.MetricsBase_ENG.Enum() default: // ignored } @@ -112,18 +112,18 @@ func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { } } -func (m *Metrics) getArch(arch string) *metrics_proto.MetricsBase_ARCH { +func (m *Metrics) getArch(arch string) *soong_metrics_proto.MetricsBase_Arch { switch arch { case "arm": - return metrics_proto.MetricsBase_ARM.Enum() + return soong_metrics_proto.MetricsBase_ARM.Enum() case "arm64": - return metrics_proto.MetricsBase_ARM64.Enum() + return soong_metrics_proto.MetricsBase_ARM64.Enum() case "x86": - return metrics_proto.MetricsBase_X86.Enum() + return soong_metrics_proto.MetricsBase_X86.Enum() case "x86_64": - return metrics_proto.MetricsBase_X86_64.Enum() + return soong_metrics_proto.MetricsBase_X86_64.Enum() default: - return metrics_proto.MetricsBase_UNKNOWN.Enum() + return soong_metrics_proto.MetricsBase_UNKNOWN.Enum() } } @@ -148,7 +148,7 @@ func (m *Metrics) Dump(outputPath string) (err error) { return err } tempPath := outputPath + ".tmp" - err = ioutil.WriteFile(tempPath, []byte(data), 0777) + err = ioutil.WriteFile(tempPath, []byte(data), 0644) if err != nil { return err } diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index feefc89c..5486ec1e 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: metrics.proto -package metrics_proto +package soong_metrics_proto -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,65 +18,70 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -type MetricsBase_BUILDVARIANT int32 +type MetricsBase_BuildVariant int32 const ( - MetricsBase_USER MetricsBase_BUILDVARIANT = 0 - MetricsBase_USERDEBUG MetricsBase_BUILDVARIANT = 1 - MetricsBase_ENG MetricsBase_BUILDVARIANT = 2 + MetricsBase_USER MetricsBase_BuildVariant = 0 + MetricsBase_USERDEBUG MetricsBase_BuildVariant = 1 + MetricsBase_ENG MetricsBase_BuildVariant = 2 ) -var MetricsBase_BUILDVARIANT_name = map[int32]string{ +var MetricsBase_BuildVariant_name = map[int32]string{ 0: "USER", 1: "USERDEBUG", 2: "ENG", } -var MetricsBase_BUILDVARIANT_value = map[string]int32{ + +var MetricsBase_BuildVariant_value = map[string]int32{ "USER": 0, "USERDEBUG": 1, "ENG": 2, } -func (x MetricsBase_BUILDVARIANT) Enum() *MetricsBase_BUILDVARIANT { - p := new(MetricsBase_BUILDVARIANT) +func (x MetricsBase_BuildVariant) Enum() *MetricsBase_BuildVariant { + p := new(MetricsBase_BuildVariant) *p = x return p } -func (x MetricsBase_BUILDVARIANT) String() string { - return proto.EnumName(MetricsBase_BUILDVARIANT_name, int32(x)) + +func (x MetricsBase_BuildVariant) String() string { + return proto.EnumName(MetricsBase_BuildVariant_name, int32(x)) } -func (x *MetricsBase_BUILDVARIANT) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(MetricsBase_BUILDVARIANT_value, data, "MetricsBase_BUILDVARIANT") + +func (x *MetricsBase_BuildVariant) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(MetricsBase_BuildVariant_value, data, "MetricsBase_BuildVariant") if err != nil { return err } - *x = MetricsBase_BUILDVARIANT(value) + *x = MetricsBase_BuildVariant(value) return nil } -func (MetricsBase_BUILDVARIANT) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{0, 0} + +func (MetricsBase_BuildVariant) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{0, 0} } -type MetricsBase_ARCH int32 +type MetricsBase_Arch int32 const ( - MetricsBase_UNKNOWN MetricsBase_ARCH = 0 - MetricsBase_ARM MetricsBase_ARCH = 1 - MetricsBase_ARM64 MetricsBase_ARCH = 2 - MetricsBase_X86 MetricsBase_ARCH = 3 - MetricsBase_X86_64 MetricsBase_ARCH = 4 + MetricsBase_UNKNOWN MetricsBase_Arch = 0 + MetricsBase_ARM MetricsBase_Arch = 1 + MetricsBase_ARM64 MetricsBase_Arch = 2 + MetricsBase_X86 MetricsBase_Arch = 3 + MetricsBase_X86_64 MetricsBase_Arch = 4 ) -var MetricsBase_ARCH_name = map[int32]string{ +var MetricsBase_Arch_name = map[int32]string{ 0: "UNKNOWN", 1: "ARM", 2: "ARM64", 3: "X86", 4: "X86_64", } -var MetricsBase_ARCH_value = map[string]int32{ + +var MetricsBase_Arch_value = map[string]int32{ "UNKNOWN": 0, "ARM": 1, "ARM64": 2, @@ -82,63 +89,70 @@ var MetricsBase_ARCH_value = map[string]int32{ "X86_64": 4, } -func (x MetricsBase_ARCH) Enum() *MetricsBase_ARCH { - p := new(MetricsBase_ARCH) +func (x MetricsBase_Arch) Enum() *MetricsBase_Arch { + p := new(MetricsBase_Arch) *p = x return p } -func (x MetricsBase_ARCH) String() string { - return proto.EnumName(MetricsBase_ARCH_name, int32(x)) + +func (x MetricsBase_Arch) String() string { + return proto.EnumName(MetricsBase_Arch_name, int32(x)) } -func (x *MetricsBase_ARCH) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(MetricsBase_ARCH_value, data, "MetricsBase_ARCH") + +func (x *MetricsBase_Arch) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(MetricsBase_Arch_value, data, "MetricsBase_Arch") if err != nil { return err } - *x = MetricsBase_ARCH(value) + *x = MetricsBase_Arch(value) return nil } -func (MetricsBase_ARCH) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{0, 1} + +func (MetricsBase_Arch) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{0, 1} } -type ModuleTypeInfo_BUILDSYSTEM int32 +type ModuleTypeInfo_BuildSystem int32 const ( - ModuleTypeInfo_UNKNOWN ModuleTypeInfo_BUILDSYSTEM = 0 - ModuleTypeInfo_SOONG ModuleTypeInfo_BUILDSYSTEM = 1 - ModuleTypeInfo_MAKE ModuleTypeInfo_BUILDSYSTEM = 2 + ModuleTypeInfo_UNKNOWN ModuleTypeInfo_BuildSystem = 0 + ModuleTypeInfo_SOONG ModuleTypeInfo_BuildSystem = 1 + ModuleTypeInfo_MAKE ModuleTypeInfo_BuildSystem = 2 ) -var ModuleTypeInfo_BUILDSYSTEM_name = map[int32]string{ +var ModuleTypeInfo_BuildSystem_name = map[int32]string{ 0: "UNKNOWN", 1: "SOONG", 2: "MAKE", } -var ModuleTypeInfo_BUILDSYSTEM_value = map[string]int32{ + +var ModuleTypeInfo_BuildSystem_value = map[string]int32{ "UNKNOWN": 0, "SOONG": 1, "MAKE": 2, } -func (x ModuleTypeInfo_BUILDSYSTEM) Enum() *ModuleTypeInfo_BUILDSYSTEM { - p := new(ModuleTypeInfo_BUILDSYSTEM) +func (x ModuleTypeInfo_BuildSystem) Enum() *ModuleTypeInfo_BuildSystem { + p := new(ModuleTypeInfo_BuildSystem) *p = x return p } -func (x ModuleTypeInfo_BUILDSYSTEM) String() string { - return proto.EnumName(ModuleTypeInfo_BUILDSYSTEM_name, int32(x)) + +func (x ModuleTypeInfo_BuildSystem) String() string { + return proto.EnumName(ModuleTypeInfo_BuildSystem_name, int32(x)) } -func (x *ModuleTypeInfo_BUILDSYSTEM) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(ModuleTypeInfo_BUILDSYSTEM_value, data, "ModuleTypeInfo_BUILDSYSTEM") + +func (x *ModuleTypeInfo_BuildSystem) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(ModuleTypeInfo_BuildSystem_value, data, "ModuleTypeInfo_BuildSystem") if err != nil { return err } - *x = ModuleTypeInfo_BUILDSYSTEM(value) + *x = ModuleTypeInfo_BuildSystem(value) return nil } -func (ModuleTypeInfo_BUILDSYSTEM) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{2, 0} + +func (ModuleTypeInfo_BuildSystem) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{2, 0} } type MetricsBase struct { @@ -151,17 +165,17 @@ type MetricsBase struct { // The target product information, eg. aosp_arm. TargetProduct *string `protobuf:"bytes,4,opt,name=target_product,json=targetProduct" json:"target_product,omitempty"` // The target build variant information, eg. eng. - TargetBuildVariant *MetricsBase_BUILDVARIANT `protobuf:"varint,5,opt,name=target_build_variant,json=targetBuildVariant,enum=build_metrics.MetricsBase_BUILDVARIANT,def=2" json:"target_build_variant,omitempty"` + TargetBuildVariant *MetricsBase_BuildVariant `protobuf:"varint,5,opt,name=target_build_variant,json=targetBuildVariant,enum=soong_build_metrics.MetricsBase_BuildVariant,def=2" json:"target_build_variant,omitempty"` // The target arch information, eg. arm. - TargetArch *MetricsBase_ARCH `protobuf:"varint,6,opt,name=target_arch,json=targetArch,enum=build_metrics.MetricsBase_ARCH,def=0" json:"target_arch,omitempty"` + TargetArch *MetricsBase_Arch `protobuf:"varint,6,opt,name=target_arch,json=targetArch,enum=soong_build_metrics.MetricsBase_Arch,def=0" json:"target_arch,omitempty"` // The target arch variant information, eg. armv7-a-neon. TargetArchVariant *string `protobuf:"bytes,7,opt,name=target_arch_variant,json=targetArchVariant" json:"target_arch_variant,omitempty"` // The target cpu variant information, eg. generic. TargetCpuVariant *string `protobuf:"bytes,8,opt,name=target_cpu_variant,json=targetCpuVariant" json:"target_cpu_variant,omitempty"` // The host arch information, eg. x86_64. - HostArch *MetricsBase_ARCH `protobuf:"varint,9,opt,name=host_arch,json=hostArch,enum=build_metrics.MetricsBase_ARCH,def=0" json:"host_arch,omitempty"` + HostArch *MetricsBase_Arch `protobuf:"varint,9,opt,name=host_arch,json=hostArch,enum=soong_build_metrics.MetricsBase_Arch,def=0" json:"host_arch,omitempty"` // The host 2nd arch information, eg. x86. - Host_2NdArch *MetricsBase_ARCH `protobuf:"varint,10,opt,name=host_2nd_arch,json=host2ndArch,enum=build_metrics.MetricsBase_ARCH,def=0" json:"host_2nd_arch,omitempty"` + Host_2NdArch *MetricsBase_Arch `protobuf:"varint,10,opt,name=host_2nd_arch,json=host2ndArch,enum=soong_build_metrics.MetricsBase_Arch,def=0" json:"host_2nd_arch,omitempty"` // The host os information, eg. linux. HostOs *string `protobuf:"bytes,11,opt,name=host_os,json=hostOs" json:"host_os,omitempty"` // The host os extra information, eg. Linux-4.17.0-3rodete2-amd64-x86_64-Debian-GNU. @@ -191,16 +205,17 @@ func (m *MetricsBase) Reset() { *m = MetricsBase{} } func (m *MetricsBase) String() string { return proto.CompactTextString(m) } func (*MetricsBase) ProtoMessage() {} func (*MetricsBase) Descriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{0} + return fileDescriptor_6039342a2ba47b72, []int{0} } + func (m *MetricsBase) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MetricsBase.Unmarshal(m, b) } func (m *MetricsBase) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MetricsBase.Marshal(b, m, deterministic) } -func (dst *MetricsBase) XXX_Merge(src proto.Message) { - xxx_messageInfo_MetricsBase.Merge(dst, src) +func (m *MetricsBase) XXX_Merge(src proto.Message) { + xxx_messageInfo_MetricsBase.Merge(m, src) } func (m *MetricsBase) XXX_Size() int { return xxx_messageInfo_MetricsBase.Size(m) @@ -211,10 +226,10 @@ func (m *MetricsBase) XXX_DiscardUnknown() { var xxx_messageInfo_MetricsBase proto.InternalMessageInfo -const Default_MetricsBase_TargetBuildVariant MetricsBase_BUILDVARIANT = MetricsBase_ENG -const Default_MetricsBase_TargetArch MetricsBase_ARCH = MetricsBase_UNKNOWN -const Default_MetricsBase_HostArch MetricsBase_ARCH = MetricsBase_UNKNOWN -const Default_MetricsBase_Host_2NdArch MetricsBase_ARCH = MetricsBase_UNKNOWN +const Default_MetricsBase_TargetBuildVariant MetricsBase_BuildVariant = MetricsBase_ENG +const Default_MetricsBase_TargetArch MetricsBase_Arch = MetricsBase_UNKNOWN +const Default_MetricsBase_HostArch MetricsBase_Arch = MetricsBase_UNKNOWN +const Default_MetricsBase_Host_2NdArch MetricsBase_Arch = MetricsBase_UNKNOWN func (m *MetricsBase) GetBuildDateTimestamp() int64 { if m != nil && m.BuildDateTimestamp != nil { @@ -244,14 +259,14 @@ func (m *MetricsBase) GetTargetProduct() string { return "" } -func (m *MetricsBase) GetTargetBuildVariant() MetricsBase_BUILDVARIANT { +func (m *MetricsBase) GetTargetBuildVariant() MetricsBase_BuildVariant { if m != nil && m.TargetBuildVariant != nil { return *m.TargetBuildVariant } return Default_MetricsBase_TargetBuildVariant } -func (m *MetricsBase) GetTargetArch() MetricsBase_ARCH { +func (m *MetricsBase) GetTargetArch() MetricsBase_Arch { if m != nil && m.TargetArch != nil { return *m.TargetArch } @@ -272,14 +287,14 @@ func (m *MetricsBase) GetTargetCpuVariant() string { return "" } -func (m *MetricsBase) GetHostArch() MetricsBase_ARCH { +func (m *MetricsBase) GetHostArch() MetricsBase_Arch { if m != nil && m.HostArch != nil { return *m.HostArch } return Default_MetricsBase_HostArch } -func (m *MetricsBase) GetHost_2NdArch() MetricsBase_ARCH { +func (m *MetricsBase) GetHost_2NdArch() MetricsBase_Arch { if m != nil && m.Host_2NdArch != nil { return *m.Host_2NdArch } @@ -378,16 +393,17 @@ func (m *PerfInfo) Reset() { *m = PerfInfo{} } func (m *PerfInfo) String() string { return proto.CompactTextString(m) } func (*PerfInfo) ProtoMessage() {} func (*PerfInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{1} + return fileDescriptor_6039342a2ba47b72, []int{1} } + func (m *PerfInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PerfInfo.Unmarshal(m, b) } func (m *PerfInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PerfInfo.Marshal(b, m, deterministic) } -func (dst *PerfInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_PerfInfo.Merge(dst, src) +func (m *PerfInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_PerfInfo.Merge(m, src) } func (m *PerfInfo) XXX_Size() int { return xxx_messageInfo_PerfInfo.Size(m) @@ -435,7 +451,7 @@ func (m *PerfInfo) GetMemoryUse() uint64 { type ModuleTypeInfo struct { // The build system, eg. Soong or Make. - BuildSystem *ModuleTypeInfo_BUILDSYSTEM `protobuf:"varint,1,opt,name=build_system,json=buildSystem,enum=build_metrics.ModuleTypeInfo_BUILDSYSTEM,def=0" json:"build_system,omitempty"` + BuildSystem *ModuleTypeInfo_BuildSystem `protobuf:"varint,1,opt,name=build_system,json=buildSystem,enum=soong_build_metrics.ModuleTypeInfo_BuildSystem,def=0" json:"build_system,omitempty"` // The module type, eg. java_library, cc_binary, and etc. ModuleType *string `protobuf:"bytes,2,opt,name=module_type,json=moduleType" json:"module_type,omitempty"` // The number of logical modules. @@ -449,16 +465,17 @@ func (m *ModuleTypeInfo) Reset() { *m = ModuleTypeInfo{} } func (m *ModuleTypeInfo) String() string { return proto.CompactTextString(m) } func (*ModuleTypeInfo) ProtoMessage() {} func (*ModuleTypeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{2} + return fileDescriptor_6039342a2ba47b72, []int{2} } + func (m *ModuleTypeInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ModuleTypeInfo.Unmarshal(m, b) } func (m *ModuleTypeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ModuleTypeInfo.Marshal(b, m, deterministic) } -func (dst *ModuleTypeInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_ModuleTypeInfo.Merge(dst, src) +func (m *ModuleTypeInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_ModuleTypeInfo.Merge(m, src) } func (m *ModuleTypeInfo) XXX_Size() int { return xxx_messageInfo_ModuleTypeInfo.Size(m) @@ -469,9 +486,9 @@ func (m *ModuleTypeInfo) XXX_DiscardUnknown() { var xxx_messageInfo_ModuleTypeInfo proto.InternalMessageInfo -const Default_ModuleTypeInfo_BuildSystem ModuleTypeInfo_BUILDSYSTEM = ModuleTypeInfo_UNKNOWN +const Default_ModuleTypeInfo_BuildSystem ModuleTypeInfo_BuildSystem = ModuleTypeInfo_UNKNOWN -func (m *ModuleTypeInfo) GetBuildSystem() ModuleTypeInfo_BUILDSYSTEM { +func (m *ModuleTypeInfo) GetBuildSystem() ModuleTypeInfo_BuildSystem { if m != nil && m.BuildSystem != nil { return *m.BuildSystem } @@ -493,65 +510,65 @@ func (m *ModuleTypeInfo) GetNumOfModules() uint32 { } func init() { - proto.RegisterType((*MetricsBase)(nil), "build_metrics.MetricsBase") - proto.RegisterType((*PerfInfo)(nil), "build_metrics.PerfInfo") - proto.RegisterType((*ModuleTypeInfo)(nil), "build_metrics.ModuleTypeInfo") - proto.RegisterEnum("build_metrics.MetricsBase_BUILDVARIANT", MetricsBase_BUILDVARIANT_name, MetricsBase_BUILDVARIANT_value) - proto.RegisterEnum("build_metrics.MetricsBase_ARCH", MetricsBase_ARCH_name, MetricsBase_ARCH_value) - proto.RegisterEnum("build_metrics.ModuleTypeInfo_BUILDSYSTEM", ModuleTypeInfo_BUILDSYSTEM_name, ModuleTypeInfo_BUILDSYSTEM_value) -} - -func init() { proto.RegisterFile("metrics.proto", fileDescriptor_metrics_9e7b895801991242) } - -var fileDescriptor_metrics_9e7b895801991242 = []byte{ - // 783 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6e, 0xdb, 0x36, - 0x14, 0xae, 0x62, 0x25, 0x96, 0x8e, 0x62, 0x57, 0x61, 0x02, 0x44, 0xc5, 0x50, 0x34, 0x30, 0xf6, - 0x93, 0x01, 0x9b, 0x57, 0x18, 0x81, 0x11, 0x04, 0xbb, 0xb1, 0x13, 0xa3, 0x35, 0x5a, 0xdb, 0x85, - 0x6c, 0x67, 0xdd, 0x2e, 0x46, 0x68, 0x12, 0xdd, 0x68, 0xb3, 0x44, 0x81, 0xa4, 0x8a, 0xf9, 0x21, - 0xf6, 0x8c, 0x7b, 0x91, 0x5d, 0x0c, 0x3c, 0xb4, 0x5c, 0xa5, 0x17, 0x29, 0x72, 0x47, 0x9d, 0xef, - 0x87, 0xdf, 0x91, 0xc8, 0x23, 0x68, 0x65, 0x4c, 0x89, 0x34, 0x96, 0xdd, 0x42, 0x70, 0xc5, 0x49, - 0xeb, 0x8f, 0x32, 0x5d, 0x27, 0x74, 0x5b, 0xec, 0xfc, 0xe7, 0x80, 0x37, 0x31, 0xeb, 0x61, 0x24, - 0x19, 0x79, 0x09, 0x27, 0x86, 0x90, 0x44, 0x8a, 0x51, 0x95, 0x66, 0x4c, 0xaa, 0x28, 0x2b, 0x02, - 0xeb, 0xcc, 0x3a, 0x6f, 0x84, 0x04, 0xb1, 0x9b, 0x48, 0xb1, 0x45, 0x85, 0x90, 0x67, 0xe0, 0x18, - 0x45, 0x9a, 0x04, 0x7b, 0x67, 0xd6, 0xb9, 0x1b, 0x36, 0xf1, 0x79, 0x9c, 0x90, 0x2b, 0x78, 0x56, - 0xac, 0x23, 0xb5, 0xe2, 0x22, 0xa3, 0x1f, 0x99, 0x90, 0x29, 0xcf, 0x69, 0xcc, 0x13, 0x96, 0x47, - 0x19, 0x0b, 0x1a, 0xc8, 0x3d, 0xad, 0x08, 0xb7, 0x06, 0xbf, 0xde, 0xc2, 0xe4, 0x1b, 0x68, 0xab, - 0x48, 0x7c, 0x60, 0x8a, 0x16, 0x82, 0x27, 0x65, 0xac, 0x02, 0x1b, 0x05, 0x2d, 0x53, 0x7d, 0x67, - 0x8a, 0xe4, 0x77, 0x38, 0xd9, 0xd2, 0x4c, 0x88, 0x8f, 0x91, 0x48, 0xa3, 0x5c, 0x05, 0xfb, 0x67, - 0xd6, 0x79, 0xbb, 0xf7, 0x5d, 0xf7, 0x5e, 0xb7, 0xdd, 0x5a, 0xa7, 0xdd, 0xe1, 0x72, 0xfc, 0xf6, - 0xe6, 0x76, 0x10, 0x8e, 0x07, 0xd3, 0xc5, 0x55, 0x63, 0x34, 0x7d, 0x15, 0x12, 0xe3, 0x34, 0xd4, - 0x92, 0x5b, 0xe3, 0x43, 0xc6, 0xe0, 0x6d, 0xfd, 0x23, 0x11, 0xdf, 0x05, 0x07, 0x68, 0xfb, 0xe2, - 0x01, 0xdb, 0x41, 0x78, 0xfd, 0xfa, 0xaa, 0xb9, 0x9c, 0xbe, 0x99, 0xce, 0x7e, 0x99, 0x86, 0x60, - 0xc4, 0x03, 0x11, 0xdf, 0x91, 0x2e, 0x1c, 0xd7, 0xac, 0x76, 0x49, 0x9b, 0xd8, 0xd6, 0xd1, 0x27, - 0x62, 0xb5, 0xf5, 0x0f, 0xb0, 0x0d, 0x44, 0xe3, 0xa2, 0xdc, 0xd1, 0x1d, 0xa4, 0xfb, 0x06, 0xb9, - 0x2e, 0xca, 0x8a, 0x3d, 0x02, 0xf7, 0x8e, 0xcb, 0x6d, 0x4c, 0xf7, 0x91, 0x31, 0x1d, 0x2d, 0xc5, - 0x90, 0x6f, 0xa1, 0x85, 0x36, 0xbd, 0x3c, 0x31, 0x56, 0xf0, 0x48, 0x2b, 0x4f, 0xcb, 0x7b, 0x79, - 0x82, 0x6e, 0xa7, 0xd0, 0x44, 0x37, 0x2e, 0x03, 0x0f, 0x73, 0x1f, 0xe8, 0xc7, 0x99, 0x24, 0x9d, - 0xed, 0x36, 0x5c, 0x52, 0xf6, 0xb7, 0x12, 0x51, 0x70, 0x88, 0xb0, 0x67, 0xe0, 0x91, 0x2e, 0xed, - 0x38, 0xb1, 0xe0, 0x52, 0x6a, 0x8b, 0xd6, 0x27, 0xce, 0xb5, 0xae, 0xcd, 0x24, 0xf9, 0x16, 0x9e, - 0xd6, 0x38, 0x18, 0xb8, 0x6d, 0x8e, 0xc9, 0x8e, 0x85, 0x41, 0x7e, 0x84, 0xe3, 0x1a, 0x6f, 0xd7, - 0xdc, 0x53, 0xf3, 0x32, 0x77, 0xdc, 0x5a, 0x6e, 0x5e, 0x2a, 0x9a, 0xa4, 0x22, 0xf0, 0x4d, 0x6e, - 0x5e, 0xaa, 0x9b, 0x54, 0x90, 0x4b, 0xf0, 0x24, 0x53, 0x65, 0x41, 0x15, 0xe7, 0x6b, 0x19, 0x1c, - 0x9d, 0x35, 0xce, 0xbd, 0xde, 0xe9, 0x67, 0x2f, 0xe7, 0x1d, 0x13, 0xab, 0x71, 0xbe, 0xe2, 0x21, - 0x20, 0x77, 0xa1, 0xa9, 0xe4, 0x02, 0xdc, 0xbf, 0x22, 0x95, 0x52, 0x51, 0xe6, 0x32, 0x20, 0x0f, - 0xeb, 0x1c, 0xcd, 0x0c, 0xcb, 0x5c, 0x92, 0x3e, 0x80, 0xe4, 0x3c, 0xff, 0x60, 0x64, 0xc7, 0x0f, - 0xcb, 0x5c, 0xa4, 0x56, 0xba, 0x3c, 0xcd, 0xff, 0x8c, 0x8c, 0xee, 0xe4, 0x0b, 0x3a, 0xa4, 0x6a, - 0x5d, 0xe7, 0x25, 0x1c, 0xd6, 0xef, 0x05, 0x71, 0xc0, 0x5e, 0xce, 0x47, 0xa1, 0xff, 0x84, 0xb4, - 0xc0, 0xd5, 0xab, 0x9b, 0xd1, 0x70, 0xf9, 0xca, 0xb7, 0x48, 0x13, 0xf4, 0x95, 0xf1, 0xf7, 0x3a, - 0x3f, 0x83, 0xad, 0x0f, 0x00, 0xf1, 0xa0, 0x3a, 0x02, 0xfe, 0x13, 0x8d, 0x0e, 0xc2, 0x89, 0x6f, - 0x11, 0x17, 0xf6, 0x07, 0xe1, 0xa4, 0x7f, 0xe1, 0xef, 0xe9, 0xda, 0xfb, 0xcb, 0xbe, 0xdf, 0x20, - 0x00, 0x07, 0xef, 0x2f, 0xfb, 0xb4, 0x7f, 0xe1, 0xdb, 0x9d, 0x7f, 0x2c, 0x70, 0xaa, 0x1c, 0x84, - 0x80, 0x9d, 0x30, 0x19, 0xe3, 0xac, 0x71, 0x43, 0x5c, 0xeb, 0x1a, 0x4e, 0x0b, 0x33, 0x59, 0x70, - 0x4d, 0x9e, 0x03, 0x48, 0x15, 0x09, 0x85, 0xe3, 0x09, 0xe7, 0x88, 0x1d, 0xba, 0x58, 0xd1, 0x53, - 0x89, 0x7c, 0x05, 0xae, 0x60, 0xd1, 0xda, 0xa0, 0x36, 0xa2, 0x8e, 0x2e, 0x20, 0xf8, 0x1c, 0x20, - 0x63, 0x19, 0x17, 0x1b, 0x5a, 0x4a, 0x86, 0x53, 0xc2, 0x0e, 0x5d, 0x53, 0x59, 0x4a, 0xd6, 0xf9, - 0xd7, 0x82, 0xf6, 0x84, 0x27, 0xe5, 0x9a, 0x2d, 0x36, 0x05, 0xc3, 0x54, 0x4b, 0x38, 0x34, 0xef, - 0x4d, 0x6e, 0xa4, 0x62, 0x19, 0xa6, 0x6b, 0xf7, 0xbe, 0xff, 0xfc, 0x42, 0xdc, 0x13, 0x99, 0xe1, - 0x32, 0xff, 0x75, 0xbe, 0x18, 0x4d, 0x6a, 0x57, 0x03, 0x25, 0x73, 0xb4, 0x21, 0x2f, 0xc0, 0xcb, - 0x50, 0x43, 0xd5, 0xa6, 0xa8, 0xfa, 0x83, 0x6c, 0x67, 0x43, 0xbe, 0x86, 0x76, 0x5e, 0x66, 0x94, - 0xaf, 0xa8, 0x29, 0x4a, 0xec, 0xb4, 0x15, 0x1e, 0xe6, 0x65, 0x36, 0x5b, 0x99, 0xfd, 0x64, 0xe7, - 0x27, 0xf0, 0x6a, 0x7b, 0xdd, 0xff, 0x0a, 0x2e, 0xec, 0xcf, 0x67, 0xb3, 0xa9, 0xfe, 0x5c, 0x0e, - 0xd8, 0x93, 0xc1, 0x9b, 0x91, 0xbf, 0x37, 0x3c, 0x7a, 0xdd, 0xf8, 0xad, 0xfa, 0x25, 0x50, 0xfc, - 0x25, 0xfc, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x8d, 0x19, 0x89, 0x22, 0x06, 0x00, 0x00, + proto.RegisterEnum("soong_build_metrics.MetricsBase_BuildVariant", MetricsBase_BuildVariant_name, MetricsBase_BuildVariant_value) + proto.RegisterEnum("soong_build_metrics.MetricsBase_Arch", MetricsBase_Arch_name, MetricsBase_Arch_value) + proto.RegisterEnum("soong_build_metrics.ModuleTypeInfo_BuildSystem", ModuleTypeInfo_BuildSystem_name, ModuleTypeInfo_BuildSystem_value) + proto.RegisterType((*MetricsBase)(nil), "soong_build_metrics.MetricsBase") + proto.RegisterType((*PerfInfo)(nil), "soong_build_metrics.PerfInfo") + proto.RegisterType((*ModuleTypeInfo)(nil), "soong_build_metrics.ModuleTypeInfo") +} + +func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } + +var fileDescriptor_6039342a2ba47b72 = []byte{ + // 769 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0x6f, 0x6b, 0xdb, 0x46, + 0x18, 0xaf, 0x62, 0x25, 0x96, 0x1e, 0xc5, 0xae, 0x7a, 0xc9, 0xa8, 0xca, 0x08, 0x33, 0x66, 0x1d, + 0x7e, 0xb1, 0xba, 0xc5, 0x14, 0x53, 0x4c, 0x19, 0xd8, 0x89, 0x29, 0x25, 0xd8, 0x2e, 0x4a, 0xdc, + 0x95, 0xed, 0xc5, 0xa1, 0x4a, 0xe7, 0x46, 0x9b, 0xa5, 0x13, 0x77, 0xa7, 0x32, 0x7f, 0x88, 0x7d, + 0x93, 0x7d, 0xad, 0x7d, 0x8f, 0x71, 0xcf, 0x49, 0x8e, 0x02, 0x81, 0x85, 0xbe, 0x3b, 0x3d, 0xbf, + 0x3f, 0xf7, 0x7b, 0x4e, 0xba, 0x47, 0xd0, 0xc9, 0x98, 0x12, 0x69, 0x2c, 0x87, 0x85, 0xe0, 0x8a, + 0x93, 0x13, 0xc9, 0x79, 0xfe, 0x85, 0x7e, 0x2e, 0xd3, 0x6d, 0x42, 0x2b, 0xa8, 0xff, 0x8f, 0x0b, + 0xde, 0xc2, 0xac, 0x67, 0x91, 0x64, 0xe4, 0x15, 0x9c, 0x1a, 0x42, 0x12, 0x29, 0x46, 0x55, 0x9a, + 0x31, 0xa9, 0xa2, 0xac, 0x08, 0xac, 0x9e, 0x35, 0x68, 0x85, 0x04, 0xb1, 0x8b, 0x48, 0xb1, 0xeb, + 0x1a, 0x21, 0xcf, 0xc0, 0x31, 0x8a, 0x34, 0x09, 0x0e, 0x7a, 0xd6, 0xc0, 0x0d, 0xdb, 0xf8, 0xfc, + 0x3e, 0x21, 0x13, 0x78, 0x56, 0x6c, 0x23, 0xb5, 0xe1, 0x22, 0xa3, 0x5f, 0x99, 0x90, 0x29, 0xcf, + 0x69, 0xcc, 0x13, 0x96, 0x47, 0x19, 0x0b, 0x5a, 0xc8, 0x7d, 0x5a, 0x13, 0x3e, 0x1a, 0xfc, 0xbc, + 0x82, 0xc9, 0x73, 0xe8, 0xaa, 0x48, 0x7c, 0x61, 0x8a, 0x16, 0x82, 0x27, 0x65, 0xac, 0x02, 0x1b, + 0x05, 0x1d, 0x53, 0xfd, 0x60, 0x8a, 0x24, 0x81, 0xd3, 0x8a, 0x66, 0x42, 0x7c, 0x8d, 0x44, 0x1a, + 0xe5, 0x2a, 0x38, 0xec, 0x59, 0x83, 0xee, 0xe8, 0xc5, 0xf0, 0x9e, 0x9e, 0x87, 0x8d, 0x7e, 0x87, + 0x33, 0x8d, 0x7c, 0x34, 0xa2, 0x49, 0x6b, 0xbe, 0x7c, 0x17, 0x12, 0xe3, 0xd7, 0x04, 0xc8, 0x0a, + 0xbc, 0x6a, 0x97, 0x48, 0xc4, 0x37, 0xc1, 0x11, 0x9a, 0x3f, 0xff, 0x5f, 0xf3, 0xa9, 0x88, 0x6f, + 0x26, 0xed, 0xf5, 0xf2, 0x72, 0xb9, 0xfa, 0x75, 0x19, 0x82, 0xb1, 0xd0, 0x45, 0x32, 0x84, 0x93, + 0x86, 0xe1, 0x3e, 0x75, 0x1b, 0x5b, 0x7c, 0x72, 0x4b, 0xac, 0x03, 0xfc, 0x0c, 0x55, 0x2c, 0x1a, + 0x17, 0xe5, 0x9e, 0xee, 0x20, 0xdd, 0x37, 0xc8, 0x79, 0x51, 0xd6, 0xec, 0x4b, 0x70, 0x6f, 0xb8, + 0xac, 0xc2, 0xba, 0xdf, 0x14, 0xd6, 0xd1, 0x06, 0x18, 0x35, 0x84, 0x0e, 0x9a, 0x8d, 0xf2, 0xc4, + 0x18, 0xc2, 0x37, 0x19, 0x7a, 0xda, 0x64, 0x94, 0x27, 0xe8, 0xf9, 0x14, 0xda, 0xe8, 0xc9, 0x65, + 0xe0, 0x61, 0x0f, 0x47, 0xfa, 0x71, 0x25, 0x49, 0xbf, 0xda, 0x8c, 0x4b, 0xca, 0xfe, 0x52, 0x22, + 0x0a, 0x8e, 0x11, 0xf6, 0x0c, 0x3c, 0xd7, 0xa5, 0x3d, 0x27, 0x16, 0x5c, 0x4a, 0x6d, 0xd1, 0xb9, + 0xe5, 0x9c, 0xeb, 0xda, 0x4a, 0x92, 0x9f, 0xe0, 0x71, 0x83, 0x83, 0xb1, 0xbb, 0xe6, 0xf3, 0xd9, + 0xb3, 0x30, 0xc8, 0x0b, 0x38, 0x69, 0xf0, 0xf6, 0x2d, 0x3e, 0x36, 0x07, 0xbb, 0xe7, 0x36, 0x72, + 0xf3, 0x52, 0xd1, 0x24, 0x15, 0x81, 0x6f, 0x72, 0xf3, 0x52, 0x5d, 0xa4, 0x82, 0xfc, 0x02, 0x9e, + 0x64, 0xaa, 0x2c, 0xa8, 0xe2, 0x7c, 0x2b, 0x83, 0x27, 0xbd, 0xd6, 0xc0, 0x1b, 0x9d, 0xdd, 0x7b, + 0x44, 0x1f, 0x98, 0xd8, 0xbc, 0xcf, 0x37, 0x3c, 0x04, 0x54, 0x5c, 0x6b, 0x01, 0x99, 0x80, 0xfb, + 0x67, 0xa4, 0x52, 0x2a, 0xca, 0x5c, 0x06, 0xe4, 0x21, 0x6a, 0x47, 0xf3, 0xc3, 0x32, 0x97, 0xe4, + 0x2d, 0x80, 0x61, 0xa2, 0xf8, 0xe4, 0x21, 0x62, 0x17, 0xd1, 0x5a, 0x9d, 0xa7, 0xf9, 0x1f, 0x91, + 0x51, 0x9f, 0x3e, 0x48, 0x8d, 0x02, 0xad, 0xee, 0xbf, 0x82, 0xe3, 0x3b, 0x17, 0xc5, 0x01, 0x7b, + 0x7d, 0x35, 0x0f, 0xfd, 0x47, 0xa4, 0x03, 0xae, 0x5e, 0x5d, 0xcc, 0x67, 0xeb, 0x77, 0xbe, 0x45, + 0xda, 0xa0, 0x2f, 0x97, 0x7f, 0xd0, 0x7f, 0x0b, 0x36, 0x1e, 0xa5, 0x07, 0xf5, 0xa7, 0xe1, 0x3f, + 0xd2, 0xe8, 0x34, 0x5c, 0xf8, 0x16, 0x71, 0xe1, 0x70, 0x1a, 0x2e, 0xc6, 0xaf, 0xfd, 0x03, 0x5d, + 0xfb, 0xf4, 0x66, 0xec, 0xb7, 0x08, 0xc0, 0xd1, 0xa7, 0x37, 0x63, 0x3a, 0x7e, 0xed, 0xdb, 0xfd, + 0xbf, 0x2d, 0x70, 0xea, 0x1c, 0x84, 0x80, 0x9d, 0x30, 0x19, 0xe3, 0x6c, 0x72, 0x43, 0x5c, 0xeb, + 0x1a, 0x4e, 0x17, 0x33, 0x89, 0x70, 0x4d, 0xce, 0x00, 0xa4, 0x8a, 0x84, 0xc2, 0x71, 0x86, 0x73, + 0xc7, 0x0e, 0x5d, 0xac, 0xe8, 0x29, 0x46, 0xbe, 0x07, 0x57, 0xb0, 0x68, 0x6b, 0x50, 0x1b, 0x51, + 0x47, 0x17, 0x10, 0x3c, 0x03, 0xc8, 0x58, 0xc6, 0xc5, 0x8e, 0x96, 0x92, 0xe1, 0x54, 0xb1, 0x43, + 0xd7, 0x54, 0xd6, 0x92, 0xf5, 0xff, 0xb5, 0xa0, 0xbb, 0xe0, 0x49, 0xb9, 0x65, 0xd7, 0xbb, 0x82, + 0x61, 0xaa, 0xdf, 0xe1, 0xd8, 0x9c, 0x9b, 0xdc, 0x49, 0xc5, 0x32, 0x4c, 0xd7, 0x1d, 0xbd, 0xbc, + 0xff, 0xba, 0xdc, 0x91, 0x9a, 0x61, 0x74, 0x85, 0xb2, 0xc6, 0xc5, 0xf9, 0x7c, 0x5b, 0x25, 0x3f, + 0x80, 0x97, 0xa1, 0x86, 0xaa, 0x5d, 0x51, 0x77, 0x09, 0xd9, 0xde, 0x86, 0xfc, 0x08, 0xdd, 0xbc, + 0xcc, 0x28, 0xdf, 0x50, 0x53, 0x94, 0xd8, 0x6f, 0x27, 0x3c, 0xce, 0xcb, 0x6c, 0xb5, 0x31, 0xfb, + 0xc9, 0xfe, 0x4b, 0xf0, 0x1a, 0x7b, 0xdd, 0x7d, 0x17, 0x2e, 0x1c, 0x5e, 0xad, 0x56, 0x4b, 0xfd, + 0xd2, 0x1c, 0xb0, 0x17, 0xd3, 0xcb, 0xb9, 0x7f, 0x30, 0xfb, 0xee, 0xb7, 0xea, 0xef, 0x51, 0x25, + 0xa7, 0xf8, 0x4b, 0xf9, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x81, 0xd0, 0x84, 0x23, 0x62, 0x06, 0x00, + 0x00, } diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index b3de2f46..93034eb8 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -14,10 +14,8 @@ syntax = "proto2"; -option optimize_for = LITE_RUNTIME; - -package build_metrics; -option go_package = "metrics_proto"; +package soong_build_metrics; +option go_package = "soong_metrics_proto"; message MetricsBase { // Timestamp generated when the build starts. @@ -32,15 +30,15 @@ message MetricsBase { // The target product information, eg. aosp_arm. optional string target_product = 4; - enum BUILDVARIANT { + enum BuildVariant { USER = 0; USERDEBUG = 1; ENG = 2; } // The target build variant information, eg. eng. - optional BUILDVARIANT target_build_variant = 5 [default = ENG]; + optional BuildVariant target_build_variant = 5 [default = ENG]; - enum ARCH { + enum Arch { UNKNOWN = 0; ARM = 1; ARM64 = 2; @@ -48,7 +46,7 @@ message MetricsBase { X86_64 = 4; } // The target arch information, eg. arm. - optional ARCH target_arch = 6 [default = UNKNOWN]; + optional Arch target_arch = 6 [default = UNKNOWN]; // The target arch variant information, eg. armv7-a-neon. optional string target_arch_variant = 7; @@ -57,10 +55,10 @@ message MetricsBase { optional string target_cpu_variant = 8; // The host arch information, eg. x86_64. - optional ARCH host_arch = 9 [default = UNKNOWN]; + optional Arch host_arch = 9 [default = UNKNOWN]; // The host 2nd arch information, eg. x86. - optional ARCH host_2nd_arch = 10 [default = UNKNOWN]; + optional Arch host_2nd_arch = 10 [default = UNKNOWN]; // The host os information, eg. linux. optional string host_os = 11; @@ -113,13 +111,13 @@ message PerfInfo { } message ModuleTypeInfo { - enum BUILDSYSTEM { + enum BuildSystem { UNKNOWN = 0; SOONG = 1; MAKE = 2; } // The build system, eg. Soong or Make. - optional BUILDSYSTEM build_system = 1 [default = UNKNOWN]; + optional BuildSystem build_system = 1 [default = UNKNOWN]; // The module type, eg. java_library, cc_binary, and etc. optional string module_type = 2; diff --git a/ui/metrics/time.go b/ui/metrics/time.go index 7e8801ac..b8baf16f 100644 --- a/ui/metrics/time.go +++ b/ui/metrics/time.go @@ -30,7 +30,7 @@ type timeEvent struct { type TimeTracer interface { Begin(name, desc string, thread tracer.Thread) - End(thread tracer.Thread) metrics_proto.PerfInfo + End(thread tracer.Thread) soong_metrics_proto.PerfInfo } type timeTracerImpl struct { @@ -51,11 +51,11 @@ func (t *timeTracerImpl) beginAt(name, desc string, atNanos uint64) { t.activeEvents = append(t.activeEvents, timeEvent{name: name, desc: desc, atNanos: atNanos}) } -func (t *timeTracerImpl) End(thread tracer.Thread) metrics_proto.PerfInfo { +func (t *timeTracerImpl) End(thread tracer.Thread) soong_metrics_proto.PerfInfo { return t.endAt(t.now()) } -func (t *timeTracerImpl) endAt(atNanos uint64) metrics_proto.PerfInfo { +func (t *timeTracerImpl) endAt(atNanos uint64) soong_metrics_proto.PerfInfo { if len(t.activeEvents) < 1 { panic("Internal error: No pending events for endAt to end!") } @@ -63,7 +63,7 @@ func (t *timeTracerImpl) endAt(atNanos uint64) metrics_proto.PerfInfo { t.activeEvents = t.activeEvents[:len(t.activeEvents)-1] realTime := atNanos - lastEvent.atNanos - return metrics_proto.PerfInfo{ + return soong_metrics_proto.PerfInfo{ Desc: &lastEvent.desc, Name: &lastEvent.name, StartTime: &lastEvent.atNanos, diff --git a/ui/status/Android.bp b/ui/status/Android.bp index 901a7130..ec929b34 100644 --- a/ui/status/Android.bp +++ b/ui/status/Android.bp @@ -19,14 +19,17 @@ bootstrap_go_package { "golang-protobuf-proto", "soong-ui-logger", "soong-ui-status-ninja_frontend", + "soong-ui-status-build_error_proto", ], srcs: [ + "critical_path.go", "kati.go", "log.go", "ninja.go", "status.go", ], testSrcs: [ + "critical_path_test.go", "kati_test.go", "ninja_test.go", "status_test.go", @@ -41,3 +44,12 @@ bootstrap_go_package { "ninja_frontend/frontend.pb.go", ], } + +bootstrap_go_package { + name: "soong-ui-status-build_error_proto", + pkgPath: "android/soong/ui/status/build_error_proto", + deps: ["golang-protobuf-proto"], + srcs: [ + "build_error_proto/build_error.pb.go", + ], +} diff --git a/ui/status/build_error_proto/build_error.pb.go b/ui/status/build_error_proto/build_error.pb.go new file mode 100644 index 00000000..d4d0a6ea --- /dev/null +++ b/ui/status/build_error_proto/build_error.pb.go @@ -0,0 +1,175 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: build_error.proto + +package soong_build_error_proto + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type BuildError struct { + // List of error messages of the overall build. The error messages + // are not associated with a build action. + ErrorMessages []string `protobuf:"bytes,1,rep,name=error_messages,json=errorMessages" json:"error_messages,omitempty"` + // List of build action errors. + ActionErrors []*BuildActionError `protobuf:"bytes,2,rep,name=action_errors,json=actionErrors" json:"action_errors,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildError) Reset() { *m = BuildError{} } +func (m *BuildError) String() string { return proto.CompactTextString(m) } +func (*BuildError) ProtoMessage() {} +func (*BuildError) Descriptor() ([]byte, []int) { + return fileDescriptor_a2e15b05802a5501, []int{0} +} + +func (m *BuildError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildError.Unmarshal(m, b) +} +func (m *BuildError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildError.Marshal(b, m, deterministic) +} +func (m *BuildError) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildError.Merge(m, src) +} +func (m *BuildError) XXX_Size() int { + return xxx_messageInfo_BuildError.Size(m) +} +func (m *BuildError) XXX_DiscardUnknown() { + xxx_messageInfo_BuildError.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildError proto.InternalMessageInfo + +func (m *BuildError) GetErrorMessages() []string { + if m != nil { + return m.ErrorMessages + } + return nil +} + +func (m *BuildError) GetActionErrors() []*BuildActionError { + if m != nil { + return m.ActionErrors + } + return nil +} + +// Build is composed of a list of build action. There can be a set of build +// actions that can failed. +type BuildActionError struct { + // Description of the command. + Description *string `protobuf:"bytes,1,opt,name=description" json:"description,omitempty"` + // The command name that raised the error. + Command *string `protobuf:"bytes,2,opt,name=command" json:"command,omitempty"` + // The command output stream. + Output *string `protobuf:"bytes,3,opt,name=output" json:"output,omitempty"` + // List of artifacts (i.e. files) that was produced by the command. + Artifacts []string `protobuf:"bytes,4,rep,name=artifacts" json:"artifacts,omitempty"` + // The error string produced by the build action. + Error *string `protobuf:"bytes,5,opt,name=error" json:"error,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildActionError) Reset() { *m = BuildActionError{} } +func (m *BuildActionError) String() string { return proto.CompactTextString(m) } +func (*BuildActionError) ProtoMessage() {} +func (*BuildActionError) Descriptor() ([]byte, []int) { + return fileDescriptor_a2e15b05802a5501, []int{1} +} + +func (m *BuildActionError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildActionError.Unmarshal(m, b) +} +func (m *BuildActionError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildActionError.Marshal(b, m, deterministic) +} +func (m *BuildActionError) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildActionError.Merge(m, src) +} +func (m *BuildActionError) XXX_Size() int { + return xxx_messageInfo_BuildActionError.Size(m) +} +func (m *BuildActionError) XXX_DiscardUnknown() { + xxx_messageInfo_BuildActionError.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildActionError proto.InternalMessageInfo + +func (m *BuildActionError) GetDescription() string { + if m != nil && m.Description != nil { + return *m.Description + } + return "" +} + +func (m *BuildActionError) GetCommand() string { + if m != nil && m.Command != nil { + return *m.Command + } + return "" +} + +func (m *BuildActionError) GetOutput() string { + if m != nil && m.Output != nil { + return *m.Output + } + return "" +} + +func (m *BuildActionError) GetArtifacts() []string { + if m != nil { + return m.Artifacts + } + return nil +} + +func (m *BuildActionError) GetError() string { + if m != nil && m.Error != nil { + return *m.Error + } + return "" +} + +func init() { + proto.RegisterType((*BuildError)(nil), "soong_build_error.BuildError") + proto.RegisterType((*BuildActionError)(nil), "soong_build_error.BuildActionError") +} + +func init() { proto.RegisterFile("build_error.proto", fileDescriptor_a2e15b05802a5501) } + +var fileDescriptor_a2e15b05802a5501 = []byte{ + // 229 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xc1, 0x4a, 0xc3, 0x40, + 0x10, 0x86, 0x49, 0x63, 0x95, 0x4c, 0xad, 0xd8, 0x41, 0x74, 0x04, 0x0f, 0xa1, 0x22, 0xe4, 0x94, + 0x83, 0x6f, 0x60, 0x41, 0xf0, 0xe2, 0x25, 0x47, 0x2f, 0x61, 0xdd, 0xac, 0x65, 0xc1, 0x64, 0xc2, + 0xce, 0xe6, 0xe8, 0x8b, 0xf8, 0xb4, 0x92, 0x69, 0xa5, 0xa5, 0x39, 0x7e, 0xdf, 0x3f, 0xfb, 0xef, + 0xce, 0xc2, 0xea, 0x73, 0xf0, 0xdf, 0x4d, 0xed, 0x42, 0xe0, 0x50, 0xf6, 0x81, 0x23, 0xe3, 0x4a, + 0x98, 0xbb, 0x6d, 0x7d, 0x14, 0xac, 0x7f, 0x00, 0x36, 0x23, 0xbe, 0x8e, 0x84, 0x4f, 0x70, 0xa5, + 0xba, 0x6e, 0x9d, 0x88, 0xd9, 0x3a, 0xa1, 0x24, 0x4f, 0x8b, 0xac, 0x5a, 0xaa, 0x7d, 0xdf, 0x4b, + 0x7c, 0x83, 0xa5, 0xb1, 0xd1, 0x73, 0xb7, 0x2b, 0x11, 0x9a, 0xe5, 0x69, 0xb1, 0x78, 0x7e, 0x2c, + 0x27, 0xfd, 0xa5, 0x96, 0xbf, 0xe8, 0xb0, 0x5e, 0x51, 0x5d, 0x9a, 0x03, 0xc8, 0xfa, 0x37, 0x81, + 0xeb, 0xd3, 0x11, 0xcc, 0x61, 0xd1, 0x38, 0xb1, 0xc1, 0xf7, 0xa3, 0xa3, 0x24, 0x4f, 0x8a, 0xac, + 0x3a, 0x56, 0x48, 0x70, 0x61, 0xb9, 0x6d, 0x4d, 0xd7, 0xd0, 0x4c, 0xd3, 0x7f, 0xc4, 0x5b, 0x38, + 0xe7, 0x21, 0xf6, 0x43, 0xa4, 0x54, 0x83, 0x3d, 0xe1, 0x03, 0x64, 0x26, 0x44, 0xff, 0x65, 0x6c, + 0x14, 0x3a, 0xd3, 0xa5, 0x0e, 0x02, 0x6f, 0x60, 0xae, 0xcf, 0xa5, 0xb9, 0x1e, 0xda, 0xc1, 0xe6, + 0xfe, 0xe3, 0x6e, 0xb2, 0x50, 0xad, 0x3f, 0xf9, 0x17, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x18, 0x9e, + 0x17, 0x5d, 0x01, 0x00, 0x00, +} diff --git a/ui/status/build_error_proto/build_error.proto b/ui/status/build_error_proto/build_error.proto new file mode 100644 index 00000000..9c8470d8 --- /dev/null +++ b/ui/status/build_error_proto/build_error.proto @@ -0,0 +1,46 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_build_error; +option go_package = "soong_build_error_proto"; + +message BuildError { + // List of error messages of the overall build. The error messages + // are not associated with a build action. + repeated string error_messages = 1; + + // List of build action errors. + repeated BuildActionError action_errors = 2; +} + +// Build is composed of a list of build action. There can be a set of build +// actions that can failed. +message BuildActionError { + // Description of the command. + optional string description = 1; + + // The command name that raised the error. + optional string command = 2; + + // The command output stream. + optional string output = 3; + + // List of artifacts (i.e. files) that was produced by the command. + repeated string artifacts = 4; + + // The error string produced by the build action. + optional string error = 5; +} diff --git a/ui/status/build_error_proto/regen.sh b/ui/status/build_error_proto/regen.sh new file mode 100755 index 00000000..7c3ec8fa --- /dev/null +++ b/ui/status/build_error_proto/regen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aprotoc --go_out=paths=source_relative:. build_error.proto diff --git a/ui/status/critical_path.go b/ui/status/critical_path.go new file mode 100644 index 00000000..8065c60f --- /dev/null +++ b/ui/status/critical_path.go @@ -0,0 +1,154 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status + +import ( + "time" + + "android/soong/ui/logger" +) + +func NewCriticalPath(log logger.Logger) StatusOutput { + return &criticalPath{ + log: log, + running: make(map[*Action]time.Time), + nodes: make(map[string]*node), + clock: osClock{}, + } +} + +type criticalPath struct { + log logger.Logger + + nodes map[string]*node + running map[*Action]time.Time + + start, end time.Time + + clock clock +} + +type clock interface { + Now() time.Time +} + +type osClock struct{} + +func (osClock) Now() time.Time { return time.Now() } + +// A critical path node stores the critical path (the minimum time to build the node and all of its dependencies given +// perfect parallelism) for an node. +type node struct { + action *Action + cumulativeDuration time.Duration + duration time.Duration + input *node +} + +func (cp *criticalPath) StartAction(action *Action, counts Counts) { + start := cp.clock.Now() + if cp.start.IsZero() { + cp.start = start + } + cp.running[action] = start +} + +func (cp *criticalPath) FinishAction(result ActionResult, counts Counts) { + if start, ok := cp.running[result.Action]; ok { + delete(cp.running, result.Action) + + // Determine the input to this edge with the longest cumulative duration + var criticalPathInput *node + for _, input := range result.Action.Inputs { + if x := cp.nodes[input]; x != nil { + if criticalPathInput == nil || x.cumulativeDuration > criticalPathInput.cumulativeDuration { + criticalPathInput = x + } + } + } + + end := cp.clock.Now() + duration := end.Sub(start) + + cumulativeDuration := duration + if criticalPathInput != nil { + cumulativeDuration += criticalPathInput.cumulativeDuration + } + + node := &node{ + action: result.Action, + cumulativeDuration: cumulativeDuration, + duration: duration, + input: criticalPathInput, + } + + for _, output := range result.Action.Outputs { + cp.nodes[output] = node + } + + cp.end = end + } +} + +func (cp *criticalPath) Flush() { + criticalPath := cp.criticalPath() + + if len(criticalPath) > 0 { + // Log the critical path to the verbose log + criticalTime := criticalPath[0].cumulativeDuration.Round(time.Second) + cp.log.Verbosef("critical path took %s", criticalTime.String()) + if !cp.start.IsZero() { + elapsedTime := cp.end.Sub(cp.start).Round(time.Second) + cp.log.Verbosef("elapsed time %s", elapsedTime.String()) + if elapsedTime > 0 { + cp.log.Verbosef("perfect parallelism ratio %d%%", + int(float64(criticalTime)/float64(elapsedTime)*100)) + } + } + cp.log.Verbose("critical path:") + for i := len(criticalPath) - 1; i >= 0; i-- { + duration := criticalPath[i].duration + duration = duration.Round(time.Second) + seconds := int(duration.Seconds()) + cp.log.Verbosef(" %2d:%02d %s", + seconds/60, seconds%60, criticalPath[i].action.Description) + } + } +} + +func (cp *criticalPath) Message(level MsgLevel, msg string) {} + +func (cp *criticalPath) Write(p []byte) (n int, err error) { return len(p), nil } + +func (cp *criticalPath) criticalPath() []*node { + var max *node + + // Find the node with the longest critical path + for _, node := range cp.nodes { + if max == nil || node.cumulativeDuration > max.cumulativeDuration { + max = node + } + } + + // Follow the critical path back to the leaf node + var criticalPath []*node + node := max + for node != nil { + criticalPath = append(criticalPath, node) + node = node.input + } + + return criticalPath +} diff --git a/ui/status/critical_path_test.go b/ui/status/critical_path_test.go new file mode 100644 index 00000000..965e0ad1 --- /dev/null +++ b/ui/status/critical_path_test.go @@ -0,0 +1,166 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status + +import ( + "reflect" + "testing" + "time" +) + +type testCriticalPath struct { + *criticalPath + Counts + + actions map[int]*Action +} + +type testClock time.Time + +func (t testClock) Now() time.Time { return time.Time(t) } + +func (t *testCriticalPath) start(id int, startTime time.Duration, outputs, inputs []string) { + t.clock = testClock(time.Unix(0, 0).Add(startTime)) + action := &Action{ + Description: outputs[0], + Outputs: outputs, + Inputs: inputs, + } + + t.actions[id] = action + t.StartAction(action, t.Counts) +} + +func (t *testCriticalPath) finish(id int, endTime time.Duration) { + t.clock = testClock(time.Unix(0, 0).Add(endTime)) + t.FinishAction(ActionResult{ + Action: t.actions[id], + }, t.Counts) +} + +func TestCriticalPath(t *testing.T) { + tests := []struct { + name string + msgs func(*testCriticalPath) + want []string + wantTime time.Duration + }{ + { + name: "empty", + msgs: func(cp *testCriticalPath) {}, + }, + { + name: "duplicate", + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.start(1, 0, []string{"a"}, nil) + cp.finish(0, 1000) + cp.finish(0, 2000) + }, + want: []string{"a"}, + wantTime: 1000, + }, + { + name: "linear", + // a + // | + // b + // | + // c + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.finish(0, 1000) + cp.start(1, 1000, []string{"b"}, []string{"a"}) + cp.finish(1, 2000) + cp.start(2, 3000, []string{"c"}, []string{"b"}) + cp.finish(2, 4000) + }, + want: []string{"c", "b", "a"}, + wantTime: 3000, + }, + { + name: "diamond", + // a + // |\ + // b c + // |/ + // d + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.finish(0, 1000) + cp.start(1, 1000, []string{"b"}, []string{"a"}) + cp.start(2, 1000, []string{"c"}, []string{"a"}) + cp.finish(1, 2000) + cp.finish(2, 3000) + cp.start(3, 3000, []string{"d"}, []string{"b", "c"}) + cp.finish(3, 4000) + }, + want: []string{"d", "c", "a"}, + wantTime: 4000, + }, + { + name: "multiple", + // a d + // | | + // b e + // | + // c + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.start(3, 0, []string{"d"}, nil) + cp.finish(0, 1000) + cp.finish(3, 1000) + cp.start(1, 1000, []string{"b"}, []string{"a"}) + cp.start(4, 1000, []string{"e"}, []string{"d"}) + cp.finish(1, 2000) + cp.start(2, 2000, []string{"c"}, []string{"b"}) + cp.finish(2, 3000) + cp.finish(4, 4000) + + }, + want: []string{"e", "d"}, + wantTime: 4000, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cp := &testCriticalPath{ + criticalPath: NewCriticalPath(nil).(*criticalPath), + actions: make(map[int]*Action), + } + + tt.msgs(cp) + + criticalPath := cp.criticalPath.criticalPath() + + var descs []string + for _, x := range criticalPath { + descs = append(descs, x.action.Description) + } + + if !reflect.DeepEqual(descs, tt.want) { + t.Errorf("criticalPath.criticalPath() = %v, want %v", descs, tt.want) + } + + var gotTime time.Duration + if len(criticalPath) > 0 { + gotTime = criticalPath[0].cumulativeDuration + } + if gotTime != tt.wantTime { + t.Errorf("cumulativeDuration[0].cumulativeDuration = %v, want %v", gotTime, tt.wantTime) + } + }) + } +} diff --git a/ui/status/log.go b/ui/status/log.go index 921aa440..9090f499 100644 --- a/ui/status/log.go +++ b/ui/status/log.go @@ -15,11 +15,17 @@ package status import ( - "android/soong/ui/logger" "compress/gzip" + "errors" "fmt" "io" + "io/ioutil" "strings" + + "github.com/golang/protobuf/proto" + + "android/soong/ui/logger" + "android/soong/ui/status/build_error_proto" ) type verboseLog struct { @@ -71,9 +77,13 @@ func (v *verboseLog) Message(level MsgLevel, message string) { fmt.Fprintf(v.w, "%s%s\n", level.Prefix(), message) } -type errorLog struct { - w io.WriteCloser +func (v *verboseLog) Write(p []byte) (int, error) { + fmt.Fprint(v.w, string(p)) + return len(p), nil +} +type errorLog struct { + w io.WriteCloser empty bool } @@ -97,20 +107,17 @@ func (e *errorLog) FinishAction(result ActionResult, counts Counts) { return } - cmd := result.Command - if cmd == "" { - cmd = result.Description - } - if !e.empty { fmt.Fprintf(e.w, "\n\n") } e.empty = false fmt.Fprintf(e.w, "FAILED: %s\n", result.Description) + if len(result.Outputs) > 0 { fmt.Fprintf(e.w, "Outputs: %s\n", strings.Join(result.Outputs, " ")) } + fmt.Fprintf(e.w, "Error: %s\n", result.Error) if result.Command != "" { fmt.Fprintf(e.w, "Command: %s\n", result.Command) @@ -134,3 +141,60 @@ func (e *errorLog) Message(level MsgLevel, message string) { fmt.Fprintf(e.w, "error: %s\n", message) } + +func (e *errorLog) Write(p []byte) (int, error) { + fmt.Fprint(e.w, string(p)) + return len(p), nil +} + +type errorProtoLog struct { + errorProto soong_build_error_proto.BuildError + filename string + log logger.Logger +} + +func NewProtoErrorLog(log logger.Logger, filename string) StatusOutput { + return &errorProtoLog{ + errorProto: soong_build_error_proto.BuildError{}, + filename: filename, + log: log, + } +} + +func (e *errorProtoLog) StartAction(action *Action, counts Counts) {} + +func (e *errorProtoLog) FinishAction(result ActionResult, counts Counts) { + if result.Error == nil { + return + } + + e.errorProto.ActionErrors = append(e.errorProto.ActionErrors, &soong_build_error_proto.BuildActionError{ + Description: proto.String(result.Description), + Command: proto.String(result.Command), + Output: proto.String(result.Output), + Artifacts: result.Outputs, + Error: proto.String(result.Error.Error()), + }) +} + +func (e *errorProtoLog) Flush() { + data, err := proto.Marshal(&e.errorProto) + if err != nil { + e.log.Println("Failed to marshal build status proto: %v", err) + return + } + err = ioutil.WriteFile(e.filename, []byte(data), 0644) + if err != nil { + e.log.Println("Failed to write file %s: %v", e.errorProto, err) + } +} + +func (e *errorProtoLog) Message(level MsgLevel, message string) { + if level > ErrorLvl { + e.errorProto.ErrorMessages = append(e.errorProto.ErrorMessages, message) + } +} + +func (e *errorProtoLog) Write(p []byte) (int, error) { + return 0, errors.New("not supported") +} diff --git a/ui/status/ninja.go b/ui/status/ninja.go index ee2a2daa..9cf2f6a8 100644 --- a/ui/status/ninja.go +++ b/ui/status/ninja.go @@ -142,6 +142,7 @@ func (n *NinjaReader) run() { action := &Action{ Description: msg.EdgeStarted.GetDesc(), Outputs: msg.EdgeStarted.Outputs, + Inputs: msg.EdgeStarted.Inputs, Command: msg.EdgeStarted.GetCommand(), } n.status.StartAction(action) diff --git a/ui/status/status.go b/ui/status/status.go index 46ec72e8..df33baa8 100644 --- a/ui/status/status.go +++ b/ui/status/status.go @@ -32,6 +32,10 @@ type Action struct { // but they can be any string. Outputs []string + // Inputs is the (optional) list of inputs. Usually these are files, + // but they can be any string. + Inputs []string + // Command is the actual command line executed to perform the action. // It's optional, but one of either Description or Command should be // set. @@ -173,6 +177,9 @@ type StatusOutput interface { // Flush is called when your outputs should be flushed / closed. No // output is expected after this call. Flush() + + // Write lets StatusOutput implement io.Writer + Write(p []byte) (n int, err error) } // Status is the multiplexer / accumulator between ToolStatus instances (via diff --git a/ui/status/status_test.go b/ui/status/status_test.go index e62785f4..94945822 100644 --- a/ui/status/status_test.go +++ b/ui/status/status_test.go @@ -27,6 +27,11 @@ func (c *counterOutput) FinishAction(result ActionResult, counts Counts) { func (c counterOutput) Message(level MsgLevel, msg string) {} func (c counterOutput) Flush() {} +func (c counterOutput) Write(p []byte) (int, error) { + // Discard writes + return len(p), nil +} + func (c counterOutput) Expect(t *testing.T, counts Counts) { if Counts(c) == counts { return diff --git a/ui/terminal/Android.bp b/ui/terminal/Android.bp index 7104a504..b533b0d3 100644 --- a/ui/terminal/Android.bp +++ b/ui/terminal/Android.bp @@ -17,11 +17,15 @@ bootstrap_go_package { pkgPath: "android/soong/ui/terminal", deps: ["soong-ui-status"], srcs: [ + "dumb_status.go", + "format.go", + "smart_status.go", "status.go", - "writer.go", + "stdio.go", "util.go", ], testSrcs: [ + "status_test.go", "util_test.go", ], darwin: { diff --git a/ui/terminal/dumb_status.go b/ui/terminal/dumb_status.go new file mode 100644 index 00000000..201770fa --- /dev/null +++ b/ui/terminal/dumb_status.go @@ -0,0 +1,71 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "fmt" + "io" + + "android/soong/ui/status" +) + +type dumbStatusOutput struct { + writer io.Writer + formatter formatter +} + +// NewDumbStatusOutput returns a StatusOutput that represents the +// current build status similarly to Ninja's built-in terminal +// output. +func NewDumbStatusOutput(w io.Writer, formatter formatter) status.StatusOutput { + return &dumbStatusOutput{ + writer: w, + formatter: formatter, + } +} + +func (s *dumbStatusOutput) Message(level status.MsgLevel, message string) { + if level >= status.StatusLvl { + fmt.Fprintln(s.writer, s.formatter.message(level, message)) + } +} + +func (s *dumbStatusOutput) StartAction(action *status.Action, counts status.Counts) { +} + +func (s *dumbStatusOutput) FinishAction(result status.ActionResult, counts status.Counts) { + str := result.Description + if str == "" { + str = result.Command + } + + progress := s.formatter.progress(counts) + str + + output := s.formatter.result(result) + output = string(stripAnsiEscapes([]byte(output))) + + if output != "" { + fmt.Fprint(s.writer, progress, "\n", output) + } else { + fmt.Fprintln(s.writer, progress) + } +} + +func (s *dumbStatusOutput) Flush() {} + +func (s *dumbStatusOutput) Write(p []byte) (int, error) { + fmt.Fprint(s.writer, string(p)) + return len(p), nil +} diff --git a/ui/terminal/format.go b/ui/terminal/format.go new file mode 100644 index 00000000..4205bdc2 --- /dev/null +++ b/ui/terminal/format.go @@ -0,0 +1,123 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "fmt" + "strings" + "time" + + "android/soong/ui/status" +) + +type formatter struct { + format string + quiet bool + start time.Time +} + +// newFormatter returns a formatter for formatting output to +// the terminal in a format similar to Ninja. +// format takes nearly all the same options as NINJA_STATUS. +// %c is currently unsupported. +func newFormatter(format string, quiet bool) formatter { + return formatter{ + format: format, + quiet: quiet, + start: time.Now(), + } +} + +func (s formatter) message(level status.MsgLevel, message string) string { + if level >= status.ErrorLvl { + return fmt.Sprintf("FAILED: %s", message) + } else if level > status.StatusLvl { + return fmt.Sprintf("%s%s", level.Prefix(), message) + } else if level == status.StatusLvl { + return message + } + return "" +} + +func (s formatter) progress(counts status.Counts) string { + if s.format == "" { + return fmt.Sprintf("[%3d%% %d/%d] ", 100*counts.FinishedActions/counts.TotalActions, counts.FinishedActions, counts.TotalActions) + } + + buf := &strings.Builder{} + for i := 0; i < len(s.format); i++ { + c := s.format[i] + if c != '%' { + buf.WriteByte(c) + continue + } + + i = i + 1 + if i == len(s.format) { + buf.WriteByte(c) + break + } + + c = s.format[i] + switch c { + case '%': + buf.WriteByte(c) + case 's': + fmt.Fprintf(buf, "%d", counts.StartedActions) + case 't': + fmt.Fprintf(buf, "%d", counts.TotalActions) + case 'r': + fmt.Fprintf(buf, "%d", counts.RunningActions) + case 'u': + fmt.Fprintf(buf, "%d", counts.TotalActions-counts.StartedActions) + case 'f': + fmt.Fprintf(buf, "%d", counts.FinishedActions) + case 'o': + fmt.Fprintf(buf, "%.1f", float64(counts.FinishedActions)/time.Since(s.start).Seconds()) + case 'c': + // TODO: implement? + buf.WriteRune('?') + case 'p': + fmt.Fprintf(buf, "%3d%%", 100*counts.FinishedActions/counts.TotalActions) + case 'e': + fmt.Fprintf(buf, "%.3f", time.Since(s.start).Seconds()) + default: + buf.WriteString("unknown placeholder '") + buf.WriteByte(c) + buf.WriteString("'") + } + } + return buf.String() +} + +func (s formatter) result(result status.ActionResult) string { + var ret string + if result.Error != nil { + targets := strings.Join(result.Outputs, " ") + if s.quiet || result.Command == "" { + ret = fmt.Sprintf("FAILED: %s\n%s", targets, result.Output) + } else { + ret = fmt.Sprintf("FAILED: %s\n%s\n%s", targets, result.Command, result.Output) + } + } else if result.Output != "" { + ret = result.Output + } + + if len(ret) > 0 && ret[len(ret)-1] != '\n' { + ret += "\n" + } + + return ret +} diff --git a/ui/terminal/smart_status.go b/ui/terminal/smart_status.go new file mode 100644 index 00000000..efcfd435 --- /dev/null +++ b/ui/terminal/smart_status.go @@ -0,0 +1,430 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "fmt" + "io" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "android/soong/ui/status" +) + +const tableHeightEnVar = "SOONG_UI_TABLE_HEIGHT" + +type actionTableEntry struct { + action *status.Action + startTime time.Time +} + +type smartStatusOutput struct { + writer io.Writer + formatter formatter + + lock sync.Mutex + + haveBlankLine bool + + tableMode bool + tableHeight int + requestedTableHeight int + termWidth, termHeight int + + runningActions []actionTableEntry + ticker *time.Ticker + done chan bool + sigwinch chan os.Signal + sigwinchHandled chan bool +} + +// NewSmartStatusOutput returns a StatusOutput that represents the +// current build status similarly to Ninja's built-in terminal +// output. +func NewSmartStatusOutput(w io.Writer, formatter formatter) status.StatusOutput { + s := &smartStatusOutput{ + writer: w, + formatter: formatter, + + haveBlankLine: true, + + tableMode: true, + + done: make(chan bool), + sigwinch: make(chan os.Signal), + } + + if env, ok := os.LookupEnv(tableHeightEnVar); ok { + h, _ := strconv.Atoi(env) + s.tableMode = h > 0 + s.requestedTableHeight = h + } + + s.updateTermSize() + + if s.tableMode { + // Add empty lines at the bottom of the screen to scroll back the existing history + // and make room for the action table. + // TODO: read the cursor position to see if the empty lines are necessary? + for i := 0; i < s.tableHeight; i++ { + fmt.Fprintln(w) + } + + // Hide the cursor to prevent seeing it bouncing around + fmt.Fprintf(s.writer, ansi.hideCursor()) + + // Configure the empty action table + s.actionTable() + + // Start a tick to update the action table periodically + s.startActionTableTick() + } + + s.startSigwinch() + + return s +} + +func (s *smartStatusOutput) Message(level status.MsgLevel, message string) { + if level < status.StatusLvl { + return + } + + str := s.formatter.message(level, message) + + s.lock.Lock() + defer s.lock.Unlock() + + if level > status.StatusLvl { + s.print(str) + } else { + s.statusLine(str) + } +} + +func (s *smartStatusOutput) StartAction(action *status.Action, counts status.Counts) { + startTime := time.Now() + + str := action.Description + if str == "" { + str = action.Command + } + + progress := s.formatter.progress(counts) + + s.lock.Lock() + defer s.lock.Unlock() + + s.runningActions = append(s.runningActions, actionTableEntry{ + action: action, + startTime: startTime, + }) + + s.statusLine(progress + str) +} + +func (s *smartStatusOutput) FinishAction(result status.ActionResult, counts status.Counts) { + str := result.Description + if str == "" { + str = result.Command + } + + progress := s.formatter.progress(counts) + str + + output := s.formatter.result(result) + + s.lock.Lock() + defer s.lock.Unlock() + + for i, runningAction := range s.runningActions { + if runningAction.action == result.Action { + s.runningActions = append(s.runningActions[:i], s.runningActions[i+1:]...) + break + } + } + + if output != "" { + s.statusLine(progress) + s.requestLine() + s.print(output) + } else { + s.statusLine(progress) + } +} + +func (s *smartStatusOutput) Flush() { + s.lock.Lock() + defer s.lock.Unlock() + + s.stopSigwinch() + + s.requestLine() + + s.runningActions = nil + + if s.tableMode { + s.stopActionTableTick() + + // Update the table after clearing runningActions to clear it + s.actionTable() + + // Reset the scrolling region to the whole terminal + fmt.Fprintf(s.writer, ansi.resetScrollingMargins()) + _, height, _ := termSize(s.writer) + // Move the cursor to the top of the now-blank, previously non-scrolling region + fmt.Fprintf(s.writer, ansi.setCursor(height-s.tableHeight, 1)) + // Turn the cursor back on + fmt.Fprintf(s.writer, ansi.showCursor()) + } +} + +func (s *smartStatusOutput) Write(p []byte) (int, error) { + s.lock.Lock() + defer s.lock.Unlock() + s.print(string(p)) + return len(p), nil +} + +func (s *smartStatusOutput) requestLine() { + if !s.haveBlankLine { + fmt.Fprintln(s.writer) + s.haveBlankLine = true + } +} + +func (s *smartStatusOutput) print(str string) { + if !s.haveBlankLine { + fmt.Fprint(s.writer, "\r", ansi.clearToEndOfLine()) + s.haveBlankLine = true + } + fmt.Fprint(s.writer, str) + if len(str) == 0 || str[len(str)-1] != '\n' { + fmt.Fprint(s.writer, "\n") + } +} + +func (s *smartStatusOutput) statusLine(str string) { + idx := strings.IndexRune(str, '\n') + if idx != -1 { + str = str[0:idx] + } + + // Limit line width to the terminal width, otherwise we'll wrap onto + // another line and we won't delete the previous line. + str = elide(str, s.termWidth) + + // Move to the beginning on the line, turn on bold, print the output, + // turn off bold, then clear the rest of the line. + start := "\r" + ansi.bold() + end := ansi.regular() + ansi.clearToEndOfLine() + fmt.Fprint(s.writer, start, str, end) + s.haveBlankLine = false +} + +func elide(str string, width int) string { + if width > 0 && len(str) > width { + // TODO: Just do a max. Ninja elides the middle, but that's + // more complicated and these lines aren't that important. + str = str[:width] + } + + return str +} + +func (s *smartStatusOutput) startActionTableTick() { + s.ticker = time.NewTicker(time.Second) + go func() { + for { + select { + case <-s.ticker.C: + s.lock.Lock() + s.actionTable() + s.lock.Unlock() + case <-s.done: + return + } + } + }() +} + +func (s *smartStatusOutput) stopActionTableTick() { + s.ticker.Stop() + s.done <- true +} + +func (s *smartStatusOutput) startSigwinch() { + signal.Notify(s.sigwinch, syscall.SIGWINCH) + go func() { + for _ = range s.sigwinch { + s.lock.Lock() + s.updateTermSize() + if s.tableMode { + s.actionTable() + } + s.lock.Unlock() + if s.sigwinchHandled != nil { + s.sigwinchHandled <- true + } + } + }() +} + +func (s *smartStatusOutput) stopSigwinch() { + signal.Stop(s.sigwinch) + close(s.sigwinch) +} + +func (s *smartStatusOutput) updateTermSize() { + if w, h, ok := termSize(s.writer); ok { + firstUpdate := s.termHeight == 0 && s.termWidth == 0 + oldScrollingHeight := s.termHeight - s.tableHeight + + s.termWidth, s.termHeight = w, h + + if s.tableMode { + tableHeight := s.requestedTableHeight + if tableHeight == 0 { + tableHeight = s.termHeight / 4 + if tableHeight < 1 { + tableHeight = 1 + } else if tableHeight > 10 { + tableHeight = 10 + } + } + if tableHeight > s.termHeight-1 { + tableHeight = s.termHeight - 1 + } + s.tableHeight = tableHeight + + scrollingHeight := s.termHeight - s.tableHeight + + if !firstUpdate { + // If the scrolling region has changed, attempt to pan the existing text so that it is + // not overwritten by the table. + if scrollingHeight < oldScrollingHeight { + pan := oldScrollingHeight - scrollingHeight + if pan > s.tableHeight { + pan = s.tableHeight + } + fmt.Fprint(s.writer, ansi.panDown(pan)) + } + } + } + } +} + +func (s *smartStatusOutput) actionTable() { + scrollingHeight := s.termHeight - s.tableHeight + + // Update the scrolling region in case the height of the terminal changed + + fmt.Fprint(s.writer, ansi.setScrollingMargins(1, scrollingHeight)) + + // Write as many status lines as fit in the table + for tableLine := 0; tableLine < s.tableHeight; tableLine++ { + if tableLine >= s.tableHeight { + break + } + // Move the cursor to the correct line of the non-scrolling region + fmt.Fprint(s.writer, ansi.setCursor(scrollingHeight+1+tableLine, 1)) + + if tableLine < len(s.runningActions) { + runningAction := s.runningActions[tableLine] + + seconds := int(time.Since(runningAction.startTime).Round(time.Second).Seconds()) + + desc := runningAction.action.Description + if desc == "" { + desc = runningAction.action.Command + } + + color := "" + if seconds >= 60 { + color = ansi.red() + ansi.bold() + } else if seconds >= 30 { + color = ansi.yellow() + ansi.bold() + } + + durationStr := fmt.Sprintf(" %2d:%02d ", seconds/60, seconds%60) + desc = elide(desc, s.termWidth-len(durationStr)) + durationStr = color + durationStr + ansi.regular() + fmt.Fprint(s.writer, durationStr, desc) + } + fmt.Fprint(s.writer, ansi.clearToEndOfLine()) + } + + // Move the cursor back to the last line of the scrolling region + fmt.Fprint(s.writer, ansi.setCursor(scrollingHeight, 1)) +} + +var ansi = ansiImpl{} + +type ansiImpl struct{} + +func (ansiImpl) clearToEndOfLine() string { + return "\x1b[K" +} + +func (ansiImpl) setCursor(row, column int) string { + // Direct cursor address + return fmt.Sprintf("\x1b[%d;%dH", row, column) +} + +func (ansiImpl) setScrollingMargins(top, bottom int) string { + // Set Top and Bottom Margins DECSTBM + return fmt.Sprintf("\x1b[%d;%dr", top, bottom) +} + +func (ansiImpl) resetScrollingMargins() string { + // Set Top and Bottom Margins DECSTBM + return fmt.Sprintf("\x1b[r") +} + +func (ansiImpl) red() string { + return "\x1b[31m" +} + +func (ansiImpl) yellow() string { + return "\x1b[33m" +} + +func (ansiImpl) bold() string { + return "\x1b[1m" +} + +func (ansiImpl) regular() string { + return "\x1b[0m" +} + +func (ansiImpl) showCursor() string { + return "\x1b[?25h" +} + +func (ansiImpl) hideCursor() string { + return "\x1b[?25l" +} + +func (ansiImpl) panDown(lines int) string { + return fmt.Sprintf("\x1b[%dS", lines) +} + +func (ansiImpl) panUp(lines int) string { + return fmt.Sprintf("\x1b[%dT", lines) +} diff --git a/ui/terminal/status.go b/ui/terminal/status.go index 2445c5b2..60dfc702 100644 --- a/ui/terminal/status.go +++ b/ui/terminal/status.go @@ -15,131 +15,23 @@ package terminal import ( - "fmt" - "strings" - "time" + "io" "android/soong/ui/status" ) -type statusOutput struct { - writer Writer - format string - - start time.Time - quiet bool -} - // NewStatusOutput returns a StatusOutput that represents the // current build status similarly to Ninja's built-in terminal // output. // // statusFormat takes nearly all the same options as NINJA_STATUS. // %c is currently unsupported. -func NewStatusOutput(w Writer, statusFormat string, quietBuild bool) status.StatusOutput { - return &statusOutput{ - writer: w, - format: statusFormat, - - start: time.Now(), - quiet: quietBuild, - } -} - -func (s *statusOutput) Message(level status.MsgLevel, message string) { - if level >= status.ErrorLvl { - s.writer.Print(fmt.Sprintf("FAILED: %s", message)) - } else if level > status.StatusLvl { - s.writer.Print(fmt.Sprintf("%s%s", level.Prefix(), message)) - } else if level == status.StatusLvl { - s.writer.StatusLine(message) - } -} - -func (s *statusOutput) StartAction(action *status.Action, counts status.Counts) { - if !s.writer.isSmartTerminal() { - return - } - - str := action.Description - if str == "" { - str = action.Command - } +func NewStatusOutput(w io.Writer, statusFormat string, forceDumbOutput, quietBuild bool) status.StatusOutput { + formatter := newFormatter(statusFormat, quietBuild) - s.writer.StatusLine(s.progress(counts) + str) -} - -func (s *statusOutput) FinishAction(result status.ActionResult, counts status.Counts) { - str := result.Description - if str == "" { - str = result.Command - } - - progress := s.progress(counts) + str - - if result.Error != nil { - targets := strings.Join(result.Outputs, " ") - if s.quiet || result.Command == "" { - s.writer.StatusAndMessage(progress, fmt.Sprintf("FAILED: %s\n%s", targets, result.Output)) - } else { - s.writer.StatusAndMessage(progress, fmt.Sprintf("FAILED: %s\n%s\n%s", targets, result.Command, result.Output)) - } - } else if result.Output != "" { - s.writer.StatusAndMessage(progress, result.Output) + if !forceDumbOutput && isSmartTerminal(w) { + return NewSmartStatusOutput(w, formatter) } else { - s.writer.StatusLine(progress) - } -} - -func (s *statusOutput) Flush() {} - -func (s *statusOutput) progress(counts status.Counts) string { - if s.format == "" { - return fmt.Sprintf("[%3d%% %d/%d] ", 100*counts.FinishedActions/counts.TotalActions, counts.FinishedActions, counts.TotalActions) - } - - buf := &strings.Builder{} - for i := 0; i < len(s.format); i++ { - c := s.format[i] - if c != '%' { - buf.WriteByte(c) - continue - } - - i = i + 1 - if i == len(s.format) { - buf.WriteByte(c) - break - } - - c = s.format[i] - switch c { - case '%': - buf.WriteByte(c) - case 's': - fmt.Fprintf(buf, "%d", counts.StartedActions) - case 't': - fmt.Fprintf(buf, "%d", counts.TotalActions) - case 'r': - fmt.Fprintf(buf, "%d", counts.RunningActions) - case 'u': - fmt.Fprintf(buf, "%d", counts.TotalActions-counts.StartedActions) - case 'f': - fmt.Fprintf(buf, "%d", counts.FinishedActions) - case 'o': - fmt.Fprintf(buf, "%.1f", float64(counts.FinishedActions)/time.Since(s.start).Seconds()) - case 'c': - // TODO: implement? - buf.WriteRune('?') - case 'p': - fmt.Fprintf(buf, "%3d%%", 100*counts.FinishedActions/counts.TotalActions) - case 'e': - fmt.Fprintf(buf, "%.3f", time.Since(s.start).Seconds()) - default: - buf.WriteString("unknown placeholder '") - buf.WriteByte(c) - buf.WriteString("'") - } + return NewDumbStatusOutput(w, formatter) } - return buf.String() } diff --git a/ui/terminal/status_test.go b/ui/terminal/status_test.go new file mode 100644 index 00000000..9f608298 --- /dev/null +++ b/ui/terminal/status_test.go @@ -0,0 +1,295 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "bytes" + "fmt" + "os" + "syscall" + "testing" + + "android/soong/ui/status" +) + +func TestStatusOutput(t *testing.T) { + tests := []struct { + name string + calls func(stat status.StatusOutput) + smart string + dumb string + }{ + { + name: "two actions", + calls: twoActions, + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action2\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action1\n[100% 2/2] action2\n", + }, + { + name: "two parallel actions", + calls: twoParallelActions, + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action1\n[100% 2/2] action2\n", + }, + { + name: "action with output", + calls: actionsWithOutput, + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\noutput1\noutput2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + dumb: "[ 33% 1/3] action1\n[ 66% 2/3] action2\noutput1\noutput2\n[100% 3/3] action3\n", + }, + { + name: "action with output without newline", + calls: actionsWithOutputWithoutNewline, + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\noutput1\noutput2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + dumb: "[ 33% 1/3] action1\n[ 66% 2/3] action2\noutput1\noutput2\n[100% 3/3] action3\n", + }, + { + name: "action with error", + calls: actionsWithError, + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\nFAILED: f1 f2\ntouch f1 f2\nerror1\nerror2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + dumb: "[ 33% 1/3] action1\n[ 66% 2/3] action2\nFAILED: f1 f2\ntouch f1 f2\nerror1\nerror2\n[100% 3/3] action3\n", + }, + { + name: "action with empty description", + calls: actionWithEmptyDescription, + smart: "\r\x1b[1m[ 0% 0/1] command1\x1b[0m\x1b[K\r\x1b[1m[100% 1/1] command1\x1b[0m\x1b[K\n", + dumb: "[100% 1/1] command1\n", + }, + { + name: "messages", + calls: actionsWithMessages, + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1mstatus\x1b[0m\x1b[K\r\x1b[Kprint\nFAILED: error\n\r\x1b[1m[ 50% 1/2] action2\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action1\nstatus\nprint\nFAILED: error\n[100% 2/2] action2\n", + }, + { + name: "action with long description", + calls: actionWithLongDescription, + smart: "\r\x1b[1m[ 0% 0/2] action with very long descrip\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action with very long descrip\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action with very long description to test eliding\n", + }, + { + name: "action with output with ansi codes", + calls: actionWithOuptutWithAnsiCodes, + smart: "\r\x1b[1m[ 0% 0/1] action1\x1b[0m\x1b[K\r\x1b[1m[100% 1/1] action1\x1b[0m\x1b[K\n\x1b[31mcolor\x1b[0m\n", + dumb: "[100% 1/1] action1\ncolor\n", + }, + } + + os.Setenv(tableHeightEnVar, "") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + t.Run("smart", func(t *testing.T) { + smart := &fakeSmartTerminal{termWidth: 40} + stat := NewStatusOutput(smart, "", false, false) + tt.calls(stat) + stat.Flush() + + if g, w := smart.String(), tt.smart; g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } + }) + + t.Run("dumb", func(t *testing.T) { + dumb := &bytes.Buffer{} + stat := NewStatusOutput(dumb, "", false, false) + tt.calls(stat) + stat.Flush() + + if g, w := dumb.String(), tt.dumb; g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } + }) + + t.Run("force dumb", func(t *testing.T) { + smart := &fakeSmartTerminal{termWidth: 40} + stat := NewStatusOutput(smart, "", true, false) + tt.calls(stat) + stat.Flush() + + if g, w := smart.String(), tt.dumb; g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } + }) + }) + } +} + +type runner struct { + counts status.Counts + stat status.StatusOutput +} + +func newRunner(stat status.StatusOutput, totalActions int) *runner { + return &runner{ + counts: status.Counts{TotalActions: totalActions}, + stat: stat, + } +} + +func (r *runner) startAction(action *status.Action) { + r.counts.StartedActions++ + r.counts.RunningActions++ + r.stat.StartAction(action, r.counts) +} + +func (r *runner) finishAction(result status.ActionResult) { + r.counts.FinishedActions++ + r.counts.RunningActions-- + r.stat.FinishAction(result, r.counts) +} + +func (r *runner) finishAndStartAction(result status.ActionResult, action *status.Action) { + r.counts.FinishedActions++ + r.stat.FinishAction(result, r.counts) + + r.counts.StartedActions++ + r.stat.StartAction(action, r.counts) +} + +var ( + action1 = &status.Action{Description: "action1"} + result1 = status.ActionResult{Action: action1} + action2 = &status.Action{Description: "action2"} + result2 = status.ActionResult{Action: action2} + action3 = &status.Action{Description: "action3"} + result3 = status.ActionResult{Action: action3} +) + +func twoActions(stat status.StatusOutput) { + runner := newRunner(stat, 2) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2) + runner.finishAction(result2) +} + +func twoParallelActions(stat status.StatusOutput) { + runner := newRunner(stat, 2) + runner.startAction(action1) + runner.startAction(action2) + runner.finishAction(result1) + runner.finishAction(result2) +} + +func actionsWithOutput(stat status.StatusOutput) { + result2WithOutput := status.ActionResult{Action: action2, Output: "output1\noutput2\n"} + + runner := newRunner(stat, 3) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2) + runner.finishAction(result2WithOutput) + runner.startAction(action3) + runner.finishAction(result3) +} + +func actionsWithOutputWithoutNewline(stat status.StatusOutput) { + result2WithOutputWithoutNewline := status.ActionResult{Action: action2, Output: "output1\noutput2"} + + runner := newRunner(stat, 3) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2) + runner.finishAction(result2WithOutputWithoutNewline) + runner.startAction(action3) + runner.finishAction(result3) +} + +func actionsWithError(stat status.StatusOutput) { + action2WithError := &status.Action{Description: "action2", Outputs: []string{"f1", "f2"}, Command: "touch f1 f2"} + result2WithError := status.ActionResult{Action: action2WithError, Output: "error1\nerror2\n", Error: fmt.Errorf("error1")} + + runner := newRunner(stat, 3) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2WithError) + runner.finishAction(result2WithError) + runner.startAction(action3) + runner.finishAction(result3) +} + +func actionWithEmptyDescription(stat status.StatusOutput) { + action1 := &status.Action{Command: "command1"} + result1 := status.ActionResult{Action: action1} + + runner := newRunner(stat, 1) + runner.startAction(action1) + runner.finishAction(result1) +} + +func actionsWithMessages(stat status.StatusOutput) { + runner := newRunner(stat, 2) + + runner.startAction(action1) + runner.finishAction(result1) + + stat.Message(status.VerboseLvl, "verbose") + stat.Message(status.StatusLvl, "status") + stat.Message(status.PrintLvl, "print") + stat.Message(status.ErrorLvl, "error") + + runner.startAction(action2) + runner.finishAction(result2) +} + +func actionWithLongDescription(stat status.StatusOutput) { + action1 := &status.Action{Description: "action with very long description to test eliding"} + result1 := status.ActionResult{Action: action1} + + runner := newRunner(stat, 2) + + runner.startAction(action1) + + runner.finishAction(result1) +} + +func actionWithOuptutWithAnsiCodes(stat status.StatusOutput) { + result1WithOutputWithAnsiCodes := status.ActionResult{Action: action1, Output: "\x1b[31mcolor\x1b[0m"} + + runner := newRunner(stat, 1) + runner.startAction(action1) + runner.finishAction(result1WithOutputWithAnsiCodes) +} + +func TestSmartStatusOutputWidthChange(t *testing.T) { + os.Setenv(tableHeightEnVar, "") + + smart := &fakeSmartTerminal{termWidth: 40} + stat := NewStatusOutput(smart, "", false, false) + smartStat := stat.(*smartStatusOutput) + smartStat.sigwinchHandled = make(chan bool) + + runner := newRunner(stat, 2) + + action := &status.Action{Description: "action with very long description to test eliding"} + result := status.ActionResult{Action: action} + + runner.startAction(action) + smart.termWidth = 30 + // Fake a SIGWINCH + smartStat.sigwinch <- syscall.SIGWINCH + <-smartStat.sigwinchHandled + runner.finishAction(result) + + stat.Flush() + + w := "\r\x1b[1m[ 0% 0/2] action with very long descrip\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action with very lo\x1b[0m\x1b[K\n" + + if g := smart.String(); g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } +} diff --git a/ui/terminal/stdio.go b/ui/terminal/stdio.go new file mode 100644 index 00000000..dec29631 --- /dev/null +++ b/ui/terminal/stdio.go @@ -0,0 +1,55 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package terminal provides a set of interfaces that can be used to interact +// with the terminal (including falling back when the terminal is detected to +// be a redirect or other dumb terminal) +package terminal + +import ( + "io" + "os" +) + +// StdioInterface represents a set of stdin/stdout/stderr Reader/Writers +type StdioInterface interface { + Stdin() io.Reader + Stdout() io.Writer + Stderr() io.Writer +} + +// StdioImpl uses the OS stdin/stdout/stderr to implement StdioInterface +type StdioImpl struct{} + +func (StdioImpl) Stdin() io.Reader { return os.Stdin } +func (StdioImpl) Stdout() io.Writer { return os.Stdout } +func (StdioImpl) Stderr() io.Writer { return os.Stderr } + +var _ StdioInterface = StdioImpl{} + +type customStdio struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer +} + +func NewCustomStdio(stdin io.Reader, stdout, stderr io.Writer) StdioInterface { + return customStdio{stdin, stdout, stderr} +} + +func (c customStdio) Stdin() io.Reader { return c.stdin } +func (c customStdio) Stdout() io.Writer { return c.stdout } +func (c customStdio) Stderr() io.Writer { return c.stderr } + +var _ StdioInterface = customStdio{} diff --git a/ui/terminal/util.go b/ui/terminal/util.go index a85a517b..7a603d7f 100644 --- a/ui/terminal/util.go +++ b/ui/terminal/util.go @@ -22,18 +22,23 @@ import ( "unsafe" ) -func isTerminal(w io.Writer) bool { +func isSmartTerminal(w io.Writer) bool { if f, ok := w.(*os.File); ok { + if term, ok := os.LookupEnv("TERM"); ok && term == "dumb" { + return false + } var termios syscall.Termios _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(), ioctlGetTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) return err == 0 + } else if _, ok := w.(*fakeSmartTerminal); ok { + return true } return false } -func termWidth(w io.Writer) (int, bool) { +func termSize(w io.Writer) (width int, height int, ok bool) { if f, ok := w.(*os.File); ok { var winsize struct { ws_row, ws_column uint16 @@ -42,9 +47,11 @@ func termWidth(w io.Writer) (int, bool) { _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(), syscall.TIOCGWINSZ, uintptr(unsafe.Pointer(&winsize)), 0, 0, 0) - return int(winsize.ws_column), err == 0 + return int(winsize.ws_column), int(winsize.ws_row), err == 0 + } else if f, ok := w.(*fakeSmartTerminal); ok { + return f.termWidth, f.termHeight, true } - return 0, false + return 0, 0, false } // stripAnsiEscapes strips ANSI control codes from a byte array in place. @@ -99,3 +106,8 @@ func stripAnsiEscapes(input []byte) []byte { return input } + +type fakeSmartTerminal struct { + bytes.Buffer + termWidth, termHeight int +} diff --git a/ui/terminal/writer.go b/ui/terminal/writer.go deleted file mode 100644 index ebe4b2aa..00000000 --- a/ui/terminal/writer.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2018 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package terminal provides a set of interfaces that can be used to interact -// with the terminal (including falling back when the terminal is detected to -// be a redirect or other dumb terminal) -package terminal - -import ( - "fmt" - "io" - "os" - "strings" - "sync" -) - -// Writer provides an interface to write temporary and permanent messages to -// the terminal. -// -// The terminal is considered to be a dumb terminal if TERM==dumb, or if a -// terminal isn't detected on stdout/stderr (generally because it's a pipe or -// file). Dumb terminals will strip out all ANSI escape sequences, including -// colors. -type Writer interface { - // Print prints the string to the terminal, overwriting any current - // status being displayed. - // - // On a dumb terminal, the status messages will be kept. - Print(str string) - - // Status prints the first line of the string to the terminal, - // overwriting any previous status line. Strings longer than the width - // of the terminal will be cut off. - // - // On a dumb terminal, previous status messages will remain, and the - // entire first line of the string will be printed. - StatusLine(str string) - - // StatusAndMessage prints the first line of status to the terminal, - // similarly to StatusLine(), then prints the full msg below that. The - // status line is retained. - // - // There is guaranteed to be no other output in between the status and - // message. - StatusAndMessage(status, msg string) - - // Finish ensures that the output ends with a newline (preserving any - // current status line that is current displayed). - // - // This does nothing on dumb terminals. - Finish() - - // Write implements the io.Writer interface. This is primarily so that - // the logger can use this interface to print to stderr without - // breaking the other semantics of this interface. - // - // Try to use any of the other functions if possible. - Write(p []byte) (n int, err error) - - isSmartTerminal() bool -} - -// NewWriter creates a new Writer based on the stdio and the TERM -// environment variable. -func NewWriter(stdio StdioInterface) Writer { - w := &writerImpl{ - stdio: stdio, - - haveBlankLine: true, - } - - if term, ok := os.LookupEnv("TERM"); ok && term != "dumb" { - w.smartTerminal = isTerminal(stdio.Stdout()) - } - w.stripEscapes = !w.smartTerminal - - return w -} - -type writerImpl struct { - stdio StdioInterface - - haveBlankLine bool - - // Protecting the above, we assume that smartTerminal and stripEscapes - // does not change after initial setup. - lock sync.Mutex - - smartTerminal bool - stripEscapes bool -} - -func (w *writerImpl) isSmartTerminal() bool { - return w.smartTerminal -} - -func (w *writerImpl) requestLine() { - if !w.haveBlankLine { - fmt.Fprintln(w.stdio.Stdout()) - w.haveBlankLine = true - } -} - -func (w *writerImpl) Print(str string) { - if w.stripEscapes { - str = string(stripAnsiEscapes([]byte(str))) - } - - w.lock.Lock() - defer w.lock.Unlock() - w.print(str) -} - -func (w *writerImpl) print(str string) { - if !w.haveBlankLine { - fmt.Fprint(w.stdio.Stdout(), "\r", "\x1b[K") - w.haveBlankLine = true - } - fmt.Fprint(w.stdio.Stdout(), str) - if len(str) == 0 || str[len(str)-1] != '\n' { - fmt.Fprint(w.stdio.Stdout(), "\n") - } -} - -func (w *writerImpl) StatusLine(str string) { - w.lock.Lock() - defer w.lock.Unlock() - - w.statusLine(str) -} - -func (w *writerImpl) statusLine(str string) { - if !w.smartTerminal { - fmt.Fprintln(w.stdio.Stdout(), str) - return - } - - idx := strings.IndexRune(str, '\n') - if idx != -1 { - str = str[0:idx] - } - - // Limit line width to the terminal width, otherwise we'll wrap onto - // another line and we won't delete the previous line. - // - // Run this on every line in case the window has been resized while - // we're printing. This could be optimized to only re-run when we get - // SIGWINCH if it ever becomes too time consuming. - if max, ok := termWidth(w.stdio.Stdout()); ok { - if len(str) > max { - // TODO: Just do a max. Ninja elides the middle, but that's - // more complicated and these lines aren't that important. - str = str[:max] - } - } - - // Move to the beginning on the line, print the output, then clear - // the rest of the line. - fmt.Fprint(w.stdio.Stdout(), "\r", str, "\x1b[K") - w.haveBlankLine = false -} - -func (w *writerImpl) StatusAndMessage(status, msg string) { - if w.stripEscapes { - msg = string(stripAnsiEscapes([]byte(msg))) - } - - w.lock.Lock() - defer w.lock.Unlock() - - w.statusLine(status) - w.requestLine() - w.print(msg) -} - -func (w *writerImpl) Finish() { - w.lock.Lock() - defer w.lock.Unlock() - - w.requestLine() -} - -func (w *writerImpl) Write(p []byte) (n int, err error) { - w.Print(string(p)) - return len(p), nil -} - -// StdioInterface represents a set of stdin/stdout/stderr Reader/Writers -type StdioInterface interface { - Stdin() io.Reader - Stdout() io.Writer - Stderr() io.Writer -} - -// StdioImpl uses the OS stdin/stdout/stderr to implement StdioInterface -type StdioImpl struct{} - -func (StdioImpl) Stdin() io.Reader { return os.Stdin } -func (StdioImpl) Stdout() io.Writer { return os.Stdout } -func (StdioImpl) Stderr() io.Writer { return os.Stderr } - -var _ StdioInterface = StdioImpl{} - -type customStdio struct { - stdin io.Reader - stdout io.Writer - stderr io.Writer -} - -func NewCustomStdio(stdin io.Reader, stdout, stderr io.Writer) StdioInterface { - return customStdio{stdin, stdout, stderr} -} - -func (c customStdio) Stdin() io.Reader { return c.stdin } -func (c customStdio) Stdout() io.Writer { return c.stdout } -func (c customStdio) Stderr() io.Writer { return c.stderr } - -var _ StdioInterface = customStdio{} diff --git a/ui/tracer/status.go b/ui/tracer/status.go index af50e2d4..c8312551 100644 --- a/ui/tracer/status.go +++ b/ui/tracer/status.go @@ -85,3 +85,8 @@ func (s *statusOutput) FinishAction(result status.ActionResult, counts status.Co func (s *statusOutput) Flush() {} func (s *statusOutput) Message(level status.MsgLevel, message string) {} + +func (s *statusOutput) Write(p []byte) (int, error) { + // Discard writes + return len(p), nil +} |