diff options
| -rw-r--r-- | cmd/multiproduct_kati/Android.bp | 24 | ||||
| -rw-r--r-- | cmd/multiproduct_kati/main.go | 194 | ||||
| -rw-r--r-- | cmd/soong_ui/main.go | 8 | ||||
| -rw-r--r-- | ui/build/config.go | 12 |
4 files changed, 230 insertions, 8 deletions
diff --git a/cmd/multiproduct_kati/Android.bp b/cmd/multiproduct_kati/Android.bp new file mode 100644 index 00000000..8c1cd266 --- /dev/null +++ b/cmd/multiproduct_kati/Android.bp @@ -0,0 +1,24 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +blueprint_go_binary { + name: "multiproduct_kati", + deps: [ + "soong-ui-build", + "soong-ui-logger", + ], + srcs: [ + "main.go", + ], +} diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go new file mode 100644 index 00000000..0570c17b --- /dev/null +++ b/cmd/multiproduct_kati/main.go @@ -0,0 +1,194 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + + "android/soong/ui/build" + "android/soong/ui/logger" +) + +// We default to number of cpus / 4, which seems to be the sweet spot for my +// system. I suspect this is mostly due to memory or disk bandwidth though, and +// may depend on the size ofthe source tree, so this probably isn't a great +// default. +func detectNumJobs() int { + if runtime.NumCPU() < 4 { + return 1 + } + return runtime.NumCPU() / 4 +} + +var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") + +var keep = flag.Bool("keep", false, "keep successful output files") + +var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)") + +var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)") +var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)") + +type Product struct { + ctx build.Context + config build.Config +} + +func main() { + log := logger.New(os.Stderr) + defer log.Cleanup() + + flag.Parse() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + build.SetupSignals(log, cancel, log.Cleanup) + + buildCtx := &build.ContextImpl{ + Context: ctx, + Logger: log, + StdioInterface: build.StdioImpl{}, + } + + failed := false + + config := build.NewConfig(buildCtx) + if *outDir == "" { + var err error + *outDir, err = ioutil.TempDir(config.OutDir(), "multiproduct") + if err != nil { + log.Fatalf("Failed to create tempdir: %v", err) + } + + if !*keep { + defer func() { + if !failed { + os.RemoveAll(*outDir) + } + }() + } + } + config.Environment().Set("OUT_DIR", *outDir) + log.Println("Output directory:", *outDir) + + build.SetupOutDir(buildCtx, config) + log.SetOutput(filepath.Join(config.OutDir(), "build.log")) + + vars, err := build.DumpMakeVars(buildCtx, config, nil, nil, []string{"all_named_products"}) + if err != nil { + log.Fatal(err) + } + products := strings.Fields(vars["all_named_products"]) + log.Verbose("Got product list:", products) + + var wg sync.WaitGroup + errs := make(chan error, len(products)) + productConfigs := make(chan Product, len(products)) + + // Run the product config for every product in parallel + for _, product := range products { + wg.Add(1) + go func(product string) { + defer wg.Done() + defer logger.Recover(func(err error) { + errs <- fmt.Errorf("Error building %s: %v", product, err) + }) + + productOutDir := filepath.Join(config.OutDir(), product) + + if err := os.MkdirAll(productOutDir, 0777); err != nil { + log.Fatalf("Error creating out directory: %v", err) + } + + f, err := os.Create(filepath.Join(productOutDir, "std.log")) + if err != nil { + log.Fatalf("Error creating std.log: %v", err) + } + + productLog := logger.New(&bytes.Buffer{}) + productLog.SetOutput(filepath.Join(productOutDir, "build.log")) + + productCtx := &build.ContextImpl{ + Context: ctx, + Logger: productLog, + StdioInterface: build.NewCustomStdio(nil, f, f), + } + + productConfig := build.NewConfig(productCtx) + productConfig.Environment().Set("OUT_DIR", productOutDir) + productConfig.Lunch(productCtx, product, "eng") + + build.Build(productCtx, productConfig, build.BuildProductConfig) + productConfigs <- Product{productCtx, productConfig} + }(product) + } + go func() { + defer close(productConfigs) + wg.Wait() + }() + + var wg2 sync.WaitGroup + // Then run up to numJobs worth of Soong and Kati + for i := 0; i < *numJobs; i++ { + wg2.Add(1) + go func() { + defer wg2.Done() + for product := range productConfigs { + func() { + defer logger.Recover(func(err error) { + errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err) + }) + + buildWhat := 0 + if !*onlyConfig { + buildWhat |= build.BuildSoong + if !*onlySoong { + buildWhat |= build.BuildKati + } + } + build.Build(product.ctx, product.config, buildWhat) + if !*keep { + // TODO: kati aborts from opendir while setting up the find emulator + //os.RemoveAll(product.config.OutDir()) + } + log.Println("Finished running for", product.config.TargetProduct()) + }() + } + }() + } + go func() { + wg2.Wait() + close(errs) + }() + + for err := range errs { + failed = true + log.Print(err) + } + + if failed { + log.Fatalln("Failed") + } +} diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 994c54dc..34739b75 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -48,14 +48,6 @@ func main() { log.Fatalln("The `soong` native UI is not yet available.") } - // Precondition: the current directory is the top of the source tree - if _, err := os.Stat("build/soong/root.bp"); err != nil { - if os.IsNotExist(err) { - log.Fatalln("soong_ui should run from the root of the source directory: build/soong/root.bp not found") - } - log.Fatalln("Error verifying tree state:", err) - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/ui/build/config.go b/ui/build/config.go index b0a7d7ad..35c5213d 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -15,6 +15,8 @@ package build import ( + "log" + "os" "path/filepath" "runtime" "strconv" @@ -40,6 +42,8 @@ type configImpl struct { katiSuffix string } +const srcDirFileCheck = "build/soong/root.bp" + func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ environ: OsEnvironment(), @@ -71,6 +75,14 @@ func NewConfig(ctx Context, args ...string) Config { ret.parallel = runtime.NumCPU() + 2 ret.keepGoing = 1 + // Precondition: the current directory is the top of the source tree + if _, err := os.Stat(srcDirFileCheck); err != nil { + if os.IsNotExist(err) { + log.Fatalf("Current working directory must be the source tree. %q not found", srcDirFileCheck) + } + log.Fatalln("Error verifying tree state:", err) + } + for _, arg := range args { arg = strings.TrimSpace(arg) if arg == "--make-mode" { |