diff options
author | Dan Willemsen <dwillemsen@google.com> | 2018-09-04 18:09:47 -0700 |
---|---|---|
committer | Dan Willemsen <dwillemsen@google.com> | 2018-09-05 13:57:10 -0700 |
commit | bcc1dbf957c5b6a6acdc64c02170605ab3aa9636 (patch) | |
tree | 31576c091c63f091eeaf3eeb09bb238357bc1a77 /cmd | |
parent | 41538385e540a665bc446e71893a379a51bc1df9 (diff) | |
download | build_soong-bcc1dbf957c5b6a6acdc64c02170605ab3aa9636.tar.gz build_soong-bcc1dbf957c5b6a6acdc64c02170605ab3aa9636.tar.bz2 build_soong-bcc1dbf957c5b6a6acdc64c02170605ab3aa9636.zip |
Refactor multiproduct_kati
Move the actual per-product builds out of the main function, combining
the product configuration and build sections. This makes it a lot more
readable.
This ends up saving a lot of FDs -- a -only-config build on aosp with 77
products used to require ~500 FDs, it succeeds with a limit of 150 FDs
now. I'm leaving the code to bump our FD limit though, since 2x the
number of internal products is too close to the 1024 limit for comfort.
Bug: 70370883
Test: prlimit -n150:150 build/soong/build_test.bash -only-config
Change-Id: Ia559beadc19deb8a5b9d50af6e0276e846fd8608
Diffstat (limited to 'cmd')
-rw-r--r-- | cmd/multiproduct_kati/main.go | 284 |
1 files changed, 146 insertions, 138 deletions
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index ea671bcf..374868c1 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -28,6 +28,7 @@ import ( "syscall" "time" + "android/soong/finder" "android/soong/ui/build" "android/soong/ui/logger" "android/soong/ui/status" @@ -66,13 +67,6 @@ var includeProducts = flag.String("products", "", "comma-separated list of produ const errorLeadingLines = 20 const errorTrailingLines = 20 -type Product struct { - ctx build.Context - config build.Config - logFile string - action *status.Action -} - func errMsgFromLog(filename string) string { if filename == "" { return "" @@ -150,6 +144,17 @@ func copyFile(from, to string) error { return err } +type mpContext struct { + Context context.Context + Logger logger.Logger + Status status.ToolStatus + Tracer tracer.Tracer + Finder *finder.Finder + Config build.Config + + LogsDir string +} + func main() { writer := terminal.NewWriter(terminal.StdioImpl{}) defer writer.Finish() @@ -253,7 +258,7 @@ func main() { productsList = allProducts } - products := make([]string, 0, len(productsList)) + finalProductsList := make([]string, 0, len(productsList)) skipList := strings.Split(*skipProducts, ",") skipProduct := func(p string) bool { for _, s := range skipList { @@ -265,156 +270,54 @@ func main() { } for _, product := range productsList { if !skipProduct(product) { - products = append(products, product) + finalProductsList = append(finalProductsList, product) } else { log.Verbose("Skipping: ", product) } } - log.Verbose("Got product list: ", products) + log.Verbose("Got product list: ", finalProductsList) s := buildCtx.Status.StartTool() - s.SetTotalActions(len(products)) - - var wg sync.WaitGroup - productConfigs := make(chan Product, len(products)) - - // Run the product config for every product in parallel - for _, product := range products { - wg.Add(1) - go func(product string) { - var stdLog string - - defer wg.Done() + s.SetTotalActions(len(finalProductsList)) - action := &status.Action{ - Description: product, - Outputs: []string{product}, - } - s.StartAction(action) - defer logger.Recover(func(err error) { - s.FinishAction(status.ActionResult{ - Action: action, - Error: err, - Output: errMsgFromLog(stdLog), - }) - }) - - productOutDir := filepath.Join(config.OutDir(), product) - productLogDir := filepath.Join(logsDir, product) - - if err := os.MkdirAll(productOutDir, 0777); err != nil { - log.Fatalf("Error creating out directory: %v", err) - } - if err := os.MkdirAll(productLogDir, 0777); err != nil { - log.Fatalf("Error creating log directory: %v", err) - } + mpCtx := &mpContext{ + Context: ctx, + Logger: log, + Status: s, + Tracer: trace, - stdLog = filepath.Join(productLogDir, "std.log") - f, err := os.Create(stdLog) - if err != nil { - log.Fatalf("Error creating std.log: %v", err) - } + Finder: finder, + Config: config, - productLog := logger.New(f) - productLog.SetOutput(filepath.Join(productLogDir, "soong.log")) - - productCtx := build.Context{ContextImpl: &build.ContextImpl{ - Context: ctx, - Logger: productLog, - Tracer: trace, - Writer: terminal.NewWriter(terminal.NewCustomStdio(nil, f, f)), - Thread: trace.NewThread(product), - Status: &status.Status{}, - }} - productCtx.Status.AddOutput(terminal.NewStatusOutput(productCtx.Writer, "")) - - productConfig := build.NewConfig(productCtx, flag.Args()...) - productConfig.Environment().Set("OUT_DIR", productOutDir) - build.FindSources(productCtx, productConfig, finder) - productConfig.Lunch(productCtx, product, *buildVariant) - - build.Build(productCtx, productConfig, build.BuildProductConfig) - productConfigs <- Product{productCtx, productConfig, stdLog, action} - }(product) + LogsDir: logsDir, } + + products := make(chan string, len(productsList)) go func() { - defer close(productConfigs) - wg.Wait() + defer close(products) + for _, product := range finalProductsList { + products <- product + } }() - var wg2 sync.WaitGroup - // Then run up to numJobs worth of Soong and Kati + var wg sync.WaitGroup for i := 0; i < *numJobs; i++ { - wg2.Add(1) + wg.Add(1) go func() { - defer wg2.Done() - for product := range productConfigs { - func() { - defer logger.Recover(func(err error) { - s.FinishAction(status.ActionResult{ - Action: product.action, - Error: err, - Output: errMsgFromLog(product.logFile), - }) - }) - - defer func() { - if *keepArtifacts { - args := zip.ZipArgs{ - FileArgs: []zip.FileArg{ - { - GlobDir: product.config.OutDir(), - SourcePrefixToStrip: product.config.OutDir(), - }, - }, - OutputFilePath: filepath.Join(config.OutDir(), product.config.TargetProduct()+".zip"), - NumParallelJobs: runtime.NumCPU(), - CompressionLevel: 5, - } - if err := zip.Run(args); err != nil { - log.Fatalf("Error zipping artifacts: %v", err) - } - } - if *incremental { - // Save space, Kati doesn't notice - if f := product.config.KatiNinjaFile(); f != "" { - os.Truncate(f, 0) - } - } else { - os.RemoveAll(product.config.OutDir()) - } - }() - - buildWhat := 0 - if !*onlyConfig { - buildWhat |= build.BuildSoong - if !*onlySoong { - buildWhat |= build.BuildKati - } - } - - before := time.Now() - build.Build(product.ctx, product.config, buildWhat) - - // Save std_full.log if Kati re-read the makefiles - if buildWhat&build.BuildKati != 0 { - if after, err := os.Stat(product.config.KatiNinjaFile()); err == nil && after.ModTime().After(before) { - err := copyFile(product.logFile, filepath.Join(filepath.Dir(product.logFile), "std_full.log")) - if err != nil { - log.Fatalf("Error copying log file: %s", err) - } - } + defer wg.Done() + for { + select { + case product := <-products: + if product == "" { + return } - - s.FinishAction(status.ActionResult{ - Action: product.action, - }) - }() + buildProduct(mpCtx, product) + } } }() } - wg2.Wait() + wg.Wait() if *alternateResultDir { args := zip.ZipArgs{ @@ -441,6 +344,111 @@ func main() { } } +func buildProduct(mpctx *mpContext, product string) { + var stdLog string + + outDir := filepath.Join(mpctx.Config.OutDir(), product) + logsDir := filepath.Join(mpctx.LogsDir, product) + + if err := os.MkdirAll(outDir, 0777); err != nil { + mpctx.Logger.Fatalf("Error creating out directory: %v", err) + } + if err := os.MkdirAll(logsDir, 0777); err != nil { + mpctx.Logger.Fatalf("Error creating log directory: %v", err) + } + + stdLog = filepath.Join(logsDir, "std.log") + f, err := os.Create(stdLog) + if err != nil { + mpctx.Logger.Fatalf("Error creating std.log: %v", err) + } + defer f.Close() + + log := logger.New(f) + defer log.Cleanup() + log.SetOutput(filepath.Join(logsDir, "soong.log")) + + action := &status.Action{ + Description: product, + Outputs: []string{product}, + } + mpctx.Status.StartAction(action) + defer logger.Recover(func(err error) { + mpctx.Status.FinishAction(status.ActionResult{ + Action: action, + Error: err, + Output: errMsgFromLog(stdLog), + }) + }) + + ctx := build.Context{ContextImpl: &build.ContextImpl{ + Context: mpctx.Context, + Logger: log, + Tracer: mpctx.Tracer, + Writer: terminal.NewWriter(terminal.NewCustomStdio(nil, f, f)), + Thread: mpctx.Tracer.NewThread(product), + Status: &status.Status{}, + }} + ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "")) + + config := build.NewConfig(ctx, flag.Args()...) + config.Environment().Set("OUT_DIR", outDir) + build.FindSources(ctx, config, mpctx.Finder) + config.Lunch(ctx, product, *buildVariant) + + defer func() { + if *keepArtifacts { + args := zip.ZipArgs{ + FileArgs: []zip.FileArg{ + { + GlobDir: outDir, + SourcePrefixToStrip: outDir, + }, + }, + OutputFilePath: filepath.Join(mpctx.Config.OutDir(), product+".zip"), + NumParallelJobs: runtime.NumCPU(), + CompressionLevel: 5, + } + if err := zip.Run(args); err != nil { + log.Fatalf("Error zipping artifacts: %v", err) + } + } + if *incremental { + // Save space, Kati doesn't notice + if f := config.KatiNinjaFile(); f != "" { + os.Truncate(f, 0) + } + } else { + os.RemoveAll(outDir) + } + }() + + buildWhat := build.BuildProductConfig + if !*onlyConfig { + buildWhat |= build.BuildSoong + if !*onlySoong { + buildWhat |= build.BuildKati + } + } + + before := time.Now() + build.Build(ctx, config, buildWhat) + + // Save std_full.log if Kati re-read the makefiles + if buildWhat&build.BuildKati != 0 { + if after, err := os.Stat(config.KatiNinjaFile()); err == nil && after.ModTime().After(before) { + err := copyFile(stdLog, filepath.Join(filepath.Dir(stdLog), "std_full.log")) + if err != nil { + log.Fatalf("Error copying log file: %s", err) + } + } + } + + mpctx.Status.FinishAction(status.ActionResult{ + Action: action, + }) +} + type failureCount int func (f *failureCount) StartAction(action *status.Action, counts status.Counts) {} |