Refactor multiproduct_kati

Move the actual per-product builds out of the main function, combining
the product configuration and build sections. This makes it a lot more
readable.

This ends up saving a lot of FDs -- a -only-config build on aosp with 77
products used to require ~500 FDs, it succeeds with a limit of 150 FDs
now. I'm leaving the code to bump our FD limit though, since 2x the
number of internal products is too close to the 1024 limit for comfort.

Bug: 70370883
Test: prlimit -n150:150 build/soong/build_test.bash -only-config
Change-Id: Ia559beadc19deb8a5b9d50af6e0276e846fd8608

1 files changed, 146 insertions, 138 deletions

diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go
index ea671bcf..374868c1 100644
--- a/cmd/multiproduct_kati/main.go
+++ b/cmd/multiproduct_kati/main.go
@@ -28,6 +28,7 @@ import (
 	"syscall"
 	"time"
 
+	"android/soong/finder"
 	"android/soong/ui/build"
 	"android/soong/ui/logger"
 	"android/soong/ui/status"
@@ -66,13 +67,6 @@ var includeProducts = flag.String("products", "", "comma-separated list of produ
 const errorLeadingLines = 20
 const errorTrailingLines = 20
 
-type Product struct {
-	ctx     build.Context
-	config  build.Config
-	logFile string
-	action  *status.Action
-}
-
 func errMsgFromLog(filename string) string {
 	if filename == "" {
 		return ""
@@ -150,6 +144,17 @@ func copyFile(from, to string) error {
 	return err
 }
 
+type mpContext struct {
+	Context context.Context
+	Logger  logger.Logger
+	Status  status.ToolStatus
+	Tracer  tracer.Tracer
+	Finder  *finder.Finder
+	Config  build.Config
+
+	LogsDir string
+}
+
 func main() {
 	writer := terminal.NewWriter(terminal.StdioImpl{})
 	defer writer.Finish()
@@ -253,7 +258,7 @@ func main() {
 		productsList = allProducts
 	}
 
-	products := make([]string, 0, len(productsList))
+	finalProductsList := make([]string, 0, len(productsList))
 	skipList := strings.Split(*skipProducts, ",")
 	skipProduct := func(p string) bool {
 		for _, s := range skipList {
@@ -265,156 +270,54 @@ func main() {
 	}
 	for _, product := range productsList {
 		if !skipProduct(product) {
-			products = append(products, product)
+			finalProductsList = append(finalProductsList, product)
 		} else {
 			log.Verbose("Skipping: ", product)
 		}
 	}
 
-	log.Verbose("Got product list: ", products)
+	log.Verbose("Got product list: ", finalProductsList)
 
 	s := buildCtx.Status.StartTool()
-	s.SetTotalActions(len(products))
-
-	var wg sync.WaitGroup
-	productConfigs := make(chan Product, len(products))
-
-	// Run the product config for every product in parallel
-	for _, product := range products {
-		wg.Add(1)
-		go func(product string) {
-			var stdLog string
-
-			defer wg.Done()
+	s.SetTotalActions(len(finalProductsList))
 
-			action := &status.Action{
-				Description: product,
-				Outputs:     []string{product},
-			}
-			s.StartAction(action)
-			defer logger.Recover(func(err error) {
-				s.FinishAction(status.ActionResult{
-					Action: action,
-					Error:  err,
-					Output: errMsgFromLog(stdLog),
-				})
-			})
-
-			productOutDir := filepath.Join(config.OutDir(), product)
-			productLogDir := filepath.Join(logsDir, product)
-
-			if err := os.MkdirAll(productOutDir, 0777); err != nil {
-				log.Fatalf("Error creating out directory: %v", err)
-			}
-			if err := os.MkdirAll(productLogDir, 0777); err != nil {
-				log.Fatalf("Error creating log directory: %v", err)
-			}
+	mpCtx := &mpContext{
+		Context: ctx,
+		Logger:  log,
+		Status:  s,
+		Tracer:  trace,
 
-			stdLog = filepath.Join(productLogDir, "std.log")
-			f, err := os.Create(stdLog)
-			if err != nil {
-				log.Fatalf("Error creating std.log: %v", err)
-			}
+		Finder: finder,
+		Config: config,
 
-			productLog := logger.New(f)
-			productLog.SetOutput(filepath.Join(productLogDir, "soong.log"))
-
-			productCtx := build.Context{ContextImpl: &build.ContextImpl{
-				Context: ctx,
-				Logger:  productLog,
-				Tracer:  trace,
-				Writer:  terminal.NewWriter(terminal.NewCustomStdio(nil, f, f)),
-				Thread:  trace.NewThread(product),
-				Status:  &status.Status{},
-			}}
-			productCtx.Status.AddOutput(terminal.NewStatusOutput(productCtx.Writer, ""))
-
-			productConfig := build.NewConfig(productCtx, flag.Args()...)
-			productConfig.Environment().Set("OUT_DIR", productOutDir)
-			build.FindSources(productCtx, productConfig, finder)
-			productConfig.Lunch(productCtx, product, *buildVariant)
-
-			build.Build(productCtx, productConfig, build.BuildProductConfig)
-			productConfigs <- Product{productCtx, productConfig, stdLog, action}
-		}(product)
+		LogsDir: logsDir,
 	}
+
+	products := make(chan string, len(productsList))
 	go func() {
-		defer close(productConfigs)
-		wg.Wait()
+		defer close(products)
+		for _, product := range finalProductsList {
+			products <- product
+		}
 	}()
 
-	var wg2 sync.WaitGroup
-	// Then run up to numJobs worth of Soong and Kati
+	var wg sync.WaitGroup
 	for i := 0; i < *numJobs; i++ {
-		wg2.Add(1)
+		wg.Add(1)
 		go func() {
-			defer wg2.Done()
-			for product := range productConfigs {
-				func() {
-					defer logger.Recover(func(err error) {
-						s.FinishAction(status.ActionResult{
-							Action: product.action,
-							Error:  err,
-							Output: errMsgFromLog(product.logFile),
-						})
-					})
-
-					defer func() {
-						if *keepArtifacts {
-							args := zip.ZipArgs{
-								FileArgs: []zip.FileArg{
-									{
-										GlobDir:             product.config.OutDir(),
-										SourcePrefixToStrip: product.config.OutDir(),
-									},
-								},
-								OutputFilePath:   filepath.Join(config.OutDir(), product.config.TargetProduct()+".zip"),
-								NumParallelJobs:  runtime.NumCPU(),
-								CompressionLevel: 5,
-							}
-							if err := zip.Run(args); err != nil {
-								log.Fatalf("Error zipping artifacts: %v", err)
-							}
-						}
-						if *incremental {
-							// Save space, Kati doesn't notice
-							if f := product.config.KatiNinjaFile(); f != "" {
-								os.Truncate(f, 0)
-							}
-						} else {
-							os.RemoveAll(product.config.OutDir())
-						}
-					}()
-
-					buildWhat := 0
-					if !*onlyConfig {
-						buildWhat |= build.BuildSoong
-						if !*onlySoong {
-							buildWhat |= build.BuildKati
-						}
-					}
-
-					before := time.Now()
-					build.Build(product.ctx, product.config, buildWhat)
-
-					// Save std_full.log if Kati re-read the makefiles
-					if buildWhat&build.BuildKati != 0 {
-						if after, err := os.Stat(product.config.KatiNinjaFile()); err == nil && after.ModTime().After(before) {
-							err := copyFile(product.logFile, filepath.Join(filepath.Dir(product.logFile), "std_full.log"))
-							if err != nil {
-								log.Fatalf("Error copying log file: %s", err)
-							}
-						}
+			defer wg.Done()
+			for {
+				select {
+				case product := <-products:
+					if product == "" {
+						return
 					}
-
-					s.FinishAction(status.ActionResult{
-						Action: product.action,
-					})
-				}()
+					buildProduct(mpCtx, product)
+				}
 			}
 		}()
 	}
-	wg2.Wait()
+	wg.Wait()
 
 	if *alternateResultDir {
 		args := zip.ZipArgs{
@@ -441,6 +344,111 @@ func main() {
 	}
 }
 
+func buildProduct(mpctx *mpContext, product string) {
+	var stdLog string
+
+	outDir := filepath.Join(mpctx.Config.OutDir(), product)
+	logsDir := filepath.Join(mpctx.LogsDir, product)
+
+	if err := os.MkdirAll(outDir, 0777); err != nil {
+		mpctx.Logger.Fatalf("Error creating out directory: %v", err)
+	}
+	if err := os.MkdirAll(logsDir, 0777); err != nil {
+		mpctx.Logger.Fatalf("Error creating log directory: %v", err)
+	}
+
+	stdLog = filepath.Join(logsDir, "std.log")
+	f, err := os.Create(stdLog)
+	if err != nil {
+		mpctx.Logger.Fatalf("Error creating std.log: %v", err)
+	}
+	defer f.Close()
+
+	log := logger.New(f)
+	defer log.Cleanup()
+	log.SetOutput(filepath.Join(logsDir, "soong.log"))
+
+	action := &status.Action{
+		Description: product,
+		Outputs:     []string{product},
+	}
+	mpctx.Status.StartAction(action)
+	defer logger.Recover(func(err error) {
+		mpctx.Status.FinishAction(status.ActionResult{
+			Action: action,
+			Error:  err,
+			Output: errMsgFromLog(stdLog),
+		})
+	})
+
+	ctx := build.Context{ContextImpl: &build.ContextImpl{
+		Context: mpctx.Context,
+		Logger:  log,
+		Tracer:  mpctx.Tracer,
+		Writer:  terminal.NewWriter(terminal.NewCustomStdio(nil, f, f)),
+		Thread:  mpctx.Tracer.NewThread(product),
+		Status:  &status.Status{},
+	}}
+	ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, ""))
+
+	config := build.NewConfig(ctx, flag.Args()...)
+	config.Environment().Set("OUT_DIR", outDir)
+	build.FindSources(ctx, config, mpctx.Finder)
+	config.Lunch(ctx, product, *buildVariant)
+
+	defer func() {
+		if *keepArtifacts {
+			args := zip.ZipArgs{
+				FileArgs: []zip.FileArg{
+					{
+						GlobDir:             outDir,
+						SourcePrefixToStrip: outDir,
+					},
+				},
+				OutputFilePath:   filepath.Join(mpctx.Config.OutDir(), product+".zip"),
+				NumParallelJobs:  runtime.NumCPU(),
+				CompressionLevel: 5,
+			}
+			if err := zip.Run(args); err != nil {
+				log.Fatalf("Error zipping artifacts: %v", err)
+			}
+		}
+		if *incremental {
+			// Save space, Kati doesn't notice
+			if f := config.KatiNinjaFile(); f != "" {
+				os.Truncate(f, 0)
+			}
+		} else {
+			os.RemoveAll(outDir)
+		}
+	}()
+
+	buildWhat := build.BuildProductConfig
+	if !*onlyConfig {
+		buildWhat |= build.BuildSoong
+		if !*onlySoong {
+			buildWhat |= build.BuildKati
+		}
+	}
+
+	before := time.Now()
+	build.Build(ctx, config, buildWhat)
+
+	// Save std_full.log if Kati re-read the makefiles
+	if buildWhat&build.BuildKati != 0 {
+		if after, err := os.Stat(config.KatiNinjaFile()); err == nil && after.ModTime().After(before) {
+			err := copyFile(stdLog, filepath.Join(filepath.Dir(stdLog), "std_full.log"))
+			if err != nil {
+				log.Fatalf("Error copying log file: %s", err)
+			}
+		}
+	}
+
+	mpctx.Status.FinishAction(status.ActionResult{
+		Action: action,
+	})
+}
+
 type failureCount int
 
 func (f *failureCount) StartAction(action *status.Action, counts status.Counts) {}