Change storage behavior of multiproduct_kati

Instead of deleting artifacts/logs from successful build (unless -keep
is set), and keeping unsuccessful artifacts, keep all logs and remove
all artifacts (unless -keep is set, then we'll compress the artifacts).

If -dist is passed in, we'll put an archive of the logs into the
DIST_DIR. Even compressed, the rest of the artifacts are still a bit too
large to dist (~5.6GB on AOSP).

Test: build/soong/build_test.bash
Test: build/soong/build_test.bash -keep
Test: build/soong/build_test.bash -dist
Change-Id: I87f55978c18c8ff2e517b8271554ba383003742f

2 files changed, 48 insertions, 23 deletions

diff --git a/cmd/multiproduct_kati/Android.bp b/cmd/multiproduct_kati/Android.bp
index b264c354..04a58023 100644
--- a/cmd/multiproduct_kati/Android.bp
+++ b/cmd/multiproduct_kati/Android.bp
@@ -18,6 +18,7 @@ blueprint_go_binary {
         "soong-ui-build",
         "soong-ui-logger",
         "soong-ui-tracer",
+        "soong-zip",
     ],
     srcs: [
         "main.go",
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go
index 1c853d63..e4a05fc0 100644
--- a/cmd/multiproduct_kati/main.go
+++ b/cmd/multiproduct_kati/main.go
@@ -30,6 +30,7 @@ import (
 	"android/soong/ui/build"
 	"android/soong/ui/logger"
 	"android/soong/ui/tracer"
+	"android/soong/zip"
 )
 
 // We default to number of cpus / 4, which seems to be the sweet spot for my
@@ -45,7 +46,7 @@ func detectNumJobs() int {
 
 var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs")
 
-var keep = flag.Bool("keep", false, "keep successful output files")
+var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts")
 
 var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
 var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or <out>/dist when empty)")
@@ -200,24 +201,19 @@ func main() {
 		if err := os.MkdirAll(*outDir, 0777); err != nil {
 			log.Fatalf("Failed to create tempdir: %v", err)
 		}
-
-		if !*keep {
-			defer func() {
-				if status.Finished() == 0 {
-					os.RemoveAll(*outDir)
-				}
-			}()
-		}
 	}
 	config.Environment().Set("OUT_DIR", *outDir)
 	log.Println("Output directory:", *outDir)
 
+	logsDir := filepath.Join(config.OutDir(), "logs")
+	os.MkdirAll(logsDir, 0777)
+
 	build.SetupOutDir(buildCtx, config)
 	if *alternateResultDir {
-		logsDir := filepath.Join(config.DistDir(), "logs")
-		os.MkdirAll(logsDir, 0777)
-		log.SetOutput(filepath.Join(logsDir, "soong.log"))
-		trace.SetOutput(filepath.Join(logsDir, "build.trace"))
+		distLogsDir := filepath.Join(config.DistDir(), "logs")
+		os.MkdirAll(distLogsDir, 0777)
+		log.SetOutput(filepath.Join(distLogsDir, "soong.log"))
+		trace.SetOutput(filepath.Join(distLogsDir, "build.trace"))
 	} else {
 		log.SetOutput(filepath.Join(config.OutDir(), "soong.log"))
 		trace.SetOutput(filepath.Join(config.OutDir(), "build.trace"))
@@ -269,17 +265,14 @@ func main() {
 			})
 
 			productOutDir := filepath.Join(config.OutDir(), product)
-			productLogDir := productOutDir
-			if *alternateResultDir {
-				productLogDir = filepath.Join(config.DistDir(), product)
-				if err := os.MkdirAll(productLogDir, 0777); err != nil {
-					log.Fatalf("Error creating log directory: %v", err)
-				}
-			}
+			productLogDir := filepath.Join(logsDir, product)
 
 			if err := os.MkdirAll(productOutDir, 0777); err != nil {
 				log.Fatalf("Error creating out directory: %v", err)
 			}
+			if err := os.MkdirAll(productLogDir, 0777); err != nil {
+				log.Fatalf("Error creating log directory: %v", err)
+			}
 
 			stdLog = filepath.Join(productLogDir, "std.log")
 			f, err := os.Create(stdLog)
@@ -324,6 +317,26 @@ func main() {
 						status.Fail(product.config.TargetProduct(), err, product.logFile)
 					})
 
+					defer func() {
+						if *keepArtifacts {
+							args := zip.ZipArgs{
+								FileArgs: []zip.FileArg{
+									{
+										GlobDir:             product.config.OutDir(),
+										SourcePrefixToStrip: product.config.OutDir(),
+									},
+								},
+								OutputFilePath:   filepath.Join(config.OutDir(), product.config.TargetProduct()+".zip"),
+								NumParallelJobs:  runtime.NumCPU(),
+								CompressionLevel: 5,
+							}
+							if err := zip.Run(args); err != nil {
+								log.Fatalf("Error zipping artifacts: %v", err)
+							}
+						}
+						os.RemoveAll(product.config.OutDir())
+					}()
+
 					buildWhat := 0
 					if !*onlyConfig {
 						buildWhat |= build.BuildSoong
@@ -332,9 +345,6 @@ func main() {
 						}
 					}
 					build.Build(product.ctx, product.config, buildWhat)
-					if !*keep {
-						os.RemoveAll(product.config.OutDir())
-					}
 					status.Finish(product.config.TargetProduct())
 				}()
 			}
@@ -342,6 +352,20 @@ func main() {
 	}
 	wg2.Wait()
 
+	if *alternateResultDir {
+		args := zip.ZipArgs{
+			FileArgs: []zip.FileArg{
+				{GlobDir: logsDir, SourcePrefixToStrip: logsDir},
+			},
+			OutputFilePath:   filepath.Join(config.DistDir(), "logs.zip"),
+			NumParallelJobs:  runtime.NumCPU(),
+			CompressionLevel: 5,
+		}
+		if err := zip.Run(args); err != nil {
+			log.Fatalf("Error zipping logs: %v", err)
+		}
+	}
+
 	if count := status.Finished(); count > 0 {
 		log.Fatalln(count, "products failed")
 	}