diff options
| author | Dan Willemsen <dwillemsen@google.com> | 2019-12-27 09:54:11 -0800 |
|---|---|---|
| committer | Luca Stefani <luca.stefani.ge1@gmail.com> | 2020-01-04 10:43:01 +0100 |
| commit | 3734467777b3780e7395bbc863db5b6065eea33d (patch) | |
| tree | c3163d81f5c764008bf5dbfd5746eca439f0d2a0 | |
| parent | b911055f07b215f684229410a5a5698248635225 (diff) | |
| download | build_soong-3734467777b3780e7395bbc863db5b6065eea33d.tar.gz build_soong-3734467777b3780e7395bbc863db5b6065eea33d.tar.bz2 build_soong-3734467777b3780e7395bbc863db5b6065eea33d.zip | |
Take into account RAM usage for multiproduct_kati
Apparently we have some instances of Soong taking 16GB of RAM. While
that's a problem to solve on it's own, make multiproduct_kati's
auto-parallelism detection smarter by taking into account the total RAM
on the machine instead of just the number of CPUs.
Bug: 146925549
Test: check soong.log for autodetected parallelism values
Change-Id: Ice34c2cf73ea4f8f235521cbefc8654a79a04eef
| -rw-r--r-- | cmd/multiproduct_kati/main.go | 30 |
1 files changed, 17 insertions, 13 deletions
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 2800ade2..4c6e243a 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -37,18 +37,7 @@ import ( "android/soong/zip" ) -// We default to number of cpus / 4, which seems to be the sweet spot for my -// system. I suspect this is mostly due to memory or disk bandwidth though, and -// may depend on the size ofthe source tree, so this probably isn't a great -// default. -func detectNumJobs() int { - if runtime.NumCPU() < 4 { - return 1 - } - return runtime.NumCPU() / 4 -} - -var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") +var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]") var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts") var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)") @@ -230,6 +219,21 @@ func main() { trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) } + var jobs = *numJobs + if jobs < 1 { + jobs = runtime.NumCPU() / 4 + + ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024) + if ramJobs := ramGb / 20; ramGb > 0 && jobs > ramJobs { + jobs = ramJobs + } + + if jobs < 1 { + jobs = 1 + } + } + log.Verbosef("Using %d parallel jobs", jobs) + setMaxFiles(log) finder := build.NewSourceFinder(buildCtx, config) @@ -304,7 +308,7 @@ func main() { }() var wg sync.WaitGroup - for i := 0; i < *numJobs; i++ { + for i := 0; i < jobs; i++ { wg.Add(1) go func() { defer wg.Done() |