From d9c1c8fbcb242d2aacebbd9dc28702b402fe44f4 Mon Sep 17 00:00:00 2001
From: Colin Cross <ccross@android.com>
Date: Mon, 23 Sep 2019 15:55:30 -0700
Subject: Shard gensrcs modules into multiple commands

gensrcs modules use a single command to convert all of the sources,
which can lead to command line length limits, long critical path
times, and excessive rebuilds.  Shard them into multiple rules,
defaulting to 100 commands in each.

Test: TestGenSrcs
Test: m
Fixes: 70221552
Fixes: 138438756
Fixes: 138829091
Fixes: 144948629
Change-Id: I8409e43011a4754e095ad1b368570a2ba8d75e50
Merged-In: I8409e43011a4754e095ad1b368570a2ba8d75e50
(cherry picked from commit 1a527688d67b9349f4ea5735916c35bdb7f1847f)
---
 genrule/genrule_test.go | 101 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 98 insertions(+), 3 deletions(-)

(limited to 'genrule/genrule_test.go')

diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go
index 0b6952f3..e8dc3b55 100644
--- a/genrule/genrule_test.go
+++ b/genrule/genrule_test.go
@@ -57,6 +57,7 @@ func testContext(config android.Config, bp string,
 	ctx := android.NewTestArchContext()
 	ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory))
 	ctx.RegisterModuleType("genrule", android.ModuleFactoryAdaptor(GenRuleFactory))
+	ctx.RegisterModuleType("gensrcs", android.ModuleFactoryAdaptor(GenSrcsFactory))
 	ctx.RegisterModuleType("genrule_defaults", android.ModuleFactoryAdaptor(defaultsFactory))
 	ctx.RegisterModuleType("tool", android.ModuleFactoryAdaptor(toolFactory))
 	ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
@@ -109,6 +110,9 @@ func testContext(config android.Config, bp string,
 		"tool_file2": nil,
 		"in1":        nil,
 		"in2":        nil,
+		"in1.txt":    nil,
+		"in2.txt":    nil,
+		"in3.txt":    nil,
 	}
 
 	for k, v := range fs {
@@ -491,11 +495,102 @@ func TestGenruleCmd(t *testing.T) {
 			}
 
 			gen := ctx.ModuleForTests("gen", "").Module().(*Module)
-			if g, w := gen.rawCommand, "'"+test.expect+"'"; w != g {
+			if g, w := gen.rawCommands[0], "'"+test.expect+"'"; w != g {
 				t.Errorf("want %q, got %q", w, g)
 			}
 		})
 	}
+}
+
+func TestGenSrcs(t *testing.T) {
+	testcases := []struct {
+		name string
+		prop string
+
+		allowMissingDependencies bool
+
+		err   string
+		cmds  []string
+		deps  []string
+		files []string
+	}{
+		{
+			name: "gensrcs",
+			prop: `
+				tools: ["tool"],
+				srcs: ["in1.txt", "in2.txt"],
+				cmd: "$(location) $(in) > $(out)",
+			`,
+			cmds: []string{
+				"'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''",
+			},
+			deps:  []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"},
+			files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"},
+		},
+		{
+			name: "shards",
+			prop: `
+				tools: ["tool"],
+				srcs: ["in1.txt", "in2.txt", "in3.txt"],
+				cmd: "$(location) $(in) > $(out)",
+				shard_size: 2,
+			`,
+			cmds: []string{
+				"'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''",
+				"'bash -c '\\''out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'\\'''",
+			},
+			deps:  []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"},
+			files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"},
+		},
+	}
+
+	for _, test := range testcases {
+		t.Run(test.name, func(t *testing.T) {
+			config := android.TestArchConfig(buildDir, nil)
+			bp := "gensrcs {\n"
+			bp += `name: "gen",` + "\n"
+			bp += `output_extension: "h",` + "\n"
+			bp += test.prop
+			bp += "}\n"
+
+			ctx := testContext(config, bp, nil)
+
+			_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+			if errs == nil {
+				_, errs = ctx.PrepareBuildActions(config)
+			}
+			if errs == nil && test.err != "" {
+				t.Fatalf("want error %q, got no error", test.err)
+			} else if errs != nil && test.err == "" {
+				android.FailIfErrored(t, errs)
+			} else if test.err != "" {
+				if len(errs) != 1 {
+					t.Errorf("want 1 error, got %d errors:", len(errs))
+					for _, err := range errs {
+						t.Errorf("   %s", err.Error())
+					}
+					t.FailNow()
+				}
+				if !strings.Contains(errs[0].Error(), test.err) {
+					t.Fatalf("want %q, got %q", test.err, errs[0].Error())
+				}
+				return
+			}
+
+			gen := ctx.ModuleForTests("gen", "").Module().(*Module)
+			if g, w := gen.rawCommands, test.cmds; !reflect.DeepEqual(w, g) {
+				t.Errorf("want %q, got %q", w, g)
+			}
+
+			if g, w := gen.outputDeps.Strings(), test.deps; !reflect.DeepEqual(w, g) {
+				t.Errorf("want deps %q, got %q", w, g)
+			}
+
+			if g, w := gen.outputFiles.Strings(), test.files; !reflect.DeepEqual(w, g) {
+				t.Errorf("want files %q, got %q", w, g)
+			}
+		})
+	}
 
 }
 
@@ -529,8 +624,8 @@ func TestGenruleDefaults(t *testing.T) {
 	gen := ctx.ModuleForTests("gen", "").Module().(*Module)
 
 	expectedCmd := "'cp ${in} __SBOX_OUT_FILES__'"
-	if gen.rawCommand != expectedCmd {
-		t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommand)
+	if gen.rawCommands[0] != expectedCmd {
+		t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommands[0])
 	}
 
 	expectedSrcs := []string{"in1"}
-- 
cgit v1.2.3