diff options
author | android-build-team Robot <android-build-team-robot@google.com> | 2019-11-27 02:27:10 +0000 |
---|---|---|
committer | android-build-team Robot <android-build-team-robot@google.com> | 2019-11-27 02:27:10 +0000 |
commit | 9d7a8975d9eae1fe22788707c41a67e6e3ce92c7 (patch) | |
tree | a2eccd26193e970726af5fd4e6799b79a571855a | |
parent | 36bd23552aee4fb119aff13d2d4a20b293660a2d (diff) | |
parent | d9c1c8fbcb242d2aacebbd9dc28702b402fe44f4 (diff) | |
download | build_soong-9d7a8975d9eae1fe22788707c41a67e6e3ce92c7.tar.gz build_soong-9d7a8975d9eae1fe22788707c41a67e6e3ce92c7.tar.bz2 build_soong-9d7a8975d9eae1fe22788707c41a67e6e3ce92c7.zip |
Snap for 6034952 from d9c1c8fbcb242d2aacebbd9dc28702b402fe44f4 to qt-qpr2-release
Change-Id: If50010e646664876b1e50fa9a4a49a766a60e94f
-rw-r--r-- | Android.bp | 1 | ||||
-rw-r--r-- | android/paths.go | 17 | ||||
-rw-r--r-- | android/rule_builder.go | 197 | ||||
-rw-r--r-- | android/rule_builder_test.go | 239 | ||||
-rw-r--r-- | android/util.go | 33 | ||||
-rw-r--r-- | android/util_test.go | 99 | ||||
-rw-r--r-- | cmd/dep_fixer/Android.bp | 8 | ||||
-rw-r--r-- | cmd/dep_fixer/main.go | 6 | ||||
-rw-r--r-- | cmd/sbox/Android.bp | 1 | ||||
-rw-r--r-- | cmd/sbox/sbox.go | 28 | ||||
-rw-r--r-- | genrule/genrule.go | 378 | ||||
-rw-r--r-- | genrule/genrule_test.go | 101 | ||||
-rw-r--r-- | java/aapt2.go | 2 | ||||
-rw-r--r-- | java/java.go | 14 | ||||
-rw-r--r-- | makedeps/Android.bp | 21 | ||||
-rw-r--r-- | makedeps/deps.go (renamed from cmd/dep_fixer/deps.go) | 2 | ||||
-rw-r--r-- | makedeps/deps_test.go (renamed from cmd/dep_fixer/deps_test.go) | 2 |
17 files changed, 851 insertions, 298 deletions
@@ -37,6 +37,7 @@ bootstrap_go_package { "blueprint-bootstrap", "soong", "soong-env", + "soong-shared", ], srcs: [ "android/androidmk.go", diff --git a/android/paths.go b/android/paths.go index 8cc7057a..0f20b844 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1267,16 +1267,23 @@ func Rel(ctx PathContext, basePath string, targetPath string) string { // MaybeRel performs the same function as filepath.Rel, but reports errors to a PathContext, and returns false if // targetPath is not inside basePath. func MaybeRel(ctx PathContext, basePath string, targetPath string) (string, bool) { + rel, isRel, err := maybeRelErr(basePath, targetPath) + if err != nil { + reportPathError(ctx, err) + } + return rel, isRel +} + +func maybeRelErr(basePath string, targetPath string) (string, bool, error) { // filepath.Rel returns an error if one path is absolute and the other is not, handle that case first. if filepath.IsAbs(basePath) != filepath.IsAbs(targetPath) { - return "", false + return "", false, nil } rel, err := filepath.Rel(basePath, targetPath) if err != nil { - reportPathError(ctx, err) - return "", false + return "", false, err } else if rel == ".." || strings.HasPrefix(rel, "../") || strings.HasPrefix(rel, "/") { - return "", false + return "", false, nil } - return rel, true + return rel, true, nil } diff --git a/android/rule_builder.go b/android/rule_builder.go index 2d0fac16..b3fccea3 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -21,6 +21,8 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/proptools" + + "android/soong/shared" ) // RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build @@ -30,6 +32,8 @@ type RuleBuilder struct { installs RuleBuilderInstalls temporariesSet map[WritablePath]bool restat bool + sbox bool + sboxOutDir WritablePath missingDeps []string } @@ -73,11 +77,36 @@ func (r *RuleBuilder) MissingDeps(missingDeps []string) { } // Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat. +// +// Restat is not compatible with Sbox() func (r *RuleBuilder) Restat() *RuleBuilder { + if r.sbox { + panic("Restat() is not compatible with Sbox()") + } r.restat = true return r } +// Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output +// directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure +// that all outputs have been written, and will discard any output files that were not specified. +// +// Sbox is not compatible with Restat() +func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder { + if r.sbox { + panic("Sbox() may not be called more than once") + } + if len(r.commands) > 0 { + panic("Sbox() may not be called after Command()") + } + if r.restat { + panic("Sbox() is not compatible with Restat()") + } + r.sbox = true + r.sboxOutDir = outputDir + return r +} + // Install associates an output of the rule with an install location, which can be retrieved later using // RuleBuilder.Installs. func (r *RuleBuilder) Install(from Path, to string) { @@ -88,7 +117,10 @@ func (r *RuleBuilder) Install(from Path, to string) { // created by this method. That can be mutated through their methods in any order, as long as the mutations do not // race with any call to Build. func (r *RuleBuilder) Command() *RuleBuilderCommand { - command := &RuleBuilderCommand{} + command := &RuleBuilderCommand{ + sbox: r.sbox, + sboxOutDir: r.sboxOutDir, + } r.commands = append(r.commands, command) return command } @@ -120,12 +152,16 @@ func (r *RuleBuilder) DeleteTemporaryFiles() { // that are also outputs of another command in the same RuleBuilder are filtered out. func (r *RuleBuilder) Inputs() Paths { outputs := r.outputSet() + depFiles := r.depFileSet() inputs := make(map[string]Path) for _, c := range r.commands { for _, input := range c.inputs { - if _, isOutput := outputs[input.String()]; !isOutput { - inputs[input.String()] = input + inputStr := input.String() + if _, isOutput := outputs[inputStr]; !isOutput { + if _, isDepFile := depFiles[inputStr]; !isDepFile { + inputs[input.String()] = input + } } } } @@ -171,6 +207,16 @@ func (r *RuleBuilder) Outputs() WritablePaths { return outputList } +func (r *RuleBuilder) depFileSet() map[string]WritablePath { + depFiles := make(map[string]WritablePath) + for _, c := range r.commands { + for _, depFile := range c.depFiles { + depFiles[depFile.String()] = depFile + } + } + return depFiles +} + // DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such // as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile. func (r *RuleBuilder) DepFiles() WritablePaths { @@ -237,9 +283,9 @@ var _ BuilderContext = ModuleContext(nil) var _ BuilderContext = SingletonContext(nil) func (r *RuleBuilder) depFileMergerCmd(ctx PathContext, depFiles WritablePaths) *RuleBuilderCommand { - return (&RuleBuilderCommand{}). + return r.Command(). Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")). - Flags(depFiles.Strings()) + Inputs(depFiles.Paths()) } // Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for @@ -259,9 +305,6 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string return } - tools := r.Tools() - commands := r.Commands() - var depFile WritablePath var depFormat blueprint.Deps if depFiles := r.DepFiles(); len(depFiles) > 0 { @@ -269,37 +312,76 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string depFormat = blueprint.DepsGCC if len(depFiles) > 1 { // Add a command locally that merges all depfiles together into the first depfile. - cmd := r.depFileMergerCmd(ctx, depFiles) - commands = append(commands, string(cmd.buf)) - tools = append(tools, cmd.tools...) + r.depFileMergerCmd(ctx, depFiles) + + if r.sbox { + // Check for Rel() errors, as all depfiles should be in the output dir + for _, path := range depFiles[1:] { + Rel(ctx, r.sboxOutDir.String(), path.String()) + } + } } } - // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to - // ImplicitOutputs. RuleBuilder never uses "$out", so the distinction between Outputs and ImplicitOutputs - // doesn't matter. - var output WritablePath - var implicitOutputs WritablePaths - if outputs := r.Outputs(); len(outputs) > 0 { - output = outputs[0] - implicitOutputs = outputs[1:] + tools := r.Tools() + commands := r.Commands() + outputs := r.Outputs() + + if len(commands) == 0 { + return + } + if len(outputs) == 0 { + panic("No outputs specified from any Commands") } - if len(commands) > 0 { - ctx.Build(pctx, BuildParams{ - Rule: ctx.Rule(pctx, name, blueprint.RuleParams{ - Command: strings.Join(proptools.NinjaEscapeList(commands), " && "), - CommandDeps: tools.Strings(), - Restat: r.restat, - }), - Implicits: r.Inputs(), - Output: output, - ImplicitOutputs: implicitOutputs, - Depfile: depFile, - Deps: depFormat, - Description: desc, - }) + commandString := strings.Join(proptools.NinjaEscapeList(commands), " && ") + + if r.sbox { + sboxOutputs := make([]string, len(outputs)) + for i, output := range outputs { + sboxOutputs[i] = "__SBOX_OUT_DIR__/" + Rel(ctx, r.sboxOutDir.String(), output.String()) + } + + commandString = proptools.ShellEscape(commandString) + if !strings.HasPrefix(commandString, `'`) { + commandString = `'` + commandString + `'` + } + + sboxCmd := &RuleBuilderCommand{} + sboxCmd.Tool(ctx.Config().HostToolPath(ctx, "sbox")). + Flag("-c").Text(commandString). + Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())). + Flag("--output-root").Text(r.sboxOutDir.String()) + + if depFile != nil { + sboxCmd.Flag("--depfile-out").Text(depFile.String()) + } + + sboxCmd.Flags(sboxOutputs) + + commandString = string(sboxCmd.buf) + tools = append(tools, sboxCmd.tools...) } + + // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to + // ImplicitOutputs. RuleBuilder never uses "$out", so the distinction between Outputs and ImplicitOutputs + // doesn't matter. + output := outputs[0] + implicitOutputs := outputs[1:] + + ctx.Build(pctx, BuildParams{ + Rule: ctx.Rule(pctx, name, blueprint.RuleParams{ + Command: commandString, + CommandDeps: tools.Strings(), + Restat: r.restat, + }), + Implicits: r.Inputs(), + Output: output, + ImplicitOutputs: implicitOutputs, + Depfile: depFile, + Deps: depFormat, + Description: desc, + }) } // RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the @@ -312,6 +394,28 @@ type RuleBuilderCommand struct { outputs WritablePaths depFiles WritablePaths tools Paths + + sbox bool + sboxOutDir WritablePath +} + +func (c *RuleBuilderCommand) addInput(path Path) string { + if c.sbox { + if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel { + return "__SBOX_OUT_DIR__/" + rel + } + } + c.inputs = append(c.inputs, path) + return path.String() +} + +func (c *RuleBuilderCommand) outputStr(path Path) string { + if c.sbox { + // Errors will be handled in RuleBuilder.Build where we have a context to report them + rel, _, _ := maybeRelErr(c.sboxOutDir.String(), path.String()) + return "__SBOX_OUT_DIR__/" + rel + } + return path.String() } // Text adds the specified raw text to the command line. The text should not contain input or output paths or the @@ -378,8 +482,7 @@ func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand { // Input adds the specified input path to the command line. The path will also be added to the dependencies returned by // RuleBuilder.Inputs. func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand { - c.inputs = append(c.inputs, path) - return c.Text(path.String()) + return c.Text(c.addInput(path)) } // Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the @@ -394,14 +497,16 @@ func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand { // Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the // command line. func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand { - c.inputs = append(c.inputs, path) + c.addInput(path) return c } // Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the // command line. func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand { - c.inputs = append(c.inputs, paths...) + for _, path := range paths { + c.addInput(path) + } return c } @@ -409,7 +514,7 @@ func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand { // RuleBuilder.Outputs. func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand { c.outputs = append(c.outputs, path) - return c.Text(path.String()) + return c.Text(c.outputStr(path)) } // Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to @@ -426,7 +531,7 @@ func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand { // commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together. func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand { c.depFiles = append(c.depFiles, path) - return c.Text(path.String()) + return c.Text(c.outputStr(path)) } // ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying @@ -455,16 +560,18 @@ func (c *RuleBuilderCommand) ImplicitDepFile(path WritablePath) *RuleBuilderComm // FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path // will also be added to the dependencies returned by RuleBuilder.Inputs. func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand { - c.inputs = append(c.inputs, path) - return c.Text(flag + path.String()) + return c.Text(flag + c.addInput(path)) } // FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep // and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by // RuleBuilder.Inputs. func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand { - c.inputs = append(c.inputs, paths...) - return c.FlagWithList(flag, paths.Strings(), sep) + strs := make([]string, len(paths)) + for i, path := range paths { + strs[i] = c.addInput(path) + } + return c.FlagWithList(flag, strs, sep) } // FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also @@ -481,14 +588,14 @@ func (c *RuleBuilderCommand) FlagForEachInput(flag string, paths Paths) *RuleBui // will also be added to the outputs returned by RuleBuilder.Outputs. func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand { c.outputs = append(c.outputs, path) - return c.Text(flag + path.String()) + return c.Text(flag + c.outputStr(path)) } // FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path // will also be added to the outputs returned by RuleBuilder.Outputs. func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand { c.depFiles = append(c.depFiles, path) - return c.Text(flag + path.String()) + return c.Text(flag + c.outputStr(path)) } // String returns the command line. diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index 7bad0258..d122a429 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -22,6 +22,10 @@ import ( "reflect" "strings" "testing" + + "github.com/google/blueprint" + + "android/soong/shared" ) func pathContext() PathContext { @@ -234,8 +238,6 @@ func ExampleRuleBuilderCommand_FlagWithList() { } func TestRuleBuilder(t *testing.T) { - rule := NewRuleBuilder() - fs := map[string][]byte{ "dep_fixer": nil, "input": nil, @@ -249,73 +251,114 @@ func TestRuleBuilder(t *testing.T) { ctx := PathContextForTesting(TestConfig("out", nil), fs) - cmd := rule.Command(). - DepFile(PathForOutput(ctx, "DepFile")). - Flag("Flag"). - FlagWithArg("FlagWithArg=", "arg"). - FlagWithDepFile("FlagWithDepFile=", PathForOutput(ctx, "depfile")). - FlagWithInput("FlagWithInput=", PathForSource(ctx, "input")). - FlagWithOutput("FlagWithOutput=", PathForOutput(ctx, "output")). - Implicit(PathForSource(ctx, "Implicit")). - ImplicitDepFile(PathForOutput(ctx, "ImplicitDepFile")). - ImplicitOutput(PathForOutput(ctx, "ImplicitOutput")). - Input(PathForSource(ctx, "Input")). - Output(PathForOutput(ctx, "Output")). - Text("Text"). - Tool(PathForSource(ctx, "Tool")) - - rule.Command(). - Text("command2"). - DepFile(PathForOutput(ctx, "depfile2")). - Input(PathForSource(ctx, "input2")). - Output(PathForOutput(ctx, "output2")). - Tool(PathForSource(ctx, "tool2")) - - // Test updates to the first command after the second command has been started - cmd.Text("after command2") - // Test updating a command when the previous update did not replace the cmd variable - cmd.Text("old cmd") - - // Test a command that uses the output of a previous command as an input - rule.Command(). - Text("command3"). - Input(PathForSource(ctx, "input3")). - Input(PathForOutput(ctx, "output2")). - Output(PathForOutput(ctx, "output3")) - - wantCommands := []string{ - "out/DepFile Flag FlagWithArg=arg FlagWithDepFile=out/depfile FlagWithInput=input FlagWithOutput=out/output Input out/Output Text Tool after command2 old cmd", - "command2 out/depfile2 input2 out/output2 tool2", - "command3 input3 out/output2 out/output3", + addCommands := func(rule *RuleBuilder) { + cmd := rule.Command(). + DepFile(PathForOutput(ctx, "DepFile")). + Flag("Flag"). + FlagWithArg("FlagWithArg=", "arg"). + FlagWithDepFile("FlagWithDepFile=", PathForOutput(ctx, "depfile")). + FlagWithInput("FlagWithInput=", PathForSource(ctx, "input")). + FlagWithOutput("FlagWithOutput=", PathForOutput(ctx, "output")). + Implicit(PathForSource(ctx, "Implicit")). + ImplicitDepFile(PathForOutput(ctx, "ImplicitDepFile")). + ImplicitOutput(PathForOutput(ctx, "ImplicitOutput")). + Input(PathForSource(ctx, "Input")). + Output(PathForOutput(ctx, "Output")). + Text("Text"). + Tool(PathForSource(ctx, "Tool")) + + rule.Command(). + Text("command2"). + DepFile(PathForOutput(ctx, "depfile2")). + Input(PathForSource(ctx, "input2")). + Output(PathForOutput(ctx, "output2")). + Tool(PathForSource(ctx, "tool2")) + + // Test updates to the first command after the second command has been started + cmd.Text("after command2") + // Test updating a command when the previous update did not replace the cmd variable + cmd.Text("old cmd") + + // Test a command that uses the output of a previous command as an input + rule.Command(). + Text("command3"). + Input(PathForSource(ctx, "input3")). + Input(PathForOutput(ctx, "output2")). + Output(PathForOutput(ctx, "output3")) } - wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer out/DepFile out/depfile out/ImplicitDepFile out/depfile2" - wantInputs := PathsForSource(ctx, []string{"Implicit", "Input", "input", "input2", "input3"}) wantOutputs := PathsForOutput(ctx, []string{"ImplicitOutput", "Output", "output", "output2", "output3"}) wantDepFiles := PathsForOutput(ctx, []string{"DepFile", "depfile", "ImplicitDepFile", "depfile2"}) wantTools := PathsForSource(ctx, []string{"Tool", "tool2"}) - if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) { - t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g) - } + t.Run("normal", func(t *testing.T) { + rule := NewRuleBuilder() + addCommands(rule) - if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w { - t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g) - } + wantCommands := []string{ + "out/DepFile Flag FlagWithArg=arg FlagWithDepFile=out/depfile FlagWithInput=input FlagWithOutput=out/output Input out/Output Text Tool after command2 old cmd", + "command2 out/depfile2 input2 out/output2 tool2", + "command3 input3 out/output2 out/output3", + } - if g, w := rule.Inputs(), wantInputs; !reflect.DeepEqual(w, g) { - t.Errorf("\nwant rule.Inputs() = %#v\n got %#v", w, g) - } - if g, w := rule.Outputs(), wantOutputs; !reflect.DeepEqual(w, g) { - t.Errorf("\nwant rule.Outputs() = %#v\n got %#v", w, g) - } - if g, w := rule.DepFiles(), wantDepFiles; !reflect.DeepEqual(w, g) { - t.Errorf("\nwant rule.DepFiles() = %#v\n got %#v", w, g) - } - if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) { - t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g) - } + wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer out/DepFile out/depfile out/ImplicitDepFile out/depfile2" + + if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) { + t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g) + } + + if g, w := rule.Inputs(), wantInputs; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.Inputs() = %#v\n got %#v", w, g) + } + if g, w := rule.Outputs(), wantOutputs; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.Outputs() = %#v\n got %#v", w, g) + } + if g, w := rule.DepFiles(), wantDepFiles; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.DepFiles() = %#v\n got %#v", w, g) + } + if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g) + } + + if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w { + t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g) + } + }) + + t.Run("sbox", func(t *testing.T) { + rule := NewRuleBuilder().Sbox(PathForOutput(ctx)) + addCommands(rule) + + wantCommands := []string{ + "__SBOX_OUT_DIR__/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_OUT_DIR__/depfile FlagWithInput=input FlagWithOutput=__SBOX_OUT_DIR__/output Input __SBOX_OUT_DIR__/Output Text Tool after command2 old cmd", + "command2 __SBOX_OUT_DIR__/depfile2 input2 __SBOX_OUT_DIR__/output2 tool2", + "command3 input3 __SBOX_OUT_DIR__/output2 __SBOX_OUT_DIR__/output3", + } + + wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_OUT_DIR__/DepFile __SBOX_OUT_DIR__/depfile __SBOX_OUT_DIR__/ImplicitDepFile __SBOX_OUT_DIR__/depfile2" + + if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) { + t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g) + } + + if g, w := rule.Inputs(), wantInputs; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.Inputs() = %#v\n got %#v", w, g) + } + if g, w := rule.Outputs(), wantOutputs; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.Outputs() = %#v\n got %#v", w, g) + } + if g, w := rule.DepFiles(), wantDepFiles; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.DepFiles() = %#v\n got %#v", w, g) + } + if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g) + } + + if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w { + t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g) + } + }) } func testRuleBuilderFactory() Module { @@ -329,14 +372,19 @@ type testRuleBuilderModule struct { ModuleBase properties struct { Src string + + Restat bool + Sbox bool } } func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) { in := PathForSource(ctx, t.properties.Src) out := PathForModuleOut(ctx, ctx.ModuleName()) + outDep := PathForModuleOut(ctx, ctx.ModuleName()+".d") + outDir := PathForModuleOut(ctx) - testRuleBuilder_Build(ctx, in, out) + testRuleBuilder_Build(ctx, in, out, outDep, outDir, t.properties.Restat, t.properties.Sbox) } type testRuleBuilderSingleton struct{} @@ -348,15 +396,23 @@ func testRuleBuilderSingletonFactory() Singleton { func (t *testRuleBuilderSingleton) GenerateBuildActions(ctx SingletonContext) { in := PathForSource(ctx, "bar") out := PathForOutput(ctx, "baz") - testRuleBuilder_Build(ctx, in, out) + outDep := PathForOutput(ctx, "baz.d") + outDir := PathForOutput(ctx) + testRuleBuilder_Build(ctx, in, out, outDep, outDir, true, false) } -func testRuleBuilder_Build(ctx BuilderContext, in Path, out WritablePath) { +func testRuleBuilder_Build(ctx BuilderContext, in Path, out, outDep, outDir WritablePath, restat, sbox bool) { rule := NewRuleBuilder() - rule.Command().Tool(PathForSource(ctx, "cp")).Input(in).Output(out) + if sbox { + rule.Sbox(outDir) + } + + rule.Command().Tool(PathForSource(ctx, "cp")).Input(in).Output(out).ImplicitDepFile(outDep) - rule.Restat() + if restat { + rule.Restat() + } rule.Build(pctx, ctx, "rule", "desc") } @@ -372,6 +428,12 @@ func TestRuleBuilder_Build(t *testing.T) { rule_builder_test { name: "foo", src: "bar", + restat: true, + } + rule_builder_test { + name: "foo_sbox", + src: "bar", + sbox: true, } ` @@ -391,9 +453,19 @@ func TestRuleBuilder_Build(t *testing.T) { _, errs = ctx.PrepareBuildActions(config) FailIfErrored(t, errs) - check := func(t *testing.T, params TestingBuildParams, wantOutput string) { - if len(params.RuleParams.CommandDeps) != 1 || params.RuleParams.CommandDeps[0] != "cp" { - t.Errorf("want RuleParams.CommandDeps = [%q], got %q", "cp", params.RuleParams.CommandDeps) + check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraCmdDeps []string) { + t.Helper() + if params.RuleParams.Command != wantCommand { + t.Errorf("\nwant RuleParams.Command = %q\n got %q", wantCommand, params.RuleParams.Command) + } + + wantDeps := append([]string{"cp"}, extraCmdDeps...) + if !reflect.DeepEqual(params.RuleParams.CommandDeps, wantDeps) { + t.Errorf("\nwant RuleParams.CommandDeps = %q\n got %q", wantDeps, params.RuleParams.CommandDeps) + } + + if params.RuleParams.Restat != wantRestat { + t.Errorf("want RuleParams.Restat = %v, got %v", wantRestat, params.RuleParams.Restat) } if len(params.Implicits) != 1 || params.Implicits[0].String() != "bar" { @@ -404,17 +476,40 @@ func TestRuleBuilder_Build(t *testing.T) { t.Errorf("want Output = %q, got %q", wantOutput, params.Output) } - if !params.RuleParams.Restat { - t.Errorf("want RuleParams.Restat = true, got %v", params.RuleParams.Restat) + if len(params.ImplicitOutputs) != 0 { + t.Errorf("want ImplicitOutputs = [], got %q", params.ImplicitOutputs.Strings()) + } + + if params.Depfile.String() != wantDepfile { + t.Errorf("want Depfile = %q, got %q", wantDepfile, params.Depfile) + } + + if params.Deps != blueprint.DepsGCC { + t.Errorf("want Deps = %q, got %q", blueprint.DepsGCC, params.Deps) } } t.Run("module", func(t *testing.T) { + outFile := filepath.Join(buildDir, ".intermediates", "foo", "foo") check(t, ctx.ModuleForTests("foo", "").Rule("rule"), - filepath.Join(buildDir, ".intermediates", "foo", "foo")) + "cp bar "+outFile, + outFile, outFile+".d", true, nil) + }) + t.Run("sbox", func(t *testing.T) { + outDir := filepath.Join(buildDir, ".intermediates", "foo_sbox") + outFile := filepath.Join(outDir, "foo_sbox") + depFile := filepath.Join(outDir, "foo_sbox.d") + sbox := filepath.Join(buildDir, "host", config.PrebuiltOS(), "bin/sbox") + sandboxPath := shared.TempDirForOutDir(buildDir) + + cmd := sbox + ` -c 'cp bar __SBOX_OUT_DIR__/foo_sbox' --sandbox-path ` + sandboxPath + " --output-root " + outDir + " --depfile-out " + depFile + " __SBOX_OUT_DIR__/foo_sbox" + + check(t, ctx.ModuleForTests("foo_sbox", "").Rule("rule"), + cmd, outFile, depFile, false, []string{sbox}) }) t.Run("singleton", func(t *testing.T) { + outFile := filepath.Join(buildDir, "baz") check(t, ctx.SingletonForTests("rule_builder_test").Rule("rule"), - filepath.Join(buildDir, "baz")) + "cp bar "+outFile, outFile, outFile+".d", true, nil) }) } diff --git a/android/util.go b/android/util.go index f9dce6fe..b17422df 100644 --- a/android/util.go +++ b/android/util.go @@ -243,3 +243,36 @@ func matchPattern(pat, str string) bool { } return strings.HasPrefix(str, pat[:i]) && strings.HasSuffix(str, pat[i+1:]) } + +// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. +func ShardPaths(paths Paths, shardSize int) []Paths { + if len(paths) == 0 { + return nil + } + ret := make([]Paths, 0, (len(paths)+shardSize-1)/shardSize) + for len(paths) > shardSize { + ret = append(ret, paths[0:shardSize]) + paths = paths[shardSize:] + } + if len(paths) > 0 { + ret = append(ret, paths) + } + return ret +} + +// ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize +// elements. +func ShardStrings(s []string, shardSize int) [][]string { + if len(s) == 0 { + return nil + } + ret := make([][]string, 0, (len(s)+shardSize-1)/shardSize) + for len(s) > shardSize { + ret = append(ret, s[0:shardSize]) + s = s[shardSize:] + } + if len(s) > 0 { + ret = append(ret, s) + } + return ret +} diff --git a/android/util_test.go b/android/util_test.go index 2e5eb07e..7f0d331c 100644 --- a/android/util_test.go +++ b/android/util_test.go @@ -404,3 +404,102 @@ func ExampleCopyOf_append() { // b = ["foo" "bar"] // c = ["foo" "baz"] } + +func Test_Shard(t *testing.T) { + type args struct { + strings []string + shardSize int + } + tests := []struct { + name string + args args + want [][]string + }{ + { + name: "empty", + args: args{ + strings: nil, + shardSize: 1, + }, + want: [][]string(nil), + }, + { + name: "single shard", + args: args{ + strings: []string{"a", "b"}, + shardSize: 2, + }, + want: [][]string{{"a", "b"}}, + }, + { + name: "single short shard", + args: args{ + strings: []string{"a", "b"}, + shardSize: 3, + }, + want: [][]string{{"a", "b"}}, + }, + { + name: "shard per input", + args: args{ + strings: []string{"a", "b", "c"}, + shardSize: 1, + }, + want: [][]string{{"a"}, {"b"}, {"c"}}, + }, + { + name: "balanced shards", + args: args{ + strings: []string{"a", "b", "c", "d"}, + shardSize: 2, + }, + want: [][]string{{"a", "b"}, {"c", "d"}}, + }, + { + name: "unbalanced shards", + args: args{ + strings: []string{"a", "b", "c"}, + shardSize: 2, + }, + want: [][]string{{"a", "b"}, {"c"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Run("strings", func(t *testing.T) { + if got := ShardStrings(tt.args.strings, tt.args.shardSize); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ShardStrings(%v, %v) = %v, want %v", + tt.args.strings, tt.args.shardSize, got, tt.want) + } + }) + + t.Run("paths", func(t *testing.T) { + stringsToPaths := func(strings []string) Paths { + if strings == nil { + return nil + } + paths := make(Paths, len(strings)) + for i, s := range strings { + paths[i] = PathForTesting(s) + } + return paths + } + + paths := stringsToPaths(tt.args.strings) + + var want []Paths + if sWant := tt.want; sWant != nil { + want = make([]Paths, len(sWant)) + for i, w := range sWant { + want[i] = stringsToPaths(w) + } + } + + if got := ShardPaths(paths, tt.args.shardSize); !reflect.DeepEqual(got, want) { + t.Errorf("ShardPaths(%v, %v) = %v, want %v", + paths, tt.args.shardSize, got, want) + } + }) + }) + } +} diff --git a/cmd/dep_fixer/Android.bp b/cmd/dep_fixer/Android.bp index d2d1113d..97364d58 100644 --- a/cmd/dep_fixer/Android.bp +++ b/cmd/dep_fixer/Android.bp @@ -14,10 +14,6 @@ blueprint_go_binary { name: "dep_fixer", - deps: ["androidmk-parser"], - srcs: [ - "main.go", - "deps.go", - ], - testSrcs: ["deps_test.go"], + deps: ["soong-makedeps"], + srcs: ["main.go"], } diff --git a/cmd/dep_fixer/main.go b/cmd/dep_fixer/main.go index f94cf2fd..d1bd1391 100644 --- a/cmd/dep_fixer/main.go +++ b/cmd/dep_fixer/main.go @@ -25,6 +25,8 @@ import ( "io/ioutil" "log" "os" + + "android/soong/makedeps" ) func main() { @@ -39,7 +41,7 @@ func main() { log.Fatal("Expected at least one input file as an argument") } - var mergedDeps *Deps + var mergedDeps *makedeps.Deps var firstInput []byte for i, arg := range flag.Args() { @@ -48,7 +50,7 @@ func main() { log.Fatalf("Error opening %q: %v", arg, err) } - deps, err := Parse(arg, bytes.NewBuffer(append([]byte(nil), input...))) + deps, err := makedeps.Parse(arg, bytes.NewBuffer(append([]byte(nil), input...))) if err != nil { log.Fatalf("Failed to parse: %v", err) } diff --git a/cmd/sbox/Android.bp b/cmd/sbox/Android.bp index fe4c7bbc..a706810d 100644 --- a/cmd/sbox/Android.bp +++ b/cmd/sbox/Android.bp @@ -14,6 +14,7 @@ blueprint_go_binary { name: "sbox", + deps: ["soong-makedeps"], srcs: [ "sbox.go", ], diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go index 4167edb3..7057b33f 100644 --- a/cmd/sbox/sbox.go +++ b/cmd/sbox/sbox.go @@ -15,6 +15,7 @@ package main import ( + "bytes" "errors" "flag" "fmt" @@ -25,6 +26,8 @@ import ( "path/filepath" "strings" "time" + + "android/soong/makedeps" ) var ( @@ -56,7 +59,7 @@ func usageViolation(violation string) { } fmt.Fprintf(os.Stderr, - "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> --overwrite [--depfile-out depFile] <outputFile> [<outputFile>...]\n"+ + "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> [--depfile-out depFile] <outputFile> [<outputFile>...]\n"+ "\n"+ "Deletes <outputRoot>,"+ "runs <commandToRun>,"+ @@ -152,9 +155,6 @@ func run() error { return err } allOutputs = append(allOutputs, sandboxedDepfile) - if !strings.Contains(rawCommand, "__SBOX_DEPFILE__") { - return fmt.Errorf("the --depfile-out argument only makes sense if the command contains the text __SBOX_DEPFILE__") - } rawCommand = strings.Replace(rawCommand, "__SBOX_DEPFILE__", filepath.Join(tempDir, sandboxedDepfile), -1) } @@ -281,6 +281,26 @@ func run() error { } } + // Rewrite the depfile so that it doesn't include the (randomized) sandbox directory + if depfileOut != "" { + in, err := ioutil.ReadFile(depfileOut) + if err != nil { + return err + } + + deps, err := makedeps.Parse(depfileOut, bytes.NewBuffer(in)) + if err != nil { + return err + } + + deps.Output = "outputfile" + + err = ioutil.WriteFile(depfileOut, deps.Print(), 0666) + if err != nil { + return err + } + } + // TODO(jeffrygaston) if a process creates more output files than it declares, should there be a warning? return nil } diff --git a/genrule/genrule.go b/genrule/genrule.go index 87e6747e..55c7e628 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -17,6 +17,7 @@ package genrule import ( "fmt" "io" + "strconv" "strings" "github.com/google/blueprint" @@ -37,10 +38,20 @@ func init() { var ( pctx = android.NewPackageContext("android/soong/genrule") + + gensrcsMerge = pctx.AndroidStaticRule("gensrcsMerge", blueprint.RuleParams{ + Command: "${soongZip} -o ${tmpZip} @${tmpZip}.rsp && ${zipSync} -d ${genDir} ${tmpZip}", + CommandDeps: []string{"${soongZip}", "${zipSync}"}, + Rspfile: "${tmpZip}.rsp", + RspfileContent: "${zipArgs}", + }, "tmpZip", "genDir", "zipArgs") ) func init() { pctx.HostBinToolVariable("sboxCmd", "sbox") + + pctx.HostBinToolVariable("soongZip", "soong_zip") + pctx.HostBinToolVariable("zipSync", "zipsync") } type SourceFileGenerator interface { @@ -110,9 +121,9 @@ type Module struct { taskGenerator taskFunc - deps android.Paths - rule blueprint.Rule - rawCommand string + deps android.Paths + rule blueprint.Rule + rawCommands []string exportedIncludeDirs android.Paths @@ -120,15 +131,20 @@ type Module struct { outputDeps android.Paths subName string + subDir string } -type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask +type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask type generateTask struct { in android.Paths out android.WritablePaths + copyTo android.WritablePaths + genDir android.WritablePath sandboxOuts []string cmd string + shard int + shards int } func (g *Module) GeneratedSourceFiles() android.Paths { @@ -167,10 +183,10 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { if len(g.properties.Export_include_dirs) > 0 { for _, dir := range g.properties.Export_include_dirs { g.exportedIncludeDirs = append(g.exportedIncludeDirs, - android.PathForModuleGen(ctx, ctx.ModuleDir(), dir)) + android.PathForModuleGen(ctx, g.subDir, ctx.ModuleDir(), dir)) } } else { - g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, "")) + g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, g.subDir)) } locationLabels := map[string][]string{} @@ -275,120 +291,154 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } - task := g.taskGenerator(ctx, String(g.properties.Cmd), srcFiles) - - for _, out := range task.out { - addLocationLabel(out.Rel(), []string{filepath.Join("__SBOX_OUT_DIR__", out.Rel())}) - } - - referencedDepfile := false + var copyFrom android.Paths + var outputFiles android.WritablePaths + var zipArgs strings.Builder - rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) { - // report the error directly without returning an error to android.Expand to catch multiple errors in a - // single run - reportError := func(fmt string, args ...interface{}) (string, error) { - ctx.PropertyErrorf("cmd", fmt, args...) - return "SOONG_ERROR", nil + for _, task := range g.taskGenerator(ctx, String(g.properties.Cmd), srcFiles) { + for _, out := range task.out { + addLocationLabel(out.Rel(), []string{filepath.Join("__SBOX_OUT_DIR__", out.Rel())}) } - switch name { - case "location": - if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 { - return reportError("at least one `tools` or `tool_files` is required if $(location) is used") - } - paths := locationLabels[firstLabel] - if len(paths) == 0 { - return reportError("default label %q has no files", firstLabel) - } else if len(paths) > 1 { - return reportError("default label %q has multiple files, use $(locations %s) to reference it", - firstLabel, firstLabel) - } - return locationLabels[firstLabel][0], nil - case "in": - return "${in}", nil - case "out": - return "__SBOX_OUT_FILES__", nil - case "depfile": - referencedDepfile = true - if !Bool(g.properties.Depfile) { - return reportError("$(depfile) used without depfile property") + referencedDepfile := false + + rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) { + // report the error directly without returning an error to android.Expand to catch multiple errors in a + // single run + reportError := func(fmt string, args ...interface{}) (string, error) { + ctx.PropertyErrorf("cmd", fmt, args...) + return "SOONG_ERROR", nil } - return "__SBOX_DEPFILE__", nil - case "genDir": - return "__SBOX_OUT_DIR__", nil - default: - if strings.HasPrefix(name, "location ") { - label := strings.TrimSpace(strings.TrimPrefix(name, "location ")) - if paths, ok := locationLabels[label]; ok { - if len(paths) == 0 { - return reportError("label %q has no files", label) - } else if len(paths) > 1 { - return reportError("label %q has multiple files, use $(locations %s) to reference it", - label, label) - } - return paths[0], nil - } else { - return reportError("unknown location label %q", label) + + switch name { + case "location": + if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 { + return reportError("at least one `tools` or `tool_files` is required if $(location) is used") + } + paths := locationLabels[firstLabel] + if len(paths) == 0 { + return reportError("default label %q has no files", firstLabel) + } else if len(paths) > 1 { + return reportError("default label %q has multiple files, use $(locations %s) to reference it", + firstLabel, firstLabel) + } + return locationLabels[firstLabel][0], nil + case "in": + return "${in}", nil + case "out": + return "__SBOX_OUT_FILES__", nil + case "depfile": + referencedDepfile = true + if !Bool(g.properties.Depfile) { + return reportError("$(depfile) used without depfile property") } - } else if strings.HasPrefix(name, "locations ") { - label := strings.TrimSpace(strings.TrimPrefix(name, "locations ")) - if paths, ok := locationLabels[label]; ok { - if len(paths) == 0 { - return reportError("label %q has no files", label) + return "__SBOX_DEPFILE__", nil + case "genDir": + return "__SBOX_OUT_DIR__", nil + default: + if strings.HasPrefix(name, "location ") { + label := strings.TrimSpace(strings.TrimPrefix(name, "location ")) + if paths, ok := locationLabels[label]; ok { + if len(paths) == 0 { + return reportError("label %q has no files", label) + } else if len(paths) > 1 { + return reportError("label %q has multiple files, use $(locations %s) to reference it", + label, label) + } + return paths[0], nil + } else { + return reportError("unknown location label %q", label) + } + } else if strings.HasPrefix(name, "locations ") { + label := strings.TrimSpace(strings.TrimPrefix(name, "locations ")) + if paths, ok := locationLabels[label]; ok { + if len(paths) == 0 { + return reportError("label %q has no files", label) + } + return strings.Join(paths, " "), nil + } else { + return reportError("unknown locations label %q", label) } - return strings.Join(paths, " "), nil } else { - return reportError("unknown locations label %q", label) + return reportError("unknown variable '$(%s)'", name) } - } else { - return reportError("unknown variable '$(%s)'", name) } + }) + + if err != nil { + ctx.PropertyErrorf("cmd", "%s", err.Error()) + return } - }) - if err != nil { - ctx.PropertyErrorf("cmd", "%s", err.Error()) - return - } + if Bool(g.properties.Depfile) && !referencedDepfile { + ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd") + return + } - if Bool(g.properties.Depfile) && !referencedDepfile { - ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd") - } + // tell the sbox command which directory to use as its sandbox root + buildDir := android.PathForOutput(ctx).String() + sandboxPath := shared.TempDirForOutDir(buildDir) - // tell the sbox command which directory to use as its sandbox root - buildDir := android.PathForOutput(ctx).String() - sandboxPath := shared.TempDirForOutDir(buildDir) + // recall that Sprintf replaces percent sign expressions, whereas dollar signs expressions remain as written, + // to be replaced later by ninja_strings.go + depfilePlaceholder := "" + if Bool(g.properties.Depfile) { + depfilePlaceholder = "$depfileArgs" + } - // recall that Sprintf replaces percent sign expressions, whereas dollar signs expressions remain as written, - // to be replaced later by ninja_strings.go - depfilePlaceholder := "" - if Bool(g.properties.Depfile) { - depfilePlaceholder = "$depfileArgs" - } + // Escape the command for the shell + rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'" + g.rawCommands = append(g.rawCommands, rawCommand) + sandboxCommand := fmt.Sprintf("rm -rf %s && $sboxCmd --sandbox-path %s --output-root %s -c %s %s $allouts", + task.genDir, sandboxPath, task.genDir, rawCommand, depfilePlaceholder) - genDir := android.PathForModuleGen(ctx) - // Escape the command for the shell - rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'" - g.rawCommand = rawCommand - sandboxCommand := fmt.Sprintf("$sboxCmd --sandbox-path %s --output-root %s -c %s %s $allouts", - sandboxPath, genDir, rawCommand, depfilePlaceholder) + ruleParams := blueprint.RuleParams{ + Command: sandboxCommand, + CommandDeps: []string{"$sboxCmd"}, + } + args := []string{"allouts"} + if Bool(g.properties.Depfile) { + ruleParams.Deps = blueprint.DepsGCC + args = append(args, "depfileArgs") + } + name := "generator" + if task.shards > 1 { + name += strconv.Itoa(task.shard) + } + rule := ctx.Rule(pctx, name, ruleParams, args...) - ruleParams := blueprint.RuleParams{ - Command: sandboxCommand, - CommandDeps: []string{"$sboxCmd"}, - } - args := []string{"allouts"} - if Bool(g.properties.Depfile) { - ruleParams.Deps = blueprint.DepsGCC - args = append(args, "depfileArgs") + g.generateSourceFile(ctx, task, rule) + + if len(task.copyTo) > 0 { + outputFiles = append(outputFiles, task.copyTo...) + copyFrom = append(copyFrom, task.out.Paths()...) + zipArgs.WriteString(" -C " + task.genDir.String()) + zipArgs.WriteString(android.JoinWithPrefix(task.out.Strings(), " -f ")) + } else { + outputFiles = append(outputFiles, task.out...) + } } - g.rule = ctx.Rule(pctx, "generator", ruleParams, args...) - g.generateSourceFile(ctx, task) + if len(copyFrom) > 0 { + ctx.Build(pctx, android.BuildParams{ + Rule: gensrcsMerge, + Implicits: copyFrom, + Outputs: outputFiles, + Args: map[string]string{ + "zipArgs": zipArgs.String(), + "tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(), + "genDir": android.PathForModuleGen(ctx, g.subDir).String(), + }, + }) + } + g.outputFiles = outputFiles.Paths() + if len(g.outputFiles) > 0 { + g.outputDeps = append(g.outputDeps, g.outputFiles[0]) + } } -func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask) { +func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask, rule blueprint.Rule) { desc := "generate" if len(task.out) == 0 { ctx.ModuleErrorf("must have at least one output file") @@ -403,9 +453,13 @@ func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask depFile = android.PathForModuleGen(ctx, task.out[0].Rel()+".d") } + if task.shards > 1 { + desc += " " + strconv.Itoa(task.shard) + } + params := android.BuildParams{ - Rule: g.rule, - Description: "generate", + Rule: rule, + Description: desc, Output: task.out[0], ImplicitOutputs: task.out[1:], Inputs: task.in, @@ -420,11 +474,6 @@ func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask } ctx.Build(pctx, params) - - for _, outputFile := range task.out { - g.outputFiles = append(g.outputFiles, outputFile) - } - g.outputDeps = append(g.outputDeps, task.out[0]) } // Collect information for opening IDE project files in java/jdeps.go. @@ -446,7 +495,7 @@ func (g *Module) AndroidMk() android.AndroidMkData { SubName: g.subName, Extra: []android.AndroidMkExtraFunc{ func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES :=", strings.Join(g.outputFiles.Strings(), " ")) + fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES :=", strings.Join(g.outputDeps.Strings(), " ")) }, }, Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { @@ -483,47 +532,80 @@ func pathToSandboxOut(path android.Path, genDir android.Path) string { func NewGenSrcs() *Module { properties := &genSrcsProperties{} - taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask { - commands := []string{} - outFiles := android.WritablePaths{} - genDir := android.PathForModuleGen(ctx) - sandboxOuts := []string{} - for _, in := range srcFiles { - outFile := android.GenPathWithExt(ctx, "", in, String(properties.Output_extension)) - outFiles = append(outFiles, outFile) - - sandboxOutfile := pathToSandboxOut(outFile, genDir) - sandboxOuts = append(sandboxOuts, sandboxOutfile) - - command, err := android.Expand(rawCommand, func(name string) (string, error) { - switch name { - case "in": - return in.String(), nil - case "out": - return sandboxOutfile, nil - default: - return "$(" + name + ")", nil - } - }) - if err != nil { - ctx.PropertyErrorf("cmd", err.Error()) + taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask { + genDir := android.PathForModuleGen(ctx, "gensrcs") + shardSize := defaultShardSize + if s := properties.Shard_size; s != nil { + shardSize = int(*s) + } + + shards := android.ShardPaths(srcFiles, shardSize) + var generateTasks []generateTask + + for i, shard := range shards { + var commands []string + var outFiles android.WritablePaths + var copyTo android.WritablePaths + var shardDir android.WritablePath + var sandboxOuts []string + + if len(shards) > 1 { + shardDir = android.PathForModuleGen(ctx, strconv.Itoa(i)) + } else { + shardDir = genDir } - // escape the command in case for example it contains '#', an odd number of '"', etc - command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command)) - commands = append(commands, command) - } - fullCommand := strings.Join(commands, " && ") + for _, in := range shard { + outFile := android.GenPathWithExt(ctx, "gensrcs", in, String(properties.Output_extension)) + sandboxOutfile := pathToSandboxOut(outFile, genDir) - return generateTask{ - in: srcFiles, - out: outFiles, - sandboxOuts: sandboxOuts, - cmd: fullCommand, + if len(shards) > 1 { + shardFile := android.GenPathWithExt(ctx, strconv.Itoa(i), in, String(properties.Output_extension)) + copyTo = append(copyTo, outFile) + outFile = shardFile + } + + outFiles = append(outFiles, outFile) + sandboxOuts = append(sandboxOuts, sandboxOutfile) + + command, err := android.Expand(rawCommand, func(name string) (string, error) { + switch name { + case "in": + return in.String(), nil + case "out": + return sandboxOutfile, nil + default: + return "$(" + name + ")", nil + } + }) + if err != nil { + ctx.PropertyErrorf("cmd", err.Error()) + } + + // escape the command in case for example it contains '#', an odd number of '"', etc + command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command)) + commands = append(commands, command) + } + fullCommand := strings.Join(commands, " && ") + + generateTasks = append(generateTasks, generateTask{ + in: shard, + out: outFiles, + copyTo: copyTo, + genDir: shardDir, + sandboxOuts: sandboxOuts, + cmd: fullCommand, + shard: i, + shards: len(shards), + }) } + + return generateTasks } - return generatorFactory(taskGenerator, properties) + g := generatorFactory(taskGenerator, properties) + g.subDir = "gensrcs" + return g } func GenSrcsFactory() android.Module { @@ -535,12 +617,17 @@ func GenSrcsFactory() android.Module { type genSrcsProperties struct { // extension that will be substituted for each output file Output_extension *string + + // maximum number of files that will be passed on a single command line. + Shard_size *int64 } +const defaultShardSize = 100 + func NewGenRule() *Module { properties := &genRuleProperties{} - taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask { + taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask { outs := make(android.WritablePaths, len(properties.Out)) sandboxOuts := make([]string, len(properties.Out)) genDir := android.PathForModuleGen(ctx) @@ -548,12 +635,13 @@ func NewGenRule() *Module { outs[i] = android.PathForModuleGen(ctx, out) sandboxOuts[i] = pathToSandboxOut(outs[i], genDir) } - return generateTask{ + return []generateTask{{ in: srcFiles, out: outs, + genDir: android.PathForModuleGen(ctx), sandboxOuts: sandboxOuts, cmd: rawCommand, - } + }} } return generatorFactory(taskGenerator, properties) diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index 0b6952f3..e8dc3b55 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -57,6 +57,7 @@ func testContext(config android.Config, bp string, ctx := android.NewTestArchContext() ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory)) ctx.RegisterModuleType("genrule", android.ModuleFactoryAdaptor(GenRuleFactory)) + ctx.RegisterModuleType("gensrcs", android.ModuleFactoryAdaptor(GenSrcsFactory)) ctx.RegisterModuleType("genrule_defaults", android.ModuleFactoryAdaptor(defaultsFactory)) ctx.RegisterModuleType("tool", android.ModuleFactoryAdaptor(toolFactory)) ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) @@ -109,6 +110,9 @@ func testContext(config android.Config, bp string, "tool_file2": nil, "in1": nil, "in2": nil, + "in1.txt": nil, + "in2.txt": nil, + "in3.txt": nil, } for k, v := range fs { @@ -491,11 +495,102 @@ func TestGenruleCmd(t *testing.T) { } gen := ctx.ModuleForTests("gen", "").Module().(*Module) - if g, w := gen.rawCommand, "'"+test.expect+"'"; w != g { + if g, w := gen.rawCommands[0], "'"+test.expect+"'"; w != g { t.Errorf("want %q, got %q", w, g) } }) } +} + +func TestGenSrcs(t *testing.T) { + testcases := []struct { + name string + prop string + + allowMissingDependencies bool + + err string + cmds []string + deps []string + files []string + }{ + { + name: "gensrcs", + prop: ` + tools: ["tool"], + srcs: ["in1.txt", "in2.txt"], + cmd: "$(location) $(in) > $(out)", + `, + cmds: []string{ + "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''", + }, + deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"}, + files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"}, + }, + { + name: "shards", + prop: ` + tools: ["tool"], + srcs: ["in1.txt", "in2.txt", "in3.txt"], + cmd: "$(location) $(in) > $(out)", + shard_size: 2, + `, + cmds: []string{ + "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''", + "'bash -c '\\''out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'\\'''", + }, + deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"}, + files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"}, + }, + } + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + config := android.TestArchConfig(buildDir, nil) + bp := "gensrcs {\n" + bp += `name: "gen",` + "\n" + bp += `output_extension: "h",` + "\n" + bp += test.prop + bp += "}\n" + + ctx := testContext(config, bp, nil) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + if errs == nil { + _, errs = ctx.PrepareBuildActions(config) + } + if errs == nil && test.err != "" { + t.Fatalf("want error %q, got no error", test.err) + } else if errs != nil && test.err == "" { + android.FailIfErrored(t, errs) + } else if test.err != "" { + if len(errs) != 1 { + t.Errorf("want 1 error, got %d errors:", len(errs)) + for _, err := range errs { + t.Errorf(" %s", err.Error()) + } + t.FailNow() + } + if !strings.Contains(errs[0].Error(), test.err) { + t.Fatalf("want %q, got %q", test.err, errs[0].Error()) + } + return + } + + gen := ctx.ModuleForTests("gen", "").Module().(*Module) + if g, w := gen.rawCommands, test.cmds; !reflect.DeepEqual(w, g) { + t.Errorf("want %q, got %q", w, g) + } + + if g, w := gen.outputDeps.Strings(), test.deps; !reflect.DeepEqual(w, g) { + t.Errorf("want deps %q, got %q", w, g) + } + + if g, w := gen.outputFiles.Strings(), test.files; !reflect.DeepEqual(w, g) { + t.Errorf("want files %q, got %q", w, g) + } + }) + } } @@ -529,8 +624,8 @@ func TestGenruleDefaults(t *testing.T) { gen := ctx.ModuleForTests("gen", "").Module().(*Module) expectedCmd := "'cp ${in} __SBOX_OUT_FILES__'" - if gen.rawCommand != expectedCmd { - t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommand) + if gen.rawCommands[0] != expectedCmd { + t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommands[0]) } expectedSrcs := []string{"in1"} diff --git a/java/aapt2.go b/java/aapt2.go index bcc8e976..f21408f9 100644 --- a/java/aapt2.go +++ b/java/aapt2.go @@ -61,7 +61,7 @@ var aapt2CompileRule = pctx.AndroidStaticRule("aapt2Compile", "outDir", "cFlags") func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths) android.WritablePaths { - shards := shardPaths(paths, AAPT2_SHARD_SIZE) + shards := android.ShardPaths(paths, AAPT2_SHARD_SIZE) ret := make(android.WritablePaths, 0, len(paths)) diff --git a/java/java.go b/java/java.go index bf738c4d..9ac38c92 100644 --- a/java/java.go +++ b/java/java.go @@ -530,18 +530,6 @@ func hasSrcExt(srcs []string, ext string) bool { return false } -func shardPaths(paths android.Paths, shardSize int) []android.Paths { - ret := make([]android.Paths, 0, (len(paths)+shardSize-1)/shardSize) - for len(paths) > shardSize { - ret = append(ret, paths[0:shardSize]) - paths = paths[shardSize:] - } - if len(paths) > 0 { - ret = append(ret, paths) - } - return ret -} - func (j *Module) hasSrcExt(ext string) bool { return hasSrcExt(j.properties.Srcs, ext) } @@ -1088,7 +1076,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path shardSize := int(*(j.properties.Javac_shard_size)) var shardSrcs []android.Paths if len(uniqueSrcFiles) > 0 { - shardSrcs = shardPaths(uniqueSrcFiles, shardSize) + shardSrcs = android.ShardPaths(uniqueSrcFiles, shardSize) for idx, shardSrc := range shardSrcs { classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx)) TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps) diff --git a/makedeps/Android.bp b/makedeps/Android.bp new file mode 100644 index 00000000..b77b08f0 --- /dev/null +++ b/makedeps/Android.bp @@ -0,0 +1,21 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +bootstrap_go_package { + name: "soong-makedeps", + pkgPath: "android/soong/makedeps", + deps: ["androidmk-parser"], + srcs: ["deps.go"], + testSrcs: ["deps_test.go"], +} diff --git a/cmd/dep_fixer/deps.go b/makedeps/deps.go index 64c97f52..e64e6f78 100644 --- a/cmd/dep_fixer/deps.go +++ b/makedeps/deps.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package makedeps import ( "bytes" diff --git a/cmd/dep_fixer/deps_test.go b/makedeps/deps_test.go index 0a779b76..a32df650 100644 --- a/cmd/dep_fixer/deps_test.go +++ b/makedeps/deps_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package makedeps import ( "bytes" |