Merge pull request #132 from danw/bootstrap

Fix Soong glob regeneration and other bootstrap cleanups

4 files changed, 111 insertions, 165 deletions

diff --git a/blueprint.bash b/blueprint.bash
index d51f18c..924838c 100755
--- a/blueprint.bash
+++ b/blueprint.bash
@@ -53,10 +53,10 @@ else
 fi
 
 # Build minibp and the primary build.ninja
-"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.minibootstrap/build.ninja" "${BUILDDIR}/.bootstrap/build.ninja"
+"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.minibootstrap/build.ninja"
 
 # Build the primary builder and the main build.ninja
-"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.bootstrap/build.ninja" "${BUILDDIR}/build.ninja"
+"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.bootstrap/build.ninja"
 
 # SKIP_NINJA can be used by wrappers that wish to run ninja themselves.
 if [ -z "$SKIP_NINJA" ]; then
diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go
index 41c3485..9161731 100644
--- a/bootstrap/bootstrap.go
+++ b/bootstrap/bootstrap.go
@@ -96,6 +96,16 @@ var (
 		},
 		"depfile", "generator")
 
+	generateBuildNinja = pctx.StaticRule("build.ninja",
+		blueprint.RuleParams{
+			Command:     "$builder $extra -b $buildDir -d $out.d -o $out $in",
+			CommandDeps: []string{"$builder"},
+			Description: "$builder $out",
+			Depfile:     "$out.d",
+			Restat:      true,
+		},
+		"builder", "extra", "generator")
+
 	// Work around a Ninja issue.  See https://github.com/martine/ninja/pull/634
 	phony = pctx.StaticRule("phony",
 		blueprint.RuleParams{
@@ -148,6 +158,7 @@ func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
 type goPackageProducer interface {
 	GoPkgRoot() string
 	GoPackageTarget() string
+	GoTestTargets() []string
 }
 
 func isGoPackageProducer(module blueprint.Module) bool {
@@ -155,16 +166,6 @@ func isGoPackageProducer(module blueprint.Module) bool {
 	return ok
 }
 
-type goTestProducer interface {
-	GoTestTarget() string
-	BuildStage() Stage
-}
-
-func isGoTestProducer(module blueprint.Module) bool {
-	_, ok := module.(goTestProducer)
-	return ok
-}
-
 type goPluginProvider interface {
 	GoPkgPath() string
 	IsPluginFor(string) bool
@@ -211,8 +212,8 @@ type goPackage struct {
 	// The path of the .a file that is to be built.
 	archiveFile string
 
-	// The path of the test .a file that is to be built.
-	testArchiveFile string
+	// The path of the test result file.
+	testResultFile []string
 
 	// The bootstrap Config
 	config *Config
@@ -246,8 +247,8 @@ func (g *goPackage) GoPackageTarget() string {
 	return g.archiveFile
 }
 
-func (g *goPackage) GoTestTarget() string {
-	return g.testArchiveFile
+func (g *goPackage) GoTestTargets() []string {
+	return g.testResultFile
 }
 
 func (g *goPackage) BuildStage() Stage {
@@ -283,8 +284,9 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
 	g.pkgRoot = packageRoot(ctx)
 	g.archiveFile = filepath.Join(g.pkgRoot,
 		filepath.FromSlash(g.properties.PkgPath)+".a")
+	var testArchiveFile string
 	if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
-		g.testArchiveFile = filepath.Join(testRoot(ctx),
+		testArchiveFile = filepath.Join(testRoot(ctx),
 			filepath.FromSlash(g.properties.PkgPath)+".a")
 	}
 
@@ -301,20 +303,18 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
 	// file to be built, but building a new ninja file requires the builder to
 	// be built.
 	if g.config.stage == g.BuildStage() {
-		var deps []string
-
 		if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc, g.config.stage) {
 			return
 		}
 
 		if g.config.runGoTests {
-			deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
+			g.testResultFile = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
 				g.properties.PkgPath, g.properties.Srcs, genSrcs,
 				g.properties.TestSrcs)
 		}
 
 		buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
-			g.properties.Srcs, genSrcs, deps)
+			g.properties.Srcs, genSrcs)
 	}
 }
 
@@ -331,9 +331,6 @@ type goBinary struct {
 		BuildStage Stage `blueprint:"mutated"`
 	}
 
-	// The path of the test .a file that is to be built.
-	testArchiveFile string
-
 	// The bootstrap Config
 	config *Config
 }
@@ -352,10 +349,6 @@ func (g *goBinary) DynamicDependencies(ctx blueprint.DynamicDependerModuleContex
 	return g.properties.Deps
 }
 
-func (g *goBinary) GoTestTarget() string {
-	return g.testArchiveFile
-}
-
 func (g *goBinary) BuildStage() Stage {
 	return g.properties.BuildStage
 }
@@ -373,20 +366,17 @@ func (g *goBinary) InstallPath() string {
 
 func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
 	var (
-		name        = ctx.ModuleName()
-		objDir      = moduleObjDir(ctx)
-		archiveFile = filepath.Join(objDir, name+".a")
-		aoutFile    = filepath.Join(objDir, "a.out")
-		binaryFile  = filepath.Join(g.InstallPath(), name)
-		hasPlugins  = false
-		pluginSrc   = ""
-		genSrcs     = []string{}
+		name            = ctx.ModuleName()
+		objDir          = moduleObjDir(ctx)
+		archiveFile     = filepath.Join(objDir, name+".a")
+		testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
+		aoutFile        = filepath.Join(objDir, "a.out")
+		binaryFile      = filepath.Join(g.InstallPath(), name)
+		hasPlugins      = false
+		pluginSrc       = ""
+		genSrcs         = []string{}
 	)
 
-	if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
-		g.testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
-	}
-
 	ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
 		func(module blueprint.Module) { hasPlugins = true })
 	if hasPlugins {
@@ -402,11 +392,11 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
 		}
 
 		if g.config.runGoTests {
-			deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
+			deps = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
 				name, g.properties.Srcs, genSrcs, g.properties.TestSrcs)
 		}
 
-		buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs, deps)
+		buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs)
 
 		var libDirFlags []string
 		ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
@@ -414,6 +404,7 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
 				dep := module.(goPackageProducer)
 				libDir := dep.GoPkgRoot()
 				libDirFlags = append(libDirFlags, "-L "+libDir)
+				deps = append(deps, dep.GoTestTargets()...)
 			})
 
 		linkArgs := map[string]string{}
@@ -429,9 +420,10 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
 		})
 
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:    cp,
-			Outputs: []string{binaryFile},
-			Inputs:  []string{aoutFile},
+			Rule:      cp,
+			Outputs:   []string{binaryFile},
+			Inputs:    []string{aoutFile},
+			OrderOnly: deps,
 		})
 	}
 }
@@ -465,7 +457,7 @@ func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string,
 }
 
 func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
-	pkgPath string, archiveFile string, srcs []string, genSrcs []string, orderDeps []string) {
+	pkgPath string, archiveFile string, srcs []string, genSrcs []string) {
 
 	srcDir := moduleSrcDir(ctx)
 	srcFiles := pathtools.PrefixPaths(srcs, srcDir)
@@ -494,7 +486,6 @@ func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
 		Rule:      compile,
 		Outputs:   []string{archiveFile},
 		Inputs:    srcFiles,
-		OrderOnly: orderDeps,
 		Implicits: deps,
 		Args:      compileArgs,
 	})
@@ -516,7 +507,7 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
 	testPassed := filepath.Join(testRoot, "test.passed")
 
 	buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
-		append(srcs, testSrcs...), genSrcs, nil)
+		append(srcs, testSrcs...), genSrcs)
 
 	ctx.Build(pctx, blueprint.BuildParams{
 		Rule:    goTestMain,
@@ -528,11 +519,13 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
 	})
 
 	libDirFlags := []string{"-L " + testRoot}
+	testDeps := []string{}
 	ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
 		func(module blueprint.Module) {
 			dep := module.(goPackageProducer)
 			libDir := dep.GoPkgRoot()
 			libDirFlags = append(libDirFlags, "-L "+libDir)
+			testDeps = append(testDeps, dep.GoTestTargets()...)
 		})
 
 	ctx.Build(pctx, blueprint.BuildParams{
@@ -556,9 +549,10 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
 	})
 
 	ctx.Build(pctx, blueprint.BuildParams{
-		Rule:    test,
-		Outputs: []string{testPassed},
-		Inputs:  []string{testFile},
+		Rule:      test,
+		Outputs:   []string{testPassed},
+		Inputs:    []string{testFile},
+		OrderOnly: testDeps,
 		Args: map[string]string{
 			"pkg":       pkgPath,
 			"pkgSrcDir": filepath.Dir(testFiles[0]),
@@ -586,10 +580,6 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 	// creating the binary that we'll use to generate the non-bootstrap
 	// build.ninja file.
 	var primaryBuilders []*goBinary
-	// bootstrapDeps contains modules that will be built in StageBootstrap
-	var bootstrapDeps []string
-	// primaryBootstrapDeps contains modules that will be built in StagePrimary
-	var primaryBootstrapDeps []string
 	// blueprintTools contains blueprint go binaries that will be built in StageMain
 	var blueprintTools []string
 	ctx.VisitAllModulesIf(isBootstrapBinaryModule,
@@ -598,12 +588,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 			binaryModuleName := ctx.ModuleName(binaryModule)
 			installPath := filepath.Join(binaryModule.InstallPath(), binaryModuleName)
 
-			switch binaryModule.BuildStage() {
-			case StageBootstrap:
-				bootstrapDeps = append(bootstrapDeps, installPath)
-			case StagePrimary:
-				primaryBootstrapDeps = append(primaryBootstrapDeps, installPath)
-			case StageMain:
+			if binaryModule.BuildStage() == StageMain {
 				blueprintTools = append(blueprintTools, installPath)
 			}
 			if binaryModule.properties.PrimaryBuilder {
@@ -611,6 +596,11 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 			}
 		})
 
+	var extraTestFlags string
+	if s.config.runGoTests {
+		extraTestFlags = " -t"
+	}
+
 	var primaryBuilderName, primaryBuilderExtraFlags string
 	switch len(primaryBuilders) {
 	case 0:
@@ -618,7 +608,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		// as the primary builder.  We can trigger its primary builder mode with
 		// the -p flag.
 		primaryBuilderName = "minibp"
-		primaryBuilderExtraFlags = "-p"
+		primaryBuilderExtraFlags = "-p" + extraTestFlags
 
 	case 1:
 		primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
@@ -634,43 +624,16 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 
 	primaryBuilderFile := filepath.Join("$BinDir", primaryBuilderName)
 
-	if s.config.runGoTests {
-		primaryBuilderExtraFlags += " -t"
-	}
-
 	// Get the filename of the top-level Blueprints file to pass to minibp.
 	topLevelBlueprints := filepath.Join("$srcDir",
 		filepath.Base(s.config.topLevelBlueprintsFile))
 
-	bootstrapDeps = append(bootstrapDeps, topLevelBlueprints)
-
 	mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
 	primaryBuilderNinjaFile := filepath.Join(bootstrapDir, "build.ninja")
 	bootstrapNinjaFileTemplate := filepath.Join(miniBootstrapDir, "build.ninja.in")
 	bootstrapNinjaFile := filepath.Join(miniBootstrapDir, "build.ninja")
 	docsFile := filepath.Join(docsDir, primaryBuilderName+".html")
 
-	primaryBootstrapDeps = append(primaryBootstrapDeps, docsFile)
-
-	// If the tests change, be sure to re-run them. These need to be
-	// dependencies for the ninja file so that it's updated after these
-	// run.
-	ctx.VisitAllModulesIf(isGoTestProducer,
-		func(module blueprint.Module) {
-			testModule := module.(goTestProducer)
-			target := testModule.GoTestTarget()
-			if target != "" {
-				switch testModule.BuildStage() {
-				case StageBootstrap:
-					bootstrapDeps = append(bootstrapDeps, target)
-				case StagePrimary:
-					primaryBootstrapDeps = append(primaryBootstrapDeps, target)
-				case StageMain:
-					blueprintTools = append(blueprintTools, target)
-				}
-			}
-		})
-
 	switch s.config.stage {
 	case StageBootstrap:
 		// We're generating a bootstrapper Ninja file, so we need to set things
@@ -680,56 +643,26 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		// cleanup process will remove files from the other builds.
 		ctx.SetNinjaBuildDir(pctx, miniBootstrapDir)
 
-		// Generate the Ninja file to build the primary builder. Save the
-		// timestamps and deps, so that we can come back to this stage if
-		// it needs to be regenerated.
-		primarybp := ctx.Rule(pctx, "primarybp",
-			blueprint.RuleParams{
-				Command: fmt.Sprintf("%s --build-primary $runTests "+
-					"-b $buildDir -d $out.d -o $out $in", minibpFile),
-				Description: "minibp $out",
-				Depfile:     "$out.d",
-			},
-			"runTests")
-
-		args := make(map[string]string)
-
-		if s.config.runGoTests {
-			args["runTests"] = "-t"
-		}
-
+		// Generate the Ninja file to build the primary builder.
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:      primarybp,
-			Outputs:   []string{primaryBuilderNinjaFile},
-			Inputs:    []string{topLevelBlueprints},
-			Implicits: bootstrapDeps,
-			Args:      args,
+			Rule:    generateBuildNinja,
+			Outputs: []string{primaryBuilderNinjaFile},
+			Inputs:  []string{topLevelBlueprints},
+			Args: map[string]string{
+				"builder": minibpFile,
+				"extra":   "--build-primary" + extraTestFlags,
+			},
 		})
 
 		// Rebuild the bootstrap Ninja file using the minibp that we just built.
-		minibp := ctx.Rule(pctx, "minibp",
-			blueprint.RuleParams{
-				Command: fmt.Sprintf("%s $runTests "+
-					"-b $buildDir -d $out.d -o $out $in", minibpFile),
-				CommandDeps: []string{minibpFile},
-				Description: "minibp $out",
-				Depfile:     "$out.d",
-				// So that we don't trigger a restart if this hasn't changed
-				Restat: true,
-			},
-			"runTests")
-
-		args = map[string]string{}
-
-		if s.config.runGoTests {
-			args["runTests"] = "-t"
-		}
-
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:    minibp,
+			Rule:    generateBuildNinja,
 			Outputs: []string{bootstrapNinjaFileTemplate},
 			Inputs:  []string{topLevelBlueprints},
-			Args:    args,
+			Args: map[string]string{
+				"builder": minibpFile,
+				"extra":   extraTestFlags,
+			},
 		})
 
 		ctx.Build(pctx, blueprint.BuildParams{
@@ -746,23 +679,27 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		// cleanup process will remove files from the other builds.
 		ctx.SetNinjaBuildDir(pctx, bootstrapDir)
 
-		// We generate the depfile here that includes the dependencies for all
-		// the Blueprints files that contribute to generating the big build
-		// manifest (build.ninja file).
-		bigbp := ctx.Rule(pctx, "bigbp",
-			blueprint.RuleParams{
-				Command: fmt.Sprintf("%s %s "+
-					"-b $buildDir -d $out.d -o $out $in", primaryBuilderFile,
-					primaryBuilderExtraFlags),
-				Description: fmt.Sprintf("%s $out", primaryBuilderName),
-				Depfile:     "$out.d",
-			})
+		// Add a way to rebuild the primary build.ninja so that globs works
+		ctx.Build(pctx, blueprint.BuildParams{
+			Rule:    generateBuildNinja,
+			Outputs: []string{primaryBuilderNinjaFile},
+			Inputs:  []string{topLevelBlueprints},
+			Args: map[string]string{
+				"builder":   minibpFile,
+				"extra":     "--build-primary" + extraTestFlags,
+				"generator": "true",
+			},
+		})
 
+		// Build the main build.ninja
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:      bigbp,
-			Outputs:   []string{mainNinjaFile},
-			Inputs:    []string{topLevelBlueprints},
-			Implicits: primaryBootstrapDeps,
+			Rule:    generateBuildNinja,
+			Outputs: []string{mainNinjaFile},
+			Inputs:  []string{topLevelBlueprints},
+			Args: map[string]string{
+				"builder": primaryBuilderFile,
+				"extra":   primaryBuilderExtraFlags,
+			},
 		})
 
 		// Generate build system docs for the primary builder.  Generating docs reads the source
@@ -786,6 +723,19 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 	case StageMain:
 		ctx.SetNinjaBuildDir(pctx, "${buildDir}")
 
+		// Add a way to rebuild the main build.ninja in case it creates rules that
+		// it will depend on itself. (In Android, globs with soong_glob)
+		ctx.Build(pctx, blueprint.BuildParams{
+			Rule:    generateBuildNinja,
+			Outputs: []string{mainNinjaFile},
+			Inputs:  []string{topLevelBlueprints},
+			Args: map[string]string{
+				"builder":   primaryBuilderFile,
+				"extra":     primaryBuilderExtraFlags,
+				"generator": "true",
+			},
+		})
+
 		if primaryBuilderName == "minibp" {
 			// This is a standalone Blueprint build, so we copy the minibp
 			// binary to the "bin" directory to make it easier to find.
diff --git a/build.ninja.in b/build.ninja.in
index b2a214b..8b3fe98 100644
--- a/build.ninja.in
+++ b/build.ninja.in
@@ -30,6 +30,12 @@ rule g.bootstrap.bootstrap
     description = bootstrap ${in}
     generator = true
 
+rule g.bootstrap.build.ninja
+    command = ${builder} ${extra} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
+    depfile = ${out}.d
+    description = ${builder} ${out}
+    restat = true
+
 rule g.bootstrap.compile
     command = GOROOT='${g.bootstrap.goRoot}' ${g.bootstrap.compileCmd} -o ${out} -p ${pkgPath} -complete ${incFlags} -pack ${in}
     description = compile ${out}
@@ -263,26 +269,16 @@ default ${g.bootstrap.BinDir}/minibp
 # Singleton: bootstrap
 # Factory:   github.com/google/blueprint/bootstrap.newSingletonFactory.func1
 
-rule s.bootstrap.primarybp
-    command = ${g.bootstrap.BinDir}/minibp --build-primary ${runTests} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
-    depfile = ${out}.d
-    description = minibp ${out}
-
-rule s.bootstrap.minibp
-    command = ${g.bootstrap.BinDir}/minibp ${runTests} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
-    depfile = ${out}.d
-    description = minibp ${out}
-    restat = true
-
-build ${g.bootstrap.buildDir}/.bootstrap/build.ninja: s.bootstrap.primarybp $
-        ${g.bootstrap.srcDir}/Blueprints | ${g.bootstrap.BinDir}/gotestmain $
-        ${g.bootstrap.BinDir}/gotestrunner ${g.bootstrap.BinDir}/minibp $
-        ${g.bootstrap.srcDir}/Blueprints
+build ${g.bootstrap.buildDir}/.bootstrap/build.ninja: g.bootstrap.build.ninja $
+        ${g.bootstrap.srcDir}/Blueprints | ${builder}
+    builder = ${g.bootstrap.BinDir}/minibp
+    extra = --build-primary
 default ${g.bootstrap.buildDir}/.bootstrap/build.ninja
 
 build ${g.bootstrap.buildDir}/.minibootstrap/build.ninja.in: $
-        s.bootstrap.minibp ${g.bootstrap.srcDir}/Blueprints | $
-        ${g.bootstrap.BinDir}/minibp
+        g.bootstrap.build.ninja ${g.bootstrap.srcDir}/Blueprints | ${builder}
+    builder = ${g.bootstrap.BinDir}/minibp
+    extra = 
 default ${g.bootstrap.buildDir}/.minibootstrap/build.ninja.in
 
 build ${g.bootstrap.buildDir}/.minibootstrap/build.ninja: $
diff --git a/tests/test.sh b/tests/test.sh
index aefff66..6269930 100755
--- a/tests/test.sh
+++ b/tests/test.sh
@@ -17,7 +17,7 @@ if [[ -d .bootstrap/blueprint/test ]]; then
 fi
 
 sleep 2
-sed -i 's/${runTests}/-t/' src.build.ninja.in
+sed -i 's/extra =/extra = -t/' src.build.ninja.in
 ./blueprint.bash
 
 if [[ ! -d .bootstrap/blueprint/test ]]; then