diff options
| -rw-r--r-- | .travis.yml | 3 | ||||
| -rw-r--r-- | Blueprints | 5 | ||||
| -rw-r--r-- | bootstrap/bootstrap.go | 259 | ||||
| -rw-r--r-- | bootstrap/cleanup.go | 2 | ||||
| -rw-r--r-- | bootstrap/command.go | 70 | ||||
| -rw-r--r-- | bootstrap/config.go | 12 | ||||
| -rw-r--r-- | build.ninja.in | 84 | ||||
| -rw-r--r-- | choosestage/choosestage.go | 194 | ||||
| -rw-r--r-- | tests/expected_all | 2 | ||||
| -rw-r--r-- | tests/expected_manifest | 3 | ||||
| -rw-r--r-- | tests/expected_rebuild_test | 2 | ||||
| -rw-r--r-- | tests/expected_regen | 5 | ||||
| -rw-r--r-- | tests/expected_start2 | 2 | ||||
| -rw-r--r-- | tests/expected_start_add_tests | 2 |
14 files changed, 453 insertions, 192 deletions
diff --git a/.travis.yml b/.travis.yml index 9268df0..95529e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,9 @@ install: script: - go test ./... - - cp build.ninja.in build.ninja.in.orig - mkdir stage - cd stage - ../bootstrap.bash - ninja - - diff -us ../build.ninja.in ../build.ninja.in.orig + - diff -us ../build.ninja.in .bootstrap/bootstrap.ninja.in - ../tests/test.sh @@ -123,3 +123,8 @@ bootstrap_go_binary( name = "gotestmain", srcs = ["gotestmain/gotestmain.go"], ) + +bootstrap_go_binary( + name = "choosestage", + srcs = ["choosestage/choosestage.go"], +) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 66a4036..3023b08 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -28,9 +28,10 @@ const bootstrapDir = ".bootstrap" var ( pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap") - gcCmd = pctx.StaticVariable("gcCmd", "$goToolDir/${goChar}g") - linkCmd = pctx.StaticVariable("linkCmd", "$goToolDir/${goChar}l") - goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain")) + gcCmd = pctx.StaticVariable("gcCmd", "$goToolDir/${goChar}g") + linkCmd = pctx.StaticVariable("linkCmd", "$goToolDir/${goChar}l") + goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain")) + chooseStageCmd = pctx.StaticVariable("chooseStageCmd", filepath.Join(bootstrapDir, "bin", "choosestage")) gc = pctx.StaticRule("gc", blueprint.RuleParams{ @@ -70,11 +71,25 @@ var ( bootstrap = pctx.StaticRule("bootstrap", blueprint.RuleParams{ - Command: "echo \"Choosing $$(basename $in) for next stage\" && $bootstrapCmd -i $in", + Command: "$bootstrapCmd -i $in", Description: "bootstrap $in", Generator: true, }) + chooseStage = pctx.StaticRule("chooseStage", + blueprint.RuleParams{ + Command: "$chooseStageCmd --current $current --bootstrap $bootstrapManifest -o $out $in", + Description: "choosing next stage", + }, + "current", "generator") + + touch = pctx.StaticRule("touch", + blueprint.RuleParams{ + Command: "touch $out", + Description: "touch $out", + }, + "depfile", "generator") + // Work around a Ninja issue. See https://github.com/martine/ninja/pull/634 phony = pctx.StaticRule("phony", blueprint.RuleParams{ @@ -186,7 +201,8 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) { // the circular dependence that occurs when the builder requires a new Ninja // file to be built, but building a new ninja file requires the builder to // be built. - if g.config.generatingBootstrapper { + switch g.config.stage { + case StageBootstrap: var deps []string if g.config.runGoTests { @@ -197,7 +213,7 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) { buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile, g.properties.Srcs, deps) - } else { + case StageMain: if len(g.properties.TestSrcs) > 0 && g.config.runGoTests { phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil) } @@ -251,7 +267,8 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) { // the circular dependence that occurs when the builder requires a new Ninja // file to be built, but building a new ninja file requires the builder to // be built. - if g.config.generatingBootstrapper { + switch g.config.stage { + case StageBootstrap: var deps []string if g.config.runGoTests { @@ -287,7 +304,7 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) { Outputs: []string{binaryFile}, Inputs: []string{aoutFile}, }) - } else { + case StageMain: if len(g.properties.TestSrcs) > 0 && g.config.runGoTests { phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil) } @@ -506,18 +523,34 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) { } // Get the filename of the top-level Blueprints file to pass to minibp. - // This comes stored in a global variable that's set by Main. topLevelBlueprints := filepath.Join("$srcDir", filepath.Base(s.config.topLevelBlueprintsFile)) + rebootstrapDeps = append(rebootstrapDeps, topLevelBlueprints) + mainNinjaFile := filepath.Join(bootstrapDir, "main.ninja.in") - mainNinjaDepFile := mainNinjaFile + ".d" + mainNinjaTimestampFile := mainNinjaFile + ".timestamp" + mainNinjaTimestampDepFile := mainNinjaTimestampFile + ".d" bootstrapNinjaFile := filepath.Join(bootstrapDir, "bootstrap.ninja.in") docsFile := filepath.Join(docsDir, primaryBuilderName+".html") rebootstrapDeps = append(rebootstrapDeps, docsFile) - if s.config.generatingBootstrapper { + // If the tests change, be sure to re-run them. These need to be + // dependencies for the ninja file so that it's updated after these + // run. Otherwise we'd never leave the bootstrap stage, since the + // timestamp file would be newer than the ninja file. + ctx.VisitAllModulesIf(isGoTestProducer, + func(module blueprint.Module) { + testModule := module.(goTestProducer) + target := testModule.GoTestTarget() + if target != "" { + rebootstrapDeps = append(rebootstrapDeps, target) + } + }) + + switch s.config.stage { + case StageBootstrap: // We're generating a bootstrapper Ninja file, so we need to set things // up to rebuild the build.ninja file using the primary builder. @@ -525,6 +558,64 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) { // otherwise the cleanup process will remove files from the other build. ctx.SetBuildDir(pctx, bootstrapDir) + // Rebuild the bootstrap Ninja file using the minibp that we just built. + // If this produces a difference, choosestage will retrigger this stage. + minibp := ctx.Rule(pctx, "minibp", + blueprint.RuleParams{ + Command: fmt.Sprintf("%s $runTests -m $bootstrapManifest "+ + "-d $out.d -o $out $in", minibpFile), + Description: "minibp $out", + Generator: true, + Depfile: "$out.d", + }, + "runTests") + + args := map[string]string{} + + if s.config.runGoTests { + args["runTests"] = "-t" + } + + ctx.Build(pctx, blueprint.BuildParams{ + Rule: minibp, + Outputs: []string{bootstrapNinjaFile}, + Inputs: []string{topLevelBlueprints}, + // $bootstrapManifest is here so that when it is updated, we + // force a rebuild of bootstrap.ninja.in. chooseStage should + // have already copied the new version over, but kept the old + // timestamps to force this regeneration. + Implicits: []string{"$bootstrapManifest", minibpFile}, + Args: args, + }) + + // We generate the depfile here that includes the dependencies for all + // the Blueprints files that contribute to generating the big build + // manifest (build.ninja file). This depfile will be used by the non- + // bootstrap build manifest to determine whether it should touch the + // timestamp file to trigger a re-bootstrap. + bigbp := ctx.Rule(pctx, "bigbp", + blueprint.RuleParams{ + Command: fmt.Sprintf("%s %s -m $bootstrapManifest "+ + "--timestamp $timestamp --timestampdep $timestampdep "+ + "-d $outfile.d -o $outfile $in", primaryBuilderFile, + primaryBuilderExtraFlags), + Description: fmt.Sprintf("%s $outfile", primaryBuilderName), + Depfile: "$outfile.d", + }, + "timestamp", "timestampdep", "outfile") + + ctx.Build(pctx, blueprint.BuildParams{ + Rule: bigbp, + Outputs: []string{mainNinjaFile, mainNinjaTimestampFile}, + Inputs: []string{topLevelBlueprints}, + Implicits: rebootstrapDeps, + Args: map[string]string{ + "timestamp": mainNinjaTimestampFile, + "timestampdep": mainNinjaTimestampDepFile, + "outfile": mainNinjaFile, + }, + }) + // Generate build system docs for the primary builder. Generating docs reads the source // files used to build the primary builder, but that dependency will be picked up through // the dependency on the primary builder itself. There are no dependencies on the @@ -543,36 +634,10 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) { Implicits: []string{primaryBuilderFile}, }) - // We generate the depfile here that includes the dependencies for all - // the Blueprints files that contribute to generating the big build - // manifest (build.ninja file). This depfile will be used by the non- - // bootstrap build manifest to determine whether it should trigger a re- - // bootstrap. Because the re-bootstrap rule's output is "build.ninja" - // we need to force the depfile to have that as its "make target" - // (recall that depfiles use a subset of the Makefile syntax). - bigbp := ctx.Rule(pctx, "bigbp", - blueprint.RuleParams{ - Command: fmt.Sprintf("%s %s -d %s -m $bootstrapManifest "+ - "-o $out $in", primaryBuilderFile, - primaryBuilderExtraFlags, mainNinjaDepFile), - Description: fmt.Sprintf("%s $out", primaryBuilderName), - Depfile: mainNinjaDepFile, - }) - - ctx.Build(pctx, blueprint.BuildParams{ - Rule: bigbp, - Outputs: []string{mainNinjaFile}, - Inputs: []string{topLevelBlueprints}, - Implicits: rebootstrapDeps, - }) - // When the current build.ninja file is a bootstrapper, we always want // to have it replace itself with a non-bootstrapper build.ninja. To // accomplish that we depend on a file that should never exist and // "build" it using Ninja's built-in phony rule. - // - // We also need to add an implicit dependency on bootstrapNinjaFile so - // that it gets generated as part of the bootstrap process. notAFile := filepath.Join(bootstrapDir, "notAFile") ctx.Build(pctx, blueprint.BuildParams{ Rule: blueprint.Phony, @@ -580,103 +645,54 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) { }) ctx.Build(pctx, blueprint.BuildParams{ - Rule: bootstrap, - Outputs: []string{"build.ninja"}, - Inputs: []string{mainNinjaFile}, - Implicits: []string{"$bootstrapCmd", notAFile, bootstrapNinjaFile}, - }) - - // Rebuild the bootstrap Ninja file using the minibp that we just built. - // The checkFile tells minibp to compare the new bootstrap file to the - // current one. If the files are the same then minibp sets the new - // file's mtime to match that of the current one. If they're different - // then the new file will have a newer timestamp than the current one - // and it will trigger a reboostrap by the non-boostrap build manifest. - minibp := ctx.Rule(pctx, "minibp", - blueprint.RuleParams{ - Command: fmt.Sprintf("%s $runTests -c $checkFile -m $bootstrapManifest "+ - "-d $out.d -o $out $in", minibpFile), - Description: "minibp $out", - Generator: true, - Depfile: "$out.d", + Rule: chooseStage, + Outputs: []string{filepath.Join(bootstrapDir, "build.ninja.in")}, + Inputs: []string{bootstrapNinjaFile, mainNinjaFile}, + Implicits: []string{"$chooseStageCmd", "$bootstrapManifest", notAFile}, + Args: map[string]string{ + "current": bootstrapNinjaFile, }, - "checkFile", "runTests") - - args := map[string]string{ - "checkFile": "$bootstrapManifest", - } - - if s.config.runGoTests { - args["runTests"] = "-t" - } - - ctx.Build(pctx, blueprint.BuildParams{ - Rule: minibp, - Outputs: []string{bootstrapNinjaFile}, - Inputs: []string{topLevelBlueprints}, - Implicits: []string{minibpFile}, - Args: args, }) - } else { - ctx.VisitAllModulesIf(isGoTestProducer, - func(module blueprint.Module) { - testModule := module.(goTestProducer) - target := testModule.GoTestTarget() - if target != "" { - rebootstrapDeps = append(rebootstrapDeps, target) - } - }) + case StageMain: // We're generating a non-bootstrapper Ninja file, so we need to set it - // up to depend on the bootstrapper Ninja file. The build.ninja target - // also has an implicit dependency on the primary builder and all other - // bootstrap go binaries, which will have phony dependencies on all of - // their sources. This will cause any changes to a bootstrap binary's - // sources to trigger a re-bootstrap operation, which will rebuild the - // binary. + // up to re-bootstrap if necessary. We do this by making build.ninja.in + // depend on the various Ninja files, the source build.ninja.in, and + // on the timestamp files. // - // On top of that we need to use the depfile generated by the bigbp - // rule. We do this by depending on that file and then setting up a - // phony rule to generate it that uses the depfile. - buildNinjaDeps := []string{"$bootstrapCmd", mainNinjaFile} - buildNinjaDeps = append(buildNinjaDeps, rebootstrapDeps...) - + // The timestamp files themselves are set up with the same dependencies + // as their Ninja files, including their own depfile. If any of the + // dependencies need to be updated, we'll touch the timestamp file, + // which will tell choosestage to switch to the stage that rebuilds + // that Ninja file. ctx.Build(pctx, blueprint.BuildParams{ - Rule: bootstrap, - Outputs: []string{"build.ninja"}, - Inputs: []string{"$bootstrapManifest"}, - Implicits: buildNinjaDeps, - }) - - ctx.Build(pctx, blueprint.BuildParams{ - Rule: phony, - Outputs: []string{mainNinjaFile}, - Inputs: []string{topLevelBlueprints}, + Rule: touch, + Outputs: []string{mainNinjaTimestampFile}, + Implicits: rebootstrapDeps, Args: map[string]string{ - "depfile": mainNinjaDepFile, + "depfile": mainNinjaTimestampDepFile, + "generator": "true", }, }) ctx.Build(pctx, blueprint.BuildParams{ - Rule: phony, - Outputs: []string{docsFile}, - Implicits: []string{primaryBuilderFile}, - }) - - // If the bootstrap Ninja invocation caused a new bootstrapNinjaFile to be - // generated then that means we need to rebootstrap using it instead of - // the current bootstrap manifest. We enable the Ninja "generator" - // behavior so that Ninja doesn't invoke this build just because it's - // missing a command line log entry for the bootstrap manifest. - ctx.Build(pctx, blueprint.BuildParams{ - Rule: cp, - Outputs: []string{"$bootstrapManifest"}, - Inputs: []string{bootstrapNinjaFile}, + Rule: chooseStage, + Outputs: []string{filepath.Join(bootstrapDir, "build.ninja.in")}, + Inputs: []string{bootstrapNinjaFile, mainNinjaFile}, + Implicits: []string{"$chooseStageCmd", "$bootstrapManifest", mainNinjaTimestampFile}, Args: map[string]string{ + "current": mainNinjaFile, "generator": "true", }, }) + // Create this phony rule so that upgrades don't delete these during + // cleanup + ctx.Build(pctx, blueprint.BuildParams{ + Rule: blueprint.Phony, + Outputs: []string{mainNinjaFile, docsFile}, + }) + if primaryBuilderName == "minibp" { // This is a standalone Blueprint build, so we copy the minibp // binary to the "bin" directory to make it easier to find. @@ -688,6 +704,13 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) { }) } } + + ctx.Build(pctx, blueprint.BuildParams{ + Rule: bootstrap, + Outputs: []string{"build.ninja"}, + Inputs: []string{filepath.Join(bootstrapDir, "build.ninja.in")}, + Implicits: []string{"$bootstrapCmd"}, + }) } // packageRoot returns the module-specific package root directory path. This diff --git a/bootstrap/cleanup.go b/bootstrap/cleanup.go index f3cb634..ba9fd48 100644 --- a/bootstrap/cleanup.go +++ b/bootstrap/cleanup.go @@ -33,7 +33,7 @@ func removeAbandonedFiles(ctx *blueprint.Context, config *Config, srcDir, manifestFile string) error { buildDir := "." - if config.generatingBootstrapper { + if config.stage == StageBootstrap { buildDir = bootstrapDir } diff --git a/bootstrap/command.go b/bootstrap/command.go index 932cfd7..4fb9c51 100644 --- a/bootstrap/command.go +++ b/bootstrap/command.go @@ -29,19 +29,21 @@ import ( ) var ( - outFile string - depFile string - checkFile string - manifestFile string - docFile string - cpuprofile string - runGoTests bool + outFile string + depFile string + timestampFile string + timestampDepFile string + manifestFile string + docFile string + cpuprofile string + runGoTests bool ) func init() { flag.StringVar(&outFile, "o", "build.ninja.in", "the Ninja file to output") flag.StringVar(&depFile, "d", "", "the dependency file to output") - flag.StringVar(&checkFile, "c", "", "the existing file to check against") + flag.StringVar(×tampFile, "timestamp", "", "file to write before the output file") + flag.StringVar(×tampDepFile, "timestampdep", "", "the dependency file for the timestamp file") flag.StringVar(&manifestFile, "m", "", "the bootstrap manifest file") flag.StringVar(&docFile, "docs", "", "build documentation file to output") flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file") @@ -69,13 +71,15 @@ func Main(ctx *blueprint.Context, config interface{}, extraNinjaFileDeps ...stri fatalf("no Blueprints file specified") } - generatingBootstrapper := false + stage := StageMain if c, ok := config.(ConfigInterface); ok { - generatingBootstrapper = c.GeneratingBootstrapper() + if c.GeneratingBootstrapper() { + stage = StageBootstrap + } } bootstrapConfig := &Config{ - generatingBootstrapper: generatingBootstrapper, + stage: stage, topLevelBlueprintsFile: flag.Arg(0), runGoTests: runGoTests, } @@ -118,48 +122,34 @@ func Main(ctx *blueprint.Context, config interface{}, extraNinjaFileDeps ...stri } const outFilePermissions = 0666 - err = ioutil.WriteFile(outFile, buf.Bytes(), outFilePermissions) - if err != nil { - fatalf("error writing %s: %s", outFile, err) - } - - if checkFile != "" { - checkData, err := ioutil.ReadFile(checkFile) + if timestampFile != "" { + err := ioutil.WriteFile(timestampFile, []byte{}, outFilePermissions) if err != nil { - fatalf("error reading %s: %s", checkFile, err) + fatalf("error writing %s: %s", timestampFile, err) } - matches := buf.Len() == len(checkData) - if matches { - for i, value := range buf.Bytes() { - if value != checkData[i] { - matches = false - break - } - } - } - - if matches { - // The new file content matches the check-file content, so we set - // the new file's mtime and atime to match that of the check-file. - checkFileInfo, err := os.Stat(checkFile) - if err != nil { - fatalf("error stat'ing %s: %s", checkFile, err) - } - - time := checkFileInfo.ModTime() - err = os.Chtimes(outFile, time, time) + if timestampDepFile != "" { + err := deptools.WriteDepFile(timestampDepFile, timestampFile, deps) if err != nil { - fatalf("error setting timestamps for %s: %s", outFile, err) + fatalf("error writing depfile: %s", err) } } } + err = ioutil.WriteFile(outFile, buf.Bytes(), outFilePermissions) + if err != nil { + fatalf("error writing %s: %s", outFile, err) + } + if depFile != "" { err := deptools.WriteDepFile(depFile, outFile, deps) if err != nil { fatalf("error writing depfile: %s", err) } + err = deptools.WriteDepFile(depFile+".timestamp", outFile+".timestamp", deps) + if err != nil { + fatalf("error writing depfile: %s", err) + } } srcDir := filepath.Dir(bootstrapConfig.topLevelBlueprintsFile) diff --git a/bootstrap/config.go b/bootstrap/config.go index ce96318..ab3b160 100644 --- a/bootstrap/config.go +++ b/bootstrap/config.go @@ -38,11 +38,15 @@ type ConfigInterface interface { GeneratingBootstrapper() bool } +type Stage int + +const ( + StageBootstrap Stage = iota + StageMain +) + type Config struct { - // generatingBootstrapper should be true if this build invocation is - // creating a build.ninja.in file to be used in a build bootstrapping - // sequence. - generatingBootstrapper bool + stage Stage topLevelBlueprintsFile string diff --git a/build.ninja.in b/build.ninja.in index 77661e9..183e954 100644 --- a/build.ninja.in +++ b/build.ninja.in @@ -13,6 +13,8 @@ g.bootstrap.bootstrapCmd = @@Bootstrap@@ g.bootstrap.bootstrapManifest = @@BootstrapManifest@@ +g.bootstrap.chooseStageCmd = .bootstrap/bin/choosestage + g.bootstrap.goRoot = @@GoRoot@@ g.bootstrap.goOS = @@GoOS@@ @@ -32,10 +34,14 @@ g.bootstrap.srcDir = @@SrcDir@@ builddir = .bootstrap rule g.bootstrap.bootstrap - command = echo "Choosing $$(basename ${in}) for next stage" && ${g.bootstrap.bootstrapCmd} -i ${in} + command = ${g.bootstrap.bootstrapCmd} -i ${in} description = bootstrap ${in} generator = true +rule g.bootstrap.chooseStage + command = ${g.bootstrap.chooseStageCmd} --current ${current} --bootstrap ${g.bootstrap.bootstrapManifest} -o ${out} ${in} + description = choosing next stage + rule g.bootstrap.cp command = cp ${in} ${out} description = cp ${out} @@ -221,6 +227,26 @@ build .bootstrap/bin/bpmodify: g.bootstrap.cp .bootstrap/bpmodify/obj/a.out default .bootstrap/bin/bpmodify # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Module: choosestage +# Variant: +# Type: bootstrap_go_binary +# Factory: github.com/google/blueprint/bootstrap.func·002 +# Defined: Blueprints:127:1 + +build .bootstrap/choosestage/obj/choosestage.a: g.bootstrap.gc $ + ${g.bootstrap.srcDir}/choosestage/choosestage.go | $ + ${g.bootstrap.gcCmd} + pkgPath = choosestage +default .bootstrap/choosestage/obj/choosestage.a + +build .bootstrap/choosestage/obj/a.out: g.bootstrap.link $ + .bootstrap/choosestage/obj/choosestage.a | ${g.bootstrap.linkCmd} +default .bootstrap/choosestage/obj/a.out +build .bootstrap/bin/choosestage: g.bootstrap.cp $ + .bootstrap/choosestage/obj/a.out +default .bootstrap/bin/choosestage + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Module: gotestmain # Variant: # Type: bootstrap_go_binary @@ -271,37 +297,49 @@ default .bootstrap/bin/minibp # Singleton: bootstrap # Factory: github.com/google/blueprint/bootstrap.func·007 -rule s.bootstrap.bigbpDocs - command = .bootstrap/bin/minibp -p --docs ${out} ${g.bootstrap.srcDir}/Blueprints - description = minibp docs ${out} - -rule s.bootstrap.bigbp - command = .bootstrap/bin/minibp -p -d .bootstrap/main.ninja.in.d -m ${g.bootstrap.bootstrapManifest} -o ${out} ${in} - depfile = .bootstrap/main.ninja.in.d - description = minibp ${out} - rule s.bootstrap.minibp - command = .bootstrap/bin/minibp ${runTests} -c ${checkFile} -m ${g.bootstrap.bootstrapManifest} -d ${out}.d -o ${out} ${in} + command = .bootstrap/bin/minibp ${runTests} -m ${g.bootstrap.bootstrapManifest} -d ${out}.d -o ${out} ${in} depfile = ${out}.d description = minibp ${out} generator = true +rule s.bootstrap.bigbp + command = .bootstrap/bin/minibp -p -m ${g.bootstrap.bootstrapManifest} --timestamp ${timestamp} --timestampdep ${timestampdep} -d ${outfile}.d -o ${outfile} ${in} + depfile = ${outfile}.d + description = minibp ${outfile} + +rule s.bootstrap.bigbpDocs + command = .bootstrap/bin/minibp -p --docs ${out} ${g.bootstrap.srcDir}/Blueprints + description = minibp docs ${out} + +build .bootstrap/bootstrap.ninja.in: s.bootstrap.minibp $ + ${g.bootstrap.srcDir}/Blueprints | ${g.bootstrap.bootstrapManifest} $ + .bootstrap/bin/minibp +default .bootstrap/bootstrap.ninja.in +build .bootstrap/main.ninja.in .bootstrap/main.ninja.in.timestamp: $ + s.bootstrap.bigbp ${g.bootstrap.srcDir}/Blueprints | $ + .bootstrap/bin/bpfmt .bootstrap/bin/bpmodify $ + .bootstrap/bin/choosestage .bootstrap/bin/gotestmain $ + .bootstrap/bin/minibp ${g.bootstrap.srcDir}/Blueprints $ + .bootstrap/docs/minibp.html + outfile = .bootstrap/main.ninja.in + timestamp = .bootstrap/main.ninja.in.timestamp + timestampdep = .bootstrap/main.ninja.in.timestamp.d +default .bootstrap/main.ninja.in .bootstrap/main.ninja.in.timestamp + build .bootstrap/docs/minibp.html: s.bootstrap.bigbpDocs | $ .bootstrap/bin/minibp default .bootstrap/docs/minibp.html -build .bootstrap/main.ninja.in: s.bootstrap.bigbp $ - ${g.bootstrap.srcDir}/Blueprints | .bootstrap/bin/bpfmt $ - .bootstrap/bin/bpmodify .bootstrap/bin/gotestmain $ - .bootstrap/bin/minibp .bootstrap/docs/minibp.html -default .bootstrap/main.ninja.in build .bootstrap/notAFile: phony default .bootstrap/notAFile -build build.ninja: g.bootstrap.bootstrap .bootstrap/main.ninja.in | $ - ${g.bootstrap.bootstrapCmd} .bootstrap/notAFile $ - .bootstrap/bootstrap.ninja.in +build .bootstrap/build.ninja.in: g.bootstrap.chooseStage $ + .bootstrap/bootstrap.ninja.in .bootstrap/main.ninja.in | $ + ${g.bootstrap.chooseStageCmd} ${g.bootstrap.bootstrapManifest} $ + .bootstrap/notAFile + current = .bootstrap/bootstrap.ninja.in +default .bootstrap/build.ninja.in + +build build.ninja: g.bootstrap.bootstrap .bootstrap/build.ninja.in | $ + ${g.bootstrap.bootstrapCmd} default build.ninja -build .bootstrap/bootstrap.ninja.in: s.bootstrap.minibp $ - ${g.bootstrap.srcDir}/Blueprints | .bootstrap/bin/minibp - checkFile = ${g.bootstrap.bootstrapManifest} -default .bootstrap/bootstrap.ninja.in diff --git a/choosestage/choosestage.go b/choosestage/choosestage.go new file mode 100644 index 0000000..e1445e0 --- /dev/null +++ b/choosestage/choosestage.go @@ -0,0 +1,194 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Choose which ninja file (stage) to run next +// +// In the common case, this program takes a list of ninja files, compares their +// mtimes against their $file.timestamp mtimes, and picks the last up to date +// ninja file to output. That stage is expected to rebuild the next file in the +// list and call this program again. If none of the ninja files are considered +// dirty, the last stage is output. +// +// One exception is if the current stage's ninja file was rewritten, it will be +// run again. +// +// Another exception is if the source bootstrap file has been updated more +// recently than the first stage, the source file will be copied to the first +// stage, and output. This would be expected with a new source drop via git. +// The timestamp of the first file is not updated so that it can be regenerated +// with any local changes. + +package choosestage + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +var ( + outputFile string + currentFile string + bootstrapFile string + verbose bool +) + +func init() { + flag.StringVar(&outputFile, "o", "", "Output file") + flag.StringVar(¤tFile, "current", "", "Current stage's file") + flag.StringVar(&bootstrapFile, "bootstrap", "", "Bootstrap file checked into source") + flag.BoolVar(&verbose, "v", false, "Verbose mode") +} + +func compareFiles(a, b string) (bool, error) { + aData, err := ioutil.ReadFile(a) + if err != nil { + return false, err + } + + bData, err := ioutil.ReadFile(b) + if err != nil { + return false, err + } + + return bytes.Equal(aData, bData), nil +} + +// If the source bootstrap reference file is newer, then we may have gotten +// other source updates too. So we need to restart everything with the file +// that was checked in instead of the bootstrap that we last built. +func copyBootstrapIfNecessary(bootstrapFile, filename string) (bool, error) { + if bootstrapFile == "" { + return false, nil + } + + bootstrapStat, err := os.Stat(bootstrapFile) + if err != nil { + return false, err + } + + fileStat, err := os.Stat(filename) + if err != nil { + return false, err + } + + time := fileStat.ModTime() + if !bootstrapStat.ModTime().After(time) { + return false, nil + } + + fmt.Printf("Newer source version of %s. Copying to %s\n", filepath.Base(bootstrapFile), filepath.Base(filename)) + if verbose { + fmt.Printf("Source: %s\nBuilt: %s\n", bootstrapStat.ModTime(), time) + } + + data, err := ioutil.ReadFile(bootstrapFile) + if err != nil { + return false, err + } + + err = ioutil.WriteFile(filename, data, 0666) + if err != nil { + return false, err + } + + // Restore timestamp to force regeneration of the bootstrap.ninja.in + err = os.Chtimes(filename, time, time) + return true, err +} + +func main() { + flag.Parse() + + if flag.NArg() == 0 { + fmt.Fprintf(os.Stderr, "Must specify at least one ninja file\n") + os.Exit(1) + } + + if outputFile == "" { + fmt.Fprintf(os.Stderr, "Must specify an output file\n") + os.Exit(1) + } + + gotoFile := flag.Arg(0) + if copied, err := copyBootstrapIfNecessary(bootstrapFile, flag.Arg(0)); err != nil { + fmt.Fprintf(os.Stderr, "Failed to copy bootstrap ninja file: %s\n", err) + os.Exit(1) + } else if !copied { + for _, fileName := range flag.Args() { + timestampName := fileName + ".timestamp" + + // If we're currently running this stage, and the build.ninja.in + // file differs from the current stage file, then it has been rebuilt. + // Restart the stage. + if currentFile == fileName { + if _, err := os.Stat(outputFile); !os.IsNotExist(err) { + if ok, err := compareFiles(fileName, outputFile); err != nil { + fmt.Fprintf(os.Stderr, "Failure when comparing files: %s\n", err) + os.Exit(1) + } else if !ok { + fmt.Printf("Stage %s has changed, restarting\n", filepath.Base(fileName)) + gotoFile = fileName + break + } + } + } + + fileStat, err := os.Stat(fileName) + if err != nil { + // Regenerate this stage on error + break + } + + timestampStat, err := os.Stat(timestampName) + if err != nil { + // This file may not exist. There's no point for + // the first stage to have one, as it should be + // a subset of the second stage dependencies, + // and both will return to the first stage. + continue + } + + if verbose { + fmt.Printf("For %s:\n file: %s\n time: %s\n", fileName, fileStat.ModTime(), timestampStat.ModTime()) + } + + // If the timestamp file has a later modification time, that + // means that this stage needs to be regenerated. Break, so + // that we run the last found stage. + if timestampStat.ModTime().After(fileStat.ModTime()) { + break + } + + gotoFile = fileName + } + } + + fmt.Printf("Choosing %s for next stage\n", filepath.Base(gotoFile)) + + data, err := ioutil.ReadFile(gotoFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Can't read file: %s", err) + os.Exit(1) + } + + err = ioutil.WriteFile(outputFile, data, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "Can't write file: %s", err) + os.Exit(1) + } +} diff --git a/tests/expected_all b/tests/expected_all index 380b0cc..a92f0b3 100644 --- a/tests/expected_all +++ b/tests/expected_all @@ -1,2 +1,2 @@ -Choosing build.ninja.in for next stage +Choosing bootstrap.ninja.in for next stage Choosing main.ninja.in for next stage diff --git a/tests/expected_manifest b/tests/expected_manifest index 380b0cc..ab67d4b 100644 --- a/tests/expected_manifest +++ b/tests/expected_manifest @@ -1,2 +1,3 @@ -Choosing build.ninja.in for next stage +Newer source version of build.ninja.in. Copying to bootstrap.ninja.in +Choosing bootstrap.ninja.in for next stage Choosing main.ninja.in for next stage diff --git a/tests/expected_rebuild_test b/tests/expected_rebuild_test index 162d1db..a92f0b3 100644 --- a/tests/expected_rebuild_test +++ b/tests/expected_rebuild_test @@ -1,2 +1,2 @@ -Choosing src.build.ninja.in for next stage +Choosing bootstrap.ninja.in for next stage Choosing main.ninja.in for next stage diff --git a/tests/expected_regen b/tests/expected_regen index 162d1db..4f7adaa 100644 --- a/tests/expected_regen +++ b/tests/expected_regen @@ -1,2 +1,5 @@ -Choosing src.build.ninja.in for next stage +Newer source version of src.build.ninja.in. Copying to bootstrap.ninja.in +Choosing bootstrap.ninja.in for next stage +Stage bootstrap.ninja.in has changed, restarting +Choosing bootstrap.ninja.in for next stage Choosing main.ninja.in for next stage diff --git a/tests/expected_start2 b/tests/expected_start2 index dc55ac3..d0b3900 100644 --- a/tests/expected_start2 +++ b/tests/expected_start2 @@ -1 +1,3 @@ +Stage bootstrap.ninja.in has changed, restarting +Choosing bootstrap.ninja.in for next stage Choosing main.ninja.in for next stage diff --git a/tests/expected_start_add_tests b/tests/expected_start_add_tests index dc55ac3..d0b3900 100644 --- a/tests/expected_start_add_tests +++ b/tests/expected_start_add_tests @@ -1 +1,3 @@ +Stage bootstrap.ninja.in has changed, restarting +Choosing bootstrap.ninja.in for next stage Choosing main.ninja.in for next stage |