Import globbing from Soong

Add globbing with dependency checking to blueprint.  Calling
ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps will return
a list of files that match the globs, while also adding efficient
dependencies to rerun the primary builder if a file that matches the
glob is added or removed.

Also use the globbing support for optional_subdirs=, subdirs= and build=
lines in blueprints files.  The globbing slightly changes the behavior
of subname= lines, it no longer falls back to looking for a file called
"Blueprints".  Blueprint files that need to include a subdirectory with
a different name can use build= instead of subdir= to directly include
them.  The Blueprints file is updated to reset subname="Blueprints" in
case we want to include subdirectories inside blueprint and the primary
builder has changed the subname.

Also adds a new test directory that contains a simple primary builder
tree to test regeneration for globbing, and runs the tests in travis.

Change-Id: I83ce525fd11e11579cc58ba5308d01ca8eea7bc6

1 files changed, 144 insertions, 0 deletions

diff --git a/bootstrap/glob.go b/bootstrap/glob.go
new file mode 100644
index 0000000..2e209d9
--- /dev/null
+++ b/bootstrap/glob.go
@@ -0,0 +1,144 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bootstrap
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/blueprint"
+	"github.com/google/blueprint/deptools"
+	"github.com/google/blueprint/pathtools"
+)
+
+// This file supports globbing source files in Blueprints files.
+//
+// The build.ninja file needs to be regenerated any time a file matching the glob is added
+// or removed.  The naive solution is to have the build.ninja file depend on all the
+// traversed directories, but this will cause the regeneration step to run every time a
+// non-matching file is added to a traversed directory, including backup files created by
+// editors.
+//
+// The solution implemented here optimizes out regenerations when the directory modifications
+// don't match the glob by having the build.ninja file depend on an intermedate file that
+// is only updated when a file matching the glob is added or removed.  The intermediate file
+// depends on the traversed directories via a depfile.  The depfile is used to avoid build
+// errors if a directory is deleted - a direct dependency on the deleted directory would result
+// in a build failure with a "missing and no known rule to make it" error.
+
+var (
+	globCmd = filepath.Join("$BinDir", "bpglob")
+
+	// globRule rule traverses directories to produce a list of files that match $glob
+	// and writes it to $out if it has changed, and writes the directories to $out.d
+	GlobRule = pctx.StaticRule("GlobRule",
+		blueprint.RuleParams{
+			Command:     fmt.Sprintf(`%s -o $out $excludes "$glob"`, globCmd),
+			CommandDeps: []string{globCmd},
+			Description: "glob $glob",
+
+			Restat:  true,
+			Deps:    blueprint.DepsGCC,
+			Depfile: "$out.d",
+		},
+		"glob", "excludes")
+)
+
+// GlobFileContext is the subset of ModuleContext and SingletonContext needed by GlobFile
+type GlobFileContext interface {
+	Build(pctx blueprint.PackageContext, params blueprint.BuildParams)
+}
+
+// GlobFile creates a rule to write to fileListFile a list of the files that match the specified
+// pattern but do not match any of the patterns specified in excludes.  The file will include
+// appropriate dependencies written to depFile to regenerate the file if and only if the list of
+// matching files has changed.
+func GlobFile(ctx GlobFileContext, pattern string, excludes []string,
+	fileListFile, depFile string) {
+
+	ctx.Build(pctx, blueprint.BuildParams{
+		Rule:    GlobRule,
+		Outputs: []string{fileListFile},
+		Args: map[string]string{
+			"glob":     pattern,
+			"excludes": joinWithPrefixAndQuote(excludes, "-e "),
+		},
+	})
+}
+
+func joinWithPrefixAndQuote(strs []string, prefix string) string {
+	if len(strs) == 0 {
+		return ""
+	}
+
+	if len(strs) == 1 {
+		return prefix + `"` + strs[0] + `"`
+	}
+
+	n := len(" ") * (len(strs) - 1)
+	for _, s := range strs {
+		n += len(prefix) + len(s) + len(`""`)
+	}
+
+	ret := make([]byte, 0, n)
+	for i, s := range strs {
+		if i != 0 {
+			ret = append(ret, ' ')
+		}
+		ret = append(ret, prefix...)
+		ret = append(ret, '"')
+		ret = append(ret, s...)
+		ret = append(ret, '"')
+	}
+	return string(ret)
+}
+
+// globSingleton collects any glob patterns that were seen by Context and writes out rules to
+// re-evaluate them whenever the contents of the searched directories change, and retrigger the
+// primary builder if the results change.
+type globSingleton struct {
+	globLister func() []blueprint.GlobPath
+}
+
+func globSingletonFactory(ctx *blueprint.Context) func() blueprint.Singleton {
+	return func() blueprint.Singleton {
+		return &globSingleton{
+			globLister: ctx.Globs,
+		}
+	}
+}
+
+func (s *globSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
+	if config, ok := ctx.Config().(ConfigInterface); ok && config.GeneratingBootstrapper() {
+		// Skip singleton for bootstrap.bash -r case to avoid putting unnecessary glob lines into
+		// the bootstrap manifest
+		return
+	}
+
+	for _, g := range s.globLister() {
+		fileListFile := filepath.Join(BuildDir, ".glob", g.Name)
+		depFile := fileListFile + ".d"
+
+		fileList := strings.Join(g.Files, "\n") + "\n"
+		pathtools.WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
+		deptools.WriteDepFile(depFile, fileListFile, g.Deps)
+
+		GlobFile(ctx, g.Pattern, g.Excludes, fileListFile, depFile)
+
+		// Make build.ninja depend on the fileListFile
+		ctx.AddNinjaFileDeps(fileListFile)
+	}
+}