diff options
-rw-r--r-- | finder/finder.go | 10 | ||||
-rw-r--r-- | finder/finder_test.go | 148 |
2 files changed, 100 insertions, 58 deletions
diff --git a/finder/finder.go b/finder/finder.go index c2df9289..ffda1558 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -18,6 +18,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "fmt" "io" "os" @@ -30,7 +31,6 @@ import ( "time" "android/soong/fs" - "errors" ) // This file provides a Finder struct that can quickly search for files satisfying @@ -159,11 +159,17 @@ type Finder struct { nodes pathMap } +var defaultNumThreads = runtime.NumCPU() * 2 + // New creates a new Finder for use func New(cacheParams CacheParams, filesystem fs.FileSystem, logger Logger, dbPath string) (f *Finder, err error) { + return newImpl(cacheParams, filesystem, logger, dbPath, defaultNumThreads) +} - numThreads := runtime.NumCPU() * 2 +// newImpl is like New but accepts more params +func newImpl(cacheParams CacheParams, filesystem fs.FileSystem, + logger Logger, dbPath string, numThreads int) (f *Finder, err error) { numDbLoadingThreads := numThreads numSearchingThreads := numThreads diff --git a/finder/finder_test.go b/finder/finder_test.go index 8d1bbd7f..1522c680 100644 --- a/finder/finder_test.go +++ b/finder/finder_test.go @@ -35,14 +35,18 @@ func newFs() *fs.MockFs { } func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Finder { - f, err := newFinderAndErr(t, filesystem, cacheParams) + return newFinderWithNumThreads(t, filesystem, cacheParams, 2) +} + +func newFinderWithNumThreads(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) *Finder { + f, err := newFinderAndErr(t, filesystem, cacheParams, numThreads) if err != nil { fatal(t, err.Error()) } return f } -func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) (*Finder, error) { +func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) (*Finder, error) { cachePath := "/finder/finder-db" cacheDir := filepath.Dir(cachePath) filesystem.MkDirs(cacheDir) @@ -51,7 +55,7 @@ func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParam } logger := log.New(ioutil.Discard, "", 0) - f, err := New(cacheParams, filesystem, logger, cachePath) + f, err := newImpl(cacheParams, filesystem, logger, cachePath, numThreads) return f, err } @@ -64,11 +68,13 @@ func finderWithSameParams(t *testing.T, original *Finder) *Finder { } func finderAndErrorWithSameParams(t *testing.T, original *Finder) (*Finder, error) { - f, err := New( + f, err := newImpl( original.cacheMetadata.Config.CacheParams, original.filesystem, original.logger, - original.DbPath) + original.DbPath, + original.numDbLoadingThreads, + ) return f, err } @@ -234,6 +240,21 @@ func runSimpleTest(t *testing.T, existentPaths []string, expectedMatches []strin assertSameResponse(t, foundPaths, absoluteMatches) } +// testAgainstSeveralThreadcounts runs the given test for each threadcount that we care to test +func testAgainstSeveralThreadcounts(t *testing.T, tester func(t *testing.T, numThreads int)) { + // test singlethreaded, multithreaded, and also using the same number of threads as + // will be used on the current system + threadCounts := []int{1, 2, defaultNumThreads} + for _, numThreads := range threadCounts { + testName := fmt.Sprintf("%v threads", numThreads) + // store numThreads in a new variable to prevent numThreads from changing in each loop + localNumThreads := numThreads + t.Run(testName, func(t *testing.T) { + tester(t, localNumThreads) + }) + } +} + // end of utils, start of individual tests func TestSingleFile(t *testing.T) { @@ -285,24 +306,30 @@ func TestEmptyPath(t *testing.T) { } func TestFilesystemRoot(t *testing.T) { - filesystem := newFs() - root := "/" - createdPath := "/findme.txt" - create(t, createdPath, filesystem) - finder := newFinder( - t, - filesystem, - CacheParams{ - RootDirs: []string{root}, - IncludeFiles: []string{"findme.txt", "skipme.txt"}, - }, - ) - defer finder.Shutdown() + testWithNumThreads := func(t *testing.T, numThreads int) { + filesystem := newFs() + root := "/" + createdPath := "/findme.txt" + create(t, createdPath, filesystem) - foundPaths := finder.FindNamedAt(root, "findme.txt") + finder := newFinderWithNumThreads( + t, + filesystem, + CacheParams{ + RootDirs: []string{root}, + IncludeFiles: []string{"findme.txt", "skipme.txt"}, + }, + numThreads, + ) + defer finder.Shutdown() - assertSameResponse(t, foundPaths, []string{createdPath}) + foundPaths := finder.FindNamedAt(root, "findme.txt") + + assertSameResponse(t, foundPaths, []string{createdPath}) + } + + testAgainstSeveralThreadcounts(t, testWithNumThreads) } func TestNonexistentDir(t *testing.T) { @@ -316,6 +343,7 @@ func TestNonexistentDir(t *testing.T) { RootDirs: []string{"/tmp/IDontExist"}, IncludeFiles: []string{"findme.txt", "skipme.txt"}, }, + 1, ) if err == nil { fatal(t, "Did not fail when given a nonexistent root directory") @@ -380,6 +408,8 @@ func TestPruneFiles(t *testing.T) { "/tmp/include/findme.txt"}) } +// TestRootDir tests that the value of RootDirs is used +// tests of the filesystem root are in TestFilesystemRoot func TestRootDir(t *testing.T) { filesystem := newFs() create(t, "/tmp/a/findme.txt", filesystem) @@ -548,48 +578,54 @@ func TestFindFirst(t *testing.T) { } func TestConcurrentFindSameDirectory(t *testing.T) { - filesystem := newFs() - // create a bunch of files and directories - paths := []string{} - for i := 0; i < 10; i++ { - parentDir := fmt.Sprintf("/tmp/%v", i) - for j := 0; j < 10; j++ { - filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j)) - paths = append(paths, filePath) + testWithNumThreads := func(t *testing.T, numThreads int) { + filesystem := newFs() + + // create a bunch of files and directories + paths := []string{} + for i := 0; i < 10; i++ { + parentDir := fmt.Sprintf("/tmp/%v", i) + for j := 0; j < 10; j++ { + filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j)) + paths = append(paths, filePath) + } + } + sort.Strings(paths) + for _, path := range paths { + create(t, path, filesystem) } - } - sort.Strings(paths) - for _, path := range paths { - create(t, path, filesystem) - } - // set up a finder - finder := newFinder( - t, - filesystem, - CacheParams{ - RootDirs: []string{"/tmp"}, - IncludeFiles: []string{"findme.txt"}, - }, - ) - defer finder.Shutdown() + // set up a finder + finder := newFinderWithNumThreads( + t, + filesystem, + CacheParams{ + RootDirs: []string{"/tmp"}, + IncludeFiles: []string{"findme.txt"}, + }, + numThreads, + ) + defer finder.Shutdown() + + numTests := 20 + results := make(chan []string, numTests) + // make several parallel calls to the finder + for i := 0; i < numTests; i++ { + go func() { + foundPaths := finder.FindNamedAt("/tmp", "findme.txt") + results <- foundPaths + }() + } - numTests := 20 - results := make(chan []string, numTests) - // make several parallel calls to the finder - for i := 0; i < numTests; i++ { - go func() { - foundPaths := finder.FindNamedAt("/tmp", "findme.txt") - results <- foundPaths - }() + // check that each response was correct + for i := 0; i < numTests; i++ { + foundPaths := <-results + assertSameResponse(t, foundPaths, paths) + } } - // check that each response was correct - for i := 0; i < numTests; i++ { - foundPaths := <-results - assertSameResponse(t, foundPaths, paths) - } + testAgainstSeveralThreadcounts(t, testWithNumThreads) } func TestConcurrentFindDifferentDirectories(t *testing.T) { |