diff options
| -rw-r--r-- | cmd/fileslist/Android.bp | 20 | ||||
| -rwxr-xr-x | cmd/fileslist/fileslist.go | 165 |
2 files changed, 185 insertions, 0 deletions
diff --git a/cmd/fileslist/Android.bp b/cmd/fileslist/Android.bp new file mode 100644 index 00000000..cbf939a1 --- /dev/null +++ b/cmd/fileslist/Android.bp @@ -0,0 +1,20 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +blueprint_go_binary { + name: "fileslist", + srcs: [ + "fileslist.go", + ], +} diff --git a/cmd/fileslist/fileslist.go b/cmd/fileslist/fileslist.go new file mode 100755 index 00000000..1cf948f7 --- /dev/null +++ b/cmd/fileslist/fileslist.go @@ -0,0 +1,165 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// fileslist.py replacement written in GO, which utilizes multi-cores. + +package main + +import ( + "crypto/sha256" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" +) + +const ( + MAX_DEFAULT_PARA = 24 +) + +func defaultPara() int { + ret := runtime.NumCPU() + if ret > MAX_DEFAULT_PARA { + return MAX_DEFAULT_PARA + } + return ret +} + +var ( + para = flag.Int("para", defaultPara(), "Number of goroutines") +) + +// Represents each file. +type Node struct { + SHA256 string + Name string // device side path. + Size int64 + path string // host side path. + stat os.FileInfo +} + +func newNode(hostPath string, devicePath string, stat os.FileInfo) Node { + return Node{Name: devicePath, path: hostPath, stat: stat} +} + +// Scan a Node and returns true if it should be added to the result. +func (n *Node) scan() bool { + n.Size = n.stat.Size() + + // Calculate SHA256. + f, err := os.Open(n.path) + if err != nil { + // If the file can't be read, it's probably a symlink to an absolute path... + // Returns the following to mimic the behavior of fileslist.py. + n.SHA256 = "----------------------------------------------------------------" + return true + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + panic(err) + } + n.SHA256 = fmt.Sprintf("%x", h.Sum(nil)) + return true +} + +func main() { + flag.Parse() + + allOutput := make([]Node, 0, 1024) // Store all outputs. + mutex := &sync.Mutex{} // Guard allOutput + + ch := make(chan Node) // Pass nodes to goroutines. + + var wg sync.WaitGroup // To wait for all goroutines. + wg.Add(*para) + + // Scan files in multiple goroutines. + for i := 0; i < *para; i++ { + go func() { + defer wg.Done() + + output := make([]Node, 0, 1024) // Local output list. + for node := range ch { + if node.scan() { + output = append(output, node) + } + } + // Add to the global output list. + mutex.Lock() + allOutput = append(allOutput, output...) + mutex.Unlock() + }() + } + + // Walk the directories and find files to scan. + for _, dir := range flag.Args() { + absDir, err := filepath.Abs(dir) + if err != nil { + panic(err) + } + deviceRoot := filepath.Clean(absDir + "/..") + err = filepath.Walk(dir, func(path string, stat os.FileInfo, err error) error { + if err != nil { + panic(err) + } + if stat.IsDir() { + return nil + } + absPath, err := filepath.Abs(path) + if err != nil { + panic(err) + } + devicePath, err := filepath.Rel(deviceRoot, absPath) + if err != nil { + panic(err) + } + devicePath = "/" + devicePath + ch <- newNode(absPath, devicePath, stat) + return nil + }) + if err != nil { + panic(err) + } + } + + // Wait until all the goroutines finish. + close(ch) + wg.Wait() + + // Sort the entries and dump as json. + sort.Slice(allOutput, func(i, j int) bool { + if allOutput[i].Size > allOutput[j].Size { + return true + } + if allOutput[i].Size == allOutput[j].Size && strings.Compare(allOutput[i].Name, allOutput[j].Name) > 0 { + return true + } + return false + }) + + j, err := json.MarshalIndent(allOutput, "", " ") + if err != nil { + panic(nil) + } + + fmt.Printf("%s\n", j) +} |