1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// fileslist.py replacement written in GO, which utilizes multi-cores.
package main
import (
"crypto/sha256"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
)
const (
MAX_DEFAULT_PARA = 24
)
func defaultPara() int {
ret := runtime.NumCPU()
if ret > MAX_DEFAULT_PARA {
return MAX_DEFAULT_PARA
}
return ret
}
var (
para = flag.Int("para", defaultPara(), "Number of goroutines")
)
// Represents each file.
type Node struct {
SHA256 string
Name string // device side path.
Size int64
path string // host side path.
stat os.FileInfo
}
func newNode(hostPath string, devicePath string, stat os.FileInfo) Node {
return Node{Name: devicePath, path: hostPath, stat: stat}
}
// Scan a Node and returns true if it should be added to the result.
func (n *Node) scan() bool {
n.Size = n.stat.Size()
// Calculate SHA256.
f, err := os.Open(n.path)
if err != nil {
// If the file can't be read, it's probably a symlink to an absolute path...
// Returns the following to mimic the behavior of fileslist.py.
n.SHA256 = "----------------------------------------------------------------"
return true
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
panic(err)
}
n.SHA256 = fmt.Sprintf("%x", h.Sum(nil))
return true
}
func main() {
flag.Parse()
allOutput := make([]Node, 0, 1024) // Store all outputs.
mutex := &sync.Mutex{} // Guard allOutput
ch := make(chan Node) // Pass nodes to goroutines.
var wg sync.WaitGroup // To wait for all goroutines.
wg.Add(*para)
// Scan files in multiple goroutines.
for i := 0; i < *para; i++ {
go func() {
defer wg.Done()
output := make([]Node, 0, 1024) // Local output list.
for node := range ch {
if node.scan() {
output = append(output, node)
}
}
// Add to the global output list.
mutex.Lock()
allOutput = append(allOutput, output...)
mutex.Unlock()
}()
}
// Walk the directories and find files to scan.
for _, dir := range flag.Args() {
absDir, err := filepath.Abs(dir)
if err != nil {
panic(err)
}
deviceRoot := filepath.Clean(absDir + "/..")
err = filepath.Walk(dir, func(path string, stat os.FileInfo, err error) error {
if err != nil {
panic(err)
}
if stat.IsDir() {
return nil
}
absPath, err := filepath.Abs(path)
if err != nil {
panic(err)
}
devicePath, err := filepath.Rel(deviceRoot, absPath)
if err != nil {
panic(err)
}
devicePath = "/" + devicePath
ch <- newNode(absPath, devicePath, stat)
return nil
})
if err != nil {
panic(err)
}
}
// Wait until all the goroutines finish.
close(ch)
wg.Wait()
// Sort the entries and dump as json.
sort.Slice(allOutput, func(i, j int) bool {
if allOutput[i].Size > allOutput[j].Size {
return true
}
if allOutput[i].Size == allOutput[j].Size && strings.Compare(allOutput[i].Name, allOutput[j].Name) > 0 {
return true
}
return false
})
j, err := json.MarshalIndent(allOutput, "", " ")
if err != nil {
panic(nil)
}
fmt.Printf("%s\n", j)
}
|
