zip2zip: Support sorting globbed arguments, '**'

When '-s' is passed, any globbed arguments will have their results
sorted. When there are multiple arguments, the files will still be
inserted in argument order.

A bare '**' is now special cased to mean every file in the input zip.

This allows zip2zip to sort entire zip files efficiently by using
`zip2zip -s -i <> -o <> '**'`. This can be useful if your original zip
program used filesystem ordering which was not reproducible.

Test: m -j blueprint_tools (new tests pass)
Change-Id: Ic3512c5fe14c94c6f3e134296905121d2ff8b58a

3 files changed, 272 insertions, 42 deletions

diff --git a/cmd/zip2zip/Android.bp b/cmd/zip2zip/Android.bp
index 8cac003b..476be4f2 100644
--- a/cmd/zip2zip/Android.bp
+++ b/cmd/zip2zip/Android.bp
@@ -18,5 +18,6 @@ blueprint_go_binary {
     srcs: [
         "zip2zip.go",
     ],
+    testSrcs: ["zip2zip_test.go"],
 }
 
diff --git a/cmd/zip2zip/zip2zip.go b/cmd/zip2zip/zip2zip.go
index 8e7523ff..48c36ccc 100644
--- a/cmd/zip2zip/zip2zip.go
+++ b/cmd/zip2zip/zip2zip.go
@@ -17,49 +17,59 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 
 	"android/soong/third_party/zip"
 )
 
 var (
-	input  = flag.String("i", "", "zip file to read from")
-	output = flag.String("o", "", "output file")
-)
+	input     = flag.String("i", "", "zip file to read from")
+	output    = flag.String("o", "", "output file")
+	sortGlobs = flag.Bool("s", false, "sort matches from each glob (defaults to the order from the input zip file)")
+	setTime   = flag.Bool("t", false, "set timestamps to 2009-01-01 00:00:00")
 
-func usage() {
-	fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [filespec]...")
-	flag.PrintDefaults()
-	fmt.Fprintln(os.Stderr, "  filespec:")
-	fmt.Fprintln(os.Stderr, "    <name>")
-	fmt.Fprintln(os.Stderr, "    <in_name>:<out_name>")
-	fmt.Fprintln(os.Stderr, "    <glob>:<out_dir>/")
-	fmt.Fprintln(os.Stderr, "")
-	fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
-	fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments")
-	os.Exit(2)
-}
+	staticTime = time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
+)
 
 func main() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [-s] [-t] [filespec]...")
+		flag.PrintDefaults()
+		fmt.Fprintln(os.Stderr, "  filespec:")
+		fmt.Fprintln(os.Stderr, "    <name>")
+		fmt.Fprintln(os.Stderr, "    <in_name>:<out_name>")
+		fmt.Fprintln(os.Stderr, "    <glob>:<out_dir>/")
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://golang.org/pkg/path/filepath/#Match")
+		fmt.Fprintln(os.Stderr, "As a special exception, '**' is supported to specify all files in the input zip")
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
+		fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments")
+	}
+
 	flag.Parse()
 
 	if flag.NArg() == 0 || *input == "" || *output == "" {
-		usage()
+		flag.Usage()
+		os.Exit(1)
 	}
 
+	log.SetFlags(log.Lshortfile)
+
 	reader, err := zip.OpenReader(*input)
 	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(3)
+		log.Fatal(err)
 	}
 	defer reader.Close()
 
 	output, err := os.Create(*output)
 	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(4)
+		log.Fatal(err)
 	}
 	defer output.Close()
 
@@ -67,20 +77,24 @@ func main() {
 	defer func() {
 		err := writer.Close()
 		if err != nil {
-			fmt.Fprintln(os.Stderr, err)
-			os.Exit(5)
+			log.Fatal(err)
 		}
 	}()
 
-	for _, arg := range flag.Args() {
+	if err := zip2zip(&reader.Reader, writer, *sortGlobs, *setTime, flag.Args()); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func zip2zip(reader *zip.Reader, writer *zip.Writer, sortGlobs, setTime bool, args []string) error {
+	for _, arg := range args {
 		var input string
 		var output string
 
 		// Reserve escaping for future implementation, so make sure no
 		// one is using \ and expecting a certain behavior.
 		if strings.Contains(arg, "\\") {
-			fmt.Fprintln(os.Stderr, "\\ characters are not currently supported")
-			os.Exit(6)
+			return fmt.Errorf("\\ characters are not currently supported")
 		}
 
 		args := strings.SplitN(arg, ":", 2)
@@ -89,40 +103,67 @@ func main() {
 			output = args[1]
 		}
 
+		type pair struct {
+			*zip.File
+			newName string
+		}
+
+		matches := []pair{}
 		if strings.IndexAny(input, "*?[") >= 0 {
+			matchAll := input == "**"
+			if !matchAll && strings.Contains(input, "**") {
+				return fmt.Errorf("** is only supported on its own, not with other characters")
+			}
+
 			for _, file := range reader.File {
-				if match, err := filepath.Match(input, file.Name); err != nil {
-					fmt.Fprintln(os.Stderr, err)
-					os.Exit(7)
-				} else if match {
-					var newFileName string
+				match := matchAll
+
+				if !match {
+					var err error
+					match, err = filepath.Match(input, file.Name)
+					if err != nil {
+						return err
+					}
+				}
+
+				if match {
+					var newName string
 					if output == "" {
-						newFileName = file.Name
+						newName = file.Name
 					} else {
 						_, name := filepath.Split(file.Name)
-						newFileName = filepath.Join(output, name)
-					}
-					err = writer.CopyFrom(file, newFileName)
-					if err != nil {
-						fmt.Fprintln(os.Stderr, err)
-						os.Exit(8)
+						newName = filepath.Join(output, name)
 					}
+					matches = append(matches, pair{file, newName})
 				}
 			}
+
+			if sortGlobs {
+				sort.SliceStable(matches, func(i, j int) bool {
+					return matches[i].newName < matches[j].newName
+				})
+			}
 		} else {
 			if output == "" {
 				output = input
 			}
 			for _, file := range reader.File {
 				if input == file.Name {
-					err = writer.CopyFrom(file, output)
-					if err != nil {
-						fmt.Fprintln(os.Stderr, err)
-						os.Exit(8)
-					}
+					matches = append(matches, pair{file, output})
 					break
 				}
 			}
 		}
+
+		for _, match := range matches {
+			if setTime {
+				match.File.SetModTime(staticTime)
+			}
+			if err := writer.CopyFrom(match.File, match.newName); err != nil {
+				return err
+			}
+		}
 	}
+
+	return nil
 }
diff --git a/cmd/zip2zip/zip2zip_test.go b/cmd/zip2zip/zip2zip_test.go
new file mode 100644
index 00000000..7f2e31a4
--- /dev/null
+++ b/cmd/zip2zip/zip2zip_test.go
@@ -0,0 +1,188 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"android/soong/third_party/zip"
+)
+
+var testCases = []struct {
+	name string
+
+	inputFiles []string
+	sortGlobs  bool
+	args       []string
+
+	outputFiles []string
+	err         error
+}{
+	{
+		name: "unsupported \\",
+
+		args: []string{"a\\b:b"},
+
+		err: fmt.Errorf("\\ characters are not currently supported"),
+	},
+	{
+		name: "unsupported **",
+
+		args: []string{"a/**:b"},
+
+		err: fmt.Errorf("** is only supported on its own, not with other characters"),
+	},
+	{ // This is modelled after the update package build rules in build/make/core/Makefile
+		name: "filter globs",
+
+		inputFiles: []string{
+			"RADIO/a",
+			"IMAGES/system.img",
+			"IMAGES/b.txt",
+			"IMAGES/recovery.img",
+			"IMAGES/vendor.img",
+			"OTA/android-info.txt",
+			"OTA/b",
+		},
+		args: []string{"OTA/android-info.txt:android-info.txt", "IMAGES/*.img:."},
+
+		outputFiles: []string{
+			"android-info.txt",
+			"system.img",
+			"recovery.img",
+			"vendor.img",
+		},
+	},
+	{
+		name: "sorted filter globs",
+
+		inputFiles: []string{
+			"RADIO/a",
+			"IMAGES/system.img",
+			"IMAGES/b.txt",
+			"IMAGES/recovery.img",
+			"IMAGES/vendor.img",
+			"OTA/android-info.txt",
+			"OTA/b",
+		},
+		sortGlobs: true,
+		args:      []string{"IMAGES/*.img:.", "OTA/android-info.txt:android-info.txt"},
+
+		outputFiles: []string{
+			"recovery.img",
+			"system.img",
+			"vendor.img",
+			"android-info.txt",
+		},
+	},
+	{
+		name: "sort all",
+
+		inputFiles: []string{
+			"RADIO/a",
+			"IMAGES/system.img",
+			"IMAGES/b.txt",
+			"IMAGES/recovery.img",
+			"IMAGES/vendor.img",
+			"OTA/b",
+			"OTA/android-info.txt",
+		},
+		sortGlobs: true,
+		args:      []string{"**"},
+
+		outputFiles: []string{
+			"IMAGES/b.txt",
+			"IMAGES/recovery.img",
+			"IMAGES/system.img",
+			"IMAGES/vendor.img",
+			"OTA/android-info.txt",
+			"OTA/b",
+			"RADIO/a",
+		},
+	},
+	{
+		name: "double input",
+
+		inputFiles: []string{
+			"b",
+			"a",
+		},
+		args: []string{"a:a2", "**"},
+
+		outputFiles: []string{
+			"a2",
+			"b",
+			"a",
+		},
+	},
+}
+
+func errorString(e error) string {
+	if e == nil {
+		return ""
+	}
+	return e.Error()
+}
+
+func TestZip2Zip(t *testing.T) {
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			inputBuf := &bytes.Buffer{}
+			outputBuf := &bytes.Buffer{}
+
+			inputWriter := zip.NewWriter(inputBuf)
+			for _, file := range testCase.inputFiles {
+				w, err := inputWriter.Create(file)
+				if err != nil {
+					t.Fatal(err)
+				}
+				fmt.Fprintln(w, "test")
+			}
+			inputWriter.Close()
+			inputBytes := inputBuf.Bytes()
+			inputReader, err := zip.NewReader(bytes.NewReader(inputBytes), int64(len(inputBytes)))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			outputWriter := zip.NewWriter(outputBuf)
+			err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, false, testCase.args)
+			if errorString(testCase.err) != errorString(err) {
+				t.Fatalf("Unexpected error:\n got: %q\nwant: %q", errorString(err), errorString(testCase.err))
+			}
+
+			outputWriter.Close()
+			outputBytes := outputBuf.Bytes()
+			outputReader, err := zip.NewReader(bytes.NewReader(outputBytes), int64(len(outputBytes)))
+			if err != nil {
+				t.Fatal(err)
+			}
+			var outputFiles []string
+			if len(outputReader.File) > 0 {
+				outputFiles = make([]string, len(outputReader.File))
+				for i, file := range outputReader.File {
+					outputFiles[i] = file.Name
+				}
+			}
+
+			if !reflect.DeepEqual(testCase.outputFiles, outputFiles) {
+				t.Fatalf("Output file list does not match:\n got: %v\nwant: %v", outputFiles, testCase.outputFiles)
+			}
+		})
+	}
+}