Filter kati output to collapse verbose messages

1. Collapses the "including .../Android.mk ..." lines like ninja does,
so that we overwrite the last line if there wasn't anything else to
print.
2. Strips ansi control codes so that log files don't include unreadable
characters.

Test: m -j
Test: m -j | tee output.log   (with colored output)
Change-Id: Ib18437f6f9d37084360097a9d586800c833072c5

7 files changed, 270 insertions, 3 deletions

diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index e4044e18..51aed2c1 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -33,5 +33,12 @@ bootstrap_go_package {
     ],
     testSrcs: [
         "environment_test.go",
+        "util_test.go",
     ],
+    darwin: {
+        srcs: ["util_darwin.go"],
+    },
+    linux: {
+        srcs: ["util_linux.go"],
+    },
 }
diff --git a/ui/build/context.go b/ui/build/context.go
index 8144e586..f85bb6c0 100644
--- a/ui/build/context.go
+++ b/ui/build/context.go
@@ -95,3 +95,10 @@ func (c ContextImpl) ImportNinjaLog(filename string, startOffset time.Time) {
 		c.Tracer.ImportNinjaLog(c.Thread, filename, startOffset)
 	}
 }
+
+func (c ContextImpl) IsTerminal() bool {
+	if term, ok := os.LookupEnv("TERM"); ok {
+		return term != "dumb" && isTerminal(c.Stdout()) && isTerminal(c.Stderr())
+	}
+	return false
+}
diff --git a/ui/build/kati.go b/ui/build/kati.go
index 423bcbc9..86db8878 100644
--- a/ui/build/kati.go
+++ b/ui/build/kati.go
@@ -15,11 +15,14 @@
 package build
 
 import (
+	"bufio"
 	"crypto/md5"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -94,10 +97,20 @@ func runKati(ctx Context, config Config) {
 
 	cmd := exec.CommandContext(ctx.Context, executable, args...)
 	cmd.Env = config.Environment().Environ()
-	cmd.Stdout = ctx.Stdout()
-	cmd.Stderr = ctx.Stderr()
+	pipe, err := cmd.StdoutPipe()
+	if err != nil {
+		ctx.Fatalln("Error getting output pipe for ckati:", err)
+	}
+	cmd.Stderr = cmd.Stdout
+
 	ctx.Verboseln(cmd.Path, cmd.Args)
-	if err := cmd.Run(); err != nil {
+	if err := cmd.Start(); err != nil {
+		ctx.Fatalln("Failed to run ckati:", err)
+	}
+
+	katiRewriteOutput(ctx, pipe)
+
+	if err := cmd.Wait(); err != nil {
 		if e, ok := err.(*exec.ExitError); ok {
 			ctx.Fatalln("ckati failed with:", e.ProcessState.String())
 		} else {
@@ -105,3 +118,57 @@ func runKati(ctx Context, config Config) {
 		}
 	}
 }
+
+var katiIncludeRe = regexp.MustCompile(`^(\[\d+/\d+] )?including [^ ]+ ...$`)
+
+func katiRewriteOutput(ctx Context, pipe io.ReadCloser) {
+	haveBlankLine := true
+	smartTerminal := ctx.IsTerminal()
+
+	scanner := bufio.NewScanner(pipe)
+	for scanner.Scan() {
+		line := scanner.Text()
+		verbose := katiIncludeRe.MatchString(line)
+
+		// For verbose lines, write them on the current line without a newline,
+		// then overwrite them if the next thing we're printing is another
+		// verbose line.
+		if smartTerminal && verbose {
+			// Limit line width to the terminal width, otherwise we'll wrap onto
+			// another line and we won't delete the previous line.
+			//
+			// Run this on every line in case the window has been resized while
+			// we're printing. This could be optimized to only re-run when we
+			// get SIGWINCH if it ever becomes too time consuming.
+			if max, ok := termWidth(ctx.Stdout()); ok {
+				if len(line) > max {
+					// Just do a max. Ninja elides the middle, but that's
+					// more complicated and these lines aren't that important.
+					line = line[:max]
+				}
+			}
+
+			// Move to the beginning on the line, print the output, then clear
+			// the rest of the line.
+			fmt.Fprint(ctx.Stdout(), "\r", line, "\x1b[K")
+			haveBlankLine = false
+			continue
+		} else if smartTerminal && !haveBlankLine {
+			// If we've previously written a verbose message, send a newline to save
+			// that message instead of overwriting it.
+			fmt.Fprintln(ctx.Stdout())
+			haveBlankLine = true
+		} else if !smartTerminal {
+			// Most editors display these as garbage, so strip them out.
+			line = string(stripAnsiEscapes([]byte(line)))
+		}
+
+		// Assume that non-verbose lines are important enough for stderr
+		fmt.Fprintln(ctx.Stderr(), line)
+	}
+
+	// Save our last verbose line.
+	if !haveBlankLine {
+		fmt.Fprintln(ctx.Stdout())
+	}
+}
diff --git a/ui/build/util.go b/ui/build/util.go
index ad084da6..37ac6b9a 100644
--- a/ui/build/util.go
+++ b/ui/build/util.go
@@ -15,9 +15,13 @@
 package build
 
 import (
+	"bytes"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
+	"unsafe"
 )
 
 // indexList finds the index of a string in a []string
@@ -77,3 +81,81 @@ func decodeKeyValue(str string) (string, string, bool) {
 	}
 	return str[:idx], str[idx+1:], true
 }
+
+func isTerminal(w io.Writer) bool {
+	if f, ok := w.(*os.File); ok {
+		var termios syscall.Termios
+		_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(),
+			ioctlGetTermios, uintptr(unsafe.Pointer(&termios)),
+			0, 0, 0)
+		return err == 0
+	}
+	return false
+}
+
+func termWidth(w io.Writer) (int, bool) {
+	if f, ok := w.(*os.File); ok {
+		var winsize struct {
+			ws_row, ws_column    uint16
+			ws_xpixel, ws_ypixel uint16
+		}
+		_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(),
+			syscall.TIOCGWINSZ, uintptr(unsafe.Pointer(&winsize)),
+			0, 0, 0)
+		return int(winsize.ws_column), err == 0
+	}
+	return 0, false
+}
+
+// stripAnsiEscapes strips ANSI control codes from a byte array in place.
+func stripAnsiEscapes(input []byte) []byte {
+	// read represents the remaining part of input that needs to be processed.
+	read := input
+	// write represents where we should be writing in input.
+	// It will share the same backing store as input so that we make our modifications
+	// in place.
+	write := input
+
+	// advance will copy count bytes from read to write and advance those slices
+	advance := func(write, read []byte, count int) ([]byte, []byte) {
+		copy(write, read[:count])
+		return write[count:], read[count:]
+	}
+
+	for {
+		// Find the next escape sequence
+		i := bytes.IndexByte(read, 0x1b)
+		// If it isn't found, or if there isn't room for <ESC>[, finish
+		if i == -1 || i+1 >= len(read) {
+			copy(write, read)
+			break
+		}
+
+		// Not a CSI code, continue searching
+		if read[i+1] != '[' {
+			write, read = advance(write, read, i+1)
+			continue
+		}
+
+		// Found a CSI code, advance up to the <ESC>
+		write, read = advance(write, read, i)
+
+		// Find the end of the CSI code
+		i = bytes.IndexFunc(read, func(r rune) bool {
+			return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
+		})
+		if i == -1 {
+			// We didn't find the end of the code, just remove the rest
+			i = len(read) - 1
+		}
+
+		// Strip off the end marker too
+		i = i + 1
+
+		// Skip the reader forward and reduce final length by that amount
+		read = read[i:]
+		input = input[:len(input)-i]
+	}
+
+	return input
+}
diff --git a/ui/build/util_darwin.go b/ui/build/util_darwin.go
new file mode 100644
index 00000000..254a9b87
--- /dev/null
+++ b/ui/build/util_darwin.go
@@ -0,0 +1,21 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"syscall"
+)
+
+const ioctlGetTermios = syscall.TIOCGETA
diff --git a/ui/build/util_linux.go b/ui/build/util_linux.go
new file mode 100644
index 00000000..0a4e1d29
--- /dev/null
+++ b/ui/build/util_linux.go
@@ -0,0 +1,21 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"syscall"
+)
+
+const ioctlGetTermios = syscall.TCGETS
diff --git a/ui/build/util_test.go b/ui/build/util_test.go
new file mode 100644
index 00000000..e85eadad
--- /dev/null
+++ b/ui/build/util_test.go
@@ -0,0 +1,62 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import "testing"
+
+func TestStripAnsiEscapes(t *testing.T) {
+	testcases := []struct {
+		input  string
+		output string
+	}{
+		{
+			"",
+			"",
+		},
+		{
+			"This is a test",
+			"This is a test",
+		},
+		{
+			"interrupted: \x1b[12",
+			"interrupted: ",
+		},
+		{
+			"other \x1bescape \x1b",
+			"other \x1bescape \x1b",
+		},
+		{ // from pretty-error macro
+			"\x1b[1mart/Android.mk: \x1b[31merror:\x1b[0m\x1b[1m art: test error \x1b[0m",
+			"art/Android.mk: error: art: test error ",
+		},
+		{ // from envsetup.sh make wrapper
+			"\x1b[0;31m#### make failed to build some targets (2 seconds) ####\x1b[00m",
+			"#### make failed to build some targets (2 seconds) ####",
+		},
+		{ // from clang (via ninja testcase)
+			"\x1b[1maffixmgr.cxx:286:15: \x1b[0m\x1b[0;1;35mwarning: \x1b[0m\x1b[1musing the result... [-Wparentheses]\x1b[0m",
+			"affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]",
+		},
+	}
+	for _, tc := range testcases {
+		got := string(stripAnsiEscapes([]byte(tc.input)))
+		if got != tc.output {
+			t.Errorf("output strings didn't match\n"+
+				"input: %#v\n"+
+				" want: %#v\n"+
+				"  got: %#v", tc.input, tc.output, got)
+		}
+	}
+}