adb: create unix_isatty() function.

Our Windows code has several different ways it checks whether an FD is
a console or not, some of which aren't exactly correct as they may
treat character devices (e.g. NUL) as consoles.

This CL disallows using the builtin isatty() function and provides
unix_isatty() instead which correctly checks these corner cases.

Change-Id: I6d551c745dae691c7eb3446b585265d62c1e62fa

6 files changed, 109 insertions, 57 deletions

diff --git a/adb/adb_trace.cpp b/adb/adb_trace.cpp
index 04b82f670..9586f7c75 100644
--- a/adb/adb_trace.cpp
+++ b/adb/adb_trace.cpp
@@ -157,7 +157,7 @@ void adb_trace_init(char** argv) {
     // Don't open log file if no tracing, since this will block
     // the crypto unmount of /data
     if (!get_trace_setting().empty()) {
-        if (isatty(STDOUT_FILENO) == 0) {
+        if (unix_isatty(STDOUT_FILENO) == 0) {
             start_device_log();
         }
     }
diff --git a/adb/commandline.cpp b/adb/commandline.cpp
index 37bd77794..c912936a6 100644
--- a/adb/commandline.cpp
+++ b/adb/commandline.cpp
@@ -1388,12 +1388,12 @@ int adb_commandline(int argc, const char **argv) {
                 // things like `adb shell < my_script.sh` work as expected.
                 // Otherwise leave |shell_type_arg| blank which uses PTY for
                 // interactive shells and raw for non-interactive.
-                if (!isatty(STDIN_FILENO)) {
+                if (!unix_isatty(STDIN_FILENO)) {
                     shell_type_arg = kShellServiceArgRaw;
                 }
             } else if (t_arg_count == 1) {
                 // A single -t arg isn't enough to override implicit -T.
-                if (!isatty(STDIN_FILENO)) {
+                if (!unix_isatty(STDIN_FILENO)) {
                     fprintf(stderr,
                             "Remote PTY will not be allocated because stdin is not a terminal.\n"
                             "Use multiple -t options to force remote PTY allocation.\n");
diff --git a/adb/line_printer.cpp b/adb/line_printer.cpp
index 81b3f0a0b..aa332f7a1 100644
--- a/adb/line_printer.cpp
+++ b/adb/line_printer.cpp
@@ -46,7 +46,7 @@ string ElideMiddle(const string& str, size_t width) {
 LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) {
 #ifndef _WIN32
   const char* term = getenv("TERM");
-  smart_terminal_ = isatty(1) && term && string(term) != "dumb";
+  smart_terminal_ = unix_isatty(1) && term && string(term) != "dumb";
 #else
   // Disable output buffer.  It'd be nice to use line buffering but
   // MSDN says: "For some systems, [_IOLBF] provides line
diff --git a/adb/sysdeps.h b/adb/sysdeps.h
index 51d09a6f4..173562745 100644
--- a/adb/sysdeps.h
+++ b/adb/sysdeps.h
@@ -181,6 +181,17 @@ static __inline__ int  adb_open_mode(const char* path, int options, int mode)
 extern int unix_open(const char* path, int options, ...);
 #define  open    ___xxx_unix_open
 
+// Checks if |fd| corresponds to a console.
+// Standard Windows isatty() returns 1 for both console FDs and character
+// devices like NUL. unix_isatty() performs some extra checking to only match
+// console FDs.
+// |fd| must be a real file descriptor, meaning STDxx_FILENO or unix_open() FDs
+// will work but adb_open() FDs will not. Additionally the OS handle associated
+// with |fd| must have GENERIC_READ access (which console FDs have by default).
+// Returns 1 if |fd| is a console FD, 0 otherwise. The value of errno after
+// calling this function is unreliable and should not be used.
+int unix_isatty(int fd);
+#define  isatty  ___xxx_isatty
 
 /* normally provided by <cutils/misc.h> */
 extern void*  load_file(const char*  pathname, unsigned*  psize);
@@ -551,6 +562,11 @@ static __inline__  int  adb_creat(const char*  path, int  mode)
 #undef   creat
 #define  creat  ___xxx_creat
 
+static __inline__ int unix_isatty(int fd) {
+    return isatty(fd);
+}
+#define  isatty  ___xxx_isatty
+
 // Helper for network_* functions.
 inline int _fd_set_error_str(int fd, std::string* error) {
   if (fd == -1) {
diff --git a/adb/sysdeps_win32.cpp b/adb/sysdeps_win32.cpp
index 14d137585..d2e6cdb3f 100644
--- a/adb/sysdeps_win32.cpp
+++ b/adb/sysdeps_win32.cpp
@@ -2499,10 +2499,52 @@ adb_sysdeps_init( void )
 //
 // Code organization:
 //
+// * _get_console_handle() and unix_isatty() provide console information.
 // * stdin_raw_init() and stdin_raw_restore() reconfigure the console.
 // * unix_read() detects console windows (as opposed to pipes, files, etc.).
 // * _console_read() is the main code of the emulation.
 
+// Returns a console HANDLE if |fd| is a console, otherwise returns nullptr.
+// If a valid HANDLE is returned and |mode| is not null, |mode| is also filled
+// with the console mode. Requires GENERIC_READ access to the underlying HANDLE.
+static HANDLE _get_console_handle(int fd, DWORD* mode=nullptr) {
+    // First check isatty(); this is very fast and eliminates most non-console
+    // FDs, but returns 1 for both consoles and character devices like NUL.
+#pragma push_macro("isatty")
+#undef isatty
+    if (!isatty(fd)) {
+        return nullptr;
+    }
+#pragma pop_macro("isatty")
+
+    // To differentiate between character devices and consoles we need to get
+    // the underlying HANDLE and use GetConsoleMode(), which is what requires
+    // GENERIC_READ permissions.
+    const intptr_t intptr_handle = _get_osfhandle(fd);
+    if (intptr_handle == -1) {
+        return nullptr;
+    }
+    const HANDLE handle = reinterpret_cast<const HANDLE>(intptr_handle);
+    DWORD temp_mode = 0;
+    if (!GetConsoleMode(handle, mode ? mode : &temp_mode)) {
+        return nullptr;
+    }
+
+    return handle;
+}
+
+// Returns a console handle if |stream| is a console, otherwise returns nullptr.
+static HANDLE _get_console_handle(FILE* const stream) {
+    const int fd = fileno(stream);
+    if (fd < 0) {
+        return nullptr;
+    }
+    return _get_console_handle(fd);
+}
+
+int unix_isatty(int fd) {
+    return _get_console_handle(fd) ? 1 : 0;
+}
 
 // Read an input record from the console; one that should be processed.
 static bool _get_interesting_input_record_uncached(const HANDLE console,
@@ -3302,20 +3344,7 @@ static HANDLE _console_handle;  // when set, console mode should be restored
 
 void stdin_raw_init(const int fd) {
     if (STDIN_FILENO == fd) {
-        const HANDLE in = GetStdHandle(STD_INPUT_HANDLE);
-        if ((in == INVALID_HANDLE_VALUE) || (in == NULL)) {
-            return;
-        }
-
-        if (GetFileType(in) != FILE_TYPE_CHAR) {
-            // stdin might be a file or pipe.
-            return;
-        }
-
-        if (!GetConsoleMode(in, &_old_console_mode)) {
-            // If GetConsoleMode() fails, stdin is probably is not a console.
-            return;
-        }
+        const HANDLE in = _get_console_handle(fd, &_old_console_mode);
 
         // Disable ENABLE_PROCESSED_INPUT so that Ctrl-C is read instead of
         // calling the process Ctrl-C routine (configured by
@@ -3366,11 +3395,8 @@ int unix_read(int fd, void* buf, size_t len) {
     } else {
         // On older versions of Windows (definitely 7, definitely not 10),
         // ReadConsole() with a size >= 31367 fails, so if |fd| is a console
-        // we need to limit the read size. This may also catch devices like NUL,
-        // but that is OK as we just want to avoid capping pipes and files which
-        // don't need size limiting. This isatty() test is very simple and quick
-        // and doesn't call the OS.
-        if (isatty(fd) && len > 4096) {
+        // we need to limit the read size.
+        if (len > 4096 && unix_isatty(fd)) {
             len = 4096;
         }
         // Just call into C Runtime which can read from pipes/files and which
@@ -3725,40 +3751,6 @@ int adb_chmod(const char* path, int mode) {
     return _wchmod(widen(path).c_str(), mode);
 }
 
-// Internal function to get a Win32 console HANDLE from a C Runtime FILE*.
-static HANDLE _get_console_handle(FILE* const stream) {
-    // Get a C Runtime file descriptor number from the FILE* structure.
-    const int fd = fileno(stream);
-    if (fd < 0) {
-        return NULL;
-    }
-
-    // If it is not a "character device", it is probably a file and not a
-    // console. Do this check early because it is probably cheap. Still do more
-    // checks after this since there are devices that pass this test, but are
-    // not a console, such as NUL, the Windows /dev/null equivalent (I think).
-    if (!isatty(fd)) {
-        return NULL;
-    }
-
-    // Given a C Runtime file descriptor number, get the underlying OS
-    // file handle.
-    const intptr_t osfh = _get_osfhandle(fd);
-    if (osfh == -1) {
-        return NULL;
-    }
-
-    const HANDLE h = reinterpret_cast<const HANDLE>(osfh);
-
-    DWORD old_mode = 0;
-    if (!GetConsoleMode(h, &old_mode)) {
-        return NULL;
-    }
-
-    // If GetConsoleMode() was successful, assume this is a console.
-    return h;
-}
-
 // Internal helper function to write UTF-8 bytes to a console. Returns -1
 // on error.
 static int _console_write_utf8(const char* buf, size_t size, FILE* stream,
diff --git a/adb/sysdeps_win32_test.cpp b/adb/sysdeps_win32_test.cpp
index 66d1ba881..55b5eb462 100755
--- a/adb/sysdeps_win32_test.cpp
+++ b/adb/sysdeps_win32_test.cpp
@@ -18,6 +18,8 @@
 
 #include "sysdeps.h"
 
+#include "base/test_utils.h"
+
 TEST(sysdeps_win32, adb_getenv) {
     // Insert all test env vars before first call to adb_getenv() which will
     // read the env var block only once.
@@ -93,3 +95,45 @@ TEST(sysdeps_win32, adb_strerror) {
     // adb_strerror() returns.
     TestAdbStrError(ECONNRESET, "Connection reset by peer");
 }
+
+TEST(sysdeps_win32, unix_isatty) {
+    // stdin and stdout should be consoles. Use CONIN$ and CONOUT$ special files
+    // so that we can test this even if stdin/stdout have been redirected. Read
+    // permissions are required for unix_isatty().
+    int conin_fd = unix_open("CONIN$", O_RDONLY);
+    int conout_fd = unix_open("CONOUT$", O_RDWR);
+    for (const int fd : {conin_fd, conout_fd}) {
+        EXPECT_TRUE(fd >= 0);
+        EXPECT_EQ(1, unix_isatty(fd));
+        EXPECT_EQ(0, unix_close(fd));
+    }
+
+    // nul returns 1 from isatty(), make sure unix_isatty() corrects that.
+    for (auto flags : {O_RDONLY, O_RDWR}) {
+        int nul_fd = unix_open("nul", flags);
+        EXPECT_TRUE(nul_fd >= 0);
+        EXPECT_EQ(0, unix_isatty(nul_fd));
+        EXPECT_EQ(0, unix_close(nul_fd));
+    }
+
+    // Check a real file, both read-write and read-only.
+    TemporaryFile temp_file;
+    EXPECT_TRUE(temp_file.fd >= 0);
+    EXPECT_EQ(0, unix_isatty(temp_file.fd));
+
+    int temp_file_ro_fd = unix_open(temp_file.path, O_RDONLY);
+    EXPECT_TRUE(temp_file_ro_fd >= 0);
+    EXPECT_EQ(0, unix_isatty(temp_file_ro_fd));
+    EXPECT_EQ(0, unix_close(temp_file_ro_fd));
+
+    // Check a real OS pipe.
+    int pipe_fds[2];
+    EXPECT_EQ(0, _pipe(pipe_fds, 64, _O_BINARY));
+    EXPECT_EQ(0, unix_isatty(pipe_fds[0]));
+    EXPECT_EQ(0, unix_isatty(pipe_fds[1]));
+    EXPECT_EQ(0, _close(pipe_fds[0]));
+    EXPECT_EQ(0, _close(pipe_fds[1]));
+
+    // Make sure an invalid FD is handled correctly.
+    EXPECT_EQ(0, unix_isatty(-1));
+}