fastboot: Support larger transfers during flash

Adding methods to queue and download flashable images by fd instead of
by pointer, so that we can deal with sending large (up to 4GB) files
on windows and linux.  This gets past limitations on linux to read
more than 2GB from a file at a time, as well as memory limitations
on win32, in order to download up to 4GB in a single transfer.

Test: fastboot -w
Test: "flash-all" from nexus factory images site (incl. fastboot -w update)
Test: fastboot flash with large and small image, large and small max-download-size
Test: Sanity check flashing on win32, darwin, linux.
Test: Sanity check 3GB image download (with 3GB max-download-size)
      on win32, darwin, linux.

Bug: 36810152
Change-Id: I528d739d344eb080d59d721dadf3b3b34d4b375e

4 files changed, 91 insertions, 20 deletions

diff --git a/fastboot/engine.cpp b/fastboot/engine.cpp
index 9f7d22657..bf887c9e9 100644
--- a/fastboot/engine.cpp
+++ b/fastboot/engine.cpp
@@ -44,6 +44,7 @@
 #define OP_NOTICE     4
 #define OP_DOWNLOAD_SPARSE 5
 #define OP_WAIT_FOR_DISCONNECT 6
+#define OP_DOWNLOAD_FD 7
 
 typedef struct Action Action;
 
@@ -56,6 +57,7 @@ struct Action {
     char cmd[CMD_SIZE];
     const char* prod;
     void* data;
+    int fd;
 
     // The protocol only supports 32-bit sizes, so you'll have to break
     // anything larger into chunks.
@@ -142,7 +144,20 @@ void fb_queue_erase(const char *ptn)
     a->msg = mkmsg("erasing '%s'", ptn);
 }
 
-void fb_queue_flash(const char *ptn, void *data, unsigned sz)
+void fb_queue_flash_fd(const char *ptn, int fd, uint32_t sz)
+{
+    Action *a;
+
+    a = queue_action(OP_DOWNLOAD_FD, "");
+    a->fd = fd;
+    a->size = sz;
+    a->msg = mkmsg("sending '%s' (%d KB)", ptn, sz / 1024);
+
+    a = queue_action(OP_COMMAND, "flash:%s", ptn);
+    a->msg = mkmsg("writing '%s'", ptn);
+}
+
+void fb_queue_flash(const char *ptn, void *data, uint32_t sz)
 {
     Action *a;
 
@@ -155,7 +170,7 @@ void fb_queue_flash(const char *ptn, void *data, unsigned sz)
     a->msg = mkmsg("writing '%s'", ptn);
 }
 
-void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, unsigned sz, size_t current,
+void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, uint32_t sz, size_t current,
                            size_t total) {
     Action *a;
 
@@ -282,7 +297,7 @@ static int cb_save(Action* a, int status, const char* resp) {
     return 0;
 }
 
-void fb_queue_query_save(const char *var, char *dest, unsigned dest_size)
+void fb_queue_query_save(const char *var, char *dest, uint32_t dest_size)
 {
     Action *a;
     a = queue_action(OP_QUERY, "getvar:%s", var);
@@ -309,7 +324,7 @@ void fb_queue_command(const char *cmd, const char *msg)
     a->msg = msg;
 }
 
-void fb_queue_download(const char *name, void *data, unsigned size)
+void fb_queue_download(const char *name, void *data, uint32_t size)
 {
     Action *a = queue_action(OP_DOWNLOAD, "");
     a->data = data;
@@ -351,6 +366,10 @@ int64_t fb_execute_queue(Transport* transport)
             status = fb_download_data(transport, a->data, a->size);
             status = a->func(a, status, status ? fb_get_error().c_str() : "");
             if (status) break;
+        } else if (a->op == OP_DOWNLOAD_FD) {
+            status = fb_download_data_fd(transport, a->fd, a->size);
+            status = a->func(a, status, status ? fb_get_error().c_str() : "");
+            if (status) break;
         } else if (a->op == OP_COMMAND) {
             status = fb_command(transport, a->cmd);
             status = a->func(a, status, status ? fb_get_error().c_str() : "");
diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp
index cb8e5c026..3b524ac3c 100644
--- a/fastboot/fastboot.cpp
+++ b/fastboot/fastboot.cpp
@@ -55,6 +55,7 @@
 #include <android-base/parsenetaddress.h>
 #include <android-base/stringprintf.h>
 #include <android-base/strings.h>
+#include <android-base/unique_fd.h>
 #include <sparse/sparse.h>
 #include <ziparchive/zip_archive.h>
 
@@ -67,6 +68,8 @@
 #include "udp.h"
 #include "usb.h"
 
+using android::base::unique_fd;
+
 #ifndef O_BINARY
 #define O_BINARY 0
 #endif
@@ -95,7 +98,7 @@ static unsigned tags_offset    = 0x00000100;
 static const std::string convert_fbe_marker_filename("convert_fbe");
 
 enum fb_buffer_type {
-    FB_BUFFER,
+    FB_BUFFER_FD,
     FB_BUFFER_SPARSE,
 };
 
@@ -103,6 +106,7 @@ struct fastboot_buffer {
     enum fb_buffer_type type;
     void* data;
     int64_t sz;
+    int fd;
 };
 
 static struct {
@@ -826,10 +830,9 @@ static bool load_buf_fd(Transport* transport, int fd, struct fastboot_buffer* bu
         buf->type = FB_BUFFER_SPARSE;
         buf->data = s;
     } else {
-        void* data = load_fd(fd, &sz);
-        if (data == nullptr) return -1;
-        buf->type = FB_BUFFER;
-        buf->data = data;
+        buf->type = FB_BUFFER_FD;
+        buf->data = nullptr;
+        buf->fd = fd;
         buf->sz = sz;
     }
 
@@ -837,11 +840,22 @@ static bool load_buf_fd(Transport* transport, int fd, struct fastboot_buffer* bu
 }
 
 static bool load_buf(Transport* transport, const char* fname, struct fastboot_buffer* buf) {
-    int fd = open(fname, O_RDONLY | O_BINARY);
+    unique_fd fd(TEMP_FAILURE_RETRY(open(fname, O_RDONLY | O_BINARY)));
+
     if (fd == -1) {
         return false;
     }
-    return load_buf_fd(transport, fd, buf);
+
+    struct stat s;
+    if (fstat(fd, &s)) {
+        return false;
+    }
+    if (!S_ISREG(s.st_mode)) {
+        errno = S_ISDIR(s.st_mode) ? EISDIR : EINVAL;
+        return false;
+    }
+
+    return load_buf_fd(transport, fd.release(), buf);
 }
 
 static void flash_buf(const char *pname, struct fastboot_buffer *buf)
@@ -864,9 +878,8 @@ static void flash_buf(const char *pname, struct fastboot_buffer *buf)
             }
             break;
         }
-
-        case FB_BUFFER:
-            fb_queue_flash(pname, buf->data, buf->sz);
+        case FB_BUFFER_FD:
+            fb_queue_flash_fd(pname, buf->fd, buf->sz);
             break;
         default:
             die("unknown buffer type: %d", buf->type);
diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h
index b62a2d8d8..3f9527055 100644
--- a/fastboot/fastboot.h
+++ b/fastboot/fastboot.h
@@ -42,6 +42,7 @@ struct sparse_file;
 int fb_command(Transport* transport, const char* cmd);
 int fb_command_response(Transport* transport, const char* cmd, char* response);
 int64_t fb_download_data(Transport* transport, const void* data, uint32_t size);
+int64_t fb_download_data_fd(Transport* transport, int fd, uint32_t size);
 int fb_download_data_sparse(Transport* transport, struct sparse_file* s);
 const std::string fb_get_error();
 
@@ -51,6 +52,7 @@ const std::string fb_get_error();
 /* engine.c - high level command queue engine */
 bool fb_getvar(Transport* transport, const std::string& key, std::string* value);
 void fb_queue_flash(const char *ptn, void *data, uint32_t sz);
+void fb_queue_flash_fd(const char *ptn, int fd, uint32_t sz);
 void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, uint32_t sz, size_t current,
                            size_t total);
 void fb_queue_erase(const char *ptn);
diff --git a/fastboot/protocol.cpp b/fastboot/protocol.cpp
index bf0479e1d..334f81ffb 100644
--- a/fastboot/protocol.cpp
+++ b/fastboot/protocol.cpp
@@ -38,6 +38,7 @@
 
 #include <android-base/stringprintf.h>
 #include <sparse/sparse.h>
+#include <utils/FileMap.h>
 
 #include "fastboot.h"
 #include "transport.h"
@@ -168,6 +169,39 @@ static int64_t _command_send(Transport* transport, const char* cmd, const void*
     return size;
 }
 
+static int64_t _command_send_fd(Transport* transport, const char* cmd, int fd, uint32_t size,
+                                char* response) {
+    static constexpr uint32_t MAX_MAP_SIZE = 512 * 1024 * 1024;
+    off64_t offset = 0;
+    uint32_t remaining = size;
+
+    if (_command_start(transport, cmd, size, response) < 0) {
+        return -1;
+    }
+
+    while (remaining) {
+        android::FileMap filemap;
+        size_t len = std::min(remaining, MAX_MAP_SIZE);
+
+        if (!filemap.create(NULL, fd, offset, len, true)) {
+            return -1;
+        }
+
+        if (_command_data(transport, filemap.getDataPtr(), len) < 0) {
+            return -1;
+        }
+
+        remaining -= len;
+        offset += len;
+    }
+
+    if (_command_end(transport) < 0) {
+        return -1;
+    }
+
+    return size;
+}
+
 static int _command_send_no_data(Transport* transport, const char* cmd, char* response) {
     return _command_start(transport, cmd, 0, response);
 }
@@ -181,9 +215,13 @@ int fb_command_response(Transport* transport, const char* cmd, char* response) {
 }
 
 int64_t fb_download_data(Transport* transport, const void* data, uint32_t size) {
-    char cmd[64];
-    snprintf(cmd, sizeof(cmd), "download:%08x", size);
-    return _command_send(transport, cmd, data, size, 0) < 0 ? -1 : 0;
+    std::string cmd(android::base::StringPrintf("download:%08x", size));
+    return _command_send(transport, cmd.c_str(), data, size, 0) < 0 ? -1 : 0;
+}
+
+int64_t fb_download_data_fd(Transport* transport, int fd, uint32_t size) {
+    std::string cmd(android::base::StringPrintf("download:%08x", size));
+    return _command_send_fd(transport, cmd.c_str(), fd, size, 0) < 0 ? -1 : 0;
 }
 
 #define TRANSPORT_BUF_SIZE 1024
@@ -257,9 +295,8 @@ int fb_download_data_sparse(Transport* transport, struct sparse_file* s) {
         return -1;
     }
 
-    char cmd[64];
-    snprintf(cmd, sizeof(cmd), "download:%08x", size);
-    int r = _command_start(transport, cmd, size, 0);
+    std::string cmd(android::base::StringPrintf("download:%08x", size));
+    int r = _command_start(transport, cmd.c_str(), size, 0);
     if (r < 0) {
         return -1;
     }