Update to 7.59.0 - March 14 2018.

Changes:

  curl: add --proxy-pinnedpubkey
  added: CURLOPT_TIMEVALUE_LARGE and CURLINFO_FILETIME_T
  CURLOPT_RESOLVE: Add support for multiple IP addresses per entry
  Add option CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS
  Add new tool option --happy-eyeballs-timeout-ms
  Add CURLOPT_RESOLVER_START_FUNCTION and CURLOPT_RESOLVER_START_DATA

Bugfixes:

  openldap: check ldap_get_attribute_ber() results for NULL before using
  FTP: reject path components with control codes
  readwrite: make sure excess reads don't go beyond buffer end
  lib555: drop text conversion and encode data as ascii codes
  lib517: make variable static to avoid compiler warning
  lib544: sync ascii code data with textual data
  GSKit: restore pinnedpubkey functionality
  darwinssl: Don't import client certificates into Keychain on macOS
  parsedate: fix date parsing for systems with 32 bit long
  openssl: fix pinned public key build error in FIPS mode
  SChannel/WinSSL: Implement public key pinning
  cookies: remove verbose "cookie size:" output
  progress-bar: don't use stderr explicitly, use bar->out
  Fixes for MSDOS
  build: open VC15 projects with VS 2017
  curl_ctype: private is*() type macros and functions
  configure: set PATH_SEPARATOR to colon for PATH w/o separator
  winbuild: make linker generate proper PDB
  curl_easy_reset: clear digest auth state
  curl/curl.h: fix comment typo for CURLOPT_DNS_LOCAL_IP6
  range: commonize FTP and FILE range handling
  progress-bar docs: update to match implementation
  fnmatch: do not match the empty string with a character set
  fnmatch: accept an alphanum to be followed by a non-alphanum in char set
  build: fix termios issue on android cross-compile
  getdate: return -1 for out of range
  formdata: use the mime-content type function
  time-cond: fix reading the file modification time on Windows
  build-openssl.bat: Extend VC15 support to include Enterprise and Professional
  build-wolfssl.bat: Extend VC15 support to include Enterprise and Professional
  openssl: Don't add verify locations when verifypeer==0
  fnmatch: optimize processing of consecutive *s and ?s pattern characters
  schannel: fix compiler warnings
  content_encoding: Add "none" alias to "identity"
  get_posix_time: only check for overflows if they can happen
  http_chunks: don't write chunks twice with CURLOPT_HTTP_TRANSFER_DECODING
  README: language fix
  sha256: build with OpenSSL < 0.9.8
  smtp: fix processing of initial dot in data
  --tlsauthtype: works only if libcurl is built with TLS-SRP support
  tests: new tests for http raw mode
  libcurl-security.3: man page discussion security concerns when using libcurl
  curl_gssapi: make sure this file too uses our *printf()
  BINDINGS: fix curb link (and remove ruby-curl-multi)
  nss: use PK11_CreateManagedGenericObject() if available
  travis: add build with iconv enabled
  ssh: add two missing state names
  CURLOPT_HEADERFUNCTION.3: mention folded headers
  http: fix the max header length detection logic
  header callback: don't chop headers into smaller pieces
  CURLOPT_HEADER.3: clarify problems with different data sizes
  curl --version: show PSL if the run-time lib has it enabled
  examples/sftpuploadresume: resume upload via CURLOPT_APPEND
  Return error if called recursively from within callbacks
  sasl: prefer PLAIN mechanism over LOGIN
  winbuild: Use CALL to run batch scripts
  curl_share_setopt.3: connection cache is shared within multi handles
  winbuild: Use macros for the names of some build utilities
  projects/README: remove reference to dead IDN link/package
  lib655: silence compiler warning
  configure: Fix version check for OpenSSL 1.1.1
  docs/MANUAL: formfind.pl is not accessible on the site anymore
  unit1309: fix warning on Windows x64
  unit1307: proper cleanup on OOM to fix torture tests
  curl_ctype: fix macro redefinition warnings
  build: get CFLAGS (including -werror) used for examples and tests
  NO_PROXY: fix for IPv6 numericals in the URL
  krb5: use nondeprecated functions
  winbuild: prefer documented zlib library names
  http2: mark the connection for close on GOAWAY
  limit-rate: kick in even before "limit" data has been received
  HTTP: allow "header;" to replace an internal header with a blank one
  http2: verbose output new MAX_CONCURRENT_STREAMS values
  SECURITY: distros' max embargo time is 14 days
  curl tool: accept --compressed also if Brotli is enabled and zlib is not
  WolfSSL: adding TLSv1.3
  checksrc.pl: add -i and -m options
  CURLOPT_COOKIEFILE.3: "-" as file name means stdin

Bug: http://b/78771319
Test: builds, boots, `vendor/google/tools/fake-ota on streaming` works
Change-Id: I18731dd7c91df36ca92a5dffc8631dee22785e34

1 files changed, 83 insertions, 76 deletions

diff --git a/lib/http.c b/lib/http.c
index a5007670..841f6cc0 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -177,9 +177,9 @@ CURLcode Curl_http_setup_conn(struct connectdata *conn)
  * if proxy headers are not available, then it will lookup into http header
  * link list
  *
- * It takes a connectdata struct as input instead of the Curl_easy simply
- * to know if this is a proxy request or not, as it then might check a
- * different header list.
+ * It takes a connectdata struct as input instead of the Curl_easy simply to
+ * know if this is a proxy request or not, as it then might check a different
+ * header list. Provide the header prefix without colon!.
  */
 char *Curl_checkProxyheaders(const struct connectdata *conn,
                              const char *thisheader)
@@ -191,7 +191,8 @@ char *Curl_checkProxyheaders(const struct connectdata *conn,
   for(head = (conn->bits.proxy && data->set.sep_headers) ?
         data->set.proxyheaders : data->set.headers;
       head; head = head->next) {
-    if(strncasecompare(head->data, thisheader, thislen))
+    if(strncasecompare(head->data, thisheader, thislen) &&
+       Curl_headersep(head->data[thislen]))
       return head->data;
   }
 
@@ -614,9 +615,9 @@ output_auth_headers(struct connectdata *conn,
   if(authstatus->picked == CURLAUTH_BASIC) {
     /* Basic */
     if((proxy && conn->bits.proxy_user_passwd &&
-        !Curl_checkProxyheaders(conn, "Proxy-authorization:")) ||
+        !Curl_checkProxyheaders(conn, "Proxy-authorization")) ||
        (!proxy && conn->bits.user_passwd &&
-        !Curl_checkheaders(conn, "Authorization:"))) {
+        !Curl_checkheaders(conn, "Authorization"))) {
       auth = "Basic";
       result = http_output_basic(conn, proxy);
       if(result)
@@ -1533,7 +1534,7 @@ static CURLcode expect100(struct Curl_easy *data,
     /* if not doing HTTP 1.0 or version 2, or disabled explicitly, we add an
        Expect: 100-continue to the headers which actually speeds up post
        operations (as there is one packet coming back from the web server) */
-    ptr = Curl_checkheaders(conn, "Expect:");
+    ptr = Curl_checkheaders(conn, "Expect");
     if(ptr) {
       data->state.expect100header =
         Curl_compareheader(ptr, "Expect:", "100-continue");
@@ -1598,7 +1599,32 @@ CURLcode Curl_add_custom_headers(struct connectdata *conn,
     headers = h[i];
 
     while(headers) {
+      char *semicolonp = NULL;
       ptr = strchr(headers->data, ':');
+      if(!ptr) {
+        char *optr;
+        /* no colon, semicolon? */
+        ptr = strchr(headers->data, ';');
+        if(ptr) {
+          optr = ptr;
+          ptr++; /* pass the semicolon */
+          while(*ptr && ISSPACE(*ptr))
+            ptr++;
+
+          if(*ptr) {
+            /* this may be used for something else in the future */
+            optr = NULL;
+          }
+          else {
+            if(*(--ptr) == ';') {
+              /* send no-value custom header if terminated by semicolon */
+              *ptr = ':';
+              semicolonp = ptr;
+            }
+          }
+          ptr = optr;
+        }
+      }
       if(ptr) {
         /* we require a colon for this to be a true header */
 
@@ -1606,8 +1632,9 @@ CURLcode Curl_add_custom_headers(struct connectdata *conn,
         while(*ptr && ISSPACE(*ptr))
           ptr++;
 
-        if(*ptr) {
-          /* only send this if the contents was non-blank */
+        if(*ptr || semicolonp) {
+          /* only send this if the contents was non-blank or done special */
+          CURLcode result = CURLE_OK;
 
           if(conn->allocptr.host &&
              /* a Host: header was sent already, don't pass on any custom Host:
@@ -1645,40 +1672,12 @@ CURLcode Curl_add_custom_headers(struct connectdata *conn,
                    !strcasecompare(data->state.first_host, conn->host.name)))
             ;
           else {
-            CURLcode result = Curl_add_bufferf(req_buffer, "%s\r\n",
-                                               headers->data);
-            if(result)
-              return result;
-          }
-        }
-      }
-      else {
-        ptr = strchr(headers->data, ';');
-        if(ptr) {
-
-          ptr++; /* pass the semicolon */
-          while(*ptr && ISSPACE(*ptr))
-            ptr++;
-
-          if(*ptr) {
-            /* this may be used for something else in the future */
-          }
-          else {
-            if(*(--ptr) == ';') {
-              CURLcode result;
-
-              /* send no-value custom header if terminated by semicolon */
-              *ptr = ':';
-              result = Curl_add_bufferf(req_buffer, "%s\r\n",
-                                        headers->data);
-
-              /* restore the previous value */
-              *ptr = ';';
-
-              if(result)
-                return result;
-            }
+            result = Curl_add_bufferf(req_buffer, "%s\r\n", headers->data);
           }
+          if(semicolonp)
+            *semicolonp = ';'; /* put back the semicolon */
+          if(result)
+            return result;
         }
       }
       headers = headers->next;
@@ -1869,7 +1868,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
      it might have been used in the proxy connect, but if we have got a header
      with the user-agent string specified, we erase the previously made string
      here. */
-  if(Curl_checkheaders(conn, "User-Agent:")) {
+  if(Curl_checkheaders(conn, "User-Agent")) {
     free(conn->allocptr.uagent);
     conn->allocptr.uagent = NULL;
   }
@@ -1890,7 +1889,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
     conn->bits.authneg = FALSE;
 
   Curl_safefree(conn->allocptr.ref);
-  if(data->change.referer && !Curl_checkheaders(conn, "Referer:")) {
+  if(data->change.referer && !Curl_checkheaders(conn, "Referer")) {
     conn->allocptr.ref = aprintf("Referer: %s\r\n", data->change.referer);
     if(!conn->allocptr.ref)
       return CURLE_OUT_OF_MEMORY;
@@ -1899,11 +1898,11 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
     conn->allocptr.ref = NULL;
 
 #if !defined(CURL_DISABLE_COOKIES)
-  if(data->set.str[STRING_COOKIE] && !Curl_checkheaders(conn, "Cookie:"))
+  if(data->set.str[STRING_COOKIE] && !Curl_checkheaders(conn, "Cookie"))
     addcookies = data->set.str[STRING_COOKIE];
 #endif
 
-  if(!Curl_checkheaders(conn, "Accept-Encoding:") &&
+  if(!Curl_checkheaders(conn, "Accept-Encoding") &&
      data->set.str[STRING_ENCODING]) {
     Curl_safefree(conn->allocptr.accept_encoding);
     conn->allocptr.accept_encoding =
@@ -1919,22 +1918,29 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
 #ifdef HAVE_LIBZ
   /* we only consider transfer-encoding magic if libz support is built-in */
 
-  if(!Curl_checkheaders(conn, "TE:") &&
+  if(!Curl_checkheaders(conn, "TE") &&
      data->set.http_transfer_encoding) {
     /* When we are to insert a TE: header in the request, we must also insert
        TE in a Connection: header, so we need to merge the custom provided
        Connection: header and prevent the original to get sent. Note that if
        the user has inserted his/hers own TE: header we don't do this magic
        but then assume that the user will handle it all! */
-    char *cptr = Curl_checkheaders(conn, "Connection:");
+    char *cptr = Curl_checkheaders(conn, "Connection");
 #define TE_HEADER "TE: gzip\r\n"
 
     Curl_safefree(conn->allocptr.te);
 
+    if(cptr) {
+      cptr = Curl_copy_header_value(cptr);
+      if(!cptr)
+        return CURLE_OUT_OF_MEMORY;
+    }
+
     /* Create the (updated) Connection: header */
-    conn->allocptr.te = cptr? aprintf("%s, TE\r\n" TE_HEADER, cptr):
-      strdup("Connection: TE\r\n" TE_HEADER);
+    conn->allocptr.te = aprintf("Connection: %s%sTE\r\n" TE_HEADER,
+                                cptr ? cptr : "", (cptr && *cptr) ? ", ":"");
 
+    free(cptr);
     if(!conn->allocptr.te)
       return CURLE_OUT_OF_MEMORY;
   }
@@ -1958,7 +1964,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
   }
 
   if(http->sendit) {
-    const char *cthdr = Curl_checkheaders(conn, "Content-Type:");
+    const char *cthdr = Curl_checkheaders(conn, "Content-Type");
 
     /* Read and seek body only. */
     http->sendit->flags |= MIME_BODY_ONLY;
@@ -1982,7 +1988,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
     http->postsize = Curl_mime_size(http->sendit);
   }
 
-  ptr = Curl_checkheaders(conn, "Transfer-Encoding:");
+  ptr = Curl_checkheaders(conn, "Transfer-Encoding");
   if(ptr) {
     /* Some kind of TE is requested, check if 'chunked' is chosen */
     data->req.upload_chunky =
@@ -2016,7 +2022,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
 
   Curl_safefree(conn->allocptr.host);
 
-  ptr = Curl_checkheaders(conn, "Host:");
+  ptr = Curl_checkheaders(conn, "Host");
   if(ptr && (!data->state.this_is_a_follow ||
              strcasecompare(data->state.first_host, conn->host.name))) {
 #if !defined(CURL_DISABLE_COOKIES)
@@ -2055,7 +2061,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
 #endif
 
     if(strcmp("Host:", ptr)) {
-      conn->allocptr.host = aprintf("%s\r\n", ptr);
+      conn->allocptr.host = aprintf("Host:%s\r\n", &ptr[5]);
       if(!conn->allocptr.host)
         return CURLE_OUT_OF_MEMORY;
     }
@@ -2164,7 +2170,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
   }
 #endif /* CURL_DISABLE_PROXY */
 
-  http->p_accept = Curl_checkheaders(conn, "Accept:")?NULL:"Accept: */*\r\n";
+  http->p_accept = Curl_checkheaders(conn, "Accept")?NULL:"Accept: */*\r\n";
 
   if((HTTPREQ_POST == httpreq || HTTPREQ_PUT == httpreq) &&
      data->state.resume_from) {
@@ -2191,8 +2197,10 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
       /* Now, let's read off the proper amount of bytes from the
          input. */
       if(conn->seek_func) {
+        Curl_set_in_callback(data, true);
         seekerr = conn->seek_func(conn->seek_client, data->state.resume_from,
                                   SEEK_SET);
+        Curl_set_in_callback(data, false);
       }
 
       if(seekerr != CURL_SEEKFUNC_OK) {
@@ -2243,14 +2251,14 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
      * ones if any such are specified.
      */
     if(((httpreq == HTTPREQ_GET) || (httpreq == HTTPREQ_HEAD)) &&
-       !Curl_checkheaders(conn, "Range:")) {
+       !Curl_checkheaders(conn, "Range")) {
       /* if a line like this was already allocated, free the previous one */
       free(conn->allocptr.rangeline);
       conn->allocptr.rangeline = aprintf("Range: bytes=%s\r\n",
                                          data->state.range);
     }
     else if((httpreq == HTTPREQ_POST || httpreq == HTTPREQ_PUT) &&
-            !Curl_checkheaders(conn, "Content-Range:")) {
+            !Curl_checkheaders(conn, "Content-Range")) {
 
       /* if a line like this was already allocated, free the previous one */
       free(conn->allocptr.rangeline);
@@ -2352,7 +2360,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
                      conn->allocptr.ref:"" /* Referer: <data> */,
                      (conn->bits.httpproxy &&
                       !conn->bits.tunnel_proxy &&
-                      !Curl_checkProxyheaders(conn, "Proxy-Connection:"))?
+                      !Curl_checkProxyheaders(conn, "Proxy-Connection"))?
                      "Proxy-Connection: Keep-Alive\r\n":"",
                      te
       );
@@ -2453,7 +2461,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
       postsize = data->state.infilesize;
 
     if((postsize != -1) && !data->req.upload_chunky &&
-       (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length:"))) {
+       (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length"))) {
       /* only add Content-Length if not uploading chunked */
       result = Curl_add_bufferf(req_buffer,
                                 "Content-Length: %" CURL_FORMAT_CURL_OFF_T
@@ -2515,7 +2523,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
        we don't upload data chunked, as RFC2616 forbids us to set both
        kinds of headers (Transfer-Encoding: chunked and Content-Length) */
     if(postsize != -1 && !data->req.upload_chunky &&
-       (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length:"))) {
+       (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length"))) {
       /* we allow replacing this header if not during auth negotiation,
          although it isn't very wise to actually set your own */
       result = Curl_add_bufferf(req_buffer,
@@ -2540,7 +2548,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
        the somewhat bigger ones we allow the app to disable it. Just make
        sure that the expect100header is always set to the preferred value
        here. */
-    ptr = Curl_checkheaders(conn, "Expect:");
+    ptr = Curl_checkheaders(conn, "Expect");
     if(ptr) {
       data->state.expect100header =
         Curl_compareheader(ptr, "Expect:", "100-continue");
@@ -2594,7 +2602,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
        we don't upload data chunked, as RFC2616 forbids us to set both
        kinds of headers (Transfer-Encoding: chunked and Content-Length) */
     if((postsize != -1) && !data->req.upload_chunky &&
-       (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length:"))) {
+       (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length"))) {
       /* we allow replacing this header if not during auth negotiation,
          although it isn't very wise to actually set your own */
       result = Curl_add_bufferf(req_buffer,
@@ -2604,7 +2612,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
         return result;
     }
 
-    if(!Curl_checkheaders(conn, "Content-Type:")) {
+    if(!Curl_checkheaders(conn, "Content-Type")) {
       result = Curl_add_bufferf(req_buffer,
                                 "Content-Type: application/"
                                 "x-www-form-urlencoded\r\n");
@@ -2616,7 +2624,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
        the somewhat bigger ones we allow the app to disable it. Just make
        sure that the expect100header is always set to the preferred value
        here. */
-    ptr = Curl_checkheaders(conn, "Expect:");
+    ptr = Curl_checkheaders(conn, "Expect");
     if(ptr) {
       data->state.expect100header =
         Curl_compareheader(ptr, "Expect:", "100-continue");
@@ -2878,20 +2886,19 @@ static CURLcode header_append(struct Curl_easy *data,
                               struct SingleRequest *k,
                               size_t length)
 {
-  if(k->hbuflen + length >= data->state.headersize) {
+  size_t newsize = k->hbuflen + length;
+  if(newsize > CURL_MAX_HTTP_HEADER) {
+    /* The reason to have a max limit for this is to avoid the risk of a bad
+       server feeding libcurl with a never-ending header that will cause
+       reallocs infinitely */
+    failf(data, "Rejected %zd bytes header (max is %d)!", newsize,
+          CURL_MAX_HTTP_HEADER);
+    return CURLE_OUT_OF_MEMORY;
+  }
+  if(newsize >= data->state.headersize) {
     /* We enlarge the header buffer as it is too small */
     char *newbuff;
     size_t hbufp_index;
-    size_t newsize;
-
-    if(k->hbuflen + length > CURL_MAX_HTTP_HEADER) {
-      /* The reason to have a max limit for this is to avoid the risk of a bad
-         server feeding libcurl with a never-ending header that will cause
-         reallocs infinitely */
-      failf(data, "Avoided giant realloc for header (max is %d)!",
-            CURL_MAX_HTTP_HEADER);
-      return CURLE_OUT_OF_MEMORY;
-    }
 
     newsize = CURLMAX((k->hbuflen + length) * 3 / 2, data->state.headersize*2);
     hbufp_index = k->hbufp - data->state.headerbuff;
@@ -3692,7 +3699,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
       k->timeofdoc = curl_getdate(k->p + strlen("Last-Modified:"),
                                   &secs);
       if(data->set.get_filetime)
-        data->info.filetime = (long)k->timeofdoc;
+        data->info.filetime = k->timeofdoc;
     }
     else if((checkprefix("WWW-Authenticate:", k->p) &&
              (401 == k->httpcode)) ||