David McCreedy brought line end conversions when doing FTP ASCII

transfers. They are done on non-windows systems and translate CRLF to LF.

4 files changed, 137 insertions, 15 deletions

diff --git a/lib/ftp.c b/lib/ftp.c
index 31921e657..9b6ba2154 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -2994,6 +2994,13 @@ CURLcode Curl_ftp_done(struct connectdata *conn, CURLcode status)
   }
   else {
     if((-1 != conn->size) && (conn->size != *ftp->bytecountp) &&
+#ifdef CURL_DO_LINEEND_CONV
+       /* Most FTP servers don't adjust their file SIZE response for CRLFs, so
+        * we'll check to see if the discrepancy can be explained by the number
+        * of CRLFs we've changed to LFs.
+        */
+       ((conn->size + data->state.crlf_conversions) != *ftp->bytecountp) &&
+#endif /* CURL_DO_LINEEND_CONV */
        (conn->maxdownload != *ftp->bytecountp)) {
       failf(data, "Received only partial file: %" FORMAT_OFF_T " bytes",
             *ftp->bytecountp);
diff --git a/lib/sendf.c b/lib/sendf.c
index dda70c42b..a9cb58363 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2005, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2006, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -139,6 +139,89 @@ void curl_slist_free_all(struct curl_slist *list)
   } while (next);
 }
 
+#ifdef CURL_DO_LINEEND_CONV
+/*
+ * convert_lineends() changes CRLF (\r\n) end-of-line markers to a single LF
+ * (\n), with special processing for CRLF sequences that are split between two
+ * blocks of data.  Remaining, bare CRs are changed to LFs.  The possibly new
+ * size of the data is returned.
+ */
+static size_t convert_lineends(struct SessionHandle *data,
+                               char *startPtr, size_t size)
+{
+  char *inPtr, *outPtr;
+
+  /* sanity check */
+  if ((startPtr == NULL) || (size < 1)) {
+    return(size);
+  }
+
+  if (data->state.prev_block_had_trailing_cr == TRUE) {
+    /* The previous block of incoming data
+       had a trailing CR, which was turned into a LF. */
+    if (*startPtr == '\n') {
+      /* This block of incoming data starts with the
+         previous block's LF so get rid of it */
+      memcpy(startPtr, startPtr+1, size-1);
+      size--;
+      /* and it wasn't a bare CR but a CRLF conversion instead */
+      data->state.crlf_conversions++;
+    }
+    data->state.prev_block_had_trailing_cr = FALSE; /* reset the flag */
+  }
+
+  /* find 1st CR, if any */
+  inPtr = outPtr = memchr(startPtr, '\r', size);
+  if (inPtr) {
+    /* at least one CR, now look for CRLF */
+    while (inPtr < (startPtr+size-1)) {
+      /* note that it's size-1, so we'll never look past the last byte */
+      if (memcmp(inPtr, "\r\n", 2) == 0) {
+        /* CRLF found, bump past the CR and copy the NL */
+        inPtr++;
+        *outPtr = *inPtr;
+        /* keep track of how many CRLFs we converted */
+        data->state.crlf_conversions++;
+      }
+      else {
+        if (*inPtr == '\r') {
+          /* lone CR, move LF instead */
+          *outPtr = '\n';
+        }
+        else {
+          /* not a CRLF nor a CR, just copy whatever it is */
+          *outPtr = *inPtr;
+        }
+      }
+      outPtr++;
+      inPtr++;
+    } /* end of while loop */
+
+    if (inPtr < startPtr+size) {
+      /* handle last byte */
+      if (*inPtr == '\r') {
+        /* deal with a CR at the end of the buffer */
+        *outPtr = '\n'; /* copy a NL instead */
+        /* note that a CRLF might be split across two blocks */
+        data->state.prev_block_had_trailing_cr = TRUE;
+      }
+      else {
+        /* copy last byte */
+        *outPtr = *inPtr;
+      }
+      outPtr++;
+      inPtr++;
+    }
+    if (outPtr < startPtr+size) {
+      /* tidy up by null terminating the now shorter data */
+      *outPtr = '\0';
+    }
+    return(outPtr - startPtr);
+  }
+  return(size);
+}
+#endif /* CURL_DO_LINEEND_CONV */
+
 /* Curl_infof() is for info message along the way */
 
 void Curl_infof(struct SessionHandle *data, const char *fmt, ...)
@@ -294,36 +377,36 @@ CURLcode Curl_client_write(struct SessionHandle *data,
   if(0 == len)
     len = strlen(ptr);
 
-#ifdef CURL_DOES_CONVERSIONS
   if(type & CLIENTWRITE_BODY) {
     if(data->ftp_in_ascii_mode) {
+#ifdef CURL_DOES_CONVERSIONS
       /* convert from the network encoding */
       size_t rc;
       rc = Curl_convert_from_network(data, ptr, len);
       /* Curl_convert_from_network calls failf if unsuccessful */
-      if(rc != CURLE_OK) {
-        return(rc);
-      }
+      if(rc != CURLE_OK)
+        return rc;
+#endif /* CURL_DOES_CONVERSIONS */
+
+#ifdef CURL_DO_LINEEND_CONV
+      /* convert end-of-line markers */
+      len = convert_lineends(data, ptr, len);
+#endif /* CURL_DO_LINEEND_CONV */
     }
+    /* If the previous block of data ended with CR and this block of data is
+       just a NL, then the length might be zero */
     if (len) {
       wrote = data->set.fwrite(ptr, 1, len, data->set.out);
-    } else {
-      wrote = len;
     }
-    if(wrote != len) {
-      failf (data, "Failed writing body");
-      return CURLE_WRITE_ERROR;
+    else {
+      wrote = len;
     }
-  }
-#else
-  if(type & CLIENTWRITE_BODY) {
-    wrote = data->set.fwrite(ptr, 1, len, data->set.out);
+
     if(wrote != len) {
       failf (data, "Failed writing body");
       return CURLE_WRITE_ERROR;
     }
   }
-#endif /* CURL_DOES_CONVERSIONS */
 
   if((type & CLIENTWRITE_HEADER) &&
      (data->set.fwrite_header || data->set.writeheader) ) {
diff --git a/lib/transfer.c b/lib/transfer.c
index 6e4fe4c71..bb43c4da9 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -1281,17 +1281,33 @@ CURLcode Curl_readwrite(struct connectdata *conn,
           conn->upload_present = nread;
 
           /* convert LF to CRLF if so asked */
+#ifdef CURL_DO_LINEEND_CONV
+          /* always convert if we're FTPing in ASCII mode */
+          if ((data->set.crlf) || (data->ftp_in_ascii_mode)) {
+#else
           if (data->set.crlf) {
+#endif /* CURL_DO_LINEEND_CONV */
               if(data->state.scratch == NULL)
                 data->state.scratch = malloc(2*BUFSIZE);
               if(data->state.scratch == NULL) {
                 failf (data, "Failed to alloc scratch buffer!");
                 return CURLE_OUT_OF_MEMORY;
               }
+              /*
+               * ASCII/EBCDIC Note: This is presumably a text (not binary)
+               * transfer so the data should already be in ASCII.
+               * That means the hex values for ASCII CR (0x0d) & LF (0x0a)
+               * must be used instead of the escape sequences \r & \n.
+               */
             for(i = 0, si = 0; i < nread; i++, si++) {
               if (conn->upload_fromhere[i] == 0x0a) {
                 data->state.scratch[si++] = 0x0d;
                 data->state.scratch[si] = 0x0a;
+                if (!data->set.crlf) {
+                  /* we're here only because FTP is in ASCII mode...
+                     bump infilesize for the LF we just added */
+                  data->set.infilesize++;
+                }
               }
               else
                 data->state.scratch[si] = conn->upload_fromhere[i];
@@ -1417,6 +1433,13 @@ CURLcode Curl_readwrite(struct connectdata *conn,
 
     if(!(conn->bits.no_body) && (conn->size != -1) &&
        (k->bytecount != conn->size) &&
+#ifdef CURL_DO_LINEEND_CONV
+       /* Most FTP servers don't adjust their file SIZE response for CRLFs,
+          so we'll check to see if the discrepancy can be explained
+          by the number of CRLFs we've changed to LFs.
+        */
+       (k->bytecount != (conn->size + data->state.crlf_conversions)) &&
+#endif /* CURL_DO_LINEEND_CONV */
        !conn->newurl) {
       failf(data, "transfer closed with %" FORMAT_OFF_T
             " bytes remaining to read",
diff --git a/lib/urldata.h b/lib/urldata.h
index 457832dcf..64d883b9e 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -936,6 +936,15 @@ struct UrlState {
 
   /* a place to store the most recenlty set FTP entrypath */
   char *most_recent_ftp_entrypath;
+
+#ifndef WIN32
+/* do FTP line-end conversions on most platforms */
+#define CURL_DO_LINEEND_CONV
+  /* for FTP downloads: track CRLF sequences that span blocks */
+  bool prev_block_had_trailing_cr;
+  /* for FTP downloads: how many CRLFs did we converted to LFs? */
+  size_t crlf_conversions;
+#endif
 };