From 95152aec685c4d11de6cb25802303e4872d51b3e Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Wed, 26 Apr 2006 07:40:37 +0000 Subject: David McCreedy brought line end conversions when doing FTP ASCII transfers. They are done on non-windows systems and translate CRLF to LF. --- lib/ftp.c | 7 ++++ lib/sendf.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++-------- lib/transfer.c | 23 ++++++++++++ lib/urldata.h | 9 +++++ 4 files changed, 137 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/ftp.c b/lib/ftp.c index 31921e657..9b6ba2154 100644 --- a/lib/ftp.c +++ b/lib/ftp.c @@ -2994,6 +2994,13 @@ CURLcode Curl_ftp_done(struct connectdata *conn, CURLcode status) } else { if((-1 != conn->size) && (conn->size != *ftp->bytecountp) && +#ifdef CURL_DO_LINEEND_CONV + /* Most FTP servers don't adjust their file SIZE response for CRLFs, so + * we'll check to see if the discrepancy can be explained by the number + * of CRLFs we've changed to LFs. + */ + ((conn->size + data->state.crlf_conversions) != *ftp->bytecountp) && +#endif /* CURL_DO_LINEEND_CONV */ (conn->maxdownload != *ftp->bytecountp)) { failf(data, "Received only partial file: %" FORMAT_OFF_T " bytes", *ftp->bytecountp); diff --git a/lib/sendf.c b/lib/sendf.c index dda70c42b..a9cb58363 100644 --- a/lib/sendf.c +++ b/lib/sendf.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2005, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2006, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -139,6 +139,89 @@ void curl_slist_free_all(struct curl_slist *list) } while (next); } +#ifdef CURL_DO_LINEEND_CONV +/* + * convert_lineends() changes CRLF (\r\n) end-of-line markers to a single LF + * (\n), with special processing for CRLF sequences that are split between two + * blocks of data. Remaining, bare CRs are changed to LFs. The possibly new + * size of the data is returned. + */ +static size_t convert_lineends(struct SessionHandle *data, + char *startPtr, size_t size) +{ + char *inPtr, *outPtr; + + /* sanity check */ + if ((startPtr == NULL) || (size < 1)) { + return(size); + } + + if (data->state.prev_block_had_trailing_cr == TRUE) { + /* The previous block of incoming data + had a trailing CR, which was turned into a LF. */ + if (*startPtr == '\n') { + /* This block of incoming data starts with the + previous block's LF so get rid of it */ + memcpy(startPtr, startPtr+1, size-1); + size--; + /* and it wasn't a bare CR but a CRLF conversion instead */ + data->state.crlf_conversions++; + } + data->state.prev_block_had_trailing_cr = FALSE; /* reset the flag */ + } + + /* find 1st CR, if any */ + inPtr = outPtr = memchr(startPtr, '\r', size); + if (inPtr) { + /* at least one CR, now look for CRLF */ + while (inPtr < (startPtr+size-1)) { + /* note that it's size-1, so we'll never look past the last byte */ + if (memcmp(inPtr, "\r\n", 2) == 0) { + /* CRLF found, bump past the CR and copy the NL */ + inPtr++; + *outPtr = *inPtr; + /* keep track of how many CRLFs we converted */ + data->state.crlf_conversions++; + } + else { + if (*inPtr == '\r') { + /* lone CR, move LF instead */ + *outPtr = '\n'; + } + else { + /* not a CRLF nor a CR, just copy whatever it is */ + *outPtr = *inPtr; + } + } + outPtr++; + inPtr++; + } /* end of while loop */ + + if (inPtr < startPtr+size) { + /* handle last byte */ + if (*inPtr == '\r') { + /* deal with a CR at the end of the buffer */ + *outPtr = '\n'; /* copy a NL instead */ + /* note that a CRLF might be split across two blocks */ + data->state.prev_block_had_trailing_cr = TRUE; + } + else { + /* copy last byte */ + *outPtr = *inPtr; + } + outPtr++; + inPtr++; + } + if (outPtr < startPtr+size) { + /* tidy up by null terminating the now shorter data */ + *outPtr = '\0'; + } + return(outPtr - startPtr); + } + return(size); +} +#endif /* CURL_DO_LINEEND_CONV */ + /* Curl_infof() is for info message along the way */ void Curl_infof(struct SessionHandle *data, const char *fmt, ...) @@ -294,36 +377,36 @@ CURLcode Curl_client_write(struct SessionHandle *data, if(0 == len) len = strlen(ptr); -#ifdef CURL_DOES_CONVERSIONS if(type & CLIENTWRITE_BODY) { if(data->ftp_in_ascii_mode) { +#ifdef CURL_DOES_CONVERSIONS /* convert from the network encoding */ size_t rc; rc = Curl_convert_from_network(data, ptr, len); /* Curl_convert_from_network calls failf if unsuccessful */ - if(rc != CURLE_OK) { - return(rc); - } + if(rc != CURLE_OK) + return rc; +#endif /* CURL_DOES_CONVERSIONS */ + +#ifdef CURL_DO_LINEEND_CONV + /* convert end-of-line markers */ + len = convert_lineends(data, ptr, len); +#endif /* CURL_DO_LINEEND_CONV */ } + /* If the previous block of data ended with CR and this block of data is + just a NL, then the length might be zero */ if (len) { wrote = data->set.fwrite(ptr, 1, len, data->set.out); - } else { - wrote = len; } - if(wrote != len) { - failf (data, "Failed writing body"); - return CURLE_WRITE_ERROR; + else { + wrote = len; } - } -#else - if(type & CLIENTWRITE_BODY) { - wrote = data->set.fwrite(ptr, 1, len, data->set.out); + if(wrote != len) { failf (data, "Failed writing body"); return CURLE_WRITE_ERROR; } } -#endif /* CURL_DOES_CONVERSIONS */ if((type & CLIENTWRITE_HEADER) && (data->set.fwrite_header || data->set.writeheader) ) { diff --git a/lib/transfer.c b/lib/transfer.c index 6e4fe4c71..bb43c4da9 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -1281,17 +1281,33 @@ CURLcode Curl_readwrite(struct connectdata *conn, conn->upload_present = nread; /* convert LF to CRLF if so asked */ +#ifdef CURL_DO_LINEEND_CONV + /* always convert if we're FTPing in ASCII mode */ + if ((data->set.crlf) || (data->ftp_in_ascii_mode)) { +#else if (data->set.crlf) { +#endif /* CURL_DO_LINEEND_CONV */ if(data->state.scratch == NULL) data->state.scratch = malloc(2*BUFSIZE); if(data->state.scratch == NULL) { failf (data, "Failed to alloc scratch buffer!"); return CURLE_OUT_OF_MEMORY; } + /* + * ASCII/EBCDIC Note: This is presumably a text (not binary) + * transfer so the data should already be in ASCII. + * That means the hex values for ASCII CR (0x0d) & LF (0x0a) + * must be used instead of the escape sequences \r & \n. + */ for(i = 0, si = 0; i < nread; i++, si++) { if (conn->upload_fromhere[i] == 0x0a) { data->state.scratch[si++] = 0x0d; data->state.scratch[si] = 0x0a; + if (!data->set.crlf) { + /* we're here only because FTP is in ASCII mode... + bump infilesize for the LF we just added */ + data->set.infilesize++; + } } else data->state.scratch[si] = conn->upload_fromhere[i]; @@ -1417,6 +1433,13 @@ CURLcode Curl_readwrite(struct connectdata *conn, if(!(conn->bits.no_body) && (conn->size != -1) && (k->bytecount != conn->size) && +#ifdef CURL_DO_LINEEND_CONV + /* Most FTP servers don't adjust their file SIZE response for CRLFs, + so we'll check to see if the discrepancy can be explained + by the number of CRLFs we've changed to LFs. + */ + (k->bytecount != (conn->size + data->state.crlf_conversions)) && +#endif /* CURL_DO_LINEEND_CONV */ !conn->newurl) { failf(data, "transfer closed with %" FORMAT_OFF_T " bytes remaining to read", diff --git a/lib/urldata.h b/lib/urldata.h index 457832dcf..64d883b9e 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -936,6 +936,15 @@ struct UrlState { /* a place to store the most recenlty set FTP entrypath */ char *most_recent_ftp_entrypath; + +#ifndef WIN32 +/* do FTP line-end conversions on most platforms */ +#define CURL_DO_LINEEND_CONV + /* for FTP downloads: track CRLF sequences that span blocks */ + bool prev_block_had_trailing_cr; + /* for FTP downloads: how many CRLFs did we converted to LFs? */ + size_t crlf_conversions; +#endif }; -- cgit v1.2.3