From ab83c0fd5b29d9ae41c8948f95d224105bd00cce Mon Sep 17 00:00:00 2001 From: Dan Fandrich Date: Fri, 8 Aug 2008 20:37:54 +0000 Subject: Refactored Curl_readwrite() into a number of smaller functions. --- lib/transfer.c | 2577 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 1329 insertions(+), 1248 deletions(-) (limited to 'lib') diff --git a/lib/transfer.c b/lib/transfer.c index 330ba7df6..3457f7c26 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -110,6 +110,12 @@ #define CURL_TIMEOUT_EXPECT_100 1000 /* counting ms here */ +static CURLcode readwrite_headers(struct SessionHandle *data, + struct connectdata *conn, + struct SingleRequest *k, + ssize_t *nread, + bool *stop_reading); + /* * This function will call the read callback to fill our buffer with data * to upload. @@ -289,7 +295,7 @@ CURLcode Curl_readrewind(struct connectdata *conn) return CURLE_OK; } - /* no callback set or failure aboe, makes us fail at once */ + /* no callback set or failure above, makes us fail at once */ failf(data, "necessary data rewind wasn't possible"); return CURLE_SEND_FAIL_REWIND; } @@ -336,1216 +342,1265 @@ static void read_rewind(struct connectdata *conn, #endif } + /* - * Curl_readwrite() is the low-level function to be called when data is to - * be read and written to/from the connection. + * Go ahead and do a read if we have a readable socket or if + * the stream was rewound (in which case we have data in a + * buffer) */ -CURLcode Curl_readwrite(struct connectdata *conn, - bool *done) +static CURLcode readwrite_data(struct SessionHandle *data, + struct connectdata *conn, + struct SingleRequest *k, + int *didwhat, bool *done) { - struct SessionHandle *data = conn->data; - struct SingleRequest *k = &data->req; CURLcode result; ssize_t nread; /* number of bytes read */ - int didwhat=0; + bool is_empty_data = FALSE; - curl_socket_t fd_read; - curl_socket_t fd_write; - int select_res = conn->cselect_bits; + /* This is where we loop until we have read everything there is to + read or we get a EWOULDBLOCK */ + do { + size_t buffersize = data->set.buffer_size? + data->set.buffer_size : BUFSIZE; + size_t bytestoread = buffersize; + int readrc; + + if(k->size != -1 && !k->header) { + /* make sure we don't read "too much" if we can help it since we + might be pipelining and then someone else might want to read what + follows! */ + curl_off_t totalleft = k->size - k->bytecount; + if(totalleft < (curl_off_t)bytestoread) + bytestoread = (size_t)totalleft; + } - conn->cselect_bits = 0; + if(bytestoread) { + /* receive data from the network! */ + readrc = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread); - /* only use the proper socket if the *_HOLD bit is not set simultaneously as - then we are in rate limiting state in that transfer direction */ + /* subzero, this would've blocked */ + if(0 > readrc) + break; /* get out of loop */ - if((k->keepon & KEEP_READBITS) == KEEP_READ) { - fd_read = conn->sockfd; -#if defined(USE_LIBSSH2) - if(conn->protocol & (PROT_SCP|PROT_SFTP)) - select_res |= CURL_CSELECT_IN; -#endif /* USE_LIBSSH2 */ - } else - fd_read = CURL_SOCKET_BAD; + /* get the CURLcode from the int */ + result = (CURLcode)readrc; - if((k->keepon & KEEP_WRITEBITS) == KEEP_WRITE) - fd_write = conn->writesockfd; - else - fd_write = CURL_SOCKET_BAD; + if(result>0) + return result; + } + else { + /* read nothing but since we wanted nothing we consider this an OK + situation to proceed from */ + nread = 0; + } - if(!select_res) { /* Call for select()/poll() only, if read/write/error - status is not known. */ - select_res = Curl_socket_ready(fd_read, fd_write, 0); - } + if((k->bytecount == 0) && (k->writebytecount == 0)) { + Curl_pgrsTime(data, TIMER_STARTTRANSFER); + if(k->exp100 > EXP100_SEND_DATA) + /* set time stamp to compare with when waiting for the 100 */ + k->start100 = Curl_tvnow(); + } - if(select_res == CURL_CSELECT_ERR) { - failf(data, "select/poll returned error"); - return CURLE_SEND_ERROR; - } + *didwhat |= KEEP_READ; + /* indicates data of zero size, i.e. empty file */ + is_empty_data = (bool)((nread == 0) && (k->bodywrites == 0)); - /* We go ahead and do a read if we have a readable socket or if - the stream was rewound (in which case we have data in a - buffer) */ - if((k->keepon & KEEP_READ) && - ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) { - /* read */ - bool is_empty_data = FALSE; - - /* This is where we loop until we have read everything there is to - read or we get a EWOULDBLOCK */ - do { - size_t buffersize = data->set.buffer_size? - data->set.buffer_size : BUFSIZE; - size_t bytestoread = buffersize; - int readrc; - - if(k->size != -1 && !k->header) { - /* make sure we don't read "too much" if we can help it since we - might be pipelining and then someone else might want to read what - follows! */ - curl_off_t totalleft = k->size - k->bytecount; - if(totalleft < (curl_off_t)bytestoread) - bytestoread = (size_t)totalleft; - } + /* NUL terminate, allowing string ops to be used */ + if(0 < nread || is_empty_data) { + k->buf[nread] = 0; + } + else if(0 >= nread) { + /* if we receive 0 or less here, the server closed the connection + and we bail out from this! */ + DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n")); + k->keepon &= ~KEEP_READ; + break; + } - if(bytestoread) { - /* receive data from the network! */ - readrc = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread); + /* Default buffer to use when we write the buffer, it may be changed + in the flow below before the actual storing is done. */ + k->str = k->buf; - /* subzero, this would've blocked */ - if(0 > readrc) - break; /* get out of loop */ + /* Since this is a two-state thing, we check if we are parsing + headers at the moment or not. */ + if(k->header) { + /* we are in parse-the-header-mode */ + bool stop_reading = FALSE; + result = readwrite_headers(data, conn, k, &nread, &stop_reading); + if(result) + return result; + if(stop_reading) + /* We've stopped dealing with input, get out of the do-while loop */ + break; + } - /* get the CURLcode from the int */ - result = (CURLcode)readrc; - if(result>0) - return result; + /* This is not an 'else if' since it may be a rest from the header + parsing, where the beginning of the buffer is headers and the end + is non-headers. */ + if(k->str && !k->header && (nread > 0 || is_empty_data)) { + + if(0 == k->bodywrites && !is_empty_data) { + /* These checks are only made the first time we are about to + write a piece of the body */ + if(conn->protocol&PROT_HTTP) { + /* HTTP-only checks */ + + if(data->req.newurl) { + if(conn->bits.close) { + /* Abort after the headers if "follow Location" is set + and we're set to close anyway. */ + k->keepon &= ~KEEP_READ; + *done = TRUE; + return CURLE_OK; + } + /* We have a new url to load, but since we want to be able + to re-use this connection properly, we read the full + response in "ignore more" */ + k->ignorebody = TRUE; + infof(data, "Ignoring the response-body\n"); + } + if(data->state.resume_from && !k->content_range && + (data->set.httpreq==HTTPREQ_GET) && + !k->ignorebody) { + /* we wanted to resume a download, although the server doesn't + * seem to support this and we did this with a GET (if it + * wasn't a GET we did a POST or PUT resume) */ + failf(data, "HTTP server doesn't seem to support " + "byte ranges. Cannot resume."); + return CURLE_RANGE_ERROR; + } + + if(data->set.timecondition && !data->state.range) { + /* A time condition has been set AND no ranges have been + requested. This seems to be what chapter 13.3.4 of + RFC 2616 defines to be the correct action for a + HTTP/1.1 client */ + if((k->timeofdoc > 0) && (data->set.timevalue > 0)) { + switch(data->set.timecondition) { + case CURL_TIMECOND_IFMODSINCE: + default: + if(k->timeofdoc < data->set.timevalue) { + infof(data, + "The requested document is not new enough\n"); + *done = TRUE; + return CURLE_OK; + } + break; + case CURL_TIMECOND_IFUNMODSINCE: + if(k->timeofdoc > data->set.timevalue) { + infof(data, + "The requested document is not old enough\n"); + *done = TRUE; + return CURLE_OK; + } + break; + } /* switch */ + } /* two valid time strings */ + } /* we have a time condition */ + + } /* this is HTTP */ + } /* this is the first time we write a body part */ + k->bodywrites++; + + /* pass data to the debug function before it gets "dechunked" */ + if(data->set.verbose) { + if(k->badheader) { + Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, + (size_t)k->hbuflen, conn); + if(k->badheader == HEADER_PARTHEADER) + Curl_debug(data, CURLINFO_DATA_IN, + k->str, (size_t)nread, conn); + } + else + Curl_debug(data, CURLINFO_DATA_IN, + k->str, (size_t)nread, conn); } - else { - /* read nothing but since we wanted nothing we consider this an OK - situation to proceed from */ - nread = 0; - result = CURLE_OK; + +#ifndef CURL_DISABLE_HTTP + if(k->chunk) { + /* + * Here comes a chunked transfer flying and we need to decode this + * properly. While the name says read, this function both reads + * and writes away the data. The returned 'nread' holds the number + * of actual data it wrote to the client. + */ + + CHUNKcode res = + Curl_httpchunk_read(conn, k->str, nread, &nread); + + if(CHUNKE_OK < res) { + if(CHUNKE_WRITE_ERROR == res) { + failf(data, "Failed writing data"); + return CURLE_WRITE_ERROR; + } + failf(data, "Received problem %d in the chunky parser", res); + return CURLE_RECV_ERROR; + } + else if(CHUNKE_STOP == res) { + size_t dataleft; + /* we're done reading chunks! */ + k->keepon &= ~KEEP_READ; /* read no more */ + + /* There are now possibly N number of bytes at the end of the + str buffer that weren't written to the client. + + We DO care about this data if we are pipelining. + Push it back to be read on the next pass. */ + + dataleft = conn->chunk.dataleft; + if(dataleft != 0) { + infof(conn->data, "Leftovers after chunking. " + " Rewinding %d bytes\n",dataleft); + read_rewind(conn, dataleft); + } + } + /* If it returned OK, we just keep going */ + } +#endif /* CURL_DISABLE_HTTP */ + + if((-1 != k->maxdownload) && + (k->bytecount + nread >= k->maxdownload)) { + /* The 'excess' amount below can't be more than BUFSIZE which + always will fit in a size_t */ + size_t excess = (size_t)(k->bytecount + nread - k->maxdownload); + if(excess > 0 && !k->ignorebody) { + infof(data, + "Rewinding stream by : %d" + " bytes on url %s (size = %" FORMAT_OFF_T + ", maxdownload = %" FORMAT_OFF_T + ", bytecount = %" FORMAT_OFF_T ", nread = %d)\n", + excess, data->state.path, + k->size, k->maxdownload, k->bytecount, nread); + read_rewind(conn, excess); + } + + nread = (ssize_t) (k->maxdownload - k->bytecount); + if(nread < 0 ) /* this should be unusual */ + nread = 0; + + k->keepon &= ~KEEP_READ; /* we're done reading */ } - if((k->bytecount == 0) && (k->writebytecount == 0)) { - Curl_pgrsTime(data, TIMER_STARTTRANSFER); - if(k->exp100 > EXP100_SEND_DATA) - /* set time stamp to compare with when waiting for the 100 */ - k->start100 = Curl_tvnow(); + k->bytecount += nread; + + Curl_pgrsSetDownloadCounter(data, k->bytecount); + + if(!k->chunk && (nread || k->badheader || is_empty_data)) { + /* If this is chunky transfer, it was already written */ + + if(k->badheader && !k->ignorebody) { + /* we parsed a piece of data wrongly assuming it was a header + and now we output it as body instead */ + result = Curl_client_write(conn, CLIENTWRITE_BODY, + data->state.headerbuff, + k->hbuflen); + if(result) + return result; + } + if(k->badheader < HEADER_ALLBAD) { + /* This switch handles various content encodings. If there's an + error here, be sure to check over the almost identical code + in http_chunks.c. + Make sure that ALL_CONTENT_ENCODINGS contains all the + encodings handled here. */ +#ifdef HAVE_LIBZ + switch (conn->data->set.http_ce_skip ? + IDENTITY : k->content_encoding) { + case IDENTITY: +#endif + /* This is the default when the server sends no + Content-Encoding header. See Curl_readwrite_init; the + memset() call initializes k->content_encoding to zero. */ + if(!k->ignorebody) + result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str, + nread); +#ifdef HAVE_LIBZ + break; + + case DEFLATE: + /* Assume CLIENTWRITE_BODY; headers are not encoded. */ + if(!k->ignorebody) + result = Curl_unencode_deflate_write(conn, k, nread); + break; + + case GZIP: + /* Assume CLIENTWRITE_BODY; headers are not encoded. */ + if(!k->ignorebody) + result = Curl_unencode_gzip_write(conn, k, nread); + break; + + case COMPRESS: + default: + failf (data, "Unrecognized content encoding type. " + "libcurl understands `identity', `deflate' and `gzip' " + "content encodings."); + result = CURLE_BAD_CONTENT_ENCODING; + break; + } +#endif + } + k->badheader = HEADER_NORMAL; /* taken care of now */ + + if(result) + return result; } - didwhat |= KEEP_READ; - /* indicates data of zero size, i.e. empty file */ - is_empty_data = (bool)((nread == 0) && (k->bodywrites == 0)); + } /* if(! header and data to read ) */ - /* NULL terminate, allowing string ops to be used */ - if(0 < nread || is_empty_data) { - k->buf[nread] = 0; + if(is_empty_data) { + /* if we received nothing, the server closed the connection and we + are done */ + k->keepon &= ~KEEP_READ; + } + + } while(data_pending(conn)); + + return CURLE_OK; +} + +/* + * Read any header lines from the server and pass them to the client app. + */ +static CURLcode readwrite_headers(struct SessionHandle *data, + struct connectdata *conn, + struct SingleRequest *k, + ssize_t *nread, + bool *stop_reading) +{ + CURLcode result; + + /* header line within buffer loop */ + do { + size_t hbufp_index; + size_t rest_length; + size_t full_length; + int writetype; + + /* str_start is start of line within buf */ + k->str_start = k->str; + + /* data is in network encoding so use 0x0a instead of '\n' */ + k->end_ptr = memchr(k->str_start, 0x0a, *nread); + + if(!k->end_ptr) { + /* Not a complete header line within buffer, append the data to + the end of the headerbuff. */ + + if(k->hbuflen + *nread >= data->state.headersize) { + /* We enlarge the header buffer as it is too small */ + char *newbuff; + size_t newsize=CURLMAX((k->hbuflen+*nread)*3/2, + data->state.headersize*2); + hbufp_index = k->hbufp - data->state.headerbuff; + newbuff = (char *)realloc(data->state.headerbuff, newsize); + if(!newbuff) { + failf (data, "Failed to alloc memory for big header!"); + return CURLE_OUT_OF_MEMORY; + } + data->state.headersize=newsize; + data->state.headerbuff = newbuff; + k->hbufp = data->state.headerbuff + hbufp_index; } - else if(0 >= nread) { - /* if we receive 0 or less here, the server closed the connection - and we bail out from this! */ - DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n")); - k->keepon &= ~KEEP_READ; - break; + memcpy(k->hbufp, k->str, *nread); + k->hbufp += *nread; + k->hbuflen += *nread; + if(!k->headerline && (k->hbuflen>5)) { + /* make a first check that this looks like a HTTP header */ + if(!checkhttpprefix(data, data->state.headerbuff)) { + /* this is not the beginning of a HTTP first header line */ + k->header = FALSE; + k->badheader = HEADER_ALLBAD; + break; + } } - /* Default buffer to use when we write the buffer, it may be changed - in the flow below before the actual storing is done. */ - k->str = k->buf; - - /* Since this is a two-state thing, we check if we are parsing - headers at the moment or not. */ - if(k->header) { - /* we are in parse-the-header-mode */ - bool stop_reading = FALSE; - - /* header line within buffer loop */ - do { - size_t hbufp_index; - size_t rest_length; - size_t full_length; - int writetype; - - /* str_start is start of line within buf */ - k->str_start = k->str; - - /* data is in network encoding so use 0x0a instead of '\n' */ - k->end_ptr = memchr(k->str_start, 0x0a, nread); - - if(!k->end_ptr) { - /* Not a complete header line within buffer, append the data to - the end of the headerbuff. */ - - if(k->hbuflen + nread >= data->state.headersize) { - /* We enlarge the header buffer as it is too small */ - char *newbuff; - size_t newsize=CURLMAX((k->hbuflen+nread)*3/2, - data->state.headersize*2); - hbufp_index = k->hbufp - data->state.headerbuff; - newbuff = (char *)realloc(data->state.headerbuff, newsize); - if(!newbuff) { - failf (data, "Failed to alloc memory for big header!"); - return CURLE_OUT_OF_MEMORY; - } - data->state.headersize=newsize; - data->state.headerbuff = newbuff; - k->hbufp = data->state.headerbuff + hbufp_index; - } - memcpy(k->hbufp, k->str, nread); - k->hbufp += nread; - k->hbuflen += nread; - if(!k->headerline && (k->hbuflen>5)) { - /* make a first check that this looks like a HTTP header */ - if(!checkhttpprefix(data, data->state.headerbuff)) { - /* this is not the beginning of a HTTP first header line */ - k->header = FALSE; - k->badheader = HEADER_ALLBAD; - break; - } - } + break; /* read more and try again */ + } - break; /* read more and try again */ - } + /* decrease the size of the remaining (supposed) header line */ + rest_length = (k->end_ptr - k->str)+1; + *nread -= (ssize_t)rest_length; - /* decrease the size of the remaining (supposed) header line */ - rest_length = (k->end_ptr - k->str)+1; - nread -= (ssize_t)rest_length; - - k->str = k->end_ptr + 1; /* move past new line */ - - full_length = k->str - k->str_start; - - /* - * We're about to copy a chunk of data to the end of the - * already received header. We make sure that the full string - * fit in the allocated header buffer, or else we enlarge - * it. - */ - if(k->hbuflen + full_length >= - data->state.headersize) { - char *newbuff; - size_t newsize=CURLMAX((k->hbuflen+full_length)*3/2, - data->state.headersize*2); - hbufp_index = k->hbufp - data->state.headerbuff; - newbuff = (char *)realloc(data->state.headerbuff, newsize); - if(!newbuff) { - failf (data, "Failed to alloc memory for big header!"); - return CURLE_OUT_OF_MEMORY; - } - data->state.headersize= newsize; - data->state.headerbuff = newbuff; - k->hbufp = data->state.headerbuff + hbufp_index; - } + k->str = k->end_ptr + 1; /* move past new line */ - /* copy to end of line */ - memcpy(k->hbufp, k->str_start, full_length); - k->hbufp += full_length; - k->hbuflen += full_length; - *k->hbufp = 0; - k->end_ptr = k->hbufp; - - k->p = data->state.headerbuff; - - /**** - * We now have a FULL header line that p points to - *****/ - - if(!k->headerline) { - /* the first read header */ - if((k->hbuflen>5) && - !checkhttpprefix(data, data->state.headerbuff)) { - /* this is not the beginning of a HTTP first header line */ - k->header = FALSE; - if(nread) - /* since there's more, this is a partial bad header */ - k->badheader = HEADER_PARTHEADER; - else { - /* this was all we read so its all a bad header */ - k->badheader = HEADER_ALLBAD; - nread = (ssize_t)rest_length; - } - break; - } - } + full_length = k->str - k->str_start; - /* headers are in network encoding so - use 0x0a and 0x0d instead of '\n' and '\r' */ - if((0x0a == *k->p) || (0x0d == *k->p)) { - size_t headerlen; - /* Zero-length header line means end of headers! */ + /* + * We're about to copy a chunk of data to the end of the + * already received header. We make sure that the full string + * fit in the allocated header buffer, or else we enlarge + * it. + */ + if(k->hbuflen + full_length >= + data->state.headersize) { + char *newbuff; + size_t newsize=CURLMAX((k->hbuflen+full_length)*3/2, + data->state.headersize*2); + hbufp_index = k->hbufp - data->state.headerbuff; + newbuff = (char *)realloc(data->state.headerbuff, newsize); + if(!newbuff) { + failf (data, "Failed to alloc memory for big header!"); + return CURLE_OUT_OF_MEMORY; + } + data->state.headersize= newsize; + data->state.headerbuff = newbuff; + k->hbufp = data->state.headerbuff + hbufp_index; + } + + /* copy to end of line */ + memcpy(k->hbufp, k->str_start, full_length); + k->hbufp += full_length; + k->hbuflen += full_length; + *k->hbufp = 0; + k->end_ptr = k->hbufp; + + k->p = data->state.headerbuff; + + /**** + * We now have a FULL header line that p points to + *****/ + + if(!k->headerline) { + /* the first read header */ + if((k->hbuflen>5) && + !checkhttpprefix(data, data->state.headerbuff)) { + /* this is not the beginning of a HTTP first header line */ + k->header = FALSE; + if(*nread) + /* since there's more, this is a partial bad header */ + k->badheader = HEADER_PARTHEADER; + else { + /* this was all we read so it's all a bad header */ + k->badheader = HEADER_ALLBAD; + *nread = (ssize_t)rest_length; + } + break; + } + } + + /* headers are in network encoding so + use 0x0a and 0x0d instead of '\n' and '\r' */ + if((0x0a == *k->p) || (0x0d == *k->p)) { + size_t headerlen; + /* Zero-length header line means end of headers! */ #ifdef CURL_DOES_CONVERSIONS - if(0x0d == *k->p) { - *k->p = '\r'; /* replace with CR in host encoding */ - k->p++; /* pass the CR byte */ - } - if(0x0a == *k->p) { - *k->p = '\n'; /* replace with LF in host encoding */ - k->p++; /* pass the LF byte */ - } + if(0x0d == *k->p) { + *k->p = '\r'; /* replace with CR in host encoding */ + k->p++; /* pass the CR byte */ + } + if(0x0a == *k->p) { + *k->p = '\n'; /* replace with LF in host encoding */ + k->p++; /* pass the LF byte */ + } #else - if('\r' == *k->p) - k->p++; /* pass the \r byte */ - if('\n' == *k->p) - k->p++; /* pass the \n byte */ + if('\r' == *k->p) + k->p++; /* pass the \r byte */ + if('\n' == *k->p) + k->p++; /* pass the \n byte */ #endif /* CURL_DOES_CONVERSIONS */ #ifndef CURL_DISABLE_HTTP - if(100 <= k->httpcode && 199 >= k->httpcode) { - /* - * We have made a HTTP PUT or POST and this is 1.1-lingo - * that tells us that the server is OK with this and ready - * to receive the data. - * However, we'll get more headers now so we must get - * back into the header-parsing state! - */ - k->header = TRUE; - k->headerline = 0; /* restart the header line counter */ - - /* if we did wait for this do enable write now! */ - if(k->exp100) { - k->exp100 = EXP100_SEND_DATA; - k->keepon |= KEEP_WRITE; - } - } - else { - k->header = FALSE; /* no more header to parse! */ - - if((k->size == -1) && !k->chunk && !conn->bits.close && - (k->httpversion >= 11) ) { - /* On HTTP 1.1, when connection is not to get closed, but no - Content-Length nor Content-Encoding chunked have been - received, according to RFC2616 section 4.4 point 5, we - assume that the server will close the connection to - signal the end of the document. */ - infof(data, "no chunk, no close, no size. Assume close to " - "signal end\n"); - conn->bits.close = TRUE; - } - } + if(100 <= k->httpcode && 199 >= k->httpcode) { + /* + * We have made a HTTP PUT or POST and this is 1.1-lingo + * that tells us that the server is OK with this and ready + * to receive the data. + * However, we'll get more headers now so we must get + * back into the header-parsing state! + */ + k->header = TRUE; + k->headerline = 0; /* restart the header line counter */ + + /* if we did wait for this do enable write now! */ + if(k->exp100) { + k->exp100 = EXP100_SEND_DATA; + k->keepon |= KEEP_WRITE; + } + } + else { + k->header = FALSE; /* no more header to parse! */ + + if((k->size == -1) && !k->chunk && !conn->bits.close && + (k->httpversion >= 11) ) { + /* On HTTP 1.1, when connection is not to get closed, but no + Content-Length nor Content-Encoding chunked have been + received, according to RFC2616 section 4.4 point 5, we + assume that the server will close the connection to + signal the end of the document. */ + infof(data, "no chunk, no close, no size. Assume close to " + "signal end\n"); + conn->bits.close = TRUE; + } + } - if(417 == k->httpcode) { - /* - * we got: "417 Expectation Failed" this means: - * we have made a HTTP call and our Expect Header - * seems to cause a problem => abort the write operations - * (or prevent them from starting). - */ - k->exp100 = EXP100_FAILED; - k->keepon &= ~KEEP_WRITE; - } + if(417 == k->httpcode) { + /* + * we got: "417 Expectation Failed" this means: + * we have made a HTTP call and our Expect Header + * seems to cause a problem => abort the write operations + * (or prevent them from starting). + */ + k->exp100 = EXP100_FAILED; + k->keepon &= ~KEEP_WRITE; + } - /* - * When all the headers have been parsed, see if we should give - * up and return an error. - */ - if(Curl_http_should_fail(conn)) { - failf (data, "The requested URL returned error: %d", - k->httpcode); - return CURLE_HTTP_RETURNED_ERROR; - } + /* + * When all the headers have been parsed, see if we should give + * up and return an error. + */ + if(Curl_http_should_fail(conn)) { + failf (data, "The requested URL returned error: %d", + k->httpcode); + return CURLE_HTTP_RETURNED_ERROR; + } #endif /* CURL_DISABLE_HTTP */ - /* now, only output this if the header AND body are requested: - */ - writetype = CLIENTWRITE_HEADER; - if(data->set.include_header) - writetype |= CLIENTWRITE_BODY; + /* now, only output this if the header AND body are requested: + */ + writetype = CLIENTWRITE_HEADER; + if(data->set.include_header) + writetype |= CLIENTWRITE_BODY; - headerlen = k->p - data->state.headerbuff; + headerlen = k->p - data->state.headerbuff; - result = Curl_client_write(conn, writetype, - data->state.headerbuff, - headerlen); - if(result) - return result; + result = Curl_client_write(conn, writetype, + data->state.headerbuff, + headerlen); + if(result) + return result; - data->info.header_size += (long)headerlen; - data->req.headerbytecount += (long)headerlen; + data->info.header_size += (long)headerlen; + data->req.headerbytecount += (long)headerlen; - data->req.deductheadercount = - (100 <= k->httpcode && 199 >= k->httpcode)?data->req.headerbytecount:0; + data->req.deductheadercount = + (100 <= k->httpcode && 199 >= k->httpcode)?data->req.headerbytecount:0; - if(data->state.resume_from && - (data->set.httpreq==HTTPREQ_GET) && - (k->httpcode == 416)) { - /* "Requested Range Not Satisfiable" */ - stop_reading = TRUE; - } + if(data->state.resume_from && + (data->set.httpreq==HTTPREQ_GET) && + (k->httpcode == 416)) { + /* "Requested Range Not Satisfiable" */ + *stop_reading = TRUE; + } #ifndef CURL_DISABLE_HTTP - if(!stop_reading) { - /* Curl_http_auth_act() checks what authentication methods - * that are available and decides which one (if any) to - * use. It will set 'newurl' if an auth metod was picked. */ - result = Curl_http_auth_act(conn); - - if(result) - return result; - - if(conn->bits.rewindaftersend) { - /* We rewind after a complete send, so thus we continue - sending now */ - infof(data, "Keep sending data to get tossed away!\n"); - k->keepon |= KEEP_WRITE; - } - } + if(!*stop_reading) { + /* Curl_http_auth_act() checks what authentication methods + * that are available and decides which one (if any) to + * use. It will set 'newurl' if an auth method was picked. */ + result = Curl_http_auth_act(conn); + + if(result) + return result; + + if(conn->bits.rewindaftersend) { + /* We rewind after a complete send, so thus we continue + sending now */ + infof(data, "Keep sending data to get tossed away!\n"); + k->keepon |= KEEP_WRITE; + } + } #endif /* CURL_DISABLE_HTTP */ - if(!k->header) { - /* - * really end-of-headers. - * - * If we requested a "no body", this is a good time to get - * out and return home. - */ - if(data->set.opt_no_body) - stop_reading = TRUE; - else { - /* If we know the expected size of this document, we set the - maximum download size to the size of the expected - document or else, we won't know when to stop reading! - - Note that we set the download maximum even if we read a - "Connection: close" header, to make sure that - "Content-Length: 0" still prevents us from attempting to - read the (missing) response-body. - */ - /* According to RFC2616 section 4.4, we MUST ignore - Content-Length: headers if we are now receiving data - using chunked Transfer-Encoding. - */ - if(k->chunk) - k->size=-1; - - } - if(-1 != k->size) { - /* We do this operation even if no_body is true, since this - data might be retrieved later with curl_easy_getinfo() - and its CURLINFO_CONTENT_LENGTH_DOWNLOAD option. */ - - Curl_pgrsSetDownloadSize(data, k->size); - k->maxdownload = k->size; - } - /* If max download size is *zero* (nothing) we already - have nothing and can safely return ok now! */ - if(0 == k->maxdownload) - stop_reading = TRUE; - - if(stop_reading) { - /* we make sure that this socket isn't read more now */ - k->keepon &= ~KEEP_READ; - } - - if(data->set.verbose) - Curl_debug(data, CURLINFO_HEADER_IN, - k->str_start, headerlen, conn); - break; /* exit header line loop */ - } + if(!k->header) { + /* + * really end-of-headers. + * + * If we requested a "no body", this is a good time to get + * out and return home. + */ + if(data->set.opt_no_body) + *stop_reading = TRUE; + else { + /* If we know the expected size of this document, we set the + maximum download size to the size of the expected + document or else, we won't know when to stop reading! + + Note that we set the download maximum even if we read a + "Connection: close" header, to make sure that + "Content-Length: 0" still prevents us from attempting to + read the (missing) response-body. + */ + /* According to RFC2616 section 4.4, we MUST ignore + Content-Length: headers if we are now receiving data + using chunked Transfer-Encoding. + */ + if(k->chunk) + k->size=-1; + + } + if(-1 != k->size) { + /* We do this operation even if no_body is true, since this + data might be retrieved later with curl_easy_getinfo() + and its CURLINFO_CONTENT_LENGTH_DOWNLOAD option. */ + + Curl_pgrsSetDownloadSize(data, k->size); + k->maxdownload = k->size; + } + /* If max download size is *zero* (nothing) we already + have nothing and can safely return ok now! */ + if(0 == k->maxdownload) + *stop_reading = TRUE; + + if(*stop_reading) { + /* we make sure that this socket isn't read more now */ + k->keepon &= ~KEEP_READ; + } + + if(data->set.verbose) + Curl_debug(data, CURLINFO_HEADER_IN, + k->str_start, headerlen, conn); + break; /* exit header line loop */ + } - /* We continue reading headers, so reset the line-based - header parsing variables hbufp && hbuflen */ - k->hbufp = data->state.headerbuff; - k->hbuflen = 0; - continue; - } + /* We continue reading headers, so reset the line-based + header parsing variables hbufp && hbuflen */ + k->hbufp = data->state.headerbuff; + k->hbuflen = 0; + continue; + } #ifndef CURL_DISABLE_HTTP - /* - * Checks for special headers coming up. - */ - - if(!k->headerline++) { - /* This is the first header, it MUST be the error code line - or else we consider this to be the body right away! */ - int httpversion_major; - int nc; + /* + * Checks for special headers coming up. + */ + + if(!k->headerline++) { + /* This is the first header, it MUST be the error code line + or else we consider this to be the body right away! */ + int httpversion_major; + int nc; #ifdef CURL_DOES_CONVERSIONS #define HEADER1 scratch #define SCRATCHSIZE 21 - CURLcode res; - char scratch[SCRATCHSIZE+1]; /* "HTTP/major.minor 123" */ - /* We can't really convert this yet because we - don't know if it's the 1st header line or the body. - So we do a partial conversion into a scratch area, - leaving the data at k->p as-is. - */ - strncpy(&scratch[0], k->p, SCRATCHSIZE); - scratch[SCRATCHSIZE] = 0; /* null terminate */ - res = Curl_convert_from_network(data, - &scratch[0], - SCRATCHSIZE); - if(CURLE_OK != res) { - /* Curl_convert_from_network calls failf if unsuccessful */ - return res; - } + CURLcode res; + char scratch[SCRATCHSIZE+1]; /* "HTTP/major.minor 123" */ + /* We can't really convert this yet because we + don't know if it's the 1st header line or the body. + So we do a partial conversion into a scratch area, + leaving the data at k->p as-is. + */ + strncpy(&scratch[0], k->p, SCRATCHSIZE); + scratch[SCRATCHSIZE] = 0; /* null terminate */ + res = Curl_convert_from_network(data, + &scratch[0], + SCRATCHSIZE); + if(CURLE_OK != res) { + /* Curl_convert_from_network calls failf if unsuccessful */ + return res; + } #else #define HEADER1 k->p /* no conversion needed, just use k->p */ #endif /* CURL_DOES_CONVERSIONS */ - nc = sscanf(HEADER1, - " HTTP/%d.%d %3d", - &httpversion_major, - &k->httpversion, - &k->httpcode); - if(nc==3) { - k->httpversion += 10 * httpversion_major; - } - else { - /* this is the real world, not a Nirvana - NCSA 1.5.x returns this crap when asked for HTTP/1.1 - */ - nc=sscanf(HEADER1, " HTTP %3d", &k->httpcode); - k->httpversion = 10; - - /* If user has set option HTTP200ALIASES, - compare header line against list of aliases - */ - if(!nc) { - if(checkhttpprefix(data, k->p)) { - nc = 1; - k->httpcode = 200; - k->httpversion = 10; - } - } - } + nc = sscanf(HEADER1, + " HTTP/%d.%d %3d", + &httpversion_major, + &k->httpversion, + &k->httpcode); + if(nc==3) { + k->httpversion += 10 * httpversion_major; + } + else { + /* this is the real world, not a Nirvana + NCSA 1.5.x returns this crap when asked for HTTP/1.1 + */ + nc=sscanf(HEADER1, " HTTP %3d", &k->httpcode); + k->httpversion = 10; + + /* If user has set option HTTP200ALIASES, + compare header line against list of aliases + */ + if(!nc) { + if(checkhttpprefix(data, k->p)) { + nc = 1; + k->httpcode = 200; + k->httpversion = 10; + } + } + } - if(nc) { - data->info.httpcode = k->httpcode; - data->info.httpversion = k->httpversion; - - /* - * This code executes as part of processing the header. As a - * result, it's not totally clear how to interpret the - * response code yet as that depends on what other headers may - * be present. 401 and 407 may be errors, but may be OK - * depending on how authentication is working. Other codes - * are definitely errors, so give up here. - */ - if(data->set.http_fail_on_error && (k->httpcode >= 400) && - ((k->httpcode != 401) || !conn->bits.user_passwd) && - ((k->httpcode != 407) || !conn->bits.proxy_user_passwd) ) { - - if(data->state.resume_from && - (data->set.httpreq==HTTPREQ_GET) && - (k->httpcode == 416)) { - /* "Requested Range Not Satisfiable", just proceed and - pretend this is no error */ - } - else { - /* serious error, go home! */ - failf (data, "The requested URL returned error: %d", - k->httpcode); - return CURLE_HTTP_RETURNED_ERROR; - } - } - - if(k->httpversion == 10) { - /* Default action for HTTP/1.0 must be to close, unless - we get one of those fancy headers that tell us the - server keeps it open for us! */ - infof(data, "HTTP 1.0, assume close after body\n"); - conn->bits.close = TRUE; - } - else if(k->httpversion >= 11 && - !conn->bits.close) { - /* If HTTP version is >= 1.1 and connection is persistent - server supports pipelining. */ - DEBUGF(infof(data, - "HTTP 1.1 or later with persistent connection, " - "pipelining supported\n")); - conn->server_supports_pipelining = TRUE; - } - - switch(k->httpcode) { - case 204: - /* (quote from RFC2616, section 10.2.5): The server has - * fulfilled the request but does not need to return an - * entity-body ... The 204 response MUST NOT include a - * message-body, and thus is always terminated by the first - * empty line after the header fields. */ - /* FALLTHROUGH */ - case 416: /* Requested Range Not Satisfiable, it has the - Content-Length: set as the "real" document but no - actual response is sent. */ - case 304: - /* (quote from RFC2616, section 10.3.5): The 304 response - * MUST NOT contain a message-body, and thus is always - * terminated by the first empty line after the header - * fields. */ - k->size=0; - k->maxdownload=0; - k->ignorecl = TRUE; /* ignore Content-Length headers */ - break; - default: - /* nothing */ - break; - } - } - else { - k->header = FALSE; /* this is not a header line */ - break; - } - } + if(nc) { + data->info.httpcode = k->httpcode; + data->info.httpversion = k->httpversion; + + /* + * This code executes as part of processing the header. As a + * result, it's not totally clear how to interpret the + * response code yet as that depends on what other headers may + * be present. 401 and 407 may be errors, but may be OK + * depending on how authentication is working. Other codes + * are definitely errors, so give up here. + */ + if(data->set.http_fail_on_error && (k->httpcode >= 400) && + ((k->httpcode != 401) || !conn->bits.user_passwd) && + ((k->httpcode != 407) || !conn->bits.proxy_user_passwd) ) { + + if(data->state.resume_from && + (data->set.httpreq==HTTPREQ_GET) && + (k->httpcode == 416)) { + /* "Requested Range Not Satisfiable", just proceed and + pretend this is no error */ + } + else { + /* serious error, go home! */ + failf (data, "The requested URL returned error: %d", + k->httpcode); + return CURLE_HTTP_RETURNED_ERROR; + } + } + + if(k->httpversion == 10) { + /* Default action for HTTP/1.0 must be to close, unless + we get one of those fancy headers that tell us the + server keeps it open for us! */ + infof(data, "HTTP 1.0, assume close after body\n"); + conn->bits.close = TRUE; + } + else if(k->httpversion >= 11 && + !conn->bits.close) { + /* If HTTP version is >= 1.1 and connection is persistent + server supports pipelining. */ + DEBUGF(infof(data, + "HTTP 1.1 or later with persistent connection, " + "pipelining supported\n")); + conn->server_supports_pipelining = TRUE; + } + + switch(k->httpcode) { + case 204: + /* (quote from RFC2616, section 10.2.5): The server has + * fulfilled the request but does not need to return an + * entity-body ... The 204 response MUST NOT include a + * message-body, and thus is always terminated by the first + * empty line after the header fields. */ + /* FALLTHROUGH */ + case 416: /* Requested Range Not Satisfiable, it has the + Content-Length: set as the "real" document but no + actual response is sent. */ + case 304: + /* (quote from RFC2616, section 10.3.5): The 304 response + * MUST NOT contain a message-body, and thus is always + * terminated by the first empty line after the header + * fields. */ + k->size=0; + k->maxdownload=0; + k->ignorecl = TRUE; /* ignore Content-Length headers */ + break; + default: + /* nothing */ + break; + } + } + else { + k->header = FALSE; /* this is not a header line */ + break; + } + } #endif /* CURL_DISABLE_HTTP */ #ifdef CURL_DOES_CONVERSIONS - /* convert from the network encoding */ - result = Curl_convert_from_network(data, k->p, strlen(k->p)); - if(CURLE_OK != result) { - return(result); - } - /* Curl_convert_from_network calls failf if unsuccessful */ + /* convert from the network encoding */ + result = Curl_convert_from_network(data, k->p, strlen(k->p)); + if(CURLE_OK != result) { + return(result); + } + /* Curl_convert_from_network calls failf if unsuccessful */ #endif /* CURL_DOES_CONVERSIONS */ #ifndef CURL_DISABLE_HTTP - /* Check for Content-Length: header lines to get size. Ignore - the header completely if we get a 416 response as then we're - resuming a document that we don't get, and this header contains - info about the true size of the document we didn't get now. */ - if(!k->ignorecl && !data->set.ignorecl && - checkprefix("Content-Length:", k->p)) { - curl_off_t contentlength = curlx_strtoofft(k->p+15, NULL, 10); - if(data->set.max_filesize && - contentlength > data->set.max_filesize) { - failf(data, "Maximum file size exceeded"); - return CURLE_FILESIZE_EXCEEDED; - } - if(contentlength >= 0) { - k->size = contentlength; - k->maxdownload = k->size; - /* we set the progress download size already at this point - just to make it easier for apps/callbacks to extract this - info as soon as possible */ - Curl_pgrsSetDownloadSize(data, k->size); - } - else { - /* Negative Content-Length is really odd, and we know it - happens for example when older Apache servers send large - files */ - conn->bits.close = TRUE; - infof(data, "Negative content-length: %" FORMAT_OFF_T - ", closing after transfer\n", contentlength); - } - } - /* check for Content-Type: header lines to get the MIME-type */ - else if(checkprefix("Content-Type:", k->p)) { - char *contenttype = Curl_copy_header_value(k->p); - if (!contenttype) - return CURLE_OUT_OF_MEMORY; - if (!*contenttype) - /* ignore empty data */ - free(contenttype); - else { - Curl_safefree(data->info.contenttype); - data->info.contenttype = contenttype; - } - } - else if((k->httpversion == 10) && - conn->bits.httpproxy && - Curl_compareheader(k->p, - "Proxy-Connection:", "keep-alive")) { - /* - * When a HTTP/1.0 reply comes when using a proxy, the - * 'Proxy-Connection: keep-alive' line tells us the - * connection will be kept alive for our pleasure. - * Default action for 1.0 is to close. - */ - conn->bits.close = FALSE; /* don't close when done */ - infof(data, "HTTP/1.0 proxy connection set to keep alive!\n"); - } - else if((k->httpversion == 11) && - conn->bits.httpproxy && - Curl_compareheader(k->p, - "Proxy-Connection:", "close")) { - /* - * We get a HTTP/1.1 response from a proxy and it says it'll - * close down after this transfer. - */ - conn->bits.close = TRUE; /* close when done */ - infof(data, "HTTP/1.1 proxy connection set close!\n"); - } - else if((k->httpversion == 10) && - Curl_compareheader(k->p, "Connection:", "keep-alive")) { - /* - * A HTTP/1.0 reply with the 'Connection: keep-alive' line - * tells us the connection will be kept alive for our - * pleasure. Default action for 1.0 is to close. - * - * [RFC2068, section 19.7.1] */ - conn->bits.close = FALSE; /* don't close when done */ - infof(data, "HTTP/1.0 connection set to keep alive!\n"); - } - else if(Curl_compareheader(k->p, "Connection:", "close")) { - /* - * [RFC 2616, section 8.1.2.1] - * "Connection: close" is HTTP/1.1 language and means that - * the connection will close when this request has been - * served. - */ - conn->bits.close = TRUE; /* close when done */ - } - else if(Curl_compareheader(k->p, - "Transfer-Encoding:", "chunked")) { - /* - * [RFC 2616, section 3.6.1] A 'chunked' transfer encoding - * means that the server will send a series of "chunks". Each - * chunk starts with line with info (including size of the - * coming block) (terminated with CRLF), then a block of data - * with the previously mentioned size. There can be any amount - * of chunks, and a chunk-data set to zero signals the - * end-of-chunks. */ - k->chunk = TRUE; /* chunks coming our way */ - - /* init our chunky engine */ - Curl_httpchunk_init(conn); - } - - else if(checkprefix("Trailer:", k->p) || - checkprefix("Trailers:", k->p)) { - /* - * This test helps Curl_httpchunk_read() to determine to look - * for well formed trailers after the zero chunksize record. In - * this case a CRLF is required after the zero chunksize record - * when no trailers are sent, or after the last trailer record. - * - * It seems both Trailer: and Trailers: occur in the wild. - */ - k->trailerhdrpresent = TRUE; - } + /* Check for Content-Length: header lines to get size. Ignore + the header completely if we get a 416 response as then we're + resuming a document that we don't get, and this header contains + info about the true size of the document we didn't get now. */ + if(!k->ignorecl && !data->set.ignorecl && + checkprefix("Content-Length:", k->p)) { + curl_off_t contentlength = curlx_strtoofft(k->p+15, NULL, 10); + if(data->set.max_filesize && + contentlength > data->set.max_filesize) { + failf(data, "Maximum file size exceeded"); + return CURLE_FILESIZE_EXCEEDED; + } + if(contentlength >= 0) { + k->size = contentlength; + k->maxdownload = k->size; + /* we set the progress download size already at this point + just to make it easier for apps/callbacks to extract this + info as soon as possible */ + Curl_pgrsSetDownloadSize(data, k->size); + } + else { + /* Negative Content-Length is really odd, and we know it + happens for example when older Apache servers send large + files */ + conn->bits.close = TRUE; + infof(data, "Negative content-length: %" FORMAT_OFF_T + ", closing after transfer\n", contentlength); + } + } + /* check for Content-Type: header lines to get the MIME-type */ + else if(checkprefix("Content-Type:", k->p)) { + char *contenttype = Curl_copy_header_value(k->p); + if (!contenttype) + return CURLE_OUT_OF_MEMORY; + if (!*contenttype) + /* ignore empty data */ + free(contenttype); + else { + Curl_safefree(data->info.contenttype); + data->info.contenttype = contenttype; + } + } + else if((k->httpversion == 10) && + conn->bits.httpproxy && + Curl_compareheader(k->p, + "Proxy-Connection:", "keep-alive")) { + /* + * When a HTTP/1.0 reply comes when using a proxy, the + * 'Proxy-Connection: keep-alive' line tells us the + * connection will be kept alive for our pleasure. + * Default action for 1.0 is to close. + */ + conn->bits.close = FALSE; /* don't close when done */ + infof(data, "HTTP/1.0 proxy connection set to keep alive!\n"); + } + else if((k->httpversion == 11) && + conn->bits.httpproxy && + Curl_compareheader(k->p, + "Proxy-Connection:", "close")) { + /* + * We get a HTTP/1.1 response from a proxy and it says it'll + * close down after this transfer. + */ + conn->bits.close = TRUE; /* close when done */ + infof(data, "HTTP/1.1 proxy connection set close!\n"); + } + else if((k->httpversion == 10) && + Curl_compareheader(k->p, "Connection:", "keep-alive")) { + /* + * A HTTP/1.0 reply with the 'Connection: keep-alive' line + * tells us the connection will be kept alive for our + * pleasure. Default action for 1.0 is to close. + * + * [RFC2068, section 19.7.1] */ + conn->bits.close = FALSE; /* don't close when done */ + infof(data, "HTTP/1.0 connection set to keep alive!\n"); + } + else if(Curl_compareheader(k->p, "Connection:", "close")) { + /* + * [RFC 2616, section 8.1.2.1] + * "Connection: close" is HTTP/1.1 language and means that + * the connection will close when this request has been + * served. + */ + conn->bits.close = TRUE; /* close when done */ + } + else if(Curl_compareheader(k->p, + "Transfer-Encoding:", "chunked")) { + /* + * [RFC 2616, section 3.6.1] A 'chunked' transfer encoding + * means that the server will send a series of "chunks". Each + * chunk starts with line with info (including size of the + * coming block) (terminated with CRLF), then a block of data + * with the previously mentioned size. There can be any amount + * of chunks, and a chunk-data set to zero signals the + * end-of-chunks. */ + k->chunk = TRUE; /* chunks coming our way */ + + /* init our chunky engine */ + Curl_httpchunk_init(conn); + } - else if(checkprefix("Content-Encoding:", k->p) && - data->set.str[STRING_ENCODING]) { - /* - * Process Content-Encoding. Look for the values: identity, - * gzip, deflate, compress, x-gzip and x-compress. x-gzip and - * x-compress are the same as gzip and compress. (Sec 3.5 RFC - * 2616). zlib cannot handle compress. However, errors are - * handled further down when the response body is processed - */ - char *start; - - /* Find the first non-space letter */ - for(start=k->p+17; - *start && ISSPACE(*start); - start++) - ; /* empty loop */ - - /* Record the content-encoding for later use */ - if(checkprefix("identity", start)) - k->content_encoding = IDENTITY; - else if(checkprefix("deflate", start)) - k->content_encoding = DEFLATE; - else if(checkprefix("gzip", start) - || checkprefix("x-gzip", start)) - k->content_encoding = GZIP; - else if(checkprefix("compress", start) - || checkprefix("x-compress", start)) - k->content_encoding = COMPRESS; - } - else if(checkprefix("Content-Range:", k->p)) { - /* Content-Range: bytes [num]- - Content-Range: bytes: [num]- - Content-Range: [num]- + else if(checkprefix("Trailer:", k->p) || + checkprefix("Trailers:", k->p)) { + /* + * This test helps Curl_httpchunk_read() to determine to look + * for well formed trailers after the zero chunksize record. In + * this case a CRLF is required after the zero chunksize record + * when no trailers are sent, or after the last trailer record. + * + * It seems both Trailer: and Trailers: occur in the wild. + */ + k->trailerhdrpresent = TRUE; + } - The second format was added since Sun's webserver - JavaWebServer/1.1.1 obviously sends the header this way! - The third added since some servers use that! - */ + else if(checkprefix("Content-Encoding:", k->p) && + data->set.str[STRING_ENCODING]) { + /* + * Process Content-Encoding. Look for the values: identity, + * gzip, deflate, compress, x-gzip and x-compress. x-gzip and + * x-compress are the same as gzip and compress. (Sec 3.5 RFC + * 2616). zlib cannot handle compress. However, errors are + * handled further down when the response body is processed + */ + char *start; + + /* Find the first non-space letter */ + for(start=k->p+17; + *start && ISSPACE(*start); + start++) + ; /* empty loop */ + + /* Record the content-encoding for later use */ + if(checkprefix("identity", start)) + k->content_encoding = IDENTITY; + else if(checkprefix("deflate", start)) + k->content_encoding = DEFLATE; + else if(checkprefix("gzip", start) + || checkprefix("x-gzip", start)) + k->content_encoding = GZIP; + else if(checkprefix("compress", start) + || checkprefix("x-compress", start)) + k->content_encoding = COMPRESS; + } + else if(checkprefix("Content-Range:", k->p)) { + /* Content-Range: bytes [num]- + Content-Range: bytes: [num]- + Content-Range: [num]- + + The second format was added since Sun's webserver + JavaWebServer/1.1.1 obviously sends the header this way! + The third added since some servers use that! + */ - char *ptr = k->p + 14; + char *ptr = k->p + 14; - /* Move forward until first digit */ - while(*ptr && !ISDIGIT(*ptr)) - ptr++; + /* Move forward until first digit */ + while(*ptr && !ISDIGIT(*ptr)) + ptr++; - k->offset = curlx_strtoofft(ptr, NULL, 10); + k->offset = curlx_strtoofft(ptr, NULL, 10); - if(data->state.resume_from == k->offset) - /* we asked for a resume and we got it */ - k->content_range = TRUE; - } + if(data->state.resume_from == k->offset) + /* we asked for a resume and we got it */ + k->content_range = TRUE; + } #if !defined(CURL_DISABLE_COOKIES) - else if(data->cookies && - checkprefix("Set-Cookie:", k->p)) { - Curl_share_lock(data, CURL_LOCK_DATA_COOKIE, - CURL_LOCK_ACCESS_SINGLE); - Curl_cookie_add(data, - data->cookies, TRUE, k->p+11, - /* If there is a custom-set Host: name, use it - here, or else use real peer host name. */ - conn->allocptr.cookiehost? - conn->allocptr.cookiehost:conn->host.name, - data->state.path); - Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE); - } + else if(data->cookies && + checkprefix("Set-Cookie:", k->p)) { + Curl_share_lock(data, CURL_LOCK_DATA_COOKIE, + CURL_LOCK_ACCESS_SINGLE); + Curl_cookie_add(data, + data->cookies, TRUE, k->p+11, + /* If there is a custom-set Host: name, use it + here, or else use real peer host name. */ + conn->allocptr.cookiehost? + conn->allocptr.cookiehost:conn->host.name, + data->state.path); + Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE); + } #endif - else if(checkprefix("Last-Modified:", k->p) && - (data->set.timecondition || data->set.get_filetime) ) { - time_t secs=time(NULL); - k->timeofdoc = curl_getdate(k->p+strlen("Last-Modified:"), - &secs); - if(data->set.get_filetime) - data->info.filetime = (long)k->timeofdoc; - } - else if((checkprefix("WWW-Authenticate:", k->p) && - (401 == k->httpcode)) || - (checkprefix("Proxy-authenticate:", k->p) && - (407 == k->httpcode))) { - result = Curl_http_input_auth(conn, k->httpcode, k->p); - if(result) - return result; - } - else if((k->httpcode >= 300 && k->httpcode < 400) && - checkprefix("Location:", k->p)) { - /* this is the URL that the server advises us to use instead */ - char *location = Curl_copy_header_value(k->p); - if (!location) - return CURLE_OUT_OF_MEMORY; - if (!*location) - /* ignore empty data */ - free(location); - else { - DEBUGASSERT(!data->req.location); - data->req.location = location; - - if(data->set.http_follow_location) { - DEBUGASSERT(!data->req.newurl); - data->req.newurl = strdup(data->req.location); /* clone */ - if(!data->req.newurl) - return CURLE_OUT_OF_MEMORY; - - /* some cases of POST and PUT etc needs to rewind the data - stream at this point */ - result = Curl_http_perhapsrewind(conn); - if(result) - return result; - } - } - } + else if(checkprefix("Last-Modified:", k->p) && + (data->set.timecondition || data->set.get_filetime) ) { + time_t secs=time(NULL); + k->timeofdoc = curl_getdate(k->p+strlen("Last-Modified:"), + &secs); + if(data->set.get_filetime) + data->info.filetime = (long)k->timeofdoc; + } + else if((checkprefix("WWW-Authenticate:", k->p) && + (401 == k->httpcode)) || + (checkprefix("Proxy-authenticate:", k->p) && + (407 == k->httpcode))) { + result = Curl_http_input_auth(conn, k->httpcode, k->p); + if(result) + return result; + } + else if((k->httpcode >= 300 && k->httpcode < 400) && + checkprefix("Location:", k->p)) { + /* this is the URL that the server advises us to use instead */ + char *location = Curl_copy_header_value(k->p); + if (!location) + return CURLE_OUT_OF_MEMORY; + if (!*location) + /* ignore empty data */ + free(location); + else { + DEBUGASSERT(!data->req.location); + data->req.location = location; + + if(data->set.http_follow_location) { + DEBUGASSERT(!data->req.newurl); + data->req.newurl = strdup(data->req.location); /* clone */ + if(!data->req.newurl) + return CURLE_OUT_OF_MEMORY; + + /* some cases of POST and PUT etc needs to rewind the data + stream at this point */ + result = Curl_http_perhapsrewind(conn); + if(result) + return result; + } + } + } #endif /* CURL_DISABLE_HTTP */ - /* - * End of header-checks. Write them to the client. - */ - - writetype = CLIENTWRITE_HEADER; - if(data->set.include_header) - writetype |= CLIENTWRITE_BODY; - - if(data->set.verbose) - Curl_debug(data, CURLINFO_HEADER_IN, - k->p, (size_t)k->hbuflen, conn); - - result = Curl_client_write(conn, writetype, k->p, k->hbuflen); - if(result) - return result; - - data->info.header_size += (long)k->hbuflen; - data->req.headerbytecount += (long)k->hbuflen; - - /* reset hbufp pointer && hbuflen */ - k->hbufp = data->state.headerbuff; - k->hbuflen = 0; - } - while(!stop_reading && *k->str); /* header line within buffer */ - - if(stop_reading) - /* We've stopped dealing with input, get out of the do-while loop */ - break; - - /* We might have reached the end of the header part here, but - there might be a non-header part left in the end of the read - buffer. */ - - } /* end if header mode */ - - /* This is not an 'else if' since it may be a rest from the header - parsing, where the beginning of the buffer is headers and the end - is non-headers. */ - if(k->str && !k->header && (nread > 0 || is_empty_data)) { - - if(0 == k->bodywrites && !is_empty_data) { - /* These checks are only made the first time we are about to - write a piece of the body */ - if(conn->protocol&PROT_HTTP) { - /* HTTP-only checks */ + /* + * End of header-checks. Write them to the client. + */ - if(data->req.newurl) { - if(conn->bits.close) { - /* Abort after the headers if "follow Location" is set - and we're set to close anyway. */ - k->keepon &= ~KEEP_READ; - *done = TRUE; - return CURLE_OK; - } - /* We have a new url to load, but since we want to be able - to re-use this connection properly, we read the full - response in "ignore more" */ - k->ignorebody = TRUE; - infof(data, "Ignoring the response-body\n"); - } - if(data->state.resume_from && !k->content_range && - (data->set.httpreq==HTTPREQ_GET) && - !k->ignorebody) { - /* we wanted to resume a download, although the server doesn't - * seem to support this and we did this with a GET (if it - * wasn't a GET we did a POST or PUT resume) */ - failf(data, "HTTP server doesn't seem to support " - "byte ranges. Cannot resume."); - return CURLE_RANGE_ERROR; - } + writetype = CLIENTWRITE_HEADER; + if(data->set.include_header) + writetype |= CLIENTWRITE_BODY; - if(data->set.timecondition && !data->state.range) { - /* A time condition has been set AND no ranges have been - requested. This seems to be what chapter 13.3.4 of - RFC 2616 defines to be the correct action for a - HTTP/1.1 client */ - if((k->timeofdoc > 0) && (data->set.timevalue > 0)) { - switch(data->set.timecondition) { - case CURL_TIMECOND_IFMODSINCE: - default: - if(k->timeofdoc < data->set.timevalue) { - infof(data, - "The requested document is not new enough\n"); - *done = TRUE; - return CURLE_OK; - } - break; - case CURL_TIMECOND_IFUNMODSINCE: - if(k->timeofdoc > data->set.timevalue) { - infof(data, - "The requested document is not old enough\n"); - *done = TRUE; - return CURLE_OK; - } - break; - } /* switch */ - } /* two valid time strings */ - } /* we have a time condition */ - - } /* this is HTTP */ - } /* this is the first time we write a body part */ - k->bodywrites++; - - /* pass data to the debug function before it gets "dechunked" */ - if(data->set.verbose) { - if(k->badheader) { - Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, - (size_t)k->hbuflen, conn); - if(k->badheader == HEADER_PARTHEADER) - Curl_debug(data, CURLINFO_DATA_IN, - k->str, (size_t)nread, conn); - } - else - Curl_debug(data, CURLINFO_DATA_IN, - k->str, (size_t)nread, conn); - } + if(data->set.verbose) + Curl_debug(data, CURLINFO_HEADER_IN, + k->p, (size_t)k->hbuflen, conn); -#ifndef CURL_DISABLE_HTTP - if(k->chunk) { - /* - * Here comes a chunked transfer flying and we need to decode this - * properly. While the name says read, this function both reads - * and writes away the data. The returned 'nread' holds the number - * of actual data it wrote to the client. - */ - - CHUNKcode res = - Curl_httpchunk_read(conn, k->str, nread, &nread); - - if(CHUNKE_OK < res) { - if(CHUNKE_WRITE_ERROR == res) { - failf(data, "Failed writing data"); - return CURLE_WRITE_ERROR; - } - failf(data, "Received problem %d in the chunky parser", res); - return CURLE_RECV_ERROR; - } - else if(CHUNKE_STOP == res) { - size_t dataleft; - /* we're done reading chunks! */ - k->keepon &= ~KEEP_READ; /* read no more */ - - /* There are now possibly N number of bytes at the end of the - str buffer that weren't written to the client. - - We DO care about this data if we are pipelining. - Push it back to be read on the next pass. */ - - dataleft = conn->chunk.dataleft; - if(dataleft != 0) { - infof(conn->data, "Leftovers after chunking. " - " Rewinding %d bytes\n",dataleft); - read_rewind(conn, dataleft); - } - } - /* If it returned OK, we just keep going */ - } -#endif /* CURL_DISABLE_HTTP */ + result = Curl_client_write(conn, writetype, k->p, k->hbuflen); + if(result) + return result; - if((-1 != k->maxdownload) && - (k->bytecount + nread >= k->maxdownload)) { - /* The 'excess' amount below can't be more than BUFSIZE which - always will fit in a size_t */ - size_t excess = (size_t)(k->bytecount + nread - k->maxdownload); - if(excess > 0 && !k->ignorebody) { - infof(data, - "Rewinding stream by : %d" - " bytes on url %s (size = %" FORMAT_OFF_T - ", maxdownload = %" FORMAT_OFF_T - ", bytecount = %" FORMAT_OFF_T ", nread = %d)\n", - excess, data->state.path, - k->size, k->maxdownload, k->bytecount, nread); - read_rewind(conn, excess); - } + data->info.header_size += (long)k->hbuflen; + data->req.headerbytecount += (long)k->hbuflen; - nread = (ssize_t) (k->maxdownload - k->bytecount); - if(nread < 0 ) /* this should be unusual */ - nread = 0; + /* reset hbufp pointer && hbuflen */ + k->hbufp = data->state.headerbuff; + k->hbuflen = 0; + } + while(!*stop_reading && *k->str); /* header line within buffer */ - k->keepon &= ~KEEP_READ; /* we're done reading */ - } + /* We might have reached the end of the header part here, but + there might be a non-header part left in the end of the read + buffer. */ - k->bytecount += nread; + return CURLE_OK; +} - Curl_pgrsSetDownloadCounter(data, k->bytecount); +/* + * Send data to upload to the server, when the socket is writable. + */ +static CURLcode readwrite_upload(struct SessionHandle *data, + struct connectdata *conn, + struct SingleRequest *k, + int *didwhat) +{ + ssize_t i, si; + ssize_t bytes_written; + CURLcode result; + ssize_t nread; /* number of bytes read */ - if(!k->chunk && (nread || k->badheader || is_empty_data)) { - /* If this is chunky transfer, it was already written */ + if((k->bytecount == 0) && (k->writebytecount == 0)) + Curl_pgrsTime(data, TIMER_STARTTRANSFER); - if(k->badheader && !k->ignorebody) { - /* we parsed a piece of data wrongly assuming it was a header - and now we output it as body instead */ - result = Curl_client_write(conn, CLIENTWRITE_BODY, - data->state.headerbuff, - k->hbuflen); - if(result) - return result; - } - if(k->badheader < HEADER_ALLBAD) { - /* This switch handles various content encodings. If there's an - error here, be sure to check over the almost identical code - in http_chunks.c. - Make sure that ALL_CONTENT_ENCODINGS contains all the - encodings handled here. */ -#ifdef HAVE_LIBZ - switch (conn->data->set.http_ce_skip ? - IDENTITY : k->content_encoding) { - case IDENTITY: -#endif - /* This is the default when the server sends no - Content-Encoding header. See Curl_readwrite_init; the - memset() call initializes k->content_encoding to zero. */ - if(!k->ignorebody) - result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str, - nread); -#ifdef HAVE_LIBZ - break; - - case DEFLATE: - /* Assume CLIENTWRITE_BODY; headers are not encoded. */ - if(!k->ignorebody) - result = Curl_unencode_deflate_write(conn, k, nread); - break; - - case GZIP: - /* Assume CLIENTWRITE_BODY; headers are not encoded. */ - if(!k->ignorebody) - result = Curl_unencode_gzip_write(conn, k, nread); - break; - - case COMPRESS: - default: - failf (data, "Unrecognized content encoding type. " - "libcurl understands `identity', `deflate' and `gzip' " - "content encodings."); - result = CURLE_BAD_CONTENT_ENCODING; - break; - } -#endif - } - k->badheader = HEADER_NORMAL; /* taken care of now */ + *didwhat |= KEEP_WRITE; - if(result) - return result; - } + /* + * We loop here to do the READ and SEND loop until we run out of + * data to send or until we get EWOULDBLOCK back + */ + do { - } /* if(! header and data to read ) */ + /* only read more data if there's no upload data already + present in the upload buffer */ + if(0 == data->req.upload_present) { + /* init the "upload from here" pointer */ + data->req.upload_fromhere = k->uploadbuf; + + if(!k->upload_done) { + /* HTTP pollution, this should be written nicer to become more + protocol agnostic. */ + int fillcount; + + if((k->exp100 == EXP100_SENDING_REQUEST) && + (data->state.proto.http->sending == HTTPSEND_BODY)) { + /* If this call is to send body data, we must take some action: + We have sent off the full HTTP 1.1 request, and we shall now + go into the Expect: 100 state and await such a header */ + k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */ + k->keepon &= ~KEEP_WRITE; /* disable writing */ + k->start100 = Curl_tvnow(); /* timeout count starts now */ + *didwhat &= ~KEEP_WRITE; /* we didn't write anything actually */ + break; + } + + result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount); + if(result) + return result; + + nread = (ssize_t)fillcount; + } + else + nread = 0; /* we're done uploading/reading */ - if(is_empty_data) { - /* if we received nothing, the server closed the connection and we - are done */ - k->keepon &= ~KEEP_READ; + if(!nread && (k->keepon & KEEP_WRITE_PAUSE)) { + /* this is a paused transfer */ + break; + } + else if(nread<=0) { + /* done */ + k->keepon &= ~KEEP_WRITE; /* we're done writing */ + + if(conn->bits.rewindaftersend) { + result = Curl_readrewind(conn); + if(result) + return result; + } + break; } - } while(data_pending(conn)); + /* store number of bytes available for upload */ + data->req.upload_present = nread; - } /* if( read from socket ) */ + /* convert LF to CRLF if so asked */ +#ifdef CURL_DO_LINEEND_CONV + /* always convert if we're FTPing in ASCII mode */ + if((data->set.crlf) || (data->set.prefer_ascii)) +#else + if(data->set.crlf) +#endif /* CURL_DO_LINEEND_CONV */ + { + if(data->state.scratch == NULL) + data->state.scratch = malloc(2*BUFSIZE); + if(data->state.scratch == NULL) { + failf (data, "Failed to alloc scratch buffer!"); + return CURLE_OUT_OF_MEMORY; + } + /* + * ASCII/EBCDIC Note: This is presumably a text (not binary) + * transfer so the data should already be in ASCII. + * That means the hex values for ASCII CR (0x0d) & LF (0x0a) + * must be used instead of the escape sequences \r & \n. + */ + for(i = 0, si = 0; i < nread; i++, si++) { + if(data->req.upload_fromhere[i] == 0x0a) { + data->state.scratch[si++] = 0x0d; + data->state.scratch[si] = 0x0a; + if(!data->set.crlf) { + /* we're here only because FTP is in ASCII mode... + bump infilesize for the LF we just added */ + data->set.infilesize++; + } + } + else + data->state.scratch[si] = data->req.upload_fromhere[i]; + } + if(si != nread) { + /* only perform the special operation if we really did replace + anything */ + nread = si; + + /* upload from the new (replaced) buffer instead */ + data->req.upload_fromhere = data->state.scratch; + + /* set the new amount too */ + data->req.upload_present = nread; + } + } + } /* if 0 == data->req.upload_present */ + else { + /* We have a partial buffer left from a previous "round". Use + that instead of reading more data */ + } - /* If we still have writing to do, we check if we have a writable socket. */ - if((k->keepon & KEEP_WRITE) && (select_res & CURL_CSELECT_OUT)) { - /* write */ + /* write to socket (send away data) */ + result = Curl_write(conn, + conn->writesockfd, /* socket to send to */ + data->req.upload_fromhere, /* buffer pointer */ + data->req.upload_present, /* buffer size */ + &bytes_written); /* actually send away */ + if(result) + return result; - ssize_t i, si; - ssize_t bytes_written; + if(data->set.verbose) + /* show the data before we change the pointer upload_fromhere */ + Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere, + (size_t)bytes_written, conn); - if((k->bytecount == 0) && (k->writebytecount == 0)) - Curl_pgrsTime(data, TIMER_STARTTRANSFER); + if(data->req.upload_present != bytes_written) { + /* we only wrote a part of the buffer (if anything), deal with it! */ - didwhat |= KEEP_WRITE; + /* store the amount of bytes left in the buffer to write */ + data->req.upload_present -= bytes_written; - /* - * We loop here to do the READ and SEND loop until we run out of - * data to send or until we get EWOULDBLOCK back - */ - do { - - /* only read more data if there's no upload data already - present in the upload buffer */ - if(0 == data->req.upload_present) { - /* init the "upload from here" pointer */ - data->req.upload_fromhere = k->uploadbuf; - - if(!k->upload_done) { - /* HTTP pollution, this should be written nicer to become more - protocol agnostic. */ - int fillcount; - - if((k->exp100 == EXP100_SENDING_REQUEST) && - (data->state.proto.http->sending == HTTPSEND_BODY)) { - /* If this call is to send body data, we must take some action: - We have sent off the full HTTP 1.1 request, and we shall now - go into the Expect: 100 state and await such a header */ - k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */ - k->keepon &= ~KEEP_WRITE; /* disable writing */ - k->start100 = Curl_tvnow(); /* timeout count starts now */ - didwhat &= ~KEEP_WRITE; /* we didn't write anything actually */ - break; - } + /* advance the pointer where to find the buffer when the next send + is to happen */ + data->req.upload_fromhere += bytes_written; + } + else { + /* we've uploaded that buffer now */ + data->req.upload_fromhere = k->uploadbuf; + data->req.upload_present = 0; /* no more bytes left */ - result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount); - if(result) - return result; + if(k->upload_done) { + /* switch off writing, we're done! */ + k->keepon &= ~KEEP_WRITE; /* we're done writing */ + } + } - nread = (ssize_t)fillcount; - } - else - nread = 0; /* we're done uploading/reading */ + k->writebytecount += bytes_written; + Curl_pgrsSetUploadCounter(data, k->writebytecount); - if(!nread && (k->keepon & KEEP_WRITE_PAUSE)) { - /* this is a paused transfer */ - break; - } - else if(nread<=0) { - /* done */ - k->keepon &= ~KEEP_WRITE; /* we're done writing */ - - if(conn->bits.rewindaftersend) { - result = Curl_readrewind(conn); - if(result) - return result; - } - break; - } + } while(0); /* just to break out from! */ - /* store number of bytes available for upload */ - data->req.upload_present = nread; + return CURLE_OK; +} - /* convert LF to CRLF if so asked */ -#ifdef CURL_DO_LINEEND_CONV - /* always convert if we're FTPing in ASCII mode */ - if((data->set.crlf) || (data->set.prefer_ascii)) -#else - if(data->set.crlf) -#endif /* CURL_DO_LINEEND_CONV */ - { - if(data->state.scratch == NULL) - data->state.scratch = malloc(2*BUFSIZE); - if(data->state.scratch == NULL) { - failf (data, "Failed to alloc scratch buffer!"); - return CURLE_OUT_OF_MEMORY; - } - /* - * ASCII/EBCDIC Note: This is presumably a text (not binary) - * transfer so the data should already be in ASCII. - * That means the hex values for ASCII CR (0x0d) & LF (0x0a) - * must be used instead of the escape sequences \r & \n. - */ - for(i = 0, si = 0; i < nread; i++, si++) { - if(data->req.upload_fromhere[i] == 0x0a) { - data->state.scratch[si++] = 0x0d; - data->state.scratch[si] = 0x0a; - if(!data->set.crlf) { - /* we're here only because FTP is in ASCII mode... - bump infilesize for the LF we just added */ - data->set.infilesize++; - } - } - else - data->state.scratch[si] = data->req.upload_fromhere[i]; - } - if(si != nread) { - /* only perform the special operation if we really did replace - anything */ - nread = si; +/* + * Curl_readwrite() is the low-level function to be called when data is to + * be read and written to/from the connection. + */ +CURLcode Curl_readwrite(struct connectdata *conn, + bool *done) +{ + struct SessionHandle *data = conn->data; + struct SingleRequest *k = &data->req; + CURLcode result; + int didwhat=0; - /* upload from the new (replaced) buffer instead */ - data->req.upload_fromhere = data->state.scratch; + curl_socket_t fd_read; + curl_socket_t fd_write; + int select_res = conn->cselect_bits; - /* set the new amount too */ - data->req.upload_present = nread; - } - } - } /* if 0 == data->req.upload_present */ - else { - /* We have a partial buffer left from a previous "round". Use - that instead of reading more data */ - } + conn->cselect_bits = 0; - /* write to socket (send away data) */ - result = Curl_write(conn, - conn->writesockfd, /* socket to send to */ - data->req.upload_fromhere, /* buffer pointer */ - data->req.upload_present, /* buffer size */ - &bytes_written); /* actually send away */ - if(result) - return result; + /* only use the proper socket if the *_HOLD bit is not set simultaneously as + then we are in rate limiting state in that transfer direction */ - if(data->set.verbose) - /* show the data before we change the pointer upload_fromhere */ - Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere, - (size_t)bytes_written, conn); + if((k->keepon & KEEP_READBITS) == KEEP_READ) { + fd_read = conn->sockfd; +#if defined(USE_LIBSSH2) + if(conn->protocol & (PROT_SCP|PROT_SFTP)) + select_res |= CURL_CSELECT_IN; +#endif /* USE_LIBSSH2 */ + } else + fd_read = CURL_SOCKET_BAD; - if(data->req.upload_present != bytes_written) { - /* we only wrote a part of the buffer (if anything), deal with it! */ + if((k->keepon & KEEP_WRITEBITS) == KEEP_WRITE) + fd_write = conn->writesockfd; + else + fd_write = CURL_SOCKET_BAD; - /* store the amount of bytes left in the buffer to write */ - data->req.upload_present -= bytes_written; + if(!select_res) { /* Call for select()/poll() only, if read/write/error + status is not known. */ + select_res = Curl_socket_ready(fd_read, fd_write, 0); + } - /* advance the pointer where to find the buffer when the next send - is to happen */ - data->req.upload_fromhere += bytes_written; - } - else { - /* we've uploaded that buffer now */ - data->req.upload_fromhere = k->uploadbuf; - data->req.upload_present = 0; /* no more bytes left */ + if(select_res == CURL_CSELECT_ERR) { + failf(data, "select/poll returned error"); + return CURLE_SEND_ERROR; + } - if(k->upload_done) { - /* switch off writing, we're done! */ - k->keepon &= ~KEEP_WRITE; /* we're done writing */ - } - } + /* We go ahead and do a read if we have a readable socket or if + the stream was rewound (in which case we have data in a + buffer) */ + if((k->keepon & KEEP_READ) && + ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) { - k->writebytecount += bytes_written; - Curl_pgrsSetUploadCounter(data, k->writebytecount); + result = readwrite_data(data, conn, k, &didwhat, done); + if(result || *done) + return result; + } - } while(0); /* just to break out from! */ + /* If we still have writing to do, we check if we have a writable socket. */ + if((k->keepon & KEEP_WRITE) && (select_res & CURL_CSELECT_OUT)) { + /* write */ - } /* if(something to write) */ + result = readwrite_upload(data, conn, k, &didwhat); + if(result) + return result; + } k->now = Curl_tvnow(); if(didwhat) { @@ -1778,7 +1833,7 @@ Transfer(struct connectdata *conn) case -1: /* select() error, stop reading */ #ifdef EINTR /* The EINTR is not serious, and it seems you might get this more - ofen when using the lib in a multi-threaded environment! */ + often when using the lib in a multi-threaded environment! */ if(SOCKERRNO == EINTR) ; else @@ -1807,7 +1862,7 @@ CURLcode Curl_pretransfer(struct SessionHandle *data) { CURLcode res; if(!data->change.url) { - /* we can't do anything wihout URL */ + /* we can't do anything without URL */ failf(data, "No URL set!"); return CURLE_URL_MALFORMAT; } @@ -1935,6 +1990,163 @@ static void strcpy_url(char *output, const char *url) } +/* + * Returns true if the given URL is absolute (as opposed to relative) + */ +static bool is_absolute_url(const char *url) +{ + char prot[16]; /* URL protocol string storage */ + char letter; /* used for a silly sscanf */ + + return 2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter); +} + +/* + * Concatenate a relative URL to a base URL making it absolute. + * URL-encodes any spaces. + * The returned pointer must be freed by the caller unless NULL + * (returns NULL on out of memory). + */ +static char *concat_url(const char *base, const char *relurl) +{ + /*** + TRY to append this new path to the old URL + to the right of the host part. Oh crap, this is doomed to cause + problems in the future... + */ + char *newest; + char *protsep; + char *pathsep; + size_t newlen; + + const char *useurl = relurl; + size_t urllen; + + /* we must make our own copy of the URL to play with, as it may + point to read-only data */ + char *url_clone=strdup(base); + + if(!url_clone) + return NULL; /* skip out of this NOW */ + + /* protsep points to the start of the host name */ + protsep=strstr(url_clone, "//"); + if(!protsep) + protsep=url_clone; + else + protsep+=2; /* pass the slashes */ + + if('/' != relurl[0]) { + int level=0; + + /* First we need to find out if there's a ?-letter in the URL, + and cut it and the right-side of that off */ + pathsep = strchr(protsep, '?'); + if(pathsep) + *pathsep=0; + + /* we have a relative path to append to the last slash if there's one + available, or if the new URL is just a query string (starts with a + '?') we append the new one at the end of the entire currently worked + out URL */ + if(useurl[0] != '?') { + pathsep = strrchr(protsep, '/'); + if(pathsep) + *pathsep=0; + } + + /* Check if there's any slash after the host name, and if so, remember + that position instead */ + pathsep = strchr(protsep, '/'); + if(pathsep) + protsep = pathsep+1; + else + protsep = NULL; + + /* now deal with one "./" or any amount of "../" in the newurl + and act accordingly */ + + if((useurl[0] == '.') && (useurl[1] == '/')) + useurl+=2; /* just skip the "./" */ + + while((useurl[0] == '.') && + (useurl[1] == '.') && + (useurl[2] == '/')) { + level++; + useurl+=3; /* pass the "../" */ + } + + if(protsep) { + while(level--) { + /* cut off one more level from the right of the original URL */ + pathsep = strrchr(protsep, '/'); + if(pathsep) + *pathsep=0; + else { + *protsep=0; + break; + } + } + } + } + else { + /* We got a new absolute path for this server, cut off from the + first slash */ + pathsep = strchr(protsep, '/'); + if(pathsep) { + /* When people use badly formatted URLs, such as + "http://www.url.com?dir=/home/daniel" we must not use the first + slash, if there's a ?-letter before it! */ + char *sep = strchr(protsep, '?'); + if(sep && (sep < pathsep)) + pathsep = sep; + *pathsep=0; + } + else { + /* There was no slash. Now, since we might be operating on a badly + formatted URL, such as "http://www.url.com?id=2380" which doesn't + use a slash separator as it is supposed to, we need to check for a + ?-letter as well! */ + pathsep = strchr(protsep, '?'); + if(pathsep) + *pathsep=0; + } + } + + /* If the new part contains a space, this is a mighty stupid redirect + but we still make an effort to do "right". To the left of a '?' + letter we replace each space with %20 while it is replaced with '+' + on the right side of the '?' letter. + */ + newlen = strlen_url(useurl); + + urllen = strlen(url_clone); + + newest=(char *)malloc( urllen + 1 + /* possible slash */ + newlen + 1 /* zero byte */); + + if(!newest) { + free(url_clone); /* don't leak this */ + return NULL; + } + + /* copy over the root url part */ + memcpy(newest, url_clone, urllen); + + /* check if we need to append a slash */ + if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) + ; + else + newest[urllen++]='/'; + + /* then append the new piece on the right side */ + strcpy_url(&newest[urllen], useurl); + + free(url_clone); + + return newest; +} + /* * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string * as given by the remote server and set up the new URL to request. @@ -1946,10 +2158,6 @@ CURLcode Curl_follow(struct SessionHandle *data, followtype type) /* see transfer.h */ { /* Location: redirect */ - char prot[16]; /* URL protocol string storage */ - char letter; /* used for a silly sscanf */ - size_t newlen; - char *newest; bool disallowport = FALSE; if(type == FOLLOW_REDIR) { @@ -1978,144 +2186,16 @@ CURLcode Curl_follow(struct SessionHandle *data, } } - if(2 != sscanf(newurl, "%15[^?&/:]://%c", prot, &letter)) { + if(!is_absolute_url(newurl)) { /*** *DANG* this is an RFC 2068 violation. The URL is supposed to be absolute and this doesn't seem to be that! - *** - Instead, we have to TRY to append this new path to the old URL - to the right of the host part. Oh crap, this is doomed to cause - problems in the future... - */ - char *protsep; - char *pathsep; - - char *useurl = newurl; - size_t urllen; - - /* we must make our own copy of the URL to play with, as it may - point to read-only data */ - char *url_clone=strdup(data->change.url); - - if(!url_clone) - return CURLE_OUT_OF_MEMORY; /* skip out of this NOW */ - - /* protsep points to the start of the host name */ - protsep=strstr(url_clone, "//"); - if(!protsep) - protsep=url_clone; - else - protsep+=2; /* pass the slashes */ - - if('/' != newurl[0]) { - int level=0; - - /* First we need to find out if there's a ?-letter in the URL, - and cut it and the right-side of that off */ - pathsep = strchr(protsep, '?'); - if(pathsep) - *pathsep=0; - - /* we have a relative path to append to the last slash if there's one - available, or if the new URL is just a query string (starts with a - '?') we append the new one at the end of the entire currently worked - out URL */ - if(useurl[0] != '?') { - pathsep = strrchr(protsep, '/'); - if(pathsep) - *pathsep=0; - } - - /* Check if there's any slash after the host name, and if so, remember - that position instead */ - pathsep = strchr(protsep, '/'); - if(pathsep) - protsep = pathsep+1; - else - protsep = NULL; - - /* now deal with one "./" or any amount of "../" in the newurl - and act accordingly */ - - if((useurl[0] == '.') && (useurl[1] == '/')) - useurl+=2; /* just skip the "./" */ - - while((useurl[0] == '.') && - (useurl[1] == '.') && - (useurl[2] == '/')) { - level++; - useurl+=3; /* pass the "../" */ - } - - if(protsep) { - while(level--) { - /* cut off one more level from the right of the original URL */ - pathsep = strrchr(protsep, '/'); - if(pathsep) - *pathsep=0; - else { - *protsep=0; - break; - } - } - } - } - else { - /* We got a new absolute path for this server, cut off from the - first slash */ - pathsep = strchr(protsep, '/'); - if(pathsep) { - /* When people use badly formatted URLs, such as - "http://www.url.com?dir=/home/daniel" we must not use the first - slash, if there's a ?-letter before it! */ - char *sep = strchr(protsep, '?'); - if(sep && (sep < pathsep)) - pathsep = sep; - *pathsep=0; - } - else { - /* There was no slash. Now, since we might be operating on a badly - formatted URL, such as "http://www.url.com?id=2380" which doesn't - use a slash separator as it is supposed to, we need to check for a - ?-letter as well! */ - pathsep = strchr(protsep, '?'); - if(pathsep) - *pathsep=0; - } - } - - /* If the new part contains a space, this is a mighty stupid redirect - but we still make an effort to do "right". To the left of a '?' - letter we replace each space with %20 while it is replaced with '+' - on the right side of the '?' letter. - */ - newlen = strlen_url(useurl); - - urllen = strlen(url_clone); - - newest=(char *)malloc( urllen + 1 + /* possible slash */ - newlen + 1 /* zero byte */); - - if(!newest) { - free(url_clone); /* don't leak this */ - return CURLE_OUT_OF_MEMORY; /* go out from this */ - } - - /* copy over the root url part */ - memcpy(newest, url_clone, urllen); - - /* check if we need to append a slash */ - if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) - ; - else - newest[urllen++]='/'; - - /* then append the new piece on the right side */ - strcpy_url(&newest[urllen], useurl); - - free(newurl); /* newurl is the allocated pointer */ - free(url_clone); - newurl = newest; + */ + char *absolute = concat_url(data->change.url, newurl); + if (!absolute) + return CURLE_OUT_OF_MEMORY; + free(newurl); + newurl = absolute; } else { /* This is an absolute URL, don't allow the custom port number */ @@ -2124,15 +2204,16 @@ CURLcode Curl_follow(struct SessionHandle *data, if(strchr(newurl, ' ')) { /* This new URL contains at least one space, this is a mighty stupid redirect but we still make an effort to do "right". */ - newlen = strlen_url(newurl); + char *newest; + size_t newlen = strlen_url(newurl); newest = malloc(newlen+1); /* get memory for this */ - if(newest) { - strcpy_url(newest, newurl); /* create a space-free URL */ + if (!newest) + return CURLE_OUT_OF_MEMORY; + strcpy_url(newest, newurl); /* create a space-free URL */ - free(newurl); /* that was no good */ - newurl = newest; /* use this instead now */ - } + free(newurl); /* that was no good */ + newurl = newest; /* use this instead now */ } } @@ -2192,7 +2273,7 @@ CURLcode Curl_follow(struct SessionHandle *data, * libcurl gets the page that most user agents would get, libcurl has to * force GET. * - * This behaviour can be overriden with CURLOPT_POST301. + * This behaviour can be overridden with CURLOPT_POST301. */ if( (data->set.httpreq == HTTPREQ_POST || data->set.httpreq == HTTPREQ_POST_FORM) @@ -2389,7 +2470,7 @@ CURLcode Curl_perform(struct SessionHandle *data) } if(res != CURLE_OK) { /* The transfer phase returned error, we mark the connection to get - * closed to prevent being re-used. This is becasue we can't + * closed to prevent being re-used. This is because we can't * possibly know if the connection is in a good shape or not now. */ conn->bits.close = TRUE; @@ -2455,7 +2536,7 @@ CURLcode Curl_perform(struct SessionHandle *data) failf(data, "%s", str); } - /* run post-transfer uncondionally, but don't clobber the return code if + /* run post-transfer unconditionally, but don't clobber the return code if we already have an error code recorder */ res2 = Curl_posttransfer(data); if(!res && res2) -- cgit v1.2.3