From e6e9b006f770ef104fbcdef32dd6e7f42eb114b7 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Fri, 17 Aug 2018 00:49:37 +0200 Subject: upload: allocate upload buffer on-demand Saves 16KB on the easy handle for operations that don't need that buffer. Part 1 of #2888 --- lib/http.c | 29 ++++++++++++++++++----------- lib/multi.c | 3 +++ lib/smb.c | 21 ++++++++++++++------- lib/transfer.c | 20 ++++++++++++++++++-- lib/transfer.h | 1 + lib/url.c | 5 ++--- lib/urldata.h | 4 +++- 7 files changed, 59 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/http.c b/lib/http.c index 2e9f40719..e727ed823 100644 --- a/lib/http.c +++ b/lib/http.c @@ -1123,7 +1123,8 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, CURLcode result; char *ptr; size_t size; - struct HTTP *http = conn->data->req.protop; + struct Curl_easy *data = conn->data; + struct HTTP *http = data->req.protop; size_t sendsize; curl_socket_t sockfd; size_t headersize; @@ -1143,7 +1144,7 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, DEBUGASSERT(size > included_body_bytes); - result = Curl_convert_to_network(conn->data, ptr, headersize); + result = Curl_convert_to_network(data, ptr, headersize); /* Curl_convert_to_network calls failf if unsuccessful */ if(result) { /* conversion failed, free memory and return to the caller */ @@ -1168,8 +1169,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, must copy the data to the uploadbuffer first, since that is the buffer we will be using if this send is retried later. */ - memcpy(conn->data->state.uploadbuffer, ptr, sendsize); - ptr = conn->data->state.uploadbuffer; + result = Curl_get_upload_buffer(data); + if(result) { + /* malloc failed, free memory and return to the caller */ + Curl_add_buffer_free(in); + return result; + } + memcpy(data->state.ulbuf, ptr, sendsize); + ptr = data->state.ulbuf; } else sendsize = size; @@ -1186,13 +1193,13 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, size_t headlen = (size_t)amount>headersize ? headersize : (size_t)amount; size_t bodylen = amount - headlen; - if(conn->data->set.verbose) { + if(data->set.verbose) { /* this data _may_ contain binary stuff */ - Curl_debug(conn->data, CURLINFO_HEADER_OUT, ptr, headlen); + Curl_debug(data, CURLINFO_HEADER_OUT, ptr, headlen); if(bodylen) { /* there was body data sent beyond the initial header part, pass that on to the debug callback too */ - Curl_debug(conn->data, CURLINFO_DATA_OUT, + Curl_debug(data, CURLINFO_DATA_OUT, ptr + headlen, bodylen); } } @@ -1217,14 +1224,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, ptr = in->buffer + amount; /* backup the currently set pointers */ - http->backup.fread_func = conn->data->state.fread_func; - http->backup.fread_in = conn->data->state.in; + http->backup.fread_func = data->state.fread_func; + http->backup.fread_in = data->state.in; http->backup.postdata = http->postdata; http->backup.postsize = http->postsize; /* set the new pointers for the request-sending */ - conn->data->state.fread_func = (curl_read_callback)readmoredata; - conn->data->state.in = (void *)conn; + data->state.fread_func = (curl_read_callback)readmoredata; + data->state.in = (void *)conn; http->postdata = ptr; http->postsize = (curl_off_t)size; diff --git a/lib/multi.c b/lib/multi.c index c2c5e1c27..0caf94322 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -592,6 +592,7 @@ static CURLcode multi_done(struct connectdata **connp, conn->dns_entry = NULL; } Curl_hostcache_prune(data); + Curl_safefree(data->state.ulbuf); /* if the transfer was completed in a paused state there can be buffered data left to free */ @@ -1575,6 +1576,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, multistate(data, CURLM_STATE_SENDPROTOCONNECT); } } + else if(result) + stream_error = TRUE; break; #endif diff --git a/lib/smb.c b/lib/smb.c index 09aa8efc7..32f0ac889 100644 --- a/lib/smb.c +++ b/lib/smb.c @@ -367,7 +367,7 @@ static CURLcode smb_send(struct connectdata *conn, ssize_t len, ssize_t bytes_written; CURLcode result; - result = Curl_write(conn, FIRSTSOCKET, conn->data->state.uploadbuffer, + result = Curl_write(conn, FIRSTSOCKET, conn->data->state.ulbuf, len, &bytes_written); if(result) return result; @@ -393,7 +393,7 @@ static CURLcode smb_flush(struct connectdata *conn) return CURLE_OK; result = Curl_write(conn, FIRSTSOCKET, - conn->data->state.uploadbuffer + smbc->sent, + conn->data->state.ulbuf + smbc->sent, len, &bytes_written); if(result) return result; @@ -409,9 +409,12 @@ static CURLcode smb_flush(struct connectdata *conn) static CURLcode smb_send_message(struct connectdata *conn, unsigned char cmd, const void *msg, size_t msg_len) { - smb_format_message(conn, (struct smb_header *)conn->data->state.uploadbuffer, + CURLcode result = Curl_get_upload_buffer(conn->data); + if(result) + return result; + smb_format_message(conn, (struct smb_header *)conn->data->state.ulbuf, cmd, msg_len); - memcpy(conn->data->state.uploadbuffer + sizeof(struct smb_header), + memcpy(conn->data->state.ulbuf + sizeof(struct smb_header), msg, msg_len); return smb_send(conn, sizeof(struct smb_header) + msg_len, 0); @@ -572,11 +575,15 @@ static CURLcode smb_send_read(struct connectdata *conn) static CURLcode smb_send_write(struct connectdata *conn) { - struct smb_write *msg = (struct smb_write *)conn->data->state.uploadbuffer; + struct smb_write *msg; struct smb_request *req = conn->data->req.protop; curl_off_t offset = conn->data->req.offset; - curl_off_t upload_size = conn->data->req.size - conn->data->req.bytecount; + CURLcode result = Curl_get_upload_buffer(conn->data); + if(result) + return result; + msg = (struct smb_write *)conn->data->state.ulbuf; + if(upload_size >= MAX_PAYLOAD_SIZE - 1) /* There is one byte of padding */ upload_size = MAX_PAYLOAD_SIZE - 1; @@ -605,7 +612,7 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, void **msg) if(!smbc->send_size && smbc->upload_size) { int nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE : (int) smbc->upload_size; - conn->data->req.upload_fromhere = conn->data->state.uploadbuffer; + conn->data->req.upload_fromhere = conn->data->state.ulbuf; result = Curl_fillreadbuffer(conn, nread, &nread); if(result && result != CURLE_AGAIN) return result; diff --git a/lib/transfer.c b/lib/transfer.c index 49f32568f..96d8ab48f 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -106,6 +106,16 @@ char *Curl_checkheaders(const struct connectdata *conn, } #endif +CURLcode Curl_get_upload_buffer(struct Curl_easy *data) +{ + if(!data->state.ulbuf) { + data->state.ulbuf = malloc(data->set.upload_buffer_size); + if(!data->state.ulbuf) + return CURLE_OUT_OF_MEMORY; + } + return CURLE_OK; +} + /* * This function will call the read callback to fill our buffer with data * to upload. @@ -914,8 +924,11 @@ static CURLcode readwrite_upload(struct Curl_easy *data, /* only read more data if there's no upload data already present in the upload buffer */ if(0 == k->upload_present) { + result = Curl_get_upload_buffer(data); + if(result) + return result; /* init the "upload from here" pointer */ - k->upload_fromhere = data->state.uploadbuffer; + k->upload_fromhere = data->state.ulbuf; if(!k->upload_done) { /* HTTP pollution, this should be written nicer to become more @@ -1071,7 +1084,10 @@ static CURLcode readwrite_upload(struct Curl_easy *data, } else { /* we've uploaded that buffer now */ - k->upload_fromhere = data->state.uploadbuffer; + result = Curl_get_upload_buffer(data); + if(result) + return result; + k->upload_fromhere = data->state.ulbuf; k->upload_present = 0; /* no more bytes left */ if(k->upload_done) { diff --git a/lib/transfer.h b/lib/transfer.h index 9ba398d27..df75f9a97 100644 --- a/lib/transfer.h +++ b/lib/transfer.h @@ -54,6 +54,7 @@ CURLcode Curl_readrewind(struct connectdata *conn); CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp); CURLcode Curl_retry_request(struct connectdata *conn, char **url); bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc); +CURLcode Curl_get_upload_buffer(struct Curl_easy *data); /* This sets up a forthcoming transfer */ void diff --git a/lib/url.c b/lib/url.c index dcb7fcf6c..12f0f8ae7 100644 --- a/lib/url.c +++ b/lib/url.c @@ -367,11 +367,9 @@ CURLcode Curl_close(struct Curl_easy *data) Curl_safefree(data->state.buffer); Curl_safefree(data->state.headerbuff); - + Curl_safefree(data->state.ulbuf); Curl_flush_cookies(data, 1); - Curl_digest_cleanup(data); - Curl_safefree(data->info.contenttype); Curl_safefree(data->info.wouldredirect); @@ -534,6 +532,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data) set->expect_100_timeout = 1000L; /* Wait for a second by default. */ set->sep_headers = TRUE; /* separated header lists by default */ set->buffer_size = READBUFFER_SIZE; + set->upload_buffer_size = UPLOAD_BUFSIZE; set->happy_eyeballs_timeout = CURL_HET_DEFAULT; Curl_http2_init_userset(set); diff --git a/lib/urldata.h b/lib/urldata.h index 6119136d4..502cff362 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1220,7 +1220,7 @@ struct UrlState { size_t headersize; /* size of the allocation */ char *buffer; /* download buffer */ - char uploadbuffer[UPLOAD_BUFSIZE + 1]; /* upload buffer */ + char *ulbuf; /* alloced upload buffer or NULL */ curl_off_t current_speed; /* the ProgressShow() function sets this, bytes / second */ bool this_is_a_follow; /* this is a followed Location: request */ @@ -1557,6 +1557,8 @@ struct UserDefined { curl_proxytype proxytype; /* what kind of proxy that is in use */ long dns_cache_timeout; /* DNS cache timeout */ long buffer_size; /* size of receive buffer to use */ + long upload_buffer_size; /* size of upload buffer to use, + keep it >= CURL_MAX_WRITE_SIZE */ void *private_data; /* application-private data */ struct curl_slist *http200aliases; /* linked list of aliases for http200 */ -- cgit v1.2.3