From b88940850002a3f1c25bc6488b95ad30eb80d696 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Sat, 20 Jul 2019 19:14:00 +0200 Subject: curl: support parallel transfers This is done by making sure each individual transfer is first added to a linked list as then they can be performed serially, or at will, in parallel. Closes #3804 --- src/tool_operate.c | 1607 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 923 insertions(+), 684 deletions(-) (limited to 'src/tool_operate.c') diff --git a/src/tool_operate.c b/src/tool_operate.c index bf9a9b8d8..1b3bec9cf 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -75,6 +75,7 @@ #include "tool_vms.h" #include "tool_help.h" #include "tool_hugehelp.h" +#include "tool_progress.h" #include "memdebug.h" /* keep this as LAST include */ @@ -98,6 +99,11 @@ CURLcode curl_easy_perform_ev(CURL *easy); "this situation and\nhow to fix it, please visit the web page mentioned " \ "above.\n" +static CURLcode create_transfers(struct GlobalConfig *global, + struct OperationConfig *config, + CURLSH *share, + bool capath_from_env); + static bool is_fatal_error(CURLcode code) { switch(code) { @@ -187,135 +193,462 @@ static curl_off_t VmsSpecialSize(const char *name, #define BUFFER_SIZE (100*1024) -static CURLcode operate_do(struct GlobalConfig *global, - struct OperationConfig *config) +struct per_transfer *transfers; /* first node */ +static struct per_transfer *transfersl; /* last node */ + +static CURLcode add_transfer(struct per_transfer **per) { - char errorbuffer[CURL_ERROR_SIZE]; - struct ProgressData progressbar; - struct getout *urlnode; + struct per_transfer *p; + p = calloc(sizeof(struct per_transfer), 1); + if(!p) + return CURLE_OUT_OF_MEMORY; + if(!transfers) + /* first entry */ + transfersl = transfers = p; + else { + /* make the last node point to the new node */ + transfersl->next = p; + /* make the new node point back to the formerly last node */ + p->prev = transfersl; + /* move the last node pointer to the new entry */ + transfersl = p; + } + *per = p; + all_xfers++; /* count total number of transfers added */ + return CURLE_OK; +} - struct HdrCbData hdrcbdata; - struct OutStruct heads; +/* Remove the specified transfer from the list (and free it), return the next + in line */ +static struct per_transfer *del_transfer(struct per_transfer *per) +{ + struct per_transfer *n; + struct per_transfer *p; + DEBUGASSERT(transfers); + DEBUGASSERT(transfersl); + DEBUGASSERT(per); - metalinkfile *mlfile_last = NULL; + n = per->next; + p = per->prev; - CURL *curl = config->easy; - char *httpgetfields = NULL; + if(p) + p->next = n; + else + transfers = n; + + if(n) + n->prev = p; + else + transfersl = p; + + free(per); + return n; +} + +static CURLcode pre_transfer(struct GlobalConfig *global, + struct per_transfer *per) +{ + curl_off_t uploadfilesize = -1; + struct_stat fileinfo; CURLcode result = CURLE_OK; - unsigned long li; - bool capath_from_env; - /* Save the values of noprogress and isatty to restore them later on */ - bool orig_noprogress = global->noprogress; - bool orig_isatty = global->isatty; + if(per->separator_err) + fprintf(global->errors, "%s\n", per->separator_err); + if(per->separator) + printf("%s\n", per->separator); - errorbuffer[0] = '\0'; + if(per->uploadfile && !stdin_upload(per->uploadfile)) { + /* VMS Note: + * + * Reading binary from files can be a problem... Only FIXED, VAR + * etc WITHOUT implied CC will work Others need a \n appended to a + * line + * + * - Stat gives a size but this is UNRELIABLE in VMS As a f.e. a + * fixed file with implied CC needs to have a byte added for every + * record processed, this can by derived from Filesize & recordsize + * for VARiable record files the records need to be counted! for + * every record add 1 for linefeed and subtract 2 for the record + * header for VARIABLE header files only the bare record data needs + * to be considered with one appended if implied CC + */ +#ifdef __VMS + /* Calculate the real upload size for VMS */ + per->infd = -1; + if(stat(per->uploadfile, &fileinfo) == 0) { + fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo); + switch(fileinfo.st_fab_rfm) { + case FAB$C_VAR: + case FAB$C_VFC: + case FAB$C_STMCR: + per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); + break; + default: + per->infd = open(per->uploadfile, O_RDONLY | O_BINARY, + "rfm=stmlf", "ctx=stm"); + } + } + if(per->infd == -1) +#else + per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); + if((per->infd == -1) || fstat(per->infd, &fileinfo)) +#endif + { + helpf(global->errors, "Can't open '%s'!\n", per->uploadfile); + if(per->infd != -1) { + close(per->infd); + per->infd = STDIN_FILENO; + } + return CURLE_READ_ERROR; + } + per->infdopen = TRUE; - /* default headers output stream is stdout */ - memset(&hdrcbdata, 0, sizeof(struct HdrCbData)); - memset(&heads, 0, sizeof(struct OutStruct)); - heads.stream = stdout; - heads.config = config; + /* we ignore file size for char/block devices, sockets, etc. */ + if(S_ISREG(fileinfo.st_mode)) + uploadfilesize = fileinfo.st_size; - /* - ** Beyond this point no return'ing from this function allowed. - ** Jump to label 'quit_curl' in order to abandon this function - ** from outside of nested loops further down below. - */ + if(uploadfilesize != -1) + my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); + per->input.fd = per->infd; + } + show_error: + return result; +} - /* Check we have a url */ - if(!config->url_list || !config->url_list->url) { - helpf(global->errors, "no URL specified!\n"); - result = CURLE_FAILED_INIT; - goto quit_curl; +/* + * Call this after a transfer has completed. + */ +static CURLcode post_transfer(struct GlobalConfig *global, + CURLSH *share, + struct per_transfer *per, + CURLcode result, + bool *retryp) +{ + struct OutStruct *outs = &per->outs; + CURL *curl = per->curl; + struct OperationConfig *config = per->config; + + *retryp = FALSE; + + if(per->infdopen) + close(per->infd); + +#ifdef __VMS + if(is_vms_shell()) { + /* VMS DCL shell behavior */ + if(!global->showerror) + vms_show = VMSSTS_HIDE; } + else +#endif + if(config->synthetic_error) { + ; + } + else if(result && global->showerror) { + fprintf(global->errors, "curl: (%d) %s\n", result, + (per->errorbuffer[0]) ? per->errorbuffer : + curl_easy_strerror(result)); + if(result == CURLE_PEER_FAILED_VERIFICATION) + fputs(CURL_CA_CERT_ERRORMSG, global->errors); + } - /* On WIN32 we can't set the path to curl-ca-bundle.crt - * at compile time. So we look here for the file in two ways: - * 1: look at the environment variable CURL_CA_BUNDLE for a path - * 2: if #1 isn't found, use the windows API function SearchPath() - * to find it along the app's path (includes app's dir and CWD) - * - * We support the environment variable thing for non-Windows platforms - * too. Just for the sake of it. - */ - capath_from_env = false; - if(!config->cacert && - !config->capath && - !config->insecure_ok) { - struct curl_tlssessioninfo *tls_backend_info = NULL; + /* Set file extended attributes */ + if(!result && config->xattr && outs->fopened && outs->stream) { + int rc = fwrite_xattr(curl, fileno(outs->stream)); + if(rc) + warnf(config->global, "Error setting extended attributes: %s\n", + strerror(errno)); + } - /* With the addition of CAINFO support for Schannel, this search could find - * a certificate bundle that was previously ignored. To maintain backward - * compatibility, only perform this search if not using Schannel. - */ - result = curl_easy_getinfo(config->easy, - CURLINFO_TLS_SSL_PTR, - &tls_backend_info); - if(result) { - goto quit_curl; + if(!result && !outs->stream && !outs->bytes) { + /* we have received no data despite the transfer was successful + ==> force cration of an empty output file (if an output file + was specified) */ + long cond_unmet = 0L; + /* do not create (or even overwrite) the file in case we get no + data because of unmet condition */ + curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet); + if(!cond_unmet && !tool_create_output_file(outs)) + result = CURLE_WRITE_ERROR; + } + + if(!outs->s_isreg && outs->stream) { + /* Dump standard stream buffered data */ + int rc = fflush(outs->stream); + if(!result && rc) { + /* something went wrong in the writing process */ + result = CURLE_WRITE_ERROR; + fprintf(global->errors, "(%d) Failed writing body\n", result); } + } - /* Set the CA cert locations specified in the environment. For Windows if - * no environment-specified filename is found then check for CA bundle - * default filename curl-ca-bundle.crt in the user's PATH. - * - * If Schannel is the selected SSL backend then these locations are - * ignored. We allow setting CA location for schannel only when explicitly - * specified by the user via CURLOPT_CAINFO / --cacert. - */ - if(tls_backend_info->backend != CURLSSLBACKEND_SCHANNEL) { - char *env; - env = curlx_getenv("CURL_CA_BUNDLE"); - if(env) { - config->cacert = strdup(env); - if(!config->cacert) { - curl_free(env); - helpf(global->errors, "out of memory\n"); - result = CURLE_OUT_OF_MEMORY; - goto quit_curl; +#ifdef USE_METALINK + if(per->metalink && !per->metalink_next_res) + fprintf(global->errors, "Metalink: fetching (%s) from (%s) OK\n", + per->mlfile->filename, per->this_url); + + if(!per->metalink && config->use_metalink && result == CURLE_OK) { + int rv = parse_metalink(config, outs, per->this_url); + if(!rv) { + fprintf(config->global->errors, "Metalink: parsing (%s) OK\n", + per->this_url); + } + else if(rv == -1) + fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n", + per->this_url); + result = create_transfers(global, config, share, FALSE); + } + else if(per->metalink && result == CURLE_OK && !per->metalink_next_res) { + int rv; + (void)fflush(outs->stream); + rv = metalink_check_hash(global, per->mlfile, outs->filename); + if(!rv) + per->metalink_next_res = 1; + } +#else + (void)share; +#endif /* USE_METALINK */ + +#ifdef USE_METALINK + if(outs->metalink_parser) + metalink_parser_context_delete(outs->metalink_parser); +#endif /* USE_METALINK */ + + if(outs->is_cd_filename && outs->stream && !global->mute && + outs->filename) + printf("curl: Saved to filename '%s'\n", outs->filename); + + /* if retry-max-time is non-zero, make sure we haven't exceeded the + time */ + if(per->retry_numretries && + (!config->retry_maxtime || + (tvdiff(tvnow(), per->retrystart) < + config->retry_maxtime*1000L)) ) { + enum { + RETRY_NO, + RETRY_TIMEOUT, + RETRY_CONNREFUSED, + RETRY_HTTP, + RETRY_FTP, + RETRY_LAST /* not used */ + } retry = RETRY_NO; + long response; + if((CURLE_OPERATION_TIMEDOUT == result) || + (CURLE_COULDNT_RESOLVE_HOST == result) || + (CURLE_COULDNT_RESOLVE_PROXY == result) || + (CURLE_FTP_ACCEPT_TIMEOUT == result)) + /* retry timeout always */ + retry = RETRY_TIMEOUT; + else if(config->retry_connrefused && + (CURLE_COULDNT_CONNECT == result)) { + long oserrno; + curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno); + if(ECONNREFUSED == oserrno) + retry = RETRY_CONNREFUSED; + } + else if((CURLE_OK == result) || + (config->failonerror && + (CURLE_HTTP_RETURNED_ERROR == result))) { + /* If it returned OK. _or_ failonerror was enabled and it + returned due to such an error, check for HTTP transient + errors to retry on. */ + char *effective_url = NULL; + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + if(effective_url && + checkprefix("http", effective_url)) { + /* This was HTTP(S) */ + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + + switch(response) { + case 500: /* Internal Server Error */ + case 502: /* Bad Gateway */ + case 503: /* Service Unavailable */ + case 504: /* Gateway Timeout */ + retry = RETRY_HTTP; + /* + * At this point, we have already written data to the output + * file (or terminal). If we write to a file, we must rewind + * or close/re-open the file so that the next attempt starts + * over from the beginning. + * + * TODO: similar action for the upload case. We might need + * to start over reading from a previous point if we have + * uploaded something when this was returned. + */ + break; } } - else { - env = curlx_getenv("SSL_CERT_DIR"); - if(env) { - config->capath = strdup(env); - if(!config->capath) { - curl_free(env); - helpf(global->errors, "out of memory\n"); - result = CURLE_OUT_OF_MEMORY; - goto quit_curl; - } - capath_from_env = true; + } /* if CURLE_OK */ + else if(result) { + long protocol; + + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + curl_easy_getinfo(curl, CURLINFO_PROTOCOL, &protocol); + + if((protocol == CURLPROTO_FTP || protocol == CURLPROTO_FTPS) && + response / 100 == 4) + /* + * This is typically when the FTP server only allows a certain + * amount of users and we are not one of them. All 4xx codes + * are transient. + */ + retry = RETRY_FTP; + } + + if(retry) { + static const char * const m[]={ + NULL, + "timeout", + "connection refused", + "HTTP error", + "FTP error" + }; + + warnf(config->global, "Transient problem: %s " + "Will retry in %ld seconds. " + "%ld retries left.\n", + m[retry], per->retry_sleep/1000L, per->retry_numretries); + + tool_go_sleep(per->retry_sleep); + per->retry_numretries--; + if(!config->retry_delay) { + per->retry_sleep *= 2; + if(per->retry_sleep > RETRY_SLEEP_MAX) + per->retry_sleep = RETRY_SLEEP_MAX; + } + if(outs->bytes && outs->filename && outs->stream) { + int rc; + /* We have written data to a output file, we truncate file + */ + if(!global->mute) + fprintf(global->errors, "Throwing away %" + CURL_FORMAT_CURL_OFF_T " bytes\n", + outs->bytes); + fflush(outs->stream); + /* truncate file at the position where we started appending */ +#ifdef HAVE_FTRUNCATE + if(ftruncate(fileno(outs->stream), outs->init)) { + /* when truncate fails, we can't just append as then we'll + create something strange, bail out */ + if(!global->mute) + fprintf(global->errors, + "failed to truncate, exiting\n"); + return CURLE_WRITE_ERROR; } - else { - env = curlx_getenv("SSL_CERT_FILE"); - if(env) { - config->cacert = strdup(env); - if(!config->cacert) { - curl_free(env); - helpf(global->errors, "out of memory\n"); - result = CURLE_OUT_OF_MEMORY; - goto quit_curl; - } - } + /* now seek to the end of the file, the position where we + just truncated the file in a large file-safe way */ + rc = fseek(outs->stream, 0, SEEK_END); +#else + /* ftruncate is not available, so just reposition the file + to the location we would have truncated it. This won't + work properly with large files on 32-bit systems, but + most of those will have ftruncate. */ + rc = fseek(outs->stream, (long)outs->init, SEEK_SET); +#endif + if(rc) { + if(!global->mute) + fprintf(global->errors, + "failed seeking to end of file, exiting\n"); + return CURLE_WRITE_ERROR; } + outs->bytes = 0; /* clear for next round */ } - - if(env) - curl_free(env); -#ifdef WIN32 - else { - result = FindWin32CACert(config, tls_backend_info->backend, - "curl-ca-bundle.crt"); - if(result) - goto quit_curl; + *retryp = TRUE; /* curl_easy_perform loop */ + return CURLE_OK; + } + } /* if retry_numretries */ + else if(per->metalink) { + /* Metalink: Decide to try the next resource or not. Try the next resource + if download was not successful. */ + long response; + if(CURLE_OK == result) { + /* TODO We want to try next resource when download was + not successful. How to know that? */ + char *effective_url = NULL; + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + if(effective_url && + curl_strnequal(effective_url, "http", 4)) { + /* This was HTTP(S) */ + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + if(response != 200 && response != 206) { + per->metalink_next_res = 1; + fprintf(global->errors, + "Metalink: fetching (%s) from (%s) FAILED " + "(HTTP status code %ld)\n", + per->mlfile->filename, per->this_url, response); + } } -#endif + } + else { + per->metalink_next_res = 1; + fprintf(global->errors, + "Metalink: fetching (%s) from (%s) FAILED (%s)\n", + per->mlfile->filename, per->this_url, + curl_easy_strerror(result)); } } + if((global->progressmode == CURL_PROGRESS_BAR) && + per->progressbar.calls) + /* if the custom progress bar has been displayed, we output a + newline here */ + fputs("\n", per->progressbar.out); + + if(config->writeout) + ourWriteOut(per->curl, &per->outs, config->writeout); + + /* Close the outs file */ + if(outs->fopened && outs->stream) { + int rc = fclose(outs->stream); + if(!result && rc) { + /* something went wrong in the writing process */ + result = CURLE_WRITE_ERROR; + fprintf(global->errors, "(%d) Failed writing body\n", result); + } + } + + /* File time can only be set _after_ the file has been closed */ + if(!result && config->remote_time && outs->s_isreg && outs->filename) { + /* Ask libcurl if we got a remote file time */ + curl_off_t filetime = -1; + curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime); + setfiletime(filetime, outs->filename, config->global->errors); + } + + /* Close function-local opened file descriptors */ + if(per->heads.fopened && per->heads.stream) + fclose(per->heads.stream); + + if(per->heads.alloc_filename) + Curl_safefree(per->heads.filename); + + curl_easy_cleanup(per->curl); + if(outs->alloc_filename) + free(outs->filename); + free(per->this_url); + free(per->separator_err); + free(per->separator); + free(per->outfile); + free(per->uploadfile); + + return CURLE_OK; +} + +/* go through the list of URLs and configs and add transfers */ + +static CURLcode create_transfers(struct GlobalConfig *global, + struct OperationConfig *config, + CURLSH *share, + bool capath_from_env) +{ + CURLcode result = CURLE_OK; + struct getout *urlnode; + metalinkfile *mlfile_last = NULL; + bool orig_noprogress = global->noprogress; + bool orig_isatty = global->isatty; + char *httpgetfields = NULL; + if(config->postfields) { if(config->use_httpget) { /* Use the postfields data for a http get */ @@ -341,44 +674,14 @@ static CURLcode operate_do(struct GlobalConfig *global, } } - /* Single header file for all URLs */ - if(config->headerfile) { - /* open file for output: */ - if(strcmp(config->headerfile, "-")) { - FILE *newfile = fopen(config->headerfile, "wb"); - if(!newfile) { - warnf(config->global, "Failed to open %s\n", config->headerfile); - result = CURLE_WRITE_ERROR; - goto quit_curl; - } - else { - heads.filename = config->headerfile; - heads.s_isreg = TRUE; - heads.fopened = TRUE; - heads.stream = newfile; - } - } - else { - /* always use binary mode for protocol header output */ - set_binmode(heads.stream); - } - } - - /* - ** Nested loops start here. - */ - - /* loop through the list of given URLs */ - for(urlnode = config->url_list; urlnode; urlnode = urlnode->next) { - + unsigned long li; unsigned long up; /* upload file counter within a single upload glob */ char *infiles; /* might be a glob pattern */ char *outfiles; unsigned long infilenum; URLGlob *inglob; - - int metalink = 0; /* nonzero for metalink download. */ + bool metalink = FALSE; /* metalink download? */ metalinkfile *mlfile; metalink_resource *mlres; @@ -461,8 +764,6 @@ static CURLcode operate_do(struct GlobalConfig *global, result = CURLE_OUT_OF_MEMORY; } } - else - uploadfile = NULL; if(!uploadfile) break; } @@ -490,65 +791,102 @@ static CURLcode operate_do(struct GlobalConfig *global, /* Here's looping around each globbed URL */ for(li = 0 ; li < urlnum; li++) { + struct per_transfer *per; + struct OutStruct *outs; + struct InStruct *input; + struct OutStruct *heads; + struct HdrCbData *hdrcbdata = NULL; + CURL *curl = curl_easy_init(); + + result = add_transfer(&per); + if(result || !curl) { + free(uploadfile); + curl_easy_cleanup(curl); + result = CURLE_OUT_OF_MEMORY; + goto show_error; + } + per->config = config; + per->curl = curl; + per->uploadfile = uploadfile; + + /* default headers output stream is stdout */ + heads = &per->heads; + heads->stream = stdout; + heads->config = config; + + /* Single header file for all URLs */ + if(config->headerfile) { + /* open file for output: */ + if(strcmp(config->headerfile, "-")) { + FILE *newfile = fopen(config->headerfile, "wb"); + if(!newfile) { + warnf(config->global, "Failed to open %s\n", config->headerfile); + result = CURLE_WRITE_ERROR; + goto quit_curl; + } + else { + heads->filename = config->headerfile; + heads->s_isreg = TRUE; + heads->fopened = TRUE; + heads->stream = newfile; + } + } + else { + /* always use binary mode for protocol header output */ + set_binmode(heads->stream); + } + } + + + hdrcbdata = &per->hdrcbdata; + + outs = &per->outs; + input = &per->input; - int infd; - bool infdopen; - char *outfile; - struct OutStruct outs; - struct InStruct input; - struct timeval retrystart; - curl_off_t uploadfilesize; - long retry_numretries; - long retry_sleep_default; - long retry_sleep; - char *this_url = NULL; - int metalink_next_res = 0; - - outfile = NULL; - infdopen = FALSE; - infd = STDIN_FILENO; - uploadfilesize = -1; /* -1 means unknown */ + per->outfile = NULL; + per->infdopen = FALSE; + per->infd = STDIN_FILENO; /* default output stream is stdout */ - memset(&outs, 0, sizeof(struct OutStruct)); - outs.stream = stdout; - outs.config = config; + outs->stream = stdout; + outs->config = config; if(metalink) { /* For Metalink download, use name in Metalink file as filename. */ - outfile = strdup(mlfile->filename); - if(!outfile) { + per->outfile = strdup(mlfile->filename); + if(!per->outfile) { result = CURLE_OUT_OF_MEMORY; goto show_error; } - this_url = strdup(mlres->url); - if(!this_url) { + per->this_url = strdup(mlres->url); + if(!per->this_url) { result = CURLE_OUT_OF_MEMORY; goto show_error; } + per->mlfile = mlfile; } else { if(urls) { - result = glob_next_url(&this_url, urls); + result = glob_next_url(&per->this_url, urls); if(result) goto show_error; } else if(!li) { - this_url = strdup(urlnode->url); - if(!this_url) { + per->this_url = strdup(urlnode->url); + if(!per->this_url) { result = CURLE_OUT_OF_MEMORY; goto show_error; } } else - this_url = NULL; - if(!this_url) + per->this_url = NULL; + if(!per->this_url) break; if(outfiles) { - outfile = strdup(outfiles); - if(!outfile) { + per->outfile = strdup(outfiles); + if(!per->outfile) { result = CURLE_OUT_OF_MEMORY; goto show_error; } @@ -556,7 +894,7 @@ static CURLcode operate_do(struct GlobalConfig *global, } if(((urlnode->flags&GETOUT_USEREMOTE) || - (outfile && strcmp("-", outfile))) && + (per->outfile && strcmp("-", per->outfile))) && (metalink || !config->use_metalink)) { /* @@ -564,12 +902,12 @@ static CURLcode operate_do(struct GlobalConfig *global, * decided we want to use the remote file name. */ - if(!outfile) { + if(!per->outfile) { /* extract the file name from the URL */ - result = get_url_file_name(&outfile, this_url); + result = get_url_file_name(&per->outfile, per->this_url); if(result) goto show_error; - if(!*outfile && !config->content_disposition) { + if(!*per->outfile && !config->content_disposition) { helpf(global->errors, "Remote file name has no length!\n"); result = CURLE_WRITE_ERROR; goto quit_urls; @@ -577,8 +915,8 @@ static CURLcode operate_do(struct GlobalConfig *global, } else if(urls) { /* fill '#1' ... '#9' terms from URL pattern */ - char *storefile = outfile; - result = glob_match_url(&outfile, storefile, urls); + char *storefile = per->outfile; + result = glob_match_url(&per->outfile, storefile, urls); Curl_safefree(storefile); if(result) { /* bad globbing */ @@ -591,7 +929,7 @@ static CURLcode operate_do(struct GlobalConfig *global, file output call */ if(config->create_dirs || metalink) { - result = create_dir_hierarchy(outfile, global->errors); + result = create_dir_hierarchy(per->outfile, global->errors); /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */ if(result == CURLE_WRITE_ERROR) goto quit_urls; @@ -603,7 +941,7 @@ static CURLcode operate_do(struct GlobalConfig *global, if((urlnode->flags & GETOUT_USEREMOTE) && config->content_disposition) { /* Our header callback MIGHT set the filename */ - DEBUGASSERT(!outs.filename); + DEBUGASSERT(!outs->filename); } if(config->resume_from_current) { @@ -611,7 +949,7 @@ static CURLcode operate_do(struct GlobalConfig *global, of the file as it is now and open it for append instead */ struct_stat fileinfo; /* VMS -- Danger, the filesize is only valid for stream files */ - if(0 == stat(outfile, &fileinfo)) + if(0 == stat(per->outfile, &fileinfo)) /* set offset to current file size: */ config->resume_from = fileinfo.st_size; else @@ -627,87 +965,36 @@ static CURLcode operate_do(struct GlobalConfig *global, "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0"); #else /* open file for output: */ - FILE *file = fopen(outfile, config->resume_from?"ab":"wb"); + FILE *file = fopen(per->outfile, config->resume_from?"ab":"wb"); #endif if(!file) { - helpf(global->errors, "Can't open '%s'!\n", outfile); + helpf(global->errors, "Can't open '%s'!\n", per->outfile); result = CURLE_WRITE_ERROR; goto quit_urls; } - outs.fopened = TRUE; - outs.stream = file; - outs.init = config->resume_from; + outs->fopened = TRUE; + outs->stream = file; + outs->init = config->resume_from; } else { - outs.stream = NULL; /* open when needed */ + outs->stream = NULL; /* open when needed */ } - outs.filename = outfile; - outs.s_isreg = TRUE; + outs->filename = per->outfile; + outs->s_isreg = TRUE; } - if(uploadfile && !stdin_upload(uploadfile)) { + if(per->uploadfile && !stdin_upload(per->uploadfile)) { /* * We have specified a file to upload and it isn't "-". */ - struct_stat fileinfo; - - this_url = add_file_name_to_url(curl, this_url, uploadfile); - if(!this_url) { + char *nurl = add_file_name_to_url(per->this_url, per->uploadfile); + if(!nurl) { result = CURLE_OUT_OF_MEMORY; goto show_error; } - /* VMS Note: - * - * Reading binary from files can be a problem... Only FIXED, VAR - * etc WITHOUT implied CC will work Others need a \n appended to a - * line - * - * - Stat gives a size but this is UNRELIABLE in VMS As a f.e. a - * fixed file with implied CC needs to have a byte added for every - * record processed, this can by derived from Filesize & recordsize - * for VARiable record files the records need to be counted! for - * every record add 1 for linefeed and subtract 2 for the record - * header for VARIABLE header files only the bare record data needs - * to be considered with one appended if implied CC - */ -#ifdef __VMS - /* Calculate the real upload size for VMS */ - infd = -1; - if(stat(uploadfile, &fileinfo) == 0) { - fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo); - switch(fileinfo.st_fab_rfm) { - case FAB$C_VAR: - case FAB$C_VFC: - case FAB$C_STMCR: - infd = open(uploadfile, O_RDONLY | O_BINARY); - break; - default: - infd = open(uploadfile, O_RDONLY | O_BINARY, - "rfm=stmlf", "ctx=stm"); - } - } - if(infd == -1) -#else - infd = open(uploadfile, O_RDONLY | O_BINARY); - if((infd == -1) || fstat(infd, &fileinfo)) -#endif - { - helpf(global->errors, "Can't open '%s'!\n", uploadfile); - if(infd != -1) { - close(infd); - infd = STDIN_FILENO; - } - result = CURLE_READ_ERROR; - goto quit_urls; - } - infdopen = TRUE; - - /* we ignore file size for char/block devices, sockets, etc. */ - if(S_ISREG(fileinfo.st_mode)) - uploadfilesize = fileinfo.st_size; - + per->this_url = nurl; } - else if(uploadfile && stdin_upload(uploadfile)) { + else if(per->uploadfile && stdin_upload(per->uploadfile)) { /* count to see if there are more than one auth bit set in the authtype field */ int authbits = 0; @@ -733,22 +1020,22 @@ static CURLcode operate_do(struct GlobalConfig *global, " file or a fixed auth type instead!\n"); } - DEBUGASSERT(infdopen == FALSE); - DEBUGASSERT(infd == STDIN_FILENO); + DEBUGASSERT(per->infdopen == FALSE); + DEBUGASSERT(per->infd == STDIN_FILENO); set_binmode(stdin); - if(!strcmp(uploadfile, ".")) { - if(curlx_nonblock((curl_socket_t)infd, TRUE) < 0) + if(!strcmp(per->uploadfile, ".")) { + if(curlx_nonblock((curl_socket_t)per->infd, TRUE) < 0) warnf(config->global, - "fcntl failed on fd=%d: %s\n", infd, strerror(errno)); + "fcntl failed on fd=%d: %s\n", per->infd, strerror(errno)); } } - if(uploadfile && config->resume_from_current) + if(per->uploadfile && config->resume_from_current) config->resume_from = -1; /* -1 will then force get-it-yourself */ - if(output_expected(this_url, uploadfile) && outs.stream && - isatty(fileno(outs.stream))) + if(output_expected(per->this_url, per->uploadfile) && outs->stream && + isatty(fileno(outs->stream))) /* we send the output to a tty, therefore we switch off the progress meter */ global->noprogress = global->isatty = TRUE; @@ -760,20 +1047,22 @@ static CURLcode operate_do(struct GlobalConfig *global, } if(urlnum > 1 && !global->mute) { - fprintf(global->errors, "\n[%lu/%lu]: %s --> %s\n", - li + 1, urlnum, this_url, outfile ? outfile : ""); + per->separator_err = + aprintf("\n[%lu/%lu]: %s --> %s", + li + 1, urlnum, per->this_url, + per->outfile ? per->outfile : ""); if(separator) - printf("%s%s\n", CURLseparator, this_url); + per->separator = aprintf("%s%s", CURLseparator, per->this_url); } if(httpgetfields) { char *urlbuffer; /* Find out whether the url contains a file name */ - const char *pc = strstr(this_url, "://"); + const char *pc = strstr(per->this_url, "://"); char sep = '?'; if(pc) pc += 3; else - pc = this_url; + pc = per->this_url; pc = strrchr(pc, '/'); /* check for a slash */ @@ -789,33 +1078,39 @@ static CURLcode operate_do(struct GlobalConfig *global, * Then append ? followed by the get fields to the url. */ if(pc) - urlbuffer = aprintf("%s%c%s", this_url, sep, httpgetfields); + urlbuffer = aprintf("%s%c%s", per->this_url, sep, httpgetfields); else /* Append / before the ? to create a well-formed url if the url contains a hostname only */ - urlbuffer = aprintf("%s/?%s", this_url, httpgetfields); + urlbuffer = aprintf("%s/?%s", per->this_url, httpgetfields); if(!urlbuffer) { result = CURLE_OUT_OF_MEMORY; goto show_error; } - Curl_safefree(this_url); /* free previous URL */ - this_url = urlbuffer; /* use our new URL instead! */ + Curl_safefree(per->this_url); /* free previous URL */ + per->this_url = urlbuffer; /* use our new URL instead! */ } if(!global->errors) global->errors = stderr; - if((!outfile || !strcmp(outfile, "-")) && !config->use_ascii) { + if((!per->outfile || !strcmp(per->outfile, "-")) && + !config->use_ascii) { /* We get the output to stdout and we have not got the ASCII/text flag, then set stdout to be binary */ set_binmode(stdout); } /* explicitly passed to stdout means okaying binary gunk */ - config->terminal_binary_ok = (outfile && !strcmp(outfile, "-")); + config->terminal_binary_ok = + (per->outfile && !strcmp(per->outfile, "-")); + + /* avoid having this setopt added to the --libcurl source + output */ + curl_easy_setopt(curl, CURLOPT_SHARE, share); if(!config->tcp_nodelay) my_setopt(curl, CURLOPT_TCP_NODELAY, 0L); @@ -824,8 +1119,8 @@ static CURLcode operate_do(struct GlobalConfig *global, my_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L); /* where to store */ - my_setopt(curl, CURLOPT_WRITEDATA, &outs); - my_setopt(curl, CURLOPT_INTERLEAVEDATA, &outs); + my_setopt(curl, CURLOPT_WRITEDATA, per); + my_setopt(curl, CURLOPT_INTERLEAVEDATA, per); if(metalink || !config->use_metalink) /* what call to write */ @@ -838,8 +1133,7 @@ static CURLcode operate_do(struct GlobalConfig *global, #endif /* USE_METALINK */ /* for uploads */ - input.fd = infd; - input.config = config; + input->config = config; /* Note that if CURLOPT_READFUNCTION is fread (the default), then * lib/telnet.c will Curl_poll() on the input file descriptor * rather then calling the READFUNCTION at regular intervals. @@ -847,13 +1141,13 @@ static CURLcode operate_do(struct GlobalConfig *global, * behaviour, by omitting to set the READFUNCTION & READDATA options, * have not been determined. */ - my_setopt(curl, CURLOPT_READDATA, &input); + my_setopt(curl, CURLOPT_READDATA, input); /* what call to read */ my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb); /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */ - my_setopt(curl, CURLOPT_SEEKDATA, &input); + my_setopt(curl, CURLOPT_SEEKDATA, input); my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb); if(config->recvpersecond && @@ -863,10 +1157,7 @@ static CURLcode operate_do(struct GlobalConfig *global, else my_setopt(curl, CURLOPT_BUFFERSIZE, (long)BUFFER_SIZE); - /* size of uploaded file: */ - if(uploadfilesize != -1) - my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); - my_setopt_str(curl, CURLOPT_URL, this_url); /* what to fetch */ + my_setopt_str(curl, CURLOPT_URL, per->this_url); my_setopt(curl, CURLOPT_NOPROGRESS, global->noprogress?1L:0L); if(config->no_body) my_setopt(curl, CURLOPT_NOBODY, 1L); @@ -915,7 +1206,7 @@ static CURLcode operate_do(struct GlobalConfig *global, my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L); my_setopt(curl, CURLOPT_REQUEST_TARGET, config->request_target); - my_setopt(curl, CURLOPT_UPLOAD, uploadfile?1L:0L); + my_setopt(curl, CURLOPT_UPLOAD, per->uploadfile?1L:0L); my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly?1L:0L); my_setopt(curl, CURLOPT_APPEND, config->ftp_append?1L:0L); @@ -934,7 +1225,7 @@ static CURLcode operate_do(struct GlobalConfig *global, my_setopt_str(curl, CURLOPT_LOGIN_OPTIONS, config->login_options); my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd); my_setopt_str(curl, CURLOPT_RANGE, config->range); - my_setopt(curl, CURLOPT_ERRORBUFFER, errorbuffer); + my_setopt(curl, CURLOPT_ERRORBUFFER, per->errorbuffer); my_setopt(curl, CURLOPT_TIMEOUT_MS, (long)(config->timeout * 1000)); switch(config->httpreq) { @@ -1230,14 +1521,14 @@ static CURLcode operate_do(struct GlobalConfig *global, /* three new ones in libcurl 7.3: */ my_setopt_str(curl, CURLOPT_INTERFACE, config->iface); my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel); + progressbarinit(&per->progressbar, config); - progressbarinit(&progressbar, config); if((global->progressmode == CURL_PROGRESS_BAR) && !global->noprogress && !global->mute) { /* we want the alternative style, then we have to implement it ourselves! */ my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_progress_cb); - my_setopt(curl, CURLOPT_XFERINFODATA, &progressbar); + my_setopt(curl, CURLOPT_XFERINFODATA, &per->progressbar); } /* new in libcurl 7.24.0: */ @@ -1420,17 +1711,17 @@ static CURLcode operate_do(struct GlobalConfig *global, if(config->content_disposition && (urlnode->flags & GETOUT_USEREMOTE)) - hdrcbdata.honor_cd_filename = TRUE; + hdrcbdata->honor_cd_filename = TRUE; else - hdrcbdata.honor_cd_filename = FALSE; + hdrcbdata->honor_cd_filename = FALSE; - hdrcbdata.outs = &outs; - hdrcbdata.heads = &heads; - hdrcbdata.global = global; - hdrcbdata.config = config; + hdrcbdata->outs = outs; + hdrcbdata->heads = heads; + hdrcbdata->global = global; + hdrcbdata->config = config; my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb); - my_setopt(curl, CURLOPT_HEADERDATA, &hdrcbdata); + my_setopt(curl, CURLOPT_HEADERDATA, per); if(config->resolve) /* new in 7.21.3 */ @@ -1536,397 +1827,40 @@ static CURLcode operate_do(struct GlobalConfig *global, my_setopt_str(curl, CURLOPT_ALTSVC, config->altsvc); #endif - /* initialize retry vars for loop below */ - retry_sleep_default = (config->retry_delay) ? - config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */ - - retry_numretries = config->req_retry; - retry_sleep = retry_sleep_default; /* ms */ - retrystart = tvnow(); - -#ifndef CURL_DISABLE_LIBCURL_OPTION - if(global->libcurl) { - result = easysrc_perform(); - if(result) - goto show_error; - } -#endif - for(;;) { #ifdef USE_METALINK if(!metalink && config->use_metalink) { - /* If outs.metalink_parser is non-NULL, delete it first. */ - if(outs.metalink_parser) - metalink_parser_context_delete(outs.metalink_parser); - outs.metalink_parser = metalink_parser_context_new(); - if(outs.metalink_parser == NULL) { + outs->metalink_parser = metalink_parser_context_new(); + if(outs->metalink_parser == NULL) { result = CURLE_OUT_OF_MEMORY; goto show_error; } fprintf(config->global->errors, - "Metalink: parsing (%s) metalink/XML...\n", this_url); + "Metalink: parsing (%s) metalink/XML...\n", per->this_url); } else if(metalink) fprintf(config->global->errors, "Metalink: fetching (%s) from (%s)...\n", - mlfile->filename, this_url); + mlfile->filename, per->this_url); #endif /* USE_METALINK */ -#ifdef CURLDEBUG - if(config->test_event_based) - result = curl_easy_perform_ev(curl); - else -#endif - result = curl_easy_perform(curl); - - if(!result && !outs.stream && !outs.bytes) { - /* we have received no data despite the transfer was successful - ==> force cration of an empty output file (if an output file - was specified) */ - long cond_unmet = 0L; - /* do not create (or even overwrite) the file in case we get no - data because of unmet condition */ - curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet); - if(!cond_unmet && !tool_create_output_file(&outs)) - result = CURLE_WRITE_ERROR; - } + per->metalink = metalink; + /* initialize retry vars for loop below */ + per->retry_sleep_default = (config->retry_delay) ? + config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */ + per->retry_numretries = config->req_retry; + per->retry_sleep = per->retry_sleep_default; /* ms */ + per->retrystart = tvnow(); - if(outs.is_cd_filename && outs.stream && !global->mute && - outs.filename) - printf("curl: Saved to filename '%s'\n", outs.filename); - - /* if retry-max-time is non-zero, make sure we haven't exceeded the - time */ - if(retry_numretries && - (!config->retry_maxtime || - (tvdiff(tvnow(), retrystart) < - config->retry_maxtime*1000L)) ) { - enum { - RETRY_NO, - RETRY_TIMEOUT, - RETRY_CONNREFUSED, - RETRY_HTTP, - RETRY_FTP, - RETRY_LAST /* not used */ - } retry = RETRY_NO; - long response; - if((CURLE_OPERATION_TIMEDOUT == result) || - (CURLE_COULDNT_RESOLVE_HOST == result) || - (CURLE_COULDNT_RESOLVE_PROXY == result) || - (CURLE_FTP_ACCEPT_TIMEOUT == result)) - /* retry timeout always */ - retry = RETRY_TIMEOUT; - else if(config->retry_connrefused && - (CURLE_COULDNT_CONNECT == result)) { - long oserrno; - curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno); - if(ECONNREFUSED == oserrno) - retry = RETRY_CONNREFUSED; - } - else if((CURLE_OK == result) || - (config->failonerror && - (CURLE_HTTP_RETURNED_ERROR == result))) { - /* If it returned OK. _or_ failonerror was enabled and it - returned due to such an error, check for HTTP transient - errors to retry on. */ - char *effective_url = NULL; - curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); - if(effective_url && - checkprefix("http", effective_url)) { - /* This was HTTP(S) */ - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - - switch(response) { - case 408: /* Request Timeout */ - case 500: /* Internal Server Error */ - case 502: /* Bad Gateway */ - case 503: /* Service Unavailable */ - case 504: /* Gateway Timeout */ - retry = RETRY_HTTP; - /* - * At this point, we have already written data to the output - * file (or terminal). If we write to a file, we must rewind - * or close/re-open the file so that the next attempt starts - * over from the beginning. - */ - break; - } - } - } /* if CURLE_OK */ - else if(result) { - long protocol; - - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - curl_easy_getinfo(curl, CURLINFO_PROTOCOL, &protocol); - - if((protocol == CURLPROTO_FTP || protocol == CURLPROTO_FTPS) && - response / 100 == 4) - /* - * This is typically when the FTP server only allows a certain - * amount of users and we are not one of them. All 4xx codes - * are transient. - */ - retry = RETRY_FTP; - } - - if(retry) { - static const char * const m[]={ - NULL, - "timeout", - "connection refused", - "HTTP error", - "FTP error" - }; - - warnf(config->global, "Transient problem: %s " - "Will retry in %ld seconds. " - "%ld retries left.\n", - m[retry], retry_sleep/1000L, retry_numretries); - - tool_go_sleep(retry_sleep); - retry_numretries--; - if(!config->retry_delay) { - retry_sleep *= 2; - if(retry_sleep > RETRY_SLEEP_MAX) - retry_sleep = RETRY_SLEEP_MAX; - } - if(outs.bytes && outs.filename && outs.stream) { - int rc; - /* We have written data to a output file, we truncate file - */ - if(!global->mute) - fprintf(global->errors, "Throwing away %" - CURL_FORMAT_CURL_OFF_T " bytes\n", - outs.bytes); - fflush(outs.stream); - /* truncate file at the position where we started appending */ -#ifdef HAVE_FTRUNCATE - if(ftruncate(fileno(outs.stream), outs.init)) { - /* when truncate fails, we can't just append as then we'll - create something strange, bail out */ - if(!global->mute) - fprintf(global->errors, - "failed to truncate, exiting\n"); - result = CURLE_WRITE_ERROR; - goto quit_urls; - } - /* now seek to the end of the file, the position where we - just truncated the file in a large file-safe way */ - rc = fseek(outs.stream, 0, SEEK_END); -#else - /* ftruncate is not available, so just reposition the file - to the location we would have truncated it. This won't - work properly with large files on 32-bit systems, but - most of those will have ftruncate. */ - rc = fseek(outs.stream, (long)outs.init, SEEK_SET); -#endif - if(rc) { - if(!global->mute) - fprintf(global->errors, - "failed seeking to end of file, exiting\n"); - result = CURLE_WRITE_ERROR; - goto quit_urls; - } - outs.bytes = 0; /* clear for next round */ - } - continue; /* curl_easy_perform loop */ - } - } /* if retry_numretries */ - else if(metalink) { - /* Metalink: Decide to try the next resource or - not. Basically, we want to try the next resource if - download was not successful. */ - long response; - if(CURLE_OK == result) { - char *effective_url = NULL; - curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); - if(effective_url && - curl_strnequal(effective_url, "http", 4)) { - /* This was HTTP(S) */ - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - if(response != 200 && response != 206) { - metalink_next_res = 1; - fprintf(global->errors, - "Metalink: fetching (%s) from (%s) FAILED " - "(HTTP status code %ld)\n", - mlfile->filename, this_url, response); - } - } - } - else { - metalink_next_res = 1; - fprintf(global->errors, - "Metalink: fetching (%s) from (%s) FAILED (%s)\n", - mlfile->filename, this_url, - (errorbuffer[0]) ? - errorbuffer : curl_easy_strerror(result)); - } - } - if(metalink && !metalink_next_res) - fprintf(global->errors, "Metalink: fetching (%s) from (%s) OK\n", - mlfile->filename, this_url); /* In all ordinary cases, just break out of loop here */ break; /* curl_easy_perform loop */ } - - if((global->progressmode == CURL_PROGRESS_BAR) && - progressbar.calls) - /* if the custom progress bar has been displayed, we output a - newline here */ - fputs("\n", progressbar.out); - - if(config->writeout) - ourWriteOut(curl, &outs, config->writeout); - - /* - ** Code within this loop may jump directly here to label 'show_error' - ** in order to display an error message for CURLcode stored in 'res' - ** variable and exit loop once that necessary writing and cleanup - ** in label 'quit_urls' has been done. - */ - - show_error: - -#ifdef __VMS - if(is_vms_shell()) { - /* VMS DCL shell behavior */ - if(!global->showerror) - vms_show = VMSSTS_HIDE; - } - else -#endif - if(config->synthetic_error) { - ; - } - else if(result && global->showerror) { - fprintf(global->errors, "curl: (%d) %s\n", result, (errorbuffer[0]) ? - errorbuffer : curl_easy_strerror(result)); - if(result == CURLE_PEER_FAILED_VERIFICATION) - fputs(CURL_CA_CERT_ERRORMSG, global->errors); - } - - /* Fall through comment to 'quit_urls' label */ - - /* - ** Upon error condition and always that a message has already been - ** displayed, code within this loop may jump directly here to label - ** 'quit_urls' otherwise it should jump to 'show_error' label above. - ** - ** When 'res' variable is _not_ CURLE_OK loop will exit once that - ** all code following 'quit_urls' has been executed. Otherwise it - ** will loop to the beginning from where it may exit if there are - ** no more urls left. - */ - - quit_urls: - - /* Set file extended attributes */ - if(!result && config->xattr && outs.fopened && outs.stream) { - int rc = fwrite_xattr(curl, fileno(outs.stream)); - if(rc) - warnf(config->global, "Error setting extended attributes: %s\n", - strerror(errno)); - } - - /* Close the file */ - if(outs.fopened && outs.stream) { - int rc = fclose(outs.stream); - if(!result && rc) { - /* something went wrong in the writing process */ - result = CURLE_WRITE_ERROR; - fprintf(global->errors, "(%d) Failed writing body\n", result); - } - } - else if(!outs.s_isreg && outs.stream) { - /* Dump standard stream buffered data */ - int rc = fflush(outs.stream); - if(!result && rc) { - /* something went wrong in the writing process */ - result = CURLE_WRITE_ERROR; - fprintf(global->errors, "(%d) Failed writing body\n", result); - } - } - -#ifdef __AMIGA__ - if(!result && outs.s_isreg && outs.filename) { - /* Set the url (up to 80 chars) as comment for the file */ - if(strlen(urlnode->url) > 78) - urlnode->url[79] = '\0'; - SetComment(outs.filename, urlnode->url); - } -#endif - - /* File time can only be set _after_ the file has been closed */ - if(!result && config->remote_time && outs.s_isreg && outs.filename) { - /* Ask libcurl if we got a remote file time */ - curl_off_t filetime = -1; - curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime); - setfiletime(filetime, outs.filename, config->global->errors); - } - -#ifdef USE_METALINK - if(!metalink && config->use_metalink && result == CURLE_OK) { - int rv = parse_metalink(config, &outs, this_url); - if(rv == 0) - fprintf(config->global->errors, "Metalink: parsing (%s) OK\n", - this_url); - else if(rv == -1) - fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n", - this_url); - } - else if(metalink && result == CURLE_OK && !metalink_next_res) { - int rv = metalink_check_hash(global, mlfile, outs.filename); - if(rv == 0) { - metalink_next_res = 1; - } - } -#endif /* USE_METALINK */ - - /* No more business with this output struct */ - if(outs.alloc_filename) - Curl_safefree(outs.filename); -#ifdef USE_METALINK - if(outs.metalink_parser) - metalink_parser_context_delete(outs.metalink_parser); -#endif /* USE_METALINK */ - memset(&outs, 0, sizeof(struct OutStruct)); - hdrcbdata.outs = NULL; - - /* Free loop-local allocated memory and close loop-local opened fd */ - - Curl_safefree(outfile); - Curl_safefree(this_url); - - if(infdopen) - close(infd); - - if(metalink) { - /* Should exit if error is fatal. */ - if(is_fatal_error(result)) { - break; - } - if(!metalink_next_res) - break; - mlres = mlres->next; - if(mlres == NULL) - break; - } - else if(urlnum > 1) { - /* when url globbing, exit loop upon critical error */ - if(is_fatal_error(result)) - break; - } - else if(result) - /* when not url globbing, exit loop upon any error */ - break; - } /* loop to the next URL */ - /* Free loop-local allocated memory */ - - Curl_safefree(uploadfile); + show_error: + quit_urls: if(urls) { /* Free list of remaining URLs */ @@ -1962,41 +1896,334 @@ static CURLcode operate_do(struct GlobalConfig *global, Curl_safefree(urlnode->infile); urlnode->flags = 0; - /* - ** Bail out upon critical errors or --fail-early - */ - if(is_fatal_error(result) || (result && global->fail_early)) - goto quit_curl; - } /* for-loop through all URLs */ + quit_curl: + + /* Free function-local referenced allocated memory */ + Curl_safefree(httpgetfields); + + return result; +} + +/* portable millisecond sleep */ +static void wait_ms(int ms) +{ +#if defined(MSDOS) + delay(ms); +#elif defined(WIN32) + Sleep(ms); +#elif defined(HAVE_USLEEP) + usleep(1000 * ms); +#else + struct timeval pending_tv; + pending_tv.tv_sec = ms / 1000; + pending_tv.tv_usec = (ms % 1000) * 1000; + (void)select(0, NULL, NULL, NULL, &pending_tv); +#endif +} + +static long all_added; /* number of easy handles currently added */ + +static int add_parallel_transfers(struct GlobalConfig *global, + CURLM *multi) +{ + struct per_transfer *per; + CURLcode result; + CURLMcode mcode; + for(per = transfers; per && (all_added < global->parallel_max); + per = per->next) { + if(per->added) + /* already added */ + continue; + + result = pre_transfer(global, per); + if(result) + break; + + (void)curl_easy_setopt(per->curl, CURLOPT_PRIVATE, per); + (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb); + (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per); + + mcode = curl_multi_add_handle(multi, per->curl); + if(mcode) + return CURLE_OUT_OF_MEMORY; + per->added = TRUE; + all_added++; + } + return CURLE_OK; +} + +static CURLcode parallel_transfers(struct GlobalConfig *global, + CURLSH *share) +{ + CURLM *multi; + bool done = FALSE; + CURLMcode mcode = CURLM_OK; + CURLcode result = CURLE_OK; + int still_running = 1; + struct timeval start = tvnow(); + + multi = curl_multi_init(); + if(!multi) + return CURLE_OUT_OF_MEMORY; + + result = add_parallel_transfers(global, multi); + if(result) + return result; + + while(!done && !mcode && still_running) { + int numfds; + + mcode = curl_multi_wait(multi, NULL, 0, 1000, &numfds); + + if(!mcode) { + if(!numfds) { + long sleep_ms; + + /* If it returns without any filedescriptor instantly, we need to + avoid busy-looping during periods where it has nothing particular + to wait for */ + curl_multi_timeout(multi, &sleep_ms); + if(sleep_ms) { + if(sleep_ms > 1000) + sleep_ms = 1000; + wait_ms((int)sleep_ms); + } + } + + mcode = curl_multi_perform(multi, &still_running); + } + + progress_meter(global, &start, FALSE); + + if(!mcode) { + int rc; + CURLMsg *msg; + bool removed = FALSE; + do { + msg = curl_multi_info_read(multi, &rc); + if(msg) { + bool retry; + struct per_transfer *ended; + CURL *easy = msg->easy_handle; + result = msg->data.result; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended); + curl_multi_remove_handle(multi, easy); + + result = post_transfer(global, share, ended, result, &retry); + if(retry) + continue; + progress_finalize(ended); /* before it goes away */ + all_added--; /* one fewer added */ + removed = TRUE; + (void)del_transfer(ended); + } + } while(msg); + if(removed) + /* one or more transfers completed, add more! */ + (void)add_parallel_transfers(global, multi); + } + } + + (void)progress_meter(global, &start, TRUE); + + /* Make sure to return some kind of error if there was a multi problem */ + if(mcode) { + result = (mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY : + /* The other multi errors should never happen, so return + something suitably generic */ + CURLE_BAD_FUNCTION_ARGUMENT; + } + + curl_multi_cleanup(multi); + + return result; +} + +static CURLcode serial_transfers(struct GlobalConfig *global, + CURLSH *share) +{ + CURLcode returncode = CURLE_OK; + CURLcode result = CURLE_OK; + struct per_transfer *per; + for(per = transfers; per;) { + bool retry; + result = pre_transfer(global, per); + if(result) + break; + +#ifndef CURL_DISABLE_LIBCURL_OPTION + if(global->libcurl) { + result = easysrc_perform(); + if(result) + break; + } +#endif +#ifdef CURLDEBUG + if(global->test_event_based) + result = curl_easy_perform_ev(per->curl); + else +#endif + result = curl_easy_perform(per->curl); + + /* store the result of the actual transfer */ + returncode = result; + + result = post_transfer(global, share, per, result, &retry); + if(retry) + continue; + per = del_transfer(per); + + /* Bail out upon critical errors or --fail-early */ + if(result || is_fatal_error(returncode) || + (returncode && global->fail_early)) + break; + } + if(returncode) + /* returncode errors have priority */ + result = returncode; + return result; +} + +static CURLcode operate_do(struct GlobalConfig *global, + struct OperationConfig *config, + CURLSH *share) +{ + CURLcode result = CURLE_OK; + bool capath_from_env; /* - ** Nested loops end here. + ** Beyond this point no return'ing from this function allowed. + ** Jump to label 'quit_curl' in order to abandon this function + ** from outside of nested loops further down below. */ - quit_curl: + /* Check we have a url */ + if(!config->url_list || !config->url_list->url) { + helpf(global->errors, "no URL specified!\n"); + return CURLE_FAILED_INIT; + } - /* Reset the global config variables */ - global->noprogress = orig_noprogress; - global->isatty = orig_isatty; + /* On WIN32 we can't set the path to curl-ca-bundle.crt + * at compile time. So we look here for the file in two ways: + * 1: look at the environment variable CURL_CA_BUNDLE for a path + * 2: if #1 isn't found, use the windows API function SearchPath() + * to find it along the app's path (includes app's dir and CWD) + * + * We support the environment variable thing for non-Windows platforms + * too. Just for the sake of it. + */ + capath_from_env = false; + if(!config->cacert && + !config->capath && + !config->insecure_ok) { + CURL *curltls = curl_easy_init(); + struct curl_tlssessioninfo *tls_backend_info = NULL; - /* Free function-local referenced allocated memory */ - Curl_safefree(httpgetfields); + /* With the addition of CAINFO support for Schannel, this search could find + * a certificate bundle that was previously ignored. To maintain backward + * compatibility, only perform this search if not using Schannel. + */ + result = curl_easy_getinfo(curltls, CURLINFO_TLS_SSL_PTR, + &tls_backend_info); + if(result) + return result; - /* Free list of given URLs */ - clean_getout(config); + /* Set the CA cert locations specified in the environment. For Windows if + * no environment-specified filename is found then check for CA bundle + * default filename curl-ca-bundle.crt in the user's PATH. + * + * If Schannel is the selected SSL backend then these locations are + * ignored. We allow setting CA location for schannel only when explicitly + * specified by the user via CURLOPT_CAINFO / --cacert. + */ + if(tls_backend_info->backend != CURLSSLBACKEND_SCHANNEL) { + char *env; + env = curlx_getenv("CURL_CA_BUNDLE"); + if(env) { + config->cacert = strdup(env); + if(!config->cacert) { + curl_free(env); + helpf(global->errors, "out of memory\n"); + return CURLE_OUT_OF_MEMORY; + } + } + else { + env = curlx_getenv("SSL_CERT_DIR"); + if(env) { + config->capath = strdup(env); + if(!config->capath) { + curl_free(env); + helpf(global->errors, "out of memory\n"); + return CURLE_OUT_OF_MEMORY; + } + capath_from_env = true; + } + else { + env = curlx_getenv("SSL_CERT_FILE"); + if(env) { + config->cacert = strdup(env); + if(!config->cacert) { + curl_free(env); + helpf(global->errors, "out of memory\n"); + return CURLE_OUT_OF_MEMORY; + } + } + } + } + + if(env) + curl_free(env); +#ifdef WIN32 + else { + result = FindWin32CACert(config, tls_backend_info->backend, + "curl-ca-bundle.crt"); + } +#endif + } + curl_easy_cleanup(curltls); + } - hdrcbdata.heads = NULL; + if(!result) + /* loop through the list of given URLs */ + result = create_transfers(global, config, share, capath_from_env); - /* Close function-local opened file descriptors */ - if(heads.fopened && heads.stream) - fclose(heads.stream); + return result; +} + +static CURLcode operate_transfers(struct GlobalConfig *global, + CURLSH *share, + CURLcode result) +{ + /* Save the values of noprogress and isatty to restore them later on */ + bool orig_noprogress = global->noprogress; + bool orig_isatty = global->isatty; + struct per_transfer *per; + + /* Time to actually do the transfers */ + if(!result) { + if(global->parallel) + result = parallel_transfers(global, share); + else + result = serial_transfers(global, share); + } - if(heads.alloc_filename) - Curl_safefree(heads.filename); + /* cleanup if there are any left */ + for(per = transfers; per;) { + bool retry; + (void)post_transfer(global, share, per, result, &retry); + /* Free list of given URLs */ + clean_getout(per->config); + + /* Release metalink related resources here */ + clean_metalink(per->config); + per = del_transfer(per); + } + + /* Reset the global config variables */ + global->noprogress = orig_noprogress; + global->isatty = orig_isatty; - /* Release metalink related resources here */ - clean_metalink(config); return result; } @@ -2040,7 +2267,7 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]) tool_version_info(); /* Check if we were asked to list the SSL engines */ else if(res == PARAM_ENGINES_REQUESTED) - tool_list_engines(config->easy); + tool_list_engines(); else if(res == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL) result = CURLE_UNSUPPORTED_PROTOCOL; else @@ -2058,27 +2285,39 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]) if(!result) { size_t count = 0; struct OperationConfig *operation = config->first; + CURLSH *share = curl_share_init(); + if(!share) { + /* Cleanup the libcurl source output */ + easysrc_cleanup(); + return CURLE_OUT_OF_MEMORY; + } + + curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); + curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); + curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); + curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); + curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_PSL); /* Get the required arguments for each operation */ - while(!result && operation) { + do { result = get_args(operation, count++); operation = operation->next; - } + } while(!result && operation); /* Set the current operation pointer */ config->current = config->first; - /* Perform each operation */ + /* Setup all transfers */ while(!result && config->current) { - result = operate_do(config, config->current); - + result = operate_do(config, config->current, share); config->current = config->current->next; - - if(config->current && config->current->easy) - curl_easy_reset(config->current->easy); } + /* now run! */ + result = operate_transfers(config, share, result); + + curl_share_cleanup(share); #ifndef CURL_DISABLE_LIBCURL_OPTION if(config->libcurl) { /* Cleanup the libcurl source output */ -- cgit v1.2.3