Support Metalink.

This change adds experimental Metalink support to curl.
To enable Metalink support, run configure with --with-libmetalink.
To feed Metalink file to curl, use --metalink option like this:

  $ curl -O --metalink foo.metalink

We use libmetalink to parse Metalink files.

1 files changed, 990 insertions, 0 deletions

diff --git a/src/tool_metalink.c b/src/tool_metalink.c
new file mode 100644
index 000000000..558fa8fa0
--- /dev/null
+++ b/src/tool_metalink.c
@@ -0,0 +1,990 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at http://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "tool_setup.h"
+
+#ifdef HAVE_UNISTD_H
+#  include <unistd.h>
+#endif
+
+#ifdef HAVE_UTIME_H
+#  include <utime.h>
+#elif defined(HAVE_SYS_UTIME_H)
+#  include <sys/utime.h>
+#endif
+
+#define ENABLE_CURLX_PRINTF
+/* use our own printf() functions */
+#include "curlx.h"
+
+#include "tool_metalink.h"
+#include "tool_binmode.h"
+#include "tool_cb_prg.h"
+#include "tool_cb_wrt.h"
+#include "tool_cb_rea.h"
+#include "tool_cb_see.h"
+#include "tool_cb_dbg.h"
+#include "tool_dirhie.h"
+#include "tool_util.h"
+#include "tool_main.h"
+#include "tool_msgs.h"
+#include "tool_setopt.h"
+#include "tool_sleep.h"
+#include "tool_writeout.h"
+#include "tool_libinfo.h"
+#include "tool_homedir.h"
+#include "tool_cb_hdr.h"
+
+#include "memdebug.h" /* keep this as LAST include */
+
+static int curl_truncate_file(struct Configurable *config,
+                              struct OutStruct outs)
+{
+  int res = 0;
+  if(outs.bytes && outs.filename) {
+    /* We have written data to a output file, we truncate file
+     */
+    if(!config->mute)
+      fprintf(config->errors, "Throwing away %"
+              CURL_FORMAT_CURL_OFF_T " bytes\n",
+              outs.bytes);
+    fflush(outs.stream);
+    /* truncate file at the position where we started appending */
+#ifdef HAVE_FTRUNCATE
+    if(ftruncate( fileno(outs.stream), outs.init)) {
+      /* when truncate fails, we can't just append as then we'll
+         create something strange, bail out */
+      if(!config->mute)
+        fprintf(config->errors,
+                "failed to truncate, exiting\n");
+      res = CURLE_WRITE_ERROR;
+      assert(0);
+      /* goto quit_urls; */
+    }
+    /* now seek to the end of the file, the position where we
+       just truncated the file in a large file-safe way */
+    fseek(outs.stream, 0, SEEK_END);
+#else
+    /* ftruncate is not available, so just reposition the file
+       to the location we would have truncated it. This won't
+       work properly with large files on 32-bit systems, but
+       most of those will have ftruncate. */
+    fseek(outs.stream, (long)outs.init, SEEK_SET);
+#endif
+    outs.bytes = 0; /* clear for next round */
+  }
+  return res;
+}
+
+struct metalinkfile *new_metalinkfile(metalink_file_t *metalinkfile) {
+  struct metalinkfile *f;
+  f = (struct metalinkfile*)malloc(sizeof(struct metalinkfile));
+  f->file = metalinkfile;
+  f->next = NULL;
+  return f;
+}
+
+struct metalink *new_metalink(metalink_t *metalink) {
+  struct metalink *ml;
+  ml = (struct metalink*)malloc(sizeof(struct metalink));
+  ml->metalink = metalink;
+  ml->next = NULL;
+  return ml;
+}
+
+int operatemetalink(CURL *curl,
+                    struct getout *urlnode,
+                    long retry_sleep_default,
+                    struct OutStruct outs,
+                    struct OutStruct heads,
+                    char *outfiles,
+                    struct Configurable *config)
+{
+  long retry_numretries;
+  int infd = STDIN_FILENO;
+  bool infdopen;
+  char *outfile;
+  struct timeval retrystart;
+  struct metalinkfile *mlfile;
+  int res = 0;
+  char *filename;
+  metalink_resource_t **mlres;
+  char errorbuffer[CURL_ERROR_SIZE];
+  struct ProgressData progressbar;
+  long retry_sleep;
+  struct InStruct input;
+  curl_off_t uploadfilesize; /* -1 means unknown */
+  char *uploadfile=NULL; /* a single file, never a glob */
+
+  uploadfilesize=-1;
+
+  mlfile = config->metalinkfile_last;
+  config->metalinkfile_last = config->metalinkfile_last->next;
+
+  filename = strdup(mlfile->file->name);
+
+  outfile = outfiles?strdup(outfiles):NULL;
+
+  if((urlnode->flags & GETOUT_USEREMOTE) ||
+     (outfile && !curlx_strequal("-", outfile)) ) {
+
+    /*
+     * We have specified a file name to store the result in, or we have
+     * decided we want to use the name attribute of file element in Metalink
+     * XML.
+     */
+
+    if(!outfile) {
+      /* Find and get file name */
+      char * pc;
+      pc = strrchr(filename, '/');
+
+      if(pc) {
+        /* duplicate the string beyond the slash */
+        pc++;
+        outfile = *pc ? strdup(pc): NULL;
+      }
+      else {
+        outfile = strdup(filename);
+      }
+
+      if(!outfile || !*outfile) {
+        helpf(config->errors, "Metalink file[@name] has no length!\n");
+        res = CURLE_WRITE_ERROR;
+        free(filename);
+        /* break; */
+        return 1;
+      }
+#if defined(MSDOS)
+      {
+        /* This is for DOS, and then we do some major replacing of
+           bad characters in the file name before using it */
+        char file1[PATH_MAX];
+        if(strlen(outfile) >= PATH_MAX)
+          outfile[PATH_MAX-1]=0; /* cut it */
+        strcpy(file1, msdosify(outfile));
+        free(outfile);
+
+        outfile = strdup(rename_if_dos_device_name(file1));
+        if(!outfile) {
+          res = CURLE_OUT_OF_MEMORY;
+          break;
+        }
+      }
+#endif /* MSDOS */
+    }
+    /* Create the directory hierarchy, if not pre-existant to a multiple
+       file output call */
+
+    if(config->create_dirs &&
+       (CURLE_WRITE_ERROR == create_dir_hierarchy(outfile, config->errors)))
+      return CURLE_WRITE_ERROR;
+
+    if(config->resume_from_current) {
+      /* We're told to continue from where we are now. Get the
+         size of the file as it is now and open it for append instead */
+
+      struct_stat fileinfo;
+
+      /* VMS -- Danger, the filesize is only valid for stream files */
+      if(0 == stat(outfile, &fileinfo))
+        /* set offset to current file size: */
+        config->resume_from = fileinfo.st_size;
+      else
+        /* let offset be 0 */
+        config->resume_from = 0;
+    }
+
+    outs.filename = outfile;
+    outs.s_isreg = TRUE;
+
+    if(config->resume_from) {
+      outs.init = config->resume_from;
+      /* open file for output: */
+      outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb");
+      if(!outs.stream) {
+        helpf(config->errors, "Can't open '%s'!\n", outfile);
+        return CURLE_WRITE_ERROR;
+      }
+    }
+    else {
+      outs.stream = NULL; /* open when needed */
+    }
+  }
+  infdopen=FALSE;
+
+  if(!config->errors)
+    config->errors = stderr;
+
+  if(!outfile && !config->use_ascii) {
+    /* We get the output to stdout and we have not got the ASCII/text
+       flag, then set stdout to be binary */
+    set_binmode(stdout);
+  }
+
+  /* Loop though all resources in Metalink */
+  for(mlres = mlfile->file->resources; *mlres; ++mlres) {
+    int try_next_res = 0;
+    if(config->tcp_nodelay)
+      my_setopt(curl, CURLOPT_TCP_NODELAY, 1);
+
+    /* where to store */
+    my_setopt(curl, CURLOPT_WRITEDATA, &outs);
+    /* what call to write */
+    my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb);
+
+
+    /* for uploads */
+    input.fd = infd;
+    input.config = config;
+    /* Note that if CURLOPT_READFUNCTION is fread (the default), then
+     * lib/telnet.c will Curl_poll() on the input file descriptor
+     * rather then calling the READFUNCTION at regular intervals.
+     * The circumstances in which it is preferable to enable this
+     * behaviour, by omitting to set the READFUNCTION & READDATA options,
+     * have not been determined.
+     */
+    my_setopt(curl, CURLOPT_READDATA, &input);
+    /* what call to read */
+    my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb);
+
+    /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what
+       CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */
+    my_setopt(curl, CURLOPT_SEEKDATA, &input);
+    my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb);
+
+    if(config->recvpersecond)
+      /* tell libcurl to use a smaller sized buffer as it allows us to
+         make better sleeps! 7.9.9 stuff! */
+      my_setopt(curl, CURLOPT_BUFFERSIZE, config->recvpersecond);
+
+    /* size of uploaded file: */
+    if(uploadfilesize != -1)
+      my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
+    my_setopt(curl, CURLOPT_URL, (*mlres)->url);     /* what to fetch */
+    my_setopt(curl, CURLOPT_NOPROGRESS, config->noprogress);
+    if(config->no_body) {
+      my_setopt(curl, CURLOPT_NOBODY, 1);
+      my_setopt(curl, CURLOPT_HEADER, 1);
+    }
+    else
+      my_setopt(curl, CURLOPT_HEADER, config->include_headers);
+
+#if !defined(CURL_DISABLE_PROXY)
+    {
+      /* TODO: Make this a run-time check instead of compile-time one. */
+
+      my_setopt_str(curl, CURLOPT_PROXY, config->proxy);
+      my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd);
+
+      /* new in libcurl 7.3 */
+      my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel);
+
+      /* new in libcurl 7.5 */
+      if(config->proxy)
+        my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->proxyver);
+
+      /* new in libcurl 7.10 */
+      if(config->socksproxy) {
+        my_setopt_str(curl, CURLOPT_PROXY, config->socksproxy);
+        my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->socksver);
+      }
+
+      /* new in libcurl 7.10.6 */
+      if(config->proxyanyauth)
+        my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
+                          (long) CURLAUTH_ANY);
+      else if(config->proxynegotiate)
+        my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
+                          (long) CURLAUTH_GSSNEGOTIATE);
+      else if(config->proxyntlm)
+        my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
+                          (long) CURLAUTH_NTLM);
+      else if(config->proxydigest)
+        my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
+                          (long) CURLAUTH_DIGEST);
+      else if(config->proxybasic)
+        my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
+                          (long) CURLAUTH_BASIC);
+
+      /* new in libcurl 7.19.4 */
+      my_setopt(curl, CURLOPT_NOPROXY, config->noproxy);
+    }
+#endif
+
+    my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror);
+    my_setopt(curl, CURLOPT_UPLOAD, uploadfile?TRUE:FALSE);
+    my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly);
+    my_setopt(curl, CURLOPT_APPEND, config->ftp_append);
+
+    if(config->netrc_opt)
+      my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
+    else if(config->netrc || config->netrc_file)
+      my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_REQUIRED);
+    else
+      my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_IGNORED);
+
+    if(config->netrc_file)
+      my_setopt(curl, CURLOPT_NETRC_FILE, config->netrc_file);
+
+    my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii);
+    my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd);
+    my_setopt_str(curl, CURLOPT_RANGE, config->range);
+    my_setopt(curl, CURLOPT_ERRORBUFFER, errorbuffer);
+    my_setopt(curl, CURLOPT_TIMEOUT, config->timeout);
+
+    if(built_in_protos & CURLPROTO_HTTP) {
+
+      long postRedir = 0;
+
+      my_setopt(curl, CURLOPT_FOLLOWLOCATION,
+                config->followlocation);
+      my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH,
+                config->unrestricted_auth);
+
+      switch(config->httpreq) {
+      case HTTPREQ_SIMPLEPOST:
+        my_setopt_str(curl, CURLOPT_POSTFIELDS,
+                      config->postfields);
+        my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE,
+                  config->postfieldsize);
+        break;
+      case HTTPREQ_POST:
+        my_setopt_httppost(curl, CURLOPT_HTTPPOST, config->httppost);
+        break;
+      default:
+        break;
+      }
+
+      my_setopt_str(curl, CURLOPT_REFERER, config->referer);
+      my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer);
+      my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent);
+      my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers);
+
+      /* new in libcurl 7.5 */
+      my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs);
+
+      /* new in libcurl 7.9.1 */
+      if(config->httpversion)
+        my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion);
+
+      /* new in libcurl 7.10.6 (default is Basic) */
+      if(config->authtype)
+        my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long) config->authtype);
+
+      /* curl 7.19.1 (the 301 version existed in 7.18.2),
+         303 was added in 7.26.0 */
+      if(config->post301)
+        postRedir |= CURL_REDIR_POST_301;
+      if(config->post302)
+        postRedir |= CURL_REDIR_POST_302;
+      if(config->post303)
+        postRedir |= CURL_REDIR_POST_303;
+      my_setopt(curl, CURLOPT_POSTREDIR, postRedir);
+
+      /* new in libcurl 7.21.6 */
+      if(config->encoding)
+        my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, "");
+
+      /* new in libcurl 7.21.6 */
+      if(config->tr_encoding)
+        my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1);
+
+    } /* (built_in_protos & CURLPROTO_HTTP) */
+
+    my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport);
+    my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT,
+              config->low_speed_limit);
+    my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time);
+    my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE,
+              config->sendpersecond);
+    my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE,
+              config->recvpersecond);
+    my_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
+              config->use_resume?config->resume_from:0);
+
+    my_setopt(curl, CURLOPT_SSLCERT, config->cert);
+    my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type);
+    my_setopt(curl, CURLOPT_SSLKEY, config->key);
+    my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type);
+    my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd);
+
+    if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
+
+      /* SSH and SSL private key uses same command-line option */
+      /* new in libcurl 7.16.1 */
+      my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key);
+      /* new in libcurl 7.16.1 */
+      my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey);
+
+      /* new in libcurl 7.17.1: SSH host key md5 checking allows us
+         to fail if we are not talking to who we think we should */
+      my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5,
+                    config->hostpubmd5);
+    }
+
+    if(config->cacert)
+      my_setopt_str(curl, CURLOPT_CAINFO, config->cacert);
+    if(config->capath)
+      my_setopt_str(curl, CURLOPT_CAPATH, config->capath);
+    if(config->crlfile)
+      my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile);
+
+    if(curlinfo->features & CURL_VERSION_SSL) {
+      if(config->insecure_ok) {
+        my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
+        my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1L);
+      }
+      else {
+        my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
+        /* libcurl default is strict verifyhost -> 2L   */
+        /* my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); */
+      }
+    }
+
+    if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
+      if(!config->insecure_ok) {
+        char *home;
+        char *file;
+        res = CURLE_OUT_OF_MEMORY;
+        home = homedir();
+        if(home) {
+          file = aprintf("%s/%sssh/known_hosts", home, DOT_CHAR);
+          if(file) {
+            /* new in curl 7.19.6 */
+            res = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, file);
+            curl_free(file);
+            if(res == CURLE_UNKNOWN_OPTION)
+              /* libssh2 version older than 1.1.1 */
+              res = CURLE_OK;
+          }
+          Curl_safefree(home);
+        }
+        if(res)
+          goto show_error;
+      }
+    }
+
+    if(config->no_body || config->remote_time) {
+      /* no body or use remote time */
+      my_setopt(curl, CURLOPT_FILETIME, TRUE);
+    }
+
+    my_setopt(curl, CURLOPT_CRLF, config->crlf);
+    my_setopt_slist(curl, CURLOPT_QUOTE, config->quote);
+    my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote);
+    my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote);
+
+#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES)
+    {
+      /* TODO: Make this a run-time check instead of compile-time one. */
+
+      if(config->cookie)
+        my_setopt_str(curl, CURLOPT_COOKIE, config->cookie);
+
+      if(config->cookiefile)
+        my_setopt_str(curl, CURLOPT_COOKIEFILE, config->cookiefile);
+
+      /* new in libcurl 7.9 */
+      if(config->cookiejar)
+        my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar);
+
+      /* new in libcurl 7.9.7 */
+      my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession);
+    }
+#endif
+
+    my_setopt_enum(curl, CURLOPT_SSLVERSION, config->ssl_version);
+    my_setopt_enum(curl, CURLOPT_TIMECONDITION, config->timecond);
+    my_setopt(curl, CURLOPT_TIMEVALUE, config->condtime);
+    my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest);
+    my_setopt(curl, CURLOPT_STDERR, config->errors);
+
+    /* three new ones in libcurl 7.3: */
+    my_setopt_str(curl, CURLOPT_INTERFACE, config->iface);
+    my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel);
+
+    progressbarinit(&progressbar, config);
+    if((config->progressmode == CURL_PROGRESS_BAR) &&
+       !config->noprogress && !config->mute) {
+      /* we want the alternative style, then we have to implement it
+         ourselves! */
+      my_setopt(curl, CURLOPT_PROGRESSFUNCTION, tool_progress_cb);
+      my_setopt(curl, CURLOPT_PROGRESSDATA, &progressbar);
+    }
+
+    /* new in libcurl 7.6.2: */
+    my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options);
+
+    /* new in libcurl 7.7: */
+    my_setopt_str(curl, CURLOPT_RANDOM_FILE, config->random_file);
+    my_setopt(curl, CURLOPT_EGDSOCKET, config->egd_file);
+    my_setopt(curl, CURLOPT_CONNECTTIMEOUT, config->connecttimeout);
+
+    if(config->cipher_list)
+      my_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list);
+
+    /* new in libcurl 7.9.2: */
+    if(config->disable_epsv)
+      /* disable it */
+      my_setopt(curl, CURLOPT_FTP_USE_EPSV, FALSE);
+
+    /* new in libcurl 7.10.5 */
+    if(config->disable_eprt)
+      /* disable it */
+      my_setopt(curl, CURLOPT_FTP_USE_EPRT, FALSE);
+
+    if(config->tracetype != TRACE_NONE) {
+      my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb);
+      my_setopt(curl, CURLOPT_DEBUGDATA, config);
+      my_setopt(curl, CURLOPT_VERBOSE, TRUE);
+    }
+
+    /* new in curl 7.9.3 */
+    if(config->engine) {
+      res = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine);
+      if(res)
+        goto show_error;
+      my_setopt(curl, CURLOPT_SSLENGINE_DEFAULT, 1);
+    }
+
+    /* new in curl 7.10.7, extended in 7.19.4 but this only sets 0 or 1 */
+    my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS,
+              config->ftp_create_dirs);
+
+    /* new in curl 7.10.8 */
+    if(config->max_filesize)
+      my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
+                config->max_filesize);
+
+    if(4 == config->ip_version)
+      my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
+    else if(6 == config->ip_version)
+      my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
+    else
+      my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER);
+
+    /* new in curl 7.15.5 */
+    if(config->ftp_ssl_reqd)
+      my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL);
+
+    /* new in curl 7.11.0 */
+    else if(config->ftp_ssl)
+      my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
+
+    /* new in curl 7.16.0 */
+    else if(config->ftp_ssl_control)
+      my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_CONTROL);
+
+    /* new in curl 7.16.1 */
+    if(config->ftp_ssl_ccc)
+      my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC, config->ftp_ssl_ccc_mode);
+
+#if defined(HAVE_GSSAPI) || defined(USE_WINDOWS_SSPI)
+    {
+      /* TODO: Make this a run-time check instead of compile-time one. */
+
+      /* new in curl 7.19.4 */
+      if(config->socks5_gssapi_service)
+        my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_SERVICE,
+                      config->socks5_gssapi_service);
+
+      /* new in curl 7.19.4 */
+      if(config->socks5_gssapi_nec)
+        my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC,
+                      config->socks5_gssapi_nec);
+    }
+#endif
+    /* curl 7.13.0 */
+    my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account);
+
+    my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl);
+
+    /* curl 7.14.2 */
+    my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip);
+
+    /* curl 7.15.1 */
+    my_setopt(curl, CURLOPT_FTP_FILEMETHOD, config->ftp_filemethod);
+
+    /* curl 7.15.2 */
+    if(config->localport) {
+      my_setopt(curl, CURLOPT_LOCALPORT, config->localport);
+      my_setopt_str(curl, CURLOPT_LOCALPORTRANGE,
+                    config->localportrange);
+    }
+
+    /* curl 7.15.5 */
+    my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER,
+                  config->ftp_alternative_to_user);
+
+    /* curl 7.16.0 */
+    if(config->disable_sessionid)
+      my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE,
+                !config->disable_sessionid);
+
+    /* curl 7.16.2 */
+    if(config->raw) {
+      my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE);
+      my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE);
+    }
+
+    /* curl 7.17.1 */
+    if(!config->nokeepalive) {
+      my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
+      if(config->alivetime != 0) {
+#if !defined(TCP_KEEPIDLE) || !defined(TCP_KEEPINTVL)
+        warnf(config, "Keep-alive functionality somewhat crippled due to "
+              "missing support in your operating system!\n");
+#endif
+        my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime);
+        my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime);
+      }
+    }
+    else
+      my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L);
+
+    /* curl 7.20.0 */
+    if(config->tftp_blksize)
+      my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize);
+
+    if(config->mail_from)
+      my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from);
+
+    if(config->mail_rcpt)
+      my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt);
+
+    /* curl 7.20.x */
+    if(config->ftp_pret)
+      my_setopt(curl, CURLOPT_FTP_USE_PRET, TRUE);
+
+    if(config->proto_present)
+      my_setopt_flags(curl, CURLOPT_PROTOCOLS, config->proto);
+    if(config->proto_redir_present)
+      my_setopt_flags(curl, CURLOPT_REDIR_PROTOCOLS, config->proto_redir);
+
+    if((urlnode->flags & GETOUT_USEREMOTE)
+       && config->content_disposition) {
+      my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb);
+      my_setopt(curl, CURLOPT_HEADERDATA, &outs);
+    }
+    else {
+      /* if HEADERFUNCTION was set to something in the previous loop, it
+         is important that we set it (back) to NULL now */
+      my_setopt(curl, CURLOPT_HEADERFUNCTION, NULL);
+      my_setopt(curl, CURLOPT_HEADERDATA, config->headerfile?&heads:NULL);
+    }
+
+    if(config->resolve)
+      /* new in 7.21.3 */
+      my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve);
+
+    /* new in 7.21.4 */
+    if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) {
+      if(config->tls_username)
+        my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME,
+                      config->tls_username);
+      if(config->tls_password)
+        my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD,
+                      config->tls_password);
+      if(config->tls_authtype)
+        my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE,
+                      config->tls_authtype);
+    }
+
+    /* new in 7.22.0 */
+    if(config->gssapi_delegation)
+      my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION,
+                    config->gssapi_delegation);
+
+    /* new in 7.25.0 */
+    if(config->ssl_allow_beast)
+      my_setopt(curl, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_ALLOW_BEAST);
+
+    if(config->mail_auth)
+      my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth);
+
+    /* initialize retry vars for loop below */
+    retry_sleep_default = (config->retry_delay) ?
+      config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */
+
+    retry_numretries = config->req_retry;
+    retry_sleep = retry_sleep_default; /* ms */
+    retrystart = tvnow();
+
+#ifndef CURL_DISABLE_LIBCURL_OPTION
+    res = easysrc_perform();
+    if(res) {
+      goto show_error;
+    }
+#endif
+
+    for(;;) {
+      res = curl_easy_perform(curl);
+      /* if retry-max-time is non-zero, make sure we haven't exceeded the
+         time */
+      if(retry_numretries &&
+         (!config->retry_maxtime ||
+          (tvdiff(tvnow(), retrystart) < config->retry_maxtime*1000)) ) {
+        enum {
+          RETRY_NO,
+          RETRY_TIMEOUT,
+          RETRY_HTTP,
+          RETRY_FTP,
+          RETRY_LAST /* not used */
+        } retry = RETRY_NO;
+        long response;
+        if(CURLE_OPERATION_TIMEDOUT == res)
+          /* retry timeout always */
+          retry = RETRY_TIMEOUT;
+        else if(CURLE_OK == res) {
+          /* Check for HTTP transient errors */
+          char *this_url=NULL;
+          curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &this_url);
+          if(this_url &&
+             curlx_strnequal(this_url, "http", 4)) {
+            /* This was HTTP(S) */
+            curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
+
+            switch(response) {
+            case 500: /* Internal Server Error */
+            case 502: /* Bad Gateway */
+            case 503: /* Service Unavailable */
+            case 504: /* Gateway Timeout */
+              retry = RETRY_HTTP;
+              /*
+               * At this point, we have already written data to the output
+               * file (or terminal). If we write to a file, we must rewind
+               * or close/re-open the file so that the next attempt starts
+               * over from the beginning.
+               *
+               * TODO: similar action for the upload case. We might need
+               * to start over reading from a previous point if we have
+               * uploaded something when this was returned.
+               */
+              break;
+            }
+          }
+        } /* if CURLE_OK */
+        else if(CURLE_LOGIN_DENIED == res) {
+          curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
+
+          if(response/100 == 4)
+            /*
+             * This is typically when the FTP server only allows a certain
+             * amount of users and we are not one of them.  All 4xx codes
+             * are transient.
+             */
+            retry = RETRY_FTP;
+        }
+
+        if(retry) {
+          static const char * const m[]={NULL,
+                                         "timeout",
+                                         "HTTP error",
+                                         "FTP error"
+          };
+          warnf(config, "Transient problem: %s "
+                "Will retry in %ld seconds. "
+                "%ld retries left.\n",
+                m[retry],
+                retry_sleep/1000,
+                retry_numretries);
+
+          tool_go_sleep(retry_sleep);
+          retry_numretries--;
+          if(!config->retry_delay) {
+            retry_sleep *= 2;
+            if(retry_sleep > RETRY_SLEEP_MAX)
+              retry_sleep = RETRY_SLEEP_MAX;
+          }
+          curl_truncate_file(config, outs);
+          continue;
+        }
+      } /* if retry_numretries */
+      else {
+        /* Metalink: Decide to try the next resource or
+           not. Basically, we want to try the next resource if
+           download was not successful. */
+        long response;
+        if(CURLE_OPERATION_TIMEDOUT == res) {
+          try_next_res = 1;
+        }
+        else if(CURLE_OK == res) {
+          char *this_url=NULL;
+          curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &this_url);
+          if(this_url &&
+             curlx_strnequal(this_url, "http", 4)) {
+            /* This was HTTP(S) */
+            curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
+
+            switch(response) {
+            case 400:
+            case 401:
+            case 402:
+            case 403:
+            case 404:
+            case 405:
+            case 406:
+            case 407:
+            case 408:
+            case 409:
+            case 410:
+            case 411:
+            case 412:
+            case 413:
+            case 414:
+            case 415:
+            case 416:
+            case 417:
+              try_next_res = 1;
+              break;
+            }
+          }
+        }
+        else if(CURLE_LOGIN_DENIED == res) {
+          if(response/100 == 5)
+            /*
+             * For permanent negative return code, try the next resource.
+             */
+            try_next_res = 1;
+        }
+      }
+
+      /* In all ordinary cases, just break out of loop here */
+      retry_sleep = retry_sleep_default;
+      break;
+
+    }
+    if(try_next_res == 1) {
+      if(mlres+1) {
+        warnf(config, "The remote server returned negative response. "
+              "Will try to the next resource.");
+      }
+      curl_truncate_file(config, outs);
+    }
+    else
+      break;
+  }
+  if((config->progressmode == CURL_PROGRESS_BAR) &&
+     progressbar.calls) {
+    /* if the custom progress bar has been displayed, we output a
+       newline here */
+    fputs("\n", progressbar.out);
+  }
+
+  if(config->writeout) {
+    ourWriteOut(curl, &outs, config->writeout);
+  }
+#ifdef USE_ENVIRONMENT
+  if(config->writeenv)
+    ourWriteEnv(curl);
+#endif
+
+show_error:
+
+#ifdef  VMS
+  if(!config->showerror)  {
+    vms_show = VMSSTS_HIDE;
+  }
+#else
+  if((res!=CURLE_OK) && config->showerror) {
+    fprintf(config->errors, "curl: (%d) %s\n", res,
+            errorbuffer[0]? errorbuffer:
+            curl_easy_strerror((CURLcode)res));
+    if(CURLE_SSL_CACERT == res) {
+#define CURL_CA_CERT_ERRORMSG1                                  \
+"More details here: http://curl.haxx.se/docs/sslcerts.html\n\n" \
+"curl performs SSL certificate verification by default, using a \"bundle\"\n" \
+" of Certificate Authority (CA) public keys (CA certs). If the default\n" \
+" bundle file isn't adequate, you can specify an alternate file\n" \
+" using the --cacert option.\n"
+
+#define CURL_CA_CERT_ERRORMSG2 \
+"If this HTTPS server uses a certificate signed by a CA represented in\n" \
+" the bundle, the certificate verification probably failed due to a\n" \
+" problem with the certificate (it might be expired, or the name might\n" \
+" not match the domain name in the URL).\n" \
+"If you'd like to turn off curl's verification of the certificate, use\n" \
+" the -k (or --insecure) option.\n"
+
+      fprintf(config->errors, "%s%s",
+              CURL_CA_CERT_ERRORMSG1,
+              CURL_CA_CERT_ERRORMSG2 );
+    }
+  }
+#endif
+
+  if(outfile && !curlx_strequal(outfile, "-") && outs.stream)
+    fclose(outs.stream);
+
+#ifdef HAVE_UTIME
+  /* Important that we set the time _after_ the file has been
+     closed, as is done above here */
+  if(config->remote_time && outs.filename) {
+    /* ask libcurl if we got a time. Pretty please */
+    long filetime;
+    curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime);
+    if(filetime >= 0) {
+      struct utimbuf times;
+      times.actime = (time_t)filetime;
+      times.modtime = (time_t)filetime;
+      utime(outs.filename, &times); /* set the time we got */
+    }
+  }
+#endif
+#ifdef __AMIGA__
+  /* Set the url as comment for the file. (up to 80 chars are allowed)
+   */
+  if(strlen(url) > 78)
+    url[79] = '\0';
+
+  SetComment( outs.filename, url);
+#endif
+
+  if(filename)
+    free(filename);
+  if(outfile)
+    free(outfile);
+
+  if(infdopen)
+    close(infd);
+  if(outfiles)
+    free(outfiles);
+
+  return 0;
+}
+
+void clean_metalink(struct Configurable *config)
+{
+  while(config->metalinkfile_list) {
+    struct metalinkfile *mlfile = config->metalinkfile_list;
+    config->metalinkfile_list = config->metalinkfile_list->next;
+    free(mlfile);
+  }
+  config->metalinkfile_last = 0;
+  while(config->metalink_list) {
+    struct metalink *ml = config->metalink_list;
+    config->metalink_list = config->metalink_list->next;
+    metalink_delete(ml->metalink);
+    free(ml);
+  }
+  config->metalink_last = 0;
+}