idn: switch to libidn2 use and IDNA2008 support

CVE-2016-8625

Bug: https://curl.haxx.se/docs/adv_20161102K.html
Reported-by: Christian Heimes

6 files changed, 34 insertions, 197 deletions

diff --git a/lib/curl_setup.h b/lib/curl_setup.h
index 5d82e339b..9619a1e66 100644
--- a/lib/curl_setup.h
+++ b/lib/curl_setup.h
@@ -599,10 +599,9 @@ int netware_init(void);
 #endif
 #endif
 
-#if defined(HAVE_LIBIDN) && defined(HAVE_TLD_H)
-/* The lib was present and the tld.h header (which is missing in libidn 0.3.X
-   but we only work with libidn 0.4.1 or later) */
-#define USE_LIBIDN
+#if defined(HAVE_LIBIDN2) && defined(HAVE_IDN2_H)
+/* The lib and header are present */
+#define USE_LIBIDN2
 #endif
 
 #ifndef SIZEOF_TIME_T
diff --git a/lib/easy.c b/lib/easy.c
index 517c39ffc..cb0ac6740 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -144,28 +144,6 @@ static CURLcode win32_init(void)
   return CURLE_OK;
 }
 
-#ifdef USE_LIBIDN
-/*
- * Initialise use of IDNA library.
- * It falls back to ASCII if $CHARSET isn't defined. This doesn't work for
- * idna_to_ascii_lz().
- */
-static void idna_init (void)
-{
-#ifdef WIN32
-  char buf[60];
-  UINT cp = GetACP();
-
-  if(!getenv("CHARSET") && cp > 0) {
-    snprintf(buf, sizeof(buf), "CHARSET=cp%u", cp);
-    putenv(buf);
-  }
-#else
-  /* to do? */
-#endif
-}
-#endif  /* USE_LIBIDN */
-
 /* true globals -- for curl_global_init() and curl_global_cleanup() */
 static unsigned int  initialized;
 static long          init_flags;
@@ -262,10 +240,6 @@ static CURLcode global_init(long flags, bool memoryfuncs)
   }
 #endif
 
-#ifdef USE_LIBIDN
-  idna_init();
-#endif
-
   if(Curl_resolver_global_init()) {
     DEBUGF(fprintf(stderr, "Error: resolver_global_init failed\n"));
     return CURLE_FAILED_INIT;
diff --git a/lib/strerror.c b/lib/strerror.c
index edb96d201..db50c7d1a 100644
--- a/lib/strerror.c
+++ b/lib/strerror.c
@@ -35,8 +35,8 @@
 
 #include <curl/curl.h>
 
-#ifdef USE_LIBIDN
-#include <idna.h>
+#ifdef USE_LIBIDN2
+#include <idn2.h>
 #endif
 
 #ifdef USE_WINDOWS_SSPI
@@ -726,83 +726,6 @@ const char *Curl_strerror(struct connectdata *conn, int err)
   return buf;
 }
 
-#ifdef USE_LIBIDN
-/*
- * Return error-string for libidn status as returned from idna_to_ascii_lz().
- */
-const char *Curl_idn_strerror (struct connectdata *conn, int err)
-{
-#ifdef HAVE_IDNA_STRERROR
-  (void)conn;
-  return idna_strerror((Idna_rc) err);
-#else
-  const char *str;
-  char *buf;
-  size_t max;
-
-  DEBUGASSERT(conn);
-
-  buf = conn->syserr_buf;
-  max = sizeof(conn->syserr_buf)-1;
-  *buf = '\0';
-
-#ifndef CURL_DISABLE_VERBOSE_STRINGS
-  switch ((Idna_rc)err) {
-    case IDNA_SUCCESS:
-      str = "No error";
-      break;
-    case IDNA_STRINGPREP_ERROR:
-      str = "Error in string preparation";
-      break;
-    case IDNA_PUNYCODE_ERROR:
-      str = "Error in Punycode operation";
-      break;
-    case IDNA_CONTAINS_NON_LDH:
-      str = "Illegal ASCII characters";
-      break;
-    case IDNA_CONTAINS_MINUS:
-      str = "Contains minus";
-      break;
-    case IDNA_INVALID_LENGTH:
-      str = "Invalid output length";
-      break;
-    case IDNA_NO_ACE_PREFIX:
-      str = "No ACE prefix (\"xn--\")";
-      break;
-    case IDNA_ROUNDTRIP_VERIFY_ERROR:
-      str = "Round trip verify error";
-      break;
-    case IDNA_CONTAINS_ACE_PREFIX:
-      str = "Already have ACE prefix (\"xn--\")";
-      break;
-    case IDNA_ICONV_ERROR:
-      str = "Locale conversion failed";
-      break;
-    case IDNA_MALLOC_ERROR:
-      str = "Allocation failed";
-      break;
-    case IDNA_DLOPEN_ERROR:
-      str = "dlopen() error";
-      break;
-    default:
-      snprintf(buf, max, "error %d", err);
-      str = NULL;
-      break;
-  }
-#else
-  if((Idna_rc)err == IDNA_SUCCESS)
-    str = "No error";
-  else
-    str = "Error";
-#endif
-  if(str)
-    strncpy(buf, str, max);
-  buf[max] = '\0';
-  return (buf);
-#endif
-}
-#endif  /* USE_LIBIDN */
-
 #ifdef USE_WINDOWS_SSPI
 const char *Curl_sspi_strerror (struct connectdata *conn, int err)
 {
diff --git a/lib/strerror.h b/lib/strerror.h
index ae8c96bd4..627273eb2 100644
--- a/lib/strerror.h
+++ b/lib/strerror.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -26,7 +26,7 @@
 
 const char *Curl_strerror (struct connectdata *conn, int err);
 
-#ifdef USE_LIBIDN
+#ifdef USE_LIBIDN2
 const char *Curl_idn_strerror (struct connectdata *conn, int err);
 #endif
 
diff --git a/lib/url.c b/lib/url.c
index 98236e219..b42673163 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -59,24 +59,15 @@
 #include <limits.h>
 #endif
 
-#ifdef USE_LIBIDN
-#include <idna.h>
-#include <tld.h>
-#include <stringprep.h>
-#ifdef HAVE_IDN_FREE_H
-#include <idn-free.h>
-#else
-/* prototype from idn-free.h, not provided by libidn 0.4.5's make install! */
-void idn_free (void *ptr);
-#endif
-#ifndef HAVE_IDN_FREE
-/* if idn_free() was not found in this version of libidn use free() instead */
-#define idn_free(x) (free)(x)
-#endif
+#ifdef USE_LIBIDN2
+#include <idn2.h>
+
 #elif defined(USE_WIN32_IDN)
 /* prototype for curl_win32_idn_to_ascii() */
 bool curl_win32_idn_to_ascii(const char *in, char **out);
-#endif  /* USE_LIBIDN */
+#endif  /* USE_LIBIDN2 */
+
+#include <idn2.h>
 
 #include "urldata.h"
 #include "netrc.h"
@@ -3771,58 +3762,15 @@ static bool is_ASCII_name(const char *hostname)
   return TRUE;
 }
 
-#ifdef USE_LIBIDN
-/*
- * Check if characters in hostname is allowed in Top Level Domain.
- */
-static bool tld_check_name(struct Curl_easy *data,
-                           const char *ace_hostname)
-{
-  size_t err_pos;
-  char *uc_name = NULL;
-  int rc;
-#ifndef CURL_DISABLE_VERBOSE_STRINGS
-  const char *tld_errmsg = "<no msg>";
-#else
-  (void)data;
-#endif
-
-  /* Convert (and downcase) ACE-name back into locale's character set */
-  rc = idna_to_unicode_lzlz(ace_hostname, &uc_name, 0);
-  if(rc != IDNA_SUCCESS)
-    return FALSE;
-
-  /* Warning: err_pos receives "the decoded character offset rather than the
-     byte position in the string." And as of libidn 1.32 that character offset
-     is for UTF-8, even if the passed in string is another locale. */
-  rc = tld_check_lz(uc_name, &err_pos, NULL);
-#ifndef CURL_DISABLE_VERBOSE_STRINGS
-#ifdef HAVE_TLD_STRERROR
-  if(rc != TLD_SUCCESS)
-    tld_errmsg = tld_strerror((Tld_rc)rc);
-#endif
-  if(rc != TLD_SUCCESS)
-    infof(data, "WARNING: TLD check for %s failed; %s\n",
-          uc_name, tld_errmsg);
-#endif /* CURL_DISABLE_VERBOSE_STRINGS */
-  if(uc_name)
-     idn_free(uc_name);
-  if(rc != TLD_SUCCESS)
-    return FALSE;
-
-  return TRUE;
-}
-#endif
-
 /*
  * Perform any necessary IDN conversion of hostname
  */
-static void fix_hostname(struct Curl_easy *data,
-                         struct connectdata *conn, struct hostname *host)
+static void fix_hostname(struct connectdata *conn, struct hostname *host)
 {
   size_t len;
+  struct Curl_easy *data = conn->data;
 
-#ifndef USE_LIBIDN
+#ifndef USE_LIBIDN2
   (void)data;
   (void)conn;
 #elif defined(CURL_DISABLE_VERBOSE_STRINGS)
@@ -3840,25 +3788,18 @@ static void fix_hostname(struct Curl_easy *data,
 
   /* Check name for non-ASCII and convert hostname to ACE form if we can */
   if(!is_ASCII_name(host->name)) {
-#ifdef USE_LIBIDN
-    if(stringprep_check_version(LIBIDN_REQUIRED_VERSION)) {
+#ifdef USE_LIBIDN2
+    if(idn2_check_version(IDN2_VERSION)) {
       char *ace_hostname = NULL;
-
-      int rc = idna_to_ascii_lz(host->name, &ace_hostname, 0);
-      infof(data, "Input domain encoded as `%s'\n",
-            stringprep_locale_charset());
-      if(rc == IDNA_SUCCESS) {
-        /* tld_check_name() displays a warning if the host name contains
-           "illegal" characters for this TLD */
-        (void)tld_check_name(data, ace_hostname);
-
-        host->encalloc = ace_hostname;
+      int rc = idn2_lookup_ul((const char *)host->name, &ace_hostname, 0);
+      if(rc == IDN2_OK) {
+        host->encalloc = (char *)ace_hostname;
         /* change the name pointer to point to the encoded hostname */
         host->name = host->encalloc;
       }
       else
         infof(data, "Failed to convert %s to ACE; %s\n", host->name,
-              Curl_idn_strerror(conn, rc));
+              idn2_strerror(rc));
     }
 #elif defined(USE_WIN32_IDN)
     char *ace_hostname = NULL;
@@ -3881,9 +3822,9 @@ static void fix_hostname(struct Curl_easy *data,
  */
 static void free_fixed_hostname(struct hostname *host)
 {
-#if defined(USE_LIBIDN)
+#if defined(USE_LIBIDN2)
   if(host->encalloc) {
-    idn_free(host->encalloc); /* must be freed with idn_free() since this was
+    idn2_free(host->encalloc); /* must be freed with idn2_free() since this was
                                  allocated by libidn */
     host->encalloc = NULL;
   }
@@ -6046,11 +5987,11 @@ static CURLcode create_conn(struct Curl_easy *data,
   /*************************************************************
    * IDN-fix the hostnames
    *************************************************************/
-  fix_hostname(data, conn, &conn->host);
+  fix_hostname(conn, &conn->host);
   if(conn->bits.conn_to_host)
-    fix_hostname(data, conn, &conn->conn_to_host);
+    fix_hostname(conn, &conn->conn_to_host);
   if(conn->proxy.name && *conn->proxy.name)
-    fix_hostname(data, conn, &conn->proxy);
+    fix_hostname(conn, &conn->proxy);
 
   /*************************************************************
    * Check whether the host and the "connect to host" are equal.
diff --git a/lib/version.c b/lib/version.c
index 12924453c..a434a6287 100644
--- a/lib/version.c
+++ b/lib/version.c
@@ -36,8 +36,8 @@
 #  include <ares.h>
 #endif
 
-#ifdef USE_LIBIDN
-#include <stringprep.h>
+#ifdef USE_LIBIDN2
+#include <idn2.h>
 #endif
 
 #ifdef USE_LIBPSL
@@ -111,9 +111,9 @@ char *curl_version(void)
   left -= len;
   ptr += len;
 #endif
-#ifdef USE_LIBIDN
-  if(stringprep_check_version(LIBIDN_REQUIRED_VERSION)) {
-    len = snprintf(ptr, left, " libidn/%s", stringprep_check_version(NULL));
+#ifdef USE_LIBIDN2
+  if(idn2_check_version(IDN2_VERSION)) {
+    len = snprintf(ptr, left, " libidn2/%s", idn2_check_version(NULL));
     left -= len;
     ptr += len;
   }
@@ -365,10 +365,10 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
     version_info.ares_num = aresnum;
   }
 #endif
-#ifdef USE_LIBIDN
+#ifdef USE_LIBIDN2
   /* This returns a version string if we use the given version or later,
      otherwise it returns NULL */
-  version_info.libidn = stringprep_check_version(LIBIDN_REQUIRED_VERSION);
+  version_info.libidn = idn2_check_version(IDN2_VERSION);
   if(version_info.libidn)
     version_info.features |= CURL_VERSION_IDN;
 #elif defined(USE_WIN32_IDN)