smtp: Support the SMTPUTF8 extension in the MAIL command

Support the SMTPUTF8 extension when sending mailbox information in the
MAIL command (FROM and AUTH parameters). Non-ASCII domain names will
be ACE encoded, if IDN is supported, whilst non-ASCII characters in
the local address part are passed to the server.

Reported-by: ygthien on github
Fixes #4828

4 files changed, 111 insertions, 19 deletions

diff --git a/lib/smtp.c b/lib/smtp.c
index 6a87849bb..cdeeb7a26 100644
--- a/lib/smtp.c
+++ b/lib/smtp.c
@@ -539,6 +539,12 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
   CURLcode result = CURLE_OK;
   struct Curl_easy *data = conn->data;
 
+  /* We notify the server we are sending UTF-8 data if a) it supports the
+     SMTPUTF8 extension and b) The mailbox contains UTF-8 charaacters, in
+     either the local address or host name parts. This is regardless of
+     whether the host name is encoded using IDN ACE */
+  bool utf8 = FALSE;
+
   /* Calculate the FROM parameter */
   if(!data->set.str[STRING_MAIL_FROM])
     /* Null reverse-path, RFC-5321, sect. 3.6.3 */
@@ -554,6 +560,12 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
     if(result)
       return result;
 
+    /* Establish whether we should report SMTPUTF8 to the server for this
+       mailbox as per RFC-6531 sect. 3.1 point 4 and sect. 3.4 */
+    utf8 = (conn->proto.smtpc.utf8_supported) &&
+           ((host.encalloc) || (!Curl_is_ASCII_name(address)) ||
+            (!Curl_is_ASCII_name(host.name)));
+
     if(host.name) {
       from = aprintf("<%s@%s>", address, host.name);
 
@@ -583,6 +595,13 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
       if(result)
         return result;
 
+      /* Establish whether we should report SMTPUTF8 to the server for this
+         mailbox as per RFC-6531 sect. 3.1 point 4 and sect. 3.4 */
+      if((!utf8) && (conn->proto.smtpc.utf8_supported) &&
+         ((host.encalloc) || (!Curl_is_ASCII_name(address)) ||
+          (!Curl_is_ASCII_name(host.name))))
+        utf8 = TRUE;
+
       if(host.name) {
         from = aprintf("<%s@%s>", address, host.name);
 
@@ -653,12 +672,14 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
 
   /* Send the MAIL command */
   result = Curl_pp_sendf(&conn->proto.smtpc.pp,
-                         "MAIL FROM:%s%s%s%s%s",
-                         from,                 /* Mandatory */
-                         auth ? " AUTH=" : "", /* Optional (on AUTH support) */
-                         auth ? auth : "",
-                         size ? " SIZE=" : "", /* Optional (on SIZE support) */
-                         size ? size : "");
+                         "MAIL FROM:%s%s%s%s%s%s",
+                         from,                 /* Mandatory                 */
+                         auth ? " AUTH=" : "", /* Optional on AUTH support  */
+                         auth ? auth : "",     /*                           */
+                         size ? " SIZE=" : "", /* Optional on SIZE support  */
+                         size ? size : "",     /*                           */
+                         utf8 ? " SMTPUTF8"    /* Internationalised mailbox */
+                               : "");          /* address included          */
 
   free(from);
   free(auth);
@@ -1677,6 +1698,10 @@ static CURLcode smtp_parse_custom_request(struct connectdata *conn)
  *
  * Notes:
  *
+ * Should a UTF-8 host name require conversion to IDN ACE and we cannot honor
+ * that convertion then we shall return success. This allow the caller to send
+ * the data to the server as a U-label (as per RFC-6531 sect. 3.2).
+ *
  * If an mailbox '@' seperator cannot be located then the mailbox is considered
  * to be either a local mailbox or an invalid mailbox (depending on what the
  * calling function deems it to be) then the input will simply be returned in
@@ -1704,14 +1729,12 @@ static CURLcode smtp_parse_address(struct connectdata *conn, const char *fqma,
     *host->name = '\0';
     host->name = host->name + 1;
 
-    /* Convert the host name to IDN ACE */
-    result = Curl_idnconvert_hostname(conn, host);
-    if(result) {
-      free(dup);
-      host->name = NULL;
+    /* Attempt to convert the host name to IDN ACE */
+    (void) Curl_idnconvert_hostname(conn, host);
 
-      return result;
-    }
+    /* If Curl_idnconvert_hostname() fails then we shall attempt to continue
+       and send the host name using UTF-8 rather than as 7-bit ACE (which is
+       our preference) */
   }
   else
     host->name = NULL;
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 5215f442f..4ddfeef36 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -109,7 +109,7 @@ test927 test928 test929 test930 test931 test932 test933 test934 test935 \
 test936 test937 test938 test939 test940 test941 test942 test943 test944 \
 test945 test946 test947 test948 test949 test950 test951 test952 test953 \
 test954 test955 test956 test957 test958 test959 test960 test961 test962 \
-test963 test964 \
+test963 test964 test965 \
 \
 test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 \
 test1008 test1009 test1010 test1011 test1012 test1013 test1014 test1015 \
diff --git a/tests/data/test965 b/tests/data/test965
new file mode 100644
index 000000000..4edfd7827
--- /dev/null
+++ b/tests/data/test965
@@ -0,0 +1,65 @@
+<testcase>
+<info>
+<keywords>
+SMTP
+IDN
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<servercmd>
+CAPA SMTPUTF8
+</servercmd>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+smtp
+</server>
+<features>
+idn
+</features>
+<setenv>
+LC_ALL=en_US.UTF-8
+LC_CTYPE=en_US.UTF-8
+</setenv>
+<precheck>
+perl -MI18N::Langinfo=langinfo,CODESET -e 'die "Needs a UTF-8 locale" if (lc(langinfo(CODESET())) ne "utf-8");'
+</precheck>
+ <name>
+SMTP with SMTPUTF8 support - UTF-8 based sender
+ </name>
+<stdin>
+From: different

+To: another

+

+body

+</stdin>
+<command>
+smtp://%HOSTIP:%SMTPPORT/965 --mail-rcpt recipient@example.com --mail-from Avsändaren@åäö.se -T -
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+EHLO 965

+MAIL FROM:<Avsändaren@xn--4cab6c.se> SMTPUTF8

+RCPT TO:<recipient@example.com>

+DATA

+QUIT

+</protocol>
+<upload>
+From: different

+To: another

+

+body

+.

+</upload>
+</verify>
+</testcase>
diff --git a/tests/ftpserver.pl b/tests/ftpserver.pl
index 59a1665bd..fe74f8b07 100755
--- a/tests/ftpserver.pl
+++ b/tests/ftpserver.pl
@@ -813,6 +813,7 @@ sub MAIL_smtp {
     else {
         my $from;
         my $size;
+        my $smtputf8 = grep /^SMTPUTF8$/, @capabilities;
         my @elements = split(/ /, $args);
 
         # Get the FROM and SIZE parameters
@@ -827,11 +828,11 @@ sub MAIL_smtp {
 
         # Validate the from address (only <> and a valid email address inside
         # <> are allowed, such as <user@example.com>)
-        if ((!$from) || (($from ne "<>") && ($from !~
-            /^<([a-zA-Z0-9._%+-]+)\@(([a-zA-Z0-9-]+)\.)+([a-zA-Z]{2,4})>$/))) {
-            sendcontrol "501 Invalid address\r\n";
-        }
-        else {
+        if (($from eq "<>") ||
+            (!$smtputf8 && $from =~
+              /^<([a-zA-Z0-9._%+-]+)\@(([a-zA-Z0-9-]+)\.)+([a-zA-Z]{2,4})>$/) ||
+            ($smtputf8 && $from =~
+              /^<([a-zA-Z0-9\x{80}-\x{ff}._%+-]+)\@(([a-zA-Z0-9\x{80}-\x{ff}-]+)\.)+([a-zA-Z]{2,4})>$/)) {
             my @found;
             my $valid = 1;
 
@@ -852,6 +853,9 @@ sub MAIL_smtp {
                 sendcontrol "250 Sender OK\r\n";
             }
         }
+        else {
+            sendcontrol "501 Invalid address\r\n";
+        }
     }
 
     return 0;