- David McCreedy provided libcurl changes for doing HTTP communication on

  non-ASCII platforms. It does add some complexity, most notably with more
  #ifdefs, but I want to see this supported added and I can't see how we can
  add it without the extra stuff added.

10 files changed, 299 insertions, 61 deletions

diff --git a/CHANGES b/CHANGES
index d762b7e29..eb6c127b0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -7,6 +7,11 @@
                                   Changelog
 
 Daniel (14 January 2007)
+- David McCreedy provided libcurl changes for doing HTTP communication on
+  non-ASCII platforms. It does add some complexity, most notably with more
+  #ifdefs, but I want to see this supported added and I can't see how we can
+  add it without the extra stuff added.
+
 - Setting CURLOPT_COOKIELIST to "ALL" when no cookies at all was present,
   libcurl would crash when trying to read a NULL pointer.
 
diff --git a/RELEASE-NOTES b/RELEASE-NOTES
index dafc16e2c..5f2ec0a70 100644
--- a/RELEASE-NOTES
+++ b/RELEASE-NOTES
@@ -5,15 +5,16 @@ Curl and libcurl 7.16.1
  Available command line options:           112
  Available curl_easy_setopt() options:     133
  Number of public functions in libcurl:    54
- Amount of public web site mirrors:        38
+ Amount of public web site mirrors:        39
  Number of known libcurl bindings:         35
  Number of contributors:                   524
 
 This release includes the following changes:
  
- o Support for SCP and SFTP were added
+ o Support for SCP and SFTP were added (powered by libssh2)
  o CURLOPT_CLOSEPOLICY is now deprecated
  o --ftp-ssl-ccc and CURLOPT_FTP_SSL_CCC were added
+ o HTTP support for non-ASCII platforms
 
 This release includes the following bugfixes:
 
diff --git a/lib/formdata.c b/lib/formdata.c
index 4c53045f2..2d98870e0 100644
--- a/lib/formdata.c
+++ b/lib/formdata.c
@@ -24,7 +24,7 @@
 /*
   Debug the form generator stand-alone by compiling this source file with:
 
-  gcc -DHAVE_CONFIG_H -I../ -g -D_FORM_DEBUG -o formdata -I../include formdata.c strequal.c
+  gcc -DHAVE_CONFIG_H -I../ -g -D_FORM_DEBUG -DCURLDEBUG -o formdata -I../include formdata.c strequal.c memdebug.c mprintf.c strerror.c
 
   run the 'formdata' executable the output should end with:
   All Tests seem to have worked ...
@@ -63,7 +63,7 @@ Content-Type: multipart/mixed, boundary=curlz1s0dkticx49MV1KGcYP5cvfSsz
 Content-Disposition: attachment; filename="inet_ntoa_r.h"
 Content-Type: text/plain
 ...
-Content-Disposition: attachment; filename="Makefile.b32.resp"
+Content-Disposition: attachment; filename="Makefile.b32"
 Content-Type: text/plain
 ...
 
@@ -73,7 +73,7 @@ Content-Type: multipart/mixed, boundary=curlirkYPmPwu6FrJ1vJ1u1BmtIufh1
 Content-Disposition: attachment; filename="inet_ntoa_r.h"
 Content-Type: text/plain
 ...
-Content-Disposition: attachment; filename="Makefile.b32.resp"
+Content-Disposition: attachment; filename="Makefile.b32"
 Content-Type: text/plain
 ...
 Content-Disposition: attachment; filename="inet_ntoa_r.h"
@@ -87,7 +87,7 @@ Content-Type: multipart/mixed, boundary=curlirkYPmPwu6FrJ1vJ1u1BmtIufh1
 Content-Disposition: attachment; filename="inet_ntoa_r.h"
 Content-Type: text/plain
 ...
-Content-Disposition: attachment; filename="Makefile.b32.resp"
+Content-Disposition: attachment; filename="Makefile.b32"
 Content-Type: text/plain
 ...
 Content-Disposition: attachment; filename="inet_ntoa_r.h"
@@ -118,6 +118,8 @@ Content-Disposition: form-data; name="FILECONTENT"
 #if defined(HAVE_LIBGEN_H) && defined(HAVE_BASENAME)
 #include <libgen.h>
 #endif
+#include "urldata.h" /* for struct SessionHandle */
+#include "easyif.h" /* for Curl_convert_... prototypes */
 #include "formdata.h"
 #include "strequal.h"
 #include "memory.h"
@@ -452,7 +454,12 @@ CURLFORMcode FormAdd(struct curl_httppost **httppost,
        * Set the Name property.
        */
     case CURLFORM_PTRNAME:
+#ifdef CURL_DOES_CONVERSIONS
+      /* treat CURLFORM_PTR like CURLFORM_COPYNAME so we'll
+         have safe memory for the eventual conversion */
+#else
       current_form->flags |= HTTPPOST_PTRNAME; /* fall through */
+#endif
     case CURLFORM_COPYNAME:
       if (current_form->name)
         return_value = CURL_FORMADD_OPTION_TWICE;
@@ -835,7 +842,7 @@ static CURLcode AddFormData(struct FormData **formp,
     *formp = newform;
 
   if (size) {
-    if(type == FORM_DATA)
+    if((type == FORM_DATA) || (type == FORM_CONTENT))
       *size += length;
     else {
       /* Since this is a file to be uploaded here, add the size of the actual
@@ -872,10 +879,11 @@ static CURLcode AddFormDataf(struct FormData **formp,
  * Curl_formclean() is used from http.c, this cleans a built FormData linked
  * list
  */
-void Curl_formclean(struct FormData *form)
+void Curl_formclean(struct FormData **form_ptr)
 {
-  struct FormData *next;
+  struct FormData *next, *form;
 
+  form = *form_ptr;
   if(!form)
     return;
 
@@ -885,8 +893,40 @@ void Curl_formclean(struct FormData *form)
     free(form);       /* free the struct */
 
   } while ((form = next) != NULL); /* continue */
+
+  *form_ptr = NULL;
 }
 
+#ifdef CURL_DOES_CONVERSIONS
+/*
+ * Curl_formcovert() is used from http.c, this converts any
+   form items that need to be sent in the network encoding.
+   Returns CURLE_OK on success.
+ */
+CURLcode Curl_formconvert(struct SessionHandle *data, struct FormData *form)
+{
+  struct FormData *next;
+  CURLcode rc;
+
+  if(!form)
+    return CURLE_OK;
+
+  if(!data)
+    return CURLE_BAD_FUNCTION_ARGUMENT;
+
+  do {
+    next=form->next;  /* the following form line */
+    if (form->type == FORM_DATA) {
+      rc = Curl_convert_to_network(data, form->line, form->length);
+      /* Curl_convert_to_network calls failf if unsuccessful */
+      if (rc != CURLE_OK)
+        return rc;
+    }
+  } while ((form = next) != NULL); /* continue */
+  return CURLE_OK;
+}
+#endif /* CURL_DOES_CONVERSIONS */
+
 /*
  * curl_formget()
  * Serialize a curl_httppost struct.
@@ -917,18 +957,18 @@ int curl_formget(struct curl_httppost *form, void *arg,
           if (temp.fp) {
             fclose(temp.fp);
           }
-          Curl_formclean(data);
+          Curl_formclean(&data);
           return -1;
         }
       } while (read == sizeof(buffer));
     } else {
       if (ptr->length != append(arg, ptr->line, ptr->length)) {
-        Curl_formclean(data);
+        Curl_formclean(&data);
         return -1;
       }
     }
   }
-  Curl_formclean(data);
+  Curl_formclean(&data);
   return 0;
 }
 
@@ -1179,7 +1219,7 @@ CURLcode Curl_getFormData(struct FormData **finalform,
         curList = curList->next;
       }
       if (result) {
-        Curl_formclean(firstform);
+        Curl_formclean(&firstform);
         free(boundary);
         return result;
       }
@@ -1194,7 +1234,10 @@ CURLcode Curl_getFormData(struct FormData **finalform,
       if(file->contenttype &&
          !checkprefix("text/", file->contenttype)) {
         /* this is not a text content, mention our binary encoding */
-        size += AddFormData(&form, "\r\nContent-Transfer-Encoding: binary", 0);
+        result = AddFormDataf(&form, &size,
+                              "\r\nContent-Transfer-Encoding: binary");
+        if (result)
+          break;
       }
 #endif
 
@@ -1232,21 +1275,26 @@ CURLcode Curl_getFormData(struct FormData **finalform,
             size_t nread;
             char buffer[512];
             while ((nread = fread(buffer, 1, sizeof(buffer), fileread)) != 0) {
-              result = AddFormData(&form, FORM_DATA, buffer, nread, &size);
+              result = AddFormData(&form, FORM_CONTENT, buffer, nread, &size);
               if (result)
                 break;
             }
           }
 
           if (result) {
-            Curl_formclean(firstform);
+            Curl_formclean(&firstform);
             free(boundary);
             return result;
           }
 
         }
         else {
-          Curl_formclean(firstform);
+#ifdef _FORM_DEBUG
+          fprintf(stderr,
+                  "\n==> Curl_getFormData couldn't open/read \"%s\"\n",
+                  file->contents);
+#endif
+          Curl_formclean(&firstform);
           free(boundary);
           *finalform = NULL;
           return CURLE_READ_ERROR;
@@ -1255,7 +1303,7 @@ CURLcode Curl_getFormData(struct FormData **finalform,
       }
       else if (post->flags & HTTPPOST_BUFFER) {
         /* include contents of buffer */
-        result = AddFormData(&form, FORM_DATA, post->buffer,
+        result = AddFormData(&form, FORM_CONTENT, post->buffer,
                              post->bufferlength, &size);
           if (result)
             break;
@@ -1263,14 +1311,14 @@ CURLcode Curl_getFormData(struct FormData **finalform,
 
       else {
         /* include the contents we got */
-        result = AddFormData(&form, FORM_DATA, post->contents,
+        result = AddFormData(&form, FORM_CONTENT, post->contents,
                              post->contentslength, &size);
         if (result)
           break;
       }
     } while ((file = file->more) != NULL); /* for each specified file for this field */
     if (result) {
-      Curl_formclean(firstform);
+      Curl_formclean(&firstform);
       free(boundary);
       return result;
     }
@@ -1288,7 +1336,7 @@ CURLcode Curl_getFormData(struct FormData **finalform,
 
   } while ((post = post->next) != NULL); /* for each field */
   if (result) {
-    Curl_formclean(firstform);
+    Curl_formclean(&firstform);
     free(boundary);
     return result;
   }
@@ -1298,7 +1346,7 @@ CURLcode Curl_getFormData(struct FormData **finalform,
                        "\r\n--%s--\r\n",
                        boundary);
   if (result) {
-    Curl_formclean(firstform);
+    Curl_formclean(&firstform);
     free(boundary);
     return result;
   }
@@ -1397,7 +1445,7 @@ size_t Curl_FormReader(char *buffer,
 
     form->data = form->data->next; /* advance */
 
-  } while(form->data && (form->data->type == FORM_DATA));
+  } while(form->data && (form->data->type != FORM_FILE));
   /* If we got an empty line and we have more data, we proceed to the next
      line immediately to avoid returning zero before we've reached the end.
      This is the bug reported November 22 1999 on curl 6.3. (Daniel) */
@@ -1464,7 +1512,7 @@ int main()
   char value5[] = "value for PTRCONTENTS + CONTENTSLENGTH";
   char value6[] = "value for PTRCOTNENTS + CONTENTSLENGTH + CONTENTTYPE";
   char value7[] = "inet_ntoa_r.h";
-  char value8[] = "Makefile.b32.resp";
+  char value8[] = "Makefile.b32";
   char type2[] = "image/gif";
   char type6[] = "text/plain";
   char type7[] = "text/html";
@@ -1473,7 +1521,8 @@ int main()
   int value5length = strlen(value4);
   int value6length = strlen(value5);
   int errors = 0;
-  int size;
+  CURLcode rc;
+  size_t size;
   size_t nread;
   char buffer[4096];
   struct curl_httppost *httppost=NULL;
@@ -1549,7 +1598,14 @@ int main()
                   CURLFORM_END))
     ++errors;
 
-  form=Curl_getFormData(httppost, &size);
+  rc = Curl_getFormData(&form, httppost, NULL, &size);
+  if(rc != CURLE_OK) {
+    if(rc != CURLE_READ_ERROR) {
+      const char *errortext = curl_easy_strerror(rc);
+      fprintf(stdout, "\n==> Curl_getFormData error: %s\n", errortext);
+    }
+    return 0;
+  }
 
   Curl_FormInit(&formread, form);
 
@@ -1557,7 +1613,7 @@ int main()
     nread = Curl_FormReader(buffer, 1, sizeof(buffer),
                             (FILE *)&formread);
 
-    if(-1 == nread)
+    if(nread < 1)
       break;
     fwrite(buffer, nread, 1, stdout);
   } while(1);
diff --git a/lib/formdata.h b/lib/formdata.h
index 304d009ed..0cac4ac27 100644
--- a/lib/formdata.h
+++ b/lib/formdata.h
@@ -25,8 +25,10 @@
  ***************************************************************************/
 
 enum formtype {
-  FORM_DATA, /* regular data */
-  FORM_FILE  /* 'line' points to a file name we should read from */
+  FORM_DATA,    /* form metadata (convert to network encoding if necessary) */
+  FORM_CONTENT, /* form content  (never convert) */
+  FORM_FILE     /* 'line' points to a file name we should read from 
+                    to create the form data (never convert) */
 };
 
 /* plain and simple linked list with lines to send */
@@ -87,7 +89,9 @@ char *Curl_formpostheader(void *formp, size_t *len);
 
 char *Curl_FormBoundary(void);
 
-void Curl_formclean(struct FormData *);
+void Curl_formclean(struct FormData **);
+
+CURLcode Curl_formconvert(struct SessionHandle *, struct FormData *);
 
 #endif
 
diff --git a/lib/http.c b/lib/http.c
index ae7c3cfbc..0387298c1 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -80,6 +80,7 @@
 #include <curl/curl.h>
 #include "transfer.h"
 #include "sendf.h"
+#include "easyif.h" /* for Curl_convert_... prototypes */
 #include "formdata.h"
 #include "progress.h"
 #include "base64.h"
@@ -154,7 +155,7 @@ static CURLcode Curl_output_basic(struct connectdata *conn, bool proxy)
                         &authorization) > 0) {
     if(*userp)
       free(*userp);
-    *userp = aprintf( "%sAuthorization: Basic %s\015\012",
+    *userp = aprintf( "%sAuthorization: Basic %s\r\n",
                       proxy?"Proxy-":"",
                       authorization);
     free(authorization);
@@ -873,6 +874,20 @@ CURLcode add_buffer_send(send_buffer *in,
   ptr = in->buffer;
   size = in->size_used;
 
+#ifdef CURL_DOES_CONVERSIONS
+  if(size - included_body_bytes > 0) {
+    res = Curl_convert_to_network(conn->data, ptr, size - included_body_bytes);
+    /* Curl_convert_to_network calls failf if unsuccessful */
+    if(res != CURLE_OK) {
+      /* conversion failed, free memory and return to the caller */
+      if(in->buffer)
+        free(in->buffer);
+      free(in);
+      return res;
+    }
+  }
+#endif /* CURL_DOES_CONVERSIONS */
+
   if(conn->protocol & PROT_HTTPS) {
     /* We never send more than CURL_MAX_WRITE_SIZE bytes in one single chunk
        when we speak HTTPS, as if only a fraction of it is sent now, this data
@@ -1512,7 +1527,7 @@ CURLcode Curl_http_done(struct connectdata *conn,
   if(HTTPREQ_POST_FORM == data->set.httpreq) {
     k->bytecount = http->readbytecount + http->writebytecount;
 
-    Curl_formclean(http->sendit); /* Now free that whole lot */
+    Curl_formclean(&http->sendit); /* Now free that whole lot */
     if(http->form.fp) {
       /* a file being uploaded was left opened, close it! */
       fclose(http->form.fp);
@@ -1699,7 +1714,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
 
   Curl_safefree(conn->allocptr.ref);
   if(data->change.referer && !checkheaders(data, "Referer:"))
-    conn->allocptr.ref = aprintf("Referer: %s\015\012", data->change.referer);
+    conn->allocptr.ref = aprintf("Referer: %s\r\n", data->change.referer);
   else
     conn->allocptr.ref = NULL;
 
@@ -1710,7 +1725,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
      data->set.encoding) {
     Curl_safefree(conn->allocptr.accept_encoding);
     conn->allocptr.accept_encoding =
-      aprintf("Accept-Encoding: %s\015\012", data->set.encoding);
+      aprintf("Accept-Encoding: %s\r\n", data->set.encoding);
     if(!conn->allocptr.accept_encoding)
       return CURLE_OUT_OF_MEMORY;
   }
@@ -2194,10 +2209,19 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
                                      &http->readbytecount,
                                      FIRSTSOCKET,
                                      &http->writebytecount);
+
+      if(result) {
+        Curl_formclean(&http->sendit); /* free that whole lot */
+        return result;
+      }
+#ifdef CURL_DOES_CONVERSIONS
+/* time to convert the form data... */
+      result = Curl_formconvert(data, http->sendit);
       if(result) {
-        Curl_formclean(http->sendit); /* free that whole lot */
+        Curl_formclean(&http->sendit); /* free that whole lot */
         return result;
       }
+#endif /* CURL_DOES_CONVERSIONS */
       break;
 
     case HTTPREQ_PUT: /* Let's PUT the data to the server! */
@@ -2316,8 +2340,8 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
                                   (size_t)postsize);
             if(CURLE_OK == result)
               result = add_buffer(req_buffer,
-                                  "\r\n0\r\n\r\n", 7); /* end of a chunked
-                                                          transfer stream */
+                                  "\x0d\x0a\x30\x0d\x0a\x0d\x0a", 7);
+                                  /* CR  LF   0  CR  LF  CR  LF */
             included_body = postsize + 7;
           }
           if(result)
diff --git a/lib/http_chunks.c b/lib/http_chunks.c
index f398b100c..d9774a2c3 100644
--- a/lib/http_chunks.c
+++ b/lib/http_chunks.c
@@ -36,6 +36,7 @@
 #include "content_encoding.h"
 #include "http.h"
 #include "memory.h"
+#include "easyif.h" /* for Curl_convert_to_network prototype */
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -96,6 +97,9 @@ void Curl_httpchunk_init(struct connectdata *conn)
  * client (for byte-counting and whatever).
  *
  * The states and the state-machine is further explained in the header file.
+ *
+ * This function always uses ASCII hex values to accommodate non-ASCII hosts.
+ * For example, 0x0d and 0x0a are used instead of '\r' and '\n'.
  */
 CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
                               char *datap,
@@ -115,7 +119,11 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
   while(length) {
     switch(ch->state) {
     case CHUNK_HEX:
-      if(ISXDIGIT(*datap)) {
+       /* Check for an ASCII hex digit.
+          We avoid the use of isxdigit to accommodate non-ASCII hosts. */
+       if((*datap >= 0x30 && *datap <= 0x39)    /* 0-9 */
+       || (*datap >= 0x41 && *datap <= 0x46)    /* A-F */
+       || (*datap >= 0x61 && *datap <= 0x66)) { /* a-f */
         if(ch->hexindex < MAXNUM_SIZE) {
           ch->hexbuffer[ch->hexindex] = *datap;
           datap++;
@@ -134,6 +142,17 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
         }
         /* length and datap are unmodified */
         ch->hexbuffer[ch->hexindex]=0;
+#ifdef CURL_DOES_CONVERSIONS
+        /* convert to host encoding before calling strtoul */
+        result = Curl_convert_from_network(conn->data,
+                                           ch->hexbuffer,
+                                           ch->hexindex);
+        if(result != CURLE_OK) {
+          /* Curl_convert_from_network calls failf if unsuccessful */
+          /* Treat it as a bad hex character */
+          return(CHUNKE_ILLEGAL_HEX);
+        }
+#endif /* CURL_DOES_CONVERSIONS */
         ch->datasize=strtoul(ch->hexbuffer, NULL, 16);
         ch->state = CHUNK_POSTHEX;
       }
@@ -143,7 +162,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       /* In this state, we're waiting for CRLF to arrive. We support
          this to allow so called chunk-extensions to show up here
          before the CRLF comes. */
-      if(*datap == '\r')
+      if(*datap == 0x0d)
         ch->state = CHUNK_CR;
       length--;
       datap++;
@@ -151,7 +170,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 
     case CHUNK_CR:
       /* waiting for the LF */
-      if(*datap == '\n') {
+      if(*datap == 0x0a) {
         /* we're now expecting data to come, unless size was zero! */
         if(0 == ch->datasize) {
           if (conn->bits.trailerHdrPresent!=TRUE) {
@@ -235,7 +254,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       break;
 
     case CHUNK_POSTCR:
-      if(*datap == '\r') {
+      if(*datap == 0x0d) {
         ch->state = CHUNK_POSTLF;
         datap++;
         length--;
@@ -245,7 +264,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       break;
 
     case CHUNK_POSTLF:
-      if(*datap == '\n') {
+      if(*datap == 0x0a) {
         /*
          * The last one before we go back to hex state and start all
          * over.
@@ -277,7 +296,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       }
       conn->trailer[conn->trlPos++]=*datap;
 
-      if(*datap == '\r')
+      if(*datap == 0x0d)
         ch->state = CHUNK_TRAILER_CR;
       else {
         datap++;
@@ -286,7 +305,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       break;
 
     case CHUNK_TRAILER_CR:
-      if(*datap == '\r') {
+      if(*datap == 0x0d) {
         ch->state = CHUNK_TRAILER_POSTCR;
         datap++;
         length--;
@@ -296,14 +315,25 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       break;
 
     case CHUNK_TRAILER_POSTCR:
-      if (*datap == '\n') {
-        conn->trailer[conn->trlPos++]='\n';
+      if (*datap == 0x0a) {
+        conn->trailer[conn->trlPos++]=0x0a;
         conn->trailer[conn->trlPos]=0;
         if (conn->trlPos==2) {
           ch->state = CHUNK_STOP;
           return CHUNKE_STOP;
         }
         else {
+#ifdef CURL_DOES_CONVERSIONS
+        /* Convert to host encoding before calling Curl_client_write */
+        result = Curl_convert_from_network(conn->data,
+                                           ch->hexbuffer,
+                                           ch->hexindex);
+        if(result != CURLE_OK) {
+          /* Curl_convert_from_network calls failf if unsuccessful */
+          /* Treat it as a bad chunk */
+          return(CHUNKE_BAD_CHUNK);
+        }
+#endif /* CURL_DOES_CONVERSIONS */
           Curl_client_write(conn, CLIENTWRITE_HEADER,
                             conn->trailer, conn->trlPos);
         }
diff --git a/lib/http_digest.c b/lib/http_digest.c
index e5d8baae5..c223784f9 100644
--- a/lib/http_digest.c
+++ b/lib/http_digest.c
@@ -39,6 +39,7 @@
 #include "strtok.h"
 #include "url.h" /* for Curl_safefree() */
 #include "memory.h"
+#include "easyif.h" /* included for Curl_convert_... prototypes */
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -234,6 +235,21 @@ CURLcode Curl_output_digest(struct connectdata *conn,
 
   struct SessionHandle *data = conn->data;
   struct digestdata *d;
+#ifdef CURL_DOES_CONVERSIONS
+  CURLcode rc;
+/* The CURL_OUTPUT_DIGEST_CONV macro below is for non-ASCII machines.
+   It converts digest text to ASCII so the MD5 will be correct for 
+   what ultimately goes over the network.
+*/
+#define CURL_OUTPUT_DIGEST_CONV(a, b) \
+  rc = Curl_convert_to_network(a, (char *)b, strlen((const char*)b)); \
+  if (rc != CURLE_OK) { \
+    free(b); \
+    return rc; \
+  }
+#else
+#define CURL_OUTPUT_DIGEST_CONV(a, b)
+#endif /* CURL_DOES_CONVERSIONS */
 
   if(proxy) {
     d = &data->state.proxydigest;
@@ -291,6 +307,8 @@ CURLcode Curl_output_digest(struct connectdata *conn,
     aprintf("%s:%s:%s", userp, d->realm, passwdp);
   if(!md5this)
     return CURLE_OUT_OF_MEMORY;
+
+  CURL_OUTPUT_DIGEST_CONV(data, md5this); /* convert on non-ASCII machines */
   Curl_md5it(md5buf, md5this);
   free(md5this); /* free this again */
 
@@ -305,6 +323,7 @@ CURLcode Curl_output_digest(struct connectdata *conn,
     tmp = aprintf("%s:%s:%s", ha1, d->nonce, d->cnonce);
     if(!tmp)
       return CURLE_OUT_OF_MEMORY;
+    CURL_OUTPUT_DIGEST_CONV(data, tmp); /* convert on non-ASCII machines */
     Curl_md5it(md5buf, (unsigned char *)tmp);
     free(tmp); /* free this again */
     md5_to_ascii(md5buf, ha1);
@@ -334,6 +353,7 @@ CURLcode Curl_output_digest(struct connectdata *conn,
        entity-body here */
     /* TODO: Append H(entity-body)*/
   }
+  CURL_OUTPUT_DIGEST_CONV(data, md5this); /* convert on non-ASCII machines */
   Curl_md5it(md5buf, md5this);
   free(md5this); /* free this again */
   md5_to_ascii(md5buf, ha2);
@@ -357,6 +377,7 @@ CURLcode Curl_output_digest(struct connectdata *conn,
   if(!md5this)
     return CURLE_OUT_OF_MEMORY;
 
+  CURL_OUTPUT_DIGEST_CONV(data, md5this); /* convert on non-ASCII machines */
   Curl_md5it(md5buf, md5this);
   free(md5this); /* free this again */
   md5_to_ascii(md5buf, request_digest);
diff --git a/lib/sendf.c b/lib/sendf.c
index d0688ec21..8886b8372 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -454,7 +454,9 @@ CURLcode Curl_client_write(struct connectdata *conn,
   return CURLE_OK;
 }
 
+#ifndef MIN
 #define MIN(a,b) ((a) < (b) ? (a) : (b))
+#endif
 
 /*
  * Internal read-from-socket function. This is meant to deal with plain
@@ -563,6 +565,7 @@ static int showit(struct SessionHandle *data, curl_infotype type,
 
 #ifdef CURL_DOES_CONVERSIONS
   char buf[BUFSIZE+1];
+  int conv_size;
 
   switch(type) {
   case CURLINFO_HEADER_OUT:
@@ -572,8 +575,24 @@ static int showit(struct SessionHandle *data, curl_infotype type,
       size = BUFSIZE; /* truncate if necessary */
       buf[BUFSIZE] = '\0';
     }
+    conv_size = size;
     memcpy(buf, ptr, size);
-    Curl_convert_from_network(data, buf, size);
+    /* Special processing is needed for this block if it
+     * contains both headers and data (separated by CRLFCRLF).
+     * We want to convert just the headers, leaving the data as-is.
+     */
+    if(size > 4) {
+      int i;
+      for(i = 0; i < size-4; i++) {
+        if(memcmp(&buf[i], "\x0d\x0a\x0d\x0a", 4) == 0) {
+          /* convert everthing through this CRLFCRLF but no further */
+          conv_size = i + 4;
+          break;
+        }
+      }
+    }
+
+    Curl_convert_from_network(data, buf, conv_size);
     /* Curl_convert_from_network calls failf if unsuccessful */
     /* we might as well continue even if it fails...   */
     ptr = buf; /* switch pointer to use my buffer instead */
@@ -594,6 +613,12 @@ static int showit(struct SessionHandle *data, curl_infotype type,
   case CURLINFO_HEADER_IN:
     fwrite(s_infotype[type], 2, 1, data->set.err);
     fwrite(ptr, size, 1, data->set.err);
+#ifdef CURL_DOES_CONVERSIONS
+    if(size != conv_size) {
+      /* we had untranslated data so we need an explicit newline */
+      fwrite("\n", 1, 1, data->set.err);
+    }
+#endif
     break;
   default: /* nada */
     break;
diff --git a/lib/transfer.c b/lib/transfer.c
index ca112331e..ee3c9851c 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -189,17 +189,39 @@ checkhttpprefix(struct SessionHandle *data,
                 const char *s)
 {
   struct curl_slist *head = data->set.http200aliases;
+  bool rc = FALSE;
+#ifdef CURL_DOES_CONVERSIONS
+  /* convert from the network encoding using a scratch area */
+  char *scratch = calloc(1, strlen(s)+1);
+  if (NULL == scratch) {
+     failf (data, "Failed to calloc memory for conversion!");
+     return FALSE; /* can't return CURLE_OUT_OF_MEMORY so return FALSE */
+  }
+  strcpy(scratch, s);
+  if (CURLE_OK != Curl_convert_from_network(data, scratch, strlen(s)+1)) {
+    /* Curl_convert_from_network calls failf if unsuccessful */
+     free(scratch);
+     return FALSE; /* can't return CURLE_foobar so return FALSE */
+  }
+  s = scratch;
+#endif /* CURL_DOES_CONVERSIONS */
 
   while (head) {
-    if (checkprefix(head->data, s))
-      return TRUE;
+    if (checkprefix(head->data, s)) {
+      rc = TRUE;
+      break;
+    }
     head = head->next;
   }
 
-  if(checkprefix("HTTP/", s))
-    return TRUE;
+  if ((rc != TRUE) && (checkprefix("HTTP/", s))) {
+    rc = TRUE;
+  }
 
-  return FALSE;
+#ifdef CURL_DOES_CONVERSIONS
+  free(scratch);
+#endif /* CURL_DOES_CONVERSIONS */
+  return rc;
 }
 
 /*
@@ -412,7 +434,8 @@ CURLcode Curl_readwrite(struct connectdata *conn,
             /* str_start is start of line within buf */
             k->str_start = k->str;
 
-            k->end_ptr = memchr(k->str_start, '\n', nread);
+            /* data is in network encoding so use 0x0a instead of '\n' */
+            k->end_ptr = memchr(k->str_start, 0x0a, nread);
 
             if (!k->end_ptr) {
               /* Not a complete header line within buffer, append the data to
@@ -510,14 +533,27 @@ CURLcode Curl_readwrite(struct connectdata *conn,
               }
             }
 
-            if (('\n' == *k->p) || ('\r' == *k->p)) {
+            /* headers are in network encoding so
+               use 0x0a and 0x0d instead of '\n' and '\r' */
+            if ((0x0a == *k->p) || (0x0d == *k->p)) {
               size_t headerlen;
               /* Zero-length header line means end of headers! */
 
+#ifdef CURL_DOES_CONVERSIONS
+              if (0x0d == *k->p) {
+                *k->p = '\r'; /* replace with CR in host encoding */
+                k->p++;       /* pass the CR byte */
+              }
+              if (0x0a == *k->p) {
+                *k->p = '\n'; /* replace with LF in host encoding */
+                k->p++;       /* pass the LF byte */
+              }
+#else
               if ('\r' == *k->p)
                 k->p++; /* pass the \r byte */
               if ('\n' == *k->p)
                 k->p++; /* pass the \n byte */
+#endif /* CURL_DOES_CONVERSIONS */
 
               if(100 == k->httpcode) {
                 /*
@@ -679,12 +715,37 @@ CURLcode Curl_readwrite(struct connectdata *conn,
 
             if (!k->headerline++) {
               /* This is the first header, it MUST be the error code line
-                 or else we consiser this to be the body right away! */
+                 or else we consider this to be the body right away! */
               int httpversion_major;
-              int nc=sscanf(k->p, " HTTP/%d.%d %3d",
-                            &httpversion_major,
-                            &k->httpversion,
-                            &k->httpcode);
+              int nc;
+#ifdef CURL_DOES_CONVERSIONS
+#define HEADER1 scratch
+#define SCRATCHSIZE 21
+              CURLcode res;
+              char scratch[SCRATCHSIZE+1]; /* "HTTP/major.minor 123" */
+              /* We can't really convert this yet because we
+                 don't know if it's the 1st header line or the body.
+                 So we do a partial conversion into a scratch area,
+                 leaving the data at k->p as-is.
+              */
+              strncpy(&scratch[0], k->p, SCRATCHSIZE);
+              scratch[SCRATCHSIZE] = 0; /* null terminate */
+              res = Curl_convert_from_network(data,
+                                              &scratch[0],
+                                              SCRATCHSIZE);
+              if (CURLE_OK != res) {
+                /* Curl_convert_from_network calls failf if unsuccessful */
+                return res;
+              }
+#else
+#define HEADER1 k->p /* no conversion needed, just use k->p */
+#endif /* CURL_DOES_CONVERSIONS */
+
+              nc = sscanf(HEADER1,
+                          " HTTP/%d.%d %3d",
+                          &httpversion_major,
+                          &k->httpversion,
+                          &k->httpcode);
               if (nc==3) {
                 k->httpversion += 10 * httpversion_major;
               }
@@ -692,7 +753,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                 /* this is the real world, not a Nirvana
                    NCSA 1.5.x returns this crap when asked for HTTP/1.1
                 */
-                nc=sscanf(k->p, " HTTP %3d", &k->httpcode);
+                nc=sscanf(HEADER1, " HTTP %3d", &k->httpcode);
                 k->httpversion = 10;
 
                /* If user has set option HTTP200ALIASES,
@@ -775,6 +836,15 @@ CURLcode Curl_readwrite(struct connectdata *conn,
               }
             }
 
+#ifdef CURL_DOES_CONVERSIONS
+            /* convert from the network encoding */
+            result = Curl_convert_from_network(data, k->p, strlen(k->p));
+            if (CURLE_OK != result) {
+               return(result);
+            }
+            /* Curl_convert_from_network calls failf if unsuccessful */
+#endif /* CURL_DOES_CONVERSIONS */
+
             /* Check for Content-Length: header lines to get size. Ignore
                the header completely if we get a 416 response as then we're
                resuming a document that we don't get, and this header contains
@@ -812,6 +882,8 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                   start++)
                 ;  /* empty loop */
 
+              /* data is now in the host encoding so
+                 use '\r' and '\n' instead of 0x0d and 0x0a */
               end = strchr(start, '\r');
               if(!end)
                 end = strchr(start, '\n');
diff --git a/lib/url.c b/lib/url.c
index 0f3f85a5f..123d75b67 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -3948,7 +3948,7 @@ static CURLcode SetupConnection(struct connectdata *conn,
     if(data->set.useragent) {
       Curl_safefree(conn->allocptr.uagent);
       conn->allocptr.uagent =
-        aprintf("User-Agent: %s\015\012", data->set.useragent);
+        aprintf("User-Agent: %s\r\n", data->set.useragent);
       if(!conn->allocptr.uagent)
         return CURLE_OUT_OF_MEMORY;
     }