x509asn1: cleanup and unify code layout

- rename 'n' to buflen in functions, and use size_t for them. Don't pass
  in negative buffer lengths.

- move most function comments to above the function starts like we use
  to

- remove several unnecessary typecasts (especially of NULL)

Reviewed-by: Patrick Monnerat
Closes #3582

1 files changed, 132 insertions, 117 deletions

diff --git a/lib/x509asn1.c b/lib/x509asn1.c
index 5410e0575..25231921c 100644
--- a/lib/x509asn1.c
+++ b/lib/x509asn1.c
@@ -120,7 +120,7 @@ static const char *getASN1Element(curl_asn1Element *elem,
      if an error occurs. */
   if(!beg || !end || beg >= end || !*beg ||
      (size_t)(end - beg) > CURL_ASN1_MAX)
-    return (const char *) NULL;
+    return NULL;
 
   /* Process header byte. */
   elem->header = beg;
@@ -129,12 +129,12 @@ static const char *getASN1Element(curl_asn1Element *elem,
   elem->class = (b >> 6) & 3;
   b &= 0x1F;
   if(b == 0x1F)
-    return (const char *) NULL; /* Long tag values not supported here. */
+    return NULL; /* Long tag values not supported here. */
   elem->tag = b;
 
   /* Process length. */
   if(beg >= end)
-    return (const char *) NULL;
+    return NULL;
   b = (unsigned char) *beg++;
   if(!(b & 0x80))
     len = b;
@@ -142,69 +142,72 @@ static const char *getASN1Element(curl_asn1Element *elem,
     /* Unspecified length. Since we have all the data, we can determine the
        effective length by skipping element until an end element is found. */
     if(!elem->constructed)
-      return (const char *) NULL;
+      return NULL;
     elem->beg = beg;
     while(beg < end && *beg) {
       beg = getASN1Element(&lelem, beg, end);
       if(!beg)
-        return (const char *) NULL;
+        return NULL;
     }
     if(beg >= end)
-      return (const char *) NULL;
+      return NULL;
     elem->end = beg;
     return beg + 1;
   }
   else if((unsigned)b > (size_t)(end - beg))
-    return (const char *) NULL; /* Does not fit in source. */
+    return NULL; /* Does not fit in source. */
   else {
     /* Get long length. */
     len = 0;
     do {
       if(len & 0xFF000000L)
-        return (const char *) NULL;  /* Lengths > 32 bits are not supported. */
+        return NULL;  /* Lengths > 32 bits are not supported. */
       len = (len << 8) | (unsigned char) *beg++;
     } while(--b);
   }
   if(len > (size_t)(end - beg))
-    return (const char *) NULL;  /* Element data does not fit in source. */
+    return NULL;  /* Element data does not fit in source. */
   elem->beg = beg;
   elem->end = beg + len;
   return elem->end;
 }
 
+/*
+ * Search the null terminated OID or OID identifier in local table.
+ * Return the table entry pointer or NULL if not found.
+ */
 static const curl_OID * searchOID(const char *oid)
 {
   const curl_OID *op;
-
-  /* Search the null terminated OID or OID identifier in local table.
-     Return the table entry pointer or NULL if not found. */
-
   for(op = OIDtable; op->numoid; op++)
     if(!strcmp(op->numoid, oid) || strcasecompare(op->textoid, oid))
       return op;
 
-  return (const curl_OID *) NULL;
+  return NULL;
 }
 
+/*
+ * Convert an ASN.1 Boolean value into its string representation.  Return the
+ * dynamically allocated string, or NULL if source is not an ASN.1 Boolean
+ * value.
+ */
+
 static const char *bool2str(const char *beg, const char *end)
 {
-  /* Convert an ASN.1 Boolean value into its string representation.
-     Return the dynamically allocated string, or NULL if source is not an
-     ASN.1 Boolean value. */
-
   if(end - beg != 1)
-    return (const char *) NULL;
+    return NULL;
   return strdup(*beg? "TRUE": "FALSE");
 }
 
+/*
+ * Convert an ASN.1 octet string to a printable string.
+ * Return the dynamically allocated string, or NULL if an error occurs.
+ */
 static const char *octet2str(const char *beg, const char *end)
 {
   size_t n = end - beg;
   char *buf = NULL;
 
-  /* Convert an ASN.1 octet string to a printable string.
-     Return the dynamically allocated string, or NULL if an error occurs. */
-
   if(n <= (SIZE_T_MAX - 1) / 3) {
     buf = malloc(3 * n + 1);
     if(buf)
@@ -220,21 +223,22 @@ static const char *bit2str(const char *beg, const char *end)
      Return the dynamically allocated string, or NULL if an error occurs. */
 
   if(++beg > end)
-    return (const char *) NULL;
+    return NULL;
   return octet2str(beg, end);
 }
 
+/*
+ * Convert an ASN.1 integer value into its string representation.
+ * Return the dynamically allocated string, or NULL if source is not an
+ * ASN.1 integer value.
+ */
 static const char *int2str(const char *beg, const char *end)
 {
   unsigned long val = 0;
   size_t n = end - beg;
 
-  /* Convert an ASN.1 integer value into its string representation.
-     Return the dynamically allocated string, or NULL if source is not an
-     ASN.1 integer value. */
-
   if(!n)
-    return (const char *) NULL;
+    return NULL;
 
   if(n > 4)
     return octet2str(beg, end);
@@ -249,6 +253,13 @@ static const char *int2str(const char *beg, const char *end)
   return curl_maprintf("%s%lx", val >= 10? "0x": "", val);
 }
 
+/*
+ * Perform a lazy conversion from an ASN.1 typed string to UTF8. Allocate the
+ * destination buffer dynamically. The allocation size will normally be too
+ * large: this is to avoid buffer overflows.
+ * Terminate the string with a nul byte and return the converted
+ * string length.
+ */
 static ssize_t
 utf8asn1str(char **to, int type, const char *from, const char *end)
 {
@@ -259,13 +270,7 @@ utf8asn1str(char **to, int type, const char *from, const char *end)
   unsigned int wc;
   char *buf;
 
-  /* Perform a lazy conversion from an ASN.1 typed string to UTF8. Allocate the
-     destination buffer dynamically. The allocation size will normally be too
-     large: this is to avoid buffer overflows.
-     Terminate the string with a nul byte and return the converted
-     string length. */
-
-  *to = (char *) NULL;
+  *to = NULL;
   switch(type) {
   case CURL_ASN1_BMP_STRING:
     size = 2;
@@ -341,97 +346,105 @@ utf8asn1str(char **to, int type, const char *from, const char *end)
   return outlength;
 }
 
+/*
+ * Convert an ASN.1 String into its UTF-8 string representation.
+ * Return the dynamically allocated string, or NULL if an error occurs.
+ */
 static const char *string2str(int type, const char *beg, const char *end)
 {
   char *buf;
-
-  /* Convert an ASN.1 String into its UTF-8 string representation.
-     Return the dynamically allocated string, or NULL if an error occurs. */
-
   if(utf8asn1str(&buf, type, beg, end) < 0)
-    return (const char *) NULL;
+    return NULL;
   return buf;
 }
 
-static int encodeUint(char *buf, int n, unsigned int x)
+/*
+ * Decimal ASCII encode unsigned integer `x' into the buflen sized buffer at
+ * buf.  Return the total number of encoded digits, even if larger than
+ * `buflen'.
+ */
+static size_t encodeUint(char *buf, size_t buflen, unsigned int x)
 {
-  int i = 0;
+  size_t i = 0;
   unsigned int y = x / 10;
 
-  /* Decimal ASCII encode unsigned integer `x' in the `n'-byte buffer at `buf'.
-     Return the total number of encoded digits, even if larger than `n'. */
-
   if(y) {
-    i = encodeUint(buf, n, y);
+    i = encodeUint(buf, buflen, y);
     x -= y * 10;
   }
-  if(i < n)
+  if(i < buflen)
     buf[i] = (char) ('0' + x);
   i++;
-  if(i < n)
+  if(i < buflen)
     buf[i] = '\0';      /* Store a terminator if possible. */
   return i;
 }
 
-static int encodeOID(char *buf, int n, const char *beg, const char *end)
+/*
+ * Convert an ASN.1 OID into its dotted string representation.
+ * Store the result in th `n'-byte buffer at `buf'.
+ * Return the converted string length, or 0 on errors.
+ */
+static size_t encodeOID(char *buf, size_t buflen,
+                        const char *beg, const char *end)
 {
-  int i;
+  size_t i;
   unsigned int x;
   unsigned int y;
 
-  /* Convert an ASN.1 OID into its dotted string representation.
-     Store the result in th `n'-byte buffer at `buf'.
-     Return the converted string length, or -1 if an error occurs. */
-
   /* Process the first two numbers. */
   y = *(const unsigned char *) beg++;
   x = y / 40;
   y -= x * 40;
-  i = encodeUint(buf, n, x);
-  if(i < n)
+  i = encodeUint(buf, buflen, x);
+  if(i < buflen)
     buf[i] = '.';
   i++;
-  i += encodeUint(buf + i, n - i, y);
+  if(i >= buflen)
+    i += encodeUint(NULL, 0, y);
+  else
+    i += encodeUint(buf + i, buflen - i, y);
 
   /* Process the trailing numbers. */
   while(beg < end) {
-    if(i < n)
+    if(i < buflen)
       buf[i] = '.';
     i++;
     x = 0;
     do {
       if(x & 0xFF000000)
-        return -1;
+        return 0;
       y = *(const unsigned char *) beg++;
       x = (x << 7) | (y & 0x7F);
     } while(y & 0x80);
-    i += encodeUint(buf + i, n - i, x);
+    if(i >= buflen)
+      i += encodeUint(NULL, 0, x);
+    else
+      i += encodeUint(buf + i, buflen - i, x);
   }
-  if(i < n)
+  if(i < buflen)
     buf[i] = '\0';
   return i;
 }
 
+/*
+ * Convert an ASN.1 OID into its dotted or symbolic string representation.
+ * Return the dynamically allocated string, or NULL if an error occurs.
+ */
+
 static const char *OID2str(const char *beg, const char *end, bool symbolic)
 {
-  char *buf = (char *) NULL;
-  const curl_OID * op;
-  int n;
-  char dummy[1];
-
-  /* Convert an ASN.1 OID into its dotted or symbolic string representation.
-     Return the dynamically allocated string, or NULL if an error occurs. */
-
+  char *buf = NULL;
   if(beg < end) {
-    n = encodeOID(dummy, 0, beg, end);
-    if(n >= 0) {
-      buf = malloc(n + 1);
+    size_t buflen = encodeOID(NULL, 0, beg, end);
+    if(buflen) {
+      buf = malloc(buflen + 1); /* one extra for the zero byte */
       if(buf) {
-        encodeOID(buf, n, beg, end);
-        buf[n] = '\0';
+        encodeOID(buf, buflen, beg, end);
+        buf[buflen] = '\0';
 
         if(symbolic) {
-          op = searchOID(buf);
+          const curl_OID *op = searchOID(buf);
           if(op) {
             free(buf);
             buf = strdup(op->textoid);
@@ -471,7 +484,7 @@ static const char *GTime2str(const char *beg, const char *end)
     sec2 = fracp[-1];
     break;
   default:
-    return (const char *) NULL;
+    return NULL;
   }
 
   /* Scan for timezone, measure fractional seconds. */
@@ -507,15 +520,16 @@ static const char *GTime2str(const char *beg, const char *end)
                        sep, tzl, tzp);
 }
 
+/*
+ *  Convert an ASN.1 UTC time to a printable string.
+ * Return the dynamically allocated string, or NULL if an error occurs.
+ */
 static const char *UTime2str(const char *beg, const char *end)
 {
   const char *tzp;
   size_t tzl;
   const char *sec;
 
-  /* Convert an ASN.1 UTC time to a printable string.
-     Return the dynamically allocated string, or NULL if an error occurs. */
-
   for(tzp = beg; tzp < end && *tzp >= '0' && *tzp <= '9'; tzp++)
     ;
   /* Get the seconds. */
@@ -526,12 +540,12 @@ static const char *UTime2str(const char *beg, const char *end)
   case 2:
     break;
   default:
-    return (const char *) NULL;
+    return NULL;
   }
 
   /* Process timezone. */
   if(tzp >= end)
-    return (const char *) NULL;
+    return NULL;
   if(*tzp == 'Z') {
     tzp = "GMT";
     end = tzp + 3;
@@ -546,13 +560,14 @@ static const char *UTime2str(const char *beg, const char *end)
                        tzl, tzp);
 }
 
+/*
+ * Convert an ASN.1 element to a printable string.
+ * Return the dynamically allocated string, or NULL if an error occurs.
+ */
 static const char *ASN1tostr(curl_asn1Element *elem, int type)
 {
-  /* Convert an ASN.1 element to a printable string.
-     Return the dynamically allocated string, or NULL if an error occurs. */
-
   if(elem->constructed)
-    return (const char *) NULL; /* No conversion of structured elements. */
+    return NULL; /* No conversion of structured elements. */
 
   if(!type)
     type = elem->tag;   /* Type not forced: use element tag as type. */
@@ -586,10 +601,14 @@ static const char *ASN1tostr(curl_asn1Element *elem, int type)
     return string2str(type, elem->beg, elem->end);
   }
 
-  return (const char *) NULL;   /* Unsupported. */
+  return NULL;   /* Unsupported. */
 }
 
-static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn)
+/*
+ * ASCII encode distinguished name at `dn' into the `buflen'-sized buffer at
+ * `buf'.  Return the total string length, even if larger than `buflen'.
+ */
+static ssize_t encodeDN(char *buf, size_t buflen, curl_asn1Element *dn)
 {
   curl_asn1Element rdn;
   curl_asn1Element atv;
@@ -601,9 +620,6 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn)
   const char *p3;
   const char *str;
 
-  /* ASCII encode distinguished name at `dn' into the `n'-byte buffer at `buf'.
-     Return the total string length, even if larger than `n'. */
-
   for(p1 = dn->beg; p1 < dn->end;) {
     p1 = getASN1Element(&rdn, p1, dn->end);
     if(!p1)
@@ -627,7 +643,7 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn)
         for(p3 = str; isupper(*p3); p3++)
           ;
         for(p3 = (*p3 || p3 - str > 2)? "/": ", "; *p3; p3++) {
-          if(l < n)
+          if(l < buflen)
             buf[l] = *p3;
           l++;
         }
@@ -635,14 +651,14 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn)
 
       /* Encode attribute name. */
       for(p3 = str; *p3; p3++) {
-        if(l < n)
+        if(l < buflen)
           buf[l] = *p3;
         l++;
       }
       free((char *) str);
 
       /* Generate equal sign. */
-      if(l < n)
+      if(l < buflen)
         buf[l] = '=';
       l++;
 
@@ -651,7 +667,7 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn)
       if(!str)
         return -1;
       for(p3 = str; *p3; p3++) {
-        if(l < n)
+        if(l < buflen)
           buf[l] = *p3;
         l++;
       }
@@ -662,28 +678,30 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn)
   return l;
 }
 
+/*
+ * Convert an ASN.1 distinguished name into a printable string.
+ * Return the dynamically allocated string, or NULL if an error occurs.
+ */
 static const char *DNtostr(curl_asn1Element *dn)
 {
-  char *buf = (char *) NULL;
-  ssize_t n = encodeDN(buf, 0, dn);
-
-  /* Convert an ASN.1 distinguished name into a printable string.
-     Return the dynamically allocated string, or NULL if an error occurs. */
+  char *buf = NULL;
+  ssize_t buflen = encodeDN(NULL, 0, dn);
 
-  if(n >= 0) {
-    buf = malloc(n + 1);
+  if(buflen >= 0) {
+    buf = malloc(buflen + 1);
     if(buf) {
-      encodeDN(buf, n + 1, dn);
-      buf[n] = '\0';
+      encodeDN(buf, buflen + 1, dn);
+      buf[buflen] = '\0';
     }
   }
-  return (const char *) buf;
+  return buf;
 }
 
 /*
- * X509 parser.
+ * ASN.1 parse an X509 certificate into structure subfields.
+ * Syntax is assumed to have already been checked by the SSL backend.
+ * See RFC 5280.
  */
-
 int Curl_parseX509(curl_X509certificate *cert,
                    const char *beg, const char *end)
 {
@@ -692,10 +710,6 @@ int Curl_parseX509(curl_X509certificate *cert,
   const char *ccp;
   static const char defaultVersion = 0;  /* v1. */
 
-  /* ASN.1 parse an X509 certificate into structure subfields.
-     Syntax is assumed to have already been checked by the SSL backend.
-     See RFC 5280. */
-
   cert->certificate.header = NULL;
   cert->certificate.beg = beg;
   cert->certificate.end = end;
@@ -802,13 +816,14 @@ int Curl_parseX509(curl_X509certificate *cert,
   return 0;
 }
 
+
+/*
+ * Copy at most 64-characters, terminate with a newline and returns the
+ * effective number of stored characters.
+ */
 static size_t copySubstring(char *to, const char *from)
 {
   size_t i;
-
-  /* Copy at most 64-characters, terminate with a newline and returns the
-     effective number of stored characters. */
-
   for(i = 0; i < 64; i++) {
     to[i] = *from;
     if(!*from++)
@@ -1105,15 +1120,15 @@ static const char *checkOID(const char *beg, const char *end,
 
   ccp = getASN1Element(&e, beg, end);
   if(!ccp || e.tag != CURL_ASN1_OBJECT_IDENTIFIER)
-    return (const char *) NULL;
+    return NULL;
 
   p = OID2str(e.beg, e.end, FALSE);
   if(!p)
-    return (const char *) NULL;
+    return NULL;
 
   matched = !strcmp(p, oid);
   free((char *) p);
-  return matched? ccp: (const char *) NULL;
+  return matched? ccp: NULL;
 }
 
 CURLcode Curl_verifyhost(struct connectdata *conn,