mk-ca-bundle.vbs: Fix UTF-8 output

- Change initial message box to mention delay when downloading/parsing.

Since there is no progress meter it was somewhat unexpected that after
choosing a filename nothing appears to happen, when actually the cert
data is in the process of being downloaded and parsed.

- Warn if OpenSSL is not present.

- Use a UTF-8 stream to make the ca-bundle data.

- Save the UTF-8 ca-bundle stream as binary so that no BOM is added.

---

This is a follow-up to d2c6d15 which switched mk-ca-bundle.vbs output to
ANSI due to corrupt UTF-8 output, now fixed.

This change completes making the default certificate bundle output of
mk-ca-bundle.vbs as close as possible to that of mk-ca-bundle.pl, which
should make it easier to review any difference between their output.

Ref: https://github.com/curl/curl/pull/1012

2 files changed, 119 insertions, 55 deletions

diff --git a/lib/mk-ca-bundle.pl b/lib/mk-ca-bundle.pl
index 75278f168..9574f1dbf 100755
--- a/lib/mk-ca-bundle.pl
+++ b/lib/mk-ca-bundle.pl
@@ -30,6 +30,7 @@
 # dependency is the OpenSSL commandline tool for optional text listing.
 # Hacked by Guenter Knauf.
 #
+use Encode;
 use Getopt::Std;
 use MIME::Base64;
 use strict;
@@ -487,7 +488,7 @@ while (<TXT>) {
               . "-----END CERTIFICATE-----\n";
       print CRT "\n$caname\n";
       print CRT @precert if($opt_m);
-      my $maxStringLength = length($caname);
+      my $maxStringLength = length(decode('UTF-8', $caname, Encode::FB_CROAK));
       if ($opt_t) {
         foreach my $key (keys %trust_purposes_by_level) {
            my $string = $key . ": " . join(", ", @{$trust_purposes_by_level{$key}});
diff --git a/lib/mk-ca-bundle.vbs b/lib/mk-ca-bundle.vbs
index 38eb48aba..a9b983e9b 100755
--- a/lib/mk-ca-bundle.vbs
+++ b/lib/mk-ca-bundle.vbs
@@ -29,20 +29,33 @@ Option Explicit
 Const myVersion = "0.4.0"

 

 Const myUrl = "https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt"

-Const myOpenssl = "openssl.exe"

 

-Const myUseOpenSSL = TRUE    ' Flag: set FALSE if openssl is not present

+Const myOpenSSL = "openssl.exe"

+Dim myUseOpenSSL

+myUseOpenSSL = TRUE          ' Flag: TRUE to use OpenSSL. If TRUE and is not

+                             ' found then a warning is shown before continuing.

+

 Const myCdSavF = TRUE        ' Flag: save downloaded data to file certdata.txt

 Const myCaBakF = TRUE        ' Flag: backup existing ca-bundle certificate

 Const myAskLiF = TRUE        ' Flag: display certdata.txt license agreement

 Const myWrapLe = 76          ' Default length of base64 output lines

 

 ' cert info code doesn't work properly with any recent openssl, leave disabled.

+' Also: we want our certificate output by default to be as similar as possible

+' to mk-ca-bundle.pl and setting this TRUE changes the base64 width to

+' OpenSSL's built-in default width, which is not the same as mk-ca-bundle.pl.

 Const myAskTiF = FALSE       ' Flag: ask to include certificate text info

 

+'

 '******************* Nothing to configure below! *******************

+'

+Const adTypeBinary = 1

+Const adTypeText = 2

+Const adSaveCreateNotExist = 1

+Const adSaveCreateOverWrite = 2

 Dim objShell, objNetwork, objFSO, objHttp

-Dim myBase, mySelf, myFh, myTmpFh, myCdData, myCdFile, myCaFile, myTmpName, myBakNum, myOptTxt, i

+Dim myBase, mySelf, myStream, myTmpFh, myCdData, myCdFile

+Dim myCaFile, myTmpName, myBakNum, myOptTxt, i

 Set objNetwork = WScript.CreateObject("WScript.Network")

 Set objShell = WScript.CreateObject("WScript.Shell")

 Set objFSO = WScript.CreateObject("Scripting.FileSystemObject")

@@ -50,16 +63,58 @@ Set objHttp = WScript.CreateObject("WinHttp.WinHttpRequest.5.1")
 If objHttp Is Nothing Then Set objHttp = WScript.CreateObject("WinHttp.WinHttpRequest")

 myBase = Left(WScript.ScriptFullName, InstrRev(WScript.ScriptFullName, "\"))

 mySelf = Left(WScript.ScriptName, InstrRev(WScript.ScriptName, ".") - 1) & " " & myVersion

+

 myCdFile = Mid(myUrl, InstrRev(myUrl, "/") + 1)

 myCaFile = "ca-bundle.crt"

-myTmpName = InputBox("Enter output filename:", mySelf, myCaFile)

-If Not (myTmpName = "") Then

-  myCaFile = myTmpName

+myTmpName = InputBox("It will take a minute to download and parse the " & _

+                     "certificate data." & _

+                     vbLf & vbLf & _

+                     "Please enter the output filename:", mySelf, myCaFile)

+If (myTmpName = "") Then

+  WScript.Quit 1

 End If

+myCaFile = myTmpName

 If (myCdFile = "") Then

   MsgBox("URL does not contain filename!"), vbCritical, mySelf

   WScript.Quit 1

 End If

+

+' Don't use OpenSSL if it's not present.

+If (myUseOpenSSL = TRUE) Then

+  Dim errnum

+

+  On Error Resume Next

+  Call objShell.Run("""" & myOpenSSL & """ version", 0, TRUE)

+  errnum = Err.Number

+  On Error GoTo 0

+

+  If Not (errnum = 0) Then

+    myUseOpenSSL = FALSE

+    MsgBox("OpenSSL was not found so the certificate bundle will not " & _

+           "include the SHA256 hash of the raw certificate data file " & _

+           "that was used to generate the certificates in the bundle. " & _

+           vbLf & vbLf & _

+           "This does not have any effect on the certificate output, " & _

+           "so this script will continue." & _

+           vbLf & vbLf & _

+           "If you want to set a custom location for OpenSSL or disable " & _

+           "this message then edit the variables at the start of the " & _

+           "script."), vbInformation, mySelf

+  End If

+End If

+

+If (myAskTiF = TRUE) And (myUseOpenSSL = TRUE) Then

+  If (6 = objShell.PopUp("Do you want to include text information about " & _

+                         "each certificate?" & vbLf & _

+                         "(Requires OpenSSL.exe in the current directory " & _

+                         "or search path)",, _

+          mySelf, vbQuestion + vbYesNo + vbDefaultButton2)) Then

+    myOptTxt = TRUE

+  Else

+    myOptTxt = FALSE

+  End If

+End If

+

 ' Uncomment the line below to ignore SSL invalid cert errors

 ' objHttp.Option(4) = 256 + 512 + 4096 + 8192

 objHttp.SetTimeouts 0, 5000, 10000, 10000

@@ -75,7 +130,7 @@ If (myCdSavF = TRUE) Then
   Call SaveBinaryData(myCdFile, objHttp.ResponseBody)

 End If

 ' Convert data from ResponseBody instead of using ResponseText because of UTF-8

-myCdData = ConvertBinaryData(objHttp.ResponseBody)

+myCdData = ConvertBinaryToUTF8(objHttp.ResponseBody)

 Set objHttp = Nothing

 ' Backup exitsing ca-bundle certificate file

 If (myCaBakF = TRUE) Then

@@ -91,15 +146,7 @@ If (myCaBakF = TRUE) Then
     myTmpFh.Move myBakFile

   End If

 End If

-If (myAskTiF = TRUE) Then

-  If (6 = objShell.PopUp("Do you want to include text information about each certificate?" & vbLf & _

-          "(requires OpenSSL commandline in current directory or in search path)",, _

-          mySelf, vbQuestion + vbYesNo + vbDefaultButton2)) Then

-    myOptTxt = TRUE

-  Else

-    myOptTxt = FALSE

-  End If

-End If

+

 ' Process the received data

 Dim myLines, myPattern, myInsideCert, myInsideLicense, myLicenseText, myNumCerts, myNumSkipped

 Dim myLabel, myOctets, myData, myPem, myRev, myUntrusted, j

@@ -107,30 +154,33 @@ myNumSkipped = 0
 myNumCerts = 0

 myData = ""

 myLines = Split(myCdData, vbLf, -1)

-Set myFh = objFSO.OpenTextFile(myCaFile, 2, TRUE)

-myFh.Write "##" & vbLf

-myFh.Write "## Bundle of CA Root Certificates" & vbLf

-myFh.Write "##" & vbLf

-myFh.Write "## Certificate data from Mozilla as of: " & _

-  ConvertDateToString(LocalDateToUTC(Now)) & " GMT" & vbLf

-myFh.Write "##" & vbLf

-myFh.Write "## This is a bundle of X.509 certificates of public Certificate Authorities" & vbLf

-myFh.Write "## (CA). These were automatically extracted from Mozilla's root certificates" & vbLf

-myFh.Write "## file (certdata.txt).  This file can be found in the mozilla source tree:" & vbLf

-myFh.Write "## " & myUrl & vbLf

-myFh.Write "##" & vbLf

-myFh.Write "## It contains the certificates in PEM format and therefore" & vbLf

-myFh.Write "## can be directly used with curl / libcurl / php_curl, or with" & vbLf

-myFh.Write "## an Apache+mod_ssl webserver for SSL client authentication." & vbLf

-myFh.Write "## Just configure this file as the SSLCACertificateFile." & vbLf

-myFh.Write "##" & vbLf

-myFh.Write "## Conversion done with mk-ca-bundle.vbs version " & myVersion & "." & vbLf

+Set myStream = CreateObject("ADODB.Stream")

+myStream.Open

+myStream.Type = adTypeText

+myStream.Charset = "utf-8"

+myStream.WriteText "##" & vbLf & _

+  "## Bundle of CA Root Certificates" & vbLf & _

+  "##" & vbLf & _

+  "## Certificate data from Mozilla as of: " & _

+    ConvertDateToString(LocalDateToUTC(Now)) & " GMT" & vbLf & _

+  "##" & vbLf & _

+  "## This is a bundle of X.509 certificates of public Certificate Authorities" & vbLf & _

+  "## (CA). These were automatically extracted from Mozilla's root certificates" & vbLf & _

+  "## file (certdata.txt).  This file can be found in the mozilla source tree:" & vbLf & _

+  "## " & myUrl & vbLf & _

+  "##" & vbLf & _

+  "## It contains the certificates in PEM format and therefore" & vbLf & _

+  "## can be directly used with curl / libcurl / php_curl, or with" & vbLf & _

+  "## an Apache+mod_ssl webserver for SSL client authentication." & vbLf & _

+  "## Just configure this file as the SSLCACertificateFile." & vbLf & _

+  "##" & vbLf & _

+  "## Conversion done with mk-ca-bundle.vbs version " & myVersion & "." & vbLf

 If (myCdSavF = TRUE) And (myUseOpenSSL = TRUE) Then

-  myFh.Write "## SHA256: " & FileSHA256(myCdFile) & vbLf

+  myStream.WriteText "## SHA256: " & FileSHA256(myCdFile) & vbLf

 End If

-myFh.Write "##" & vbLf & vbLf

+myStream.WriteText "##" & vbLf & vbLf

 

-myFh.Write vbLf

+myStream.WriteText vbLf

 For i = 0 To UBound(myLines)

   If InstrRev(myLines(i), "CKA_LABEL ") Then

     myPattern = "^CKA_LABEL\s+[A-Z0-9]+\s+""(.+?)"""

@@ -148,13 +198,13 @@ For i = 0 To UBound(myLines)
       If (myUntrusted = TRUE) Then

         myNumSkipped = myNumSkipped + 1

       Else

-        myFh.Write myLabel & vbLf

-        myFh.Write String(Len(myLabel), "=") & vbLf

+        myStream.WriteText myLabel & vbLf

+        myStream.WriteText String(Len(myLabel), "=") & vbLf

         myPem = "-----BEGIN CERTIFICATE-----" & vbLf & _

                 Base64Encode(myData) & vbLf & _

                 "-----END CERTIFICATE-----" & vbLf

         If (myOptTxt = FALSE) Then

-          myFh.Write myPem & vbLf

+          myStream.WriteText myPem & vbLf

         Else

           Dim myCmd, myRval, myTmpIn, myTmpOut

           myTmpIn = objFSO.GetSpecialFolder(2).Path & "\" & objFSO.GetTempName

@@ -162,8 +212,8 @@ For i = 0 To UBound(myLines)
           Set myTmpFh = objFSO.OpenTextFile(myTmpIn, 2, TRUE)

           myTmpFh.Write myPem

           myTmpFh.Close

-          myCmd = myOpenssl & " x509 -md5 -fingerprint -text -inform PEM" & _

-                  " -in " & myTmpIn & " -out " & myTmpOut

+          myCmd = """" & myOpenSSL & """ x509 -md5 -fingerprint -text " & _

+                  "-inform PEM -in " & myTmpIn & " -out " & myTmpOut

           myRval = objShell.Run (myCmd, 0, TRUE)

           objFSO.DeleteFile myTmpIn, TRUE

           If Not (myRval = 0) Then

@@ -172,7 +222,7 @@ For i = 0 To UBound(myLines)
             WScript.Quit 3

           End If

           Set myTmpFh = objFSO.OpenTextFile(myTmpOut, 1)

-          myFh.Write myTmpFh.ReadAll & vbLf

+          myStream.WriteText myTmpFh.ReadAll & vbLf

           myTmpFh.Close

           objFSO.DeleteFile myTmpOut, TRUE

         End If

@@ -188,7 +238,7 @@ For i = 0 To UBound(myLines)
   If InstrRev(myLines(i), "CVS_ID ") Then

     myPattern = "^CVS_ID\s+""(.+?)"""

     myRev = RegExprFirst(myPattern, myLines(i))

-    myFh.Write "# " & myRev & vbLf & vbLf

+    myStream.WriteText "# " & myRev & vbLf & vbLf

   End If

   If InstrRev(myLines(i), "CKA_VALUE MULTILINE_OCTAL") Then

     myInsideCert = TRUE

@@ -199,7 +249,7 @@ For i = 0 To UBound(myLines)
     myInsideLicense = TRUE

   End If

   If (myInsideLicense = TRUE) Then

-    myFh.Write myLines(i) & vbLf

+    myStream.WriteText myLines(i) & vbLf

     myLicenseText = myLicenseText & Mid(myLines(i), 2) & vbLf

   End If

   If InstrRev(myLines(i), "***** END LICENSE BLOCK *****") Then

@@ -208,34 +258,47 @@ For i = 0 To UBound(myLines)
       If Not (6 = objShell.PopUp(myLicenseText & vbLf & _

               "Do you agree to the license shown above (required to proceed) ?",, _

               mySelf, vbQuestion + vbYesNo + vbDefaultButton1)) Then

-        myFh.Close

+        myStream.Close

         objFSO.DeleteFile myCaFile, TRUE

         WScript.Quit 2

       End If

     End If

   End If

 Next

-myFh.Close

+

+' To stop the UTF-8 BOM from being written the stream has to be copied and

+' then saved as binary.

+Dim myCopy

+Set myCopy = CreateObject("ADODB.Stream")

+myCopy.Type = adTypeBinary

+myCopy.Open

+myStream.Position = 3 ' Skip UTF-8 BOM

+myStream.CopyTo myCopy

+myCopy.SaveToFile myCaFile, adSaveCreateOverWrite

+myCopy.Close

+myStream.Close

+Set myCopy = Nothing

+Set myStream = Nothing

+

+' Done

 objShell.PopUp "Done (" & myNumCerts & " CA certs processed, " & myNumSkipped & _

                " untrusted skipped).", 20, mySelf, vbInformation

 WScript.Quit 0

 

-Function ConvertBinaryData(arrBytes)

+Function ConvertBinaryToUTF8(arrBytes)

   Dim objStream

   Set objStream = CreateObject("ADODB.Stream")

   objStream.Open

-  objStream.Type = 1

+  objStream.Type = adTypeBinary

   objStream.Write arrBytes

   objStream.Position = 0

-  objStream.Type = 2

+  objStream.Type = adTypeText

   objStream.Charset = "utf-8"

-  ConvertBinaryData = objStream.ReadText

+  ConvertBinaryToUTF8 = objStream.ReadText

   Set objStream = Nothing

 End Function

 

 Function SaveBinaryData(filename, data)

-  Const adTypeBinary = 1

-  Const adSaveCreateOverWrite = 2

   Dim objStream

   Set objStream = CreateObject("ADODB.Stream")

   objStream.Type = adTypeBinary

@@ -351,7 +414,7 @@ Function FileSHA256(filename)
   Dim cmd, rval, tmpOut, tmpFh

   if (myUseOpenSSL = TRUE) Then

     tmpOut = objFSO.GetSpecialFolder(2).Path & "\" & objFSO.GetTempName

-    cmd = """" & myOpenssl & """ dgst -r -sha256 -out """ & tmpOut & """ """ & filename & """"

+    cmd = """" & myOpenSSL & """ dgst -r -sha256 -out """ & tmpOut & """ """ & filename & """"

     rval = objShell.Run(cmd, 0, TRUE)

     If Not (rval = 0) Then

       MsgBox("Failed to get sha256 of """ & filename & """ with OpenSSL commandline!"), vbCritical, mySelf